diff --git a/.github/actions/prepare-test-host/action.yml b/.github/actions/prepare-test-host/action.yml new file mode 100644 index 0000000..c1852d5 --- /dev/null +++ b/.github/actions/prepare-test-host/action.yml @@ -0,0 +1,28 @@ +name: Prepare isolated DSP test tools +description: Satisfy the DSP's existing trusted-tool rules on a disposable hosted runner +runs: + using: composite + steps: + - name: Require an isolated GitHub-hosted runner + shell: bash + env: + RUNNER_KIND: ${{ runner.environment }} + run: test "$RUNNER_KIND" = github-hosted + - name: Prepare trusted Node and Chrome paths + shell: bash + run: | + # Hosted tool caches may have writable parents. Do not weaken runtime checks. + sudo install -d -o root -g root -m 755 /dispatch-ci /dispatch-ci/bin + sudo install -o root -g root -m 755 "$(command -v node)" /dispatch-ci/bin/node + echo /dispatch-ci/bin >> "$GITHUB_PATH" + test -x /opt/google/chrome/chrome + sudo chown root:root /opt /opt/google + sudo chown -R root:root /opt/google/chrome + sudo chmod go-w /opt /opt/google /opt/google/chrome + sudo chmod -R go-w /opt/google/chrome + sudo chmod 4755 /opt/google/chrome/chrome-sandbox + stat -Lc '%u %a %n' /opt /opt/google /opt/google/chrome /opt/google/chrome/chrome /dispatch-ci/bin/node + echo 'DISPATCH_CHROME_EXECUTABLE=/opt/google/chrome/chrome' >> "$GITHUB_ENV" + command -v Xvfb + command -v setpriv + /usr/bin/python3 -c "import ctypes; ctypes.CDLL('libX11.so.6'); ctypes.CDLL('libXtst.so.6')" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..e38c1ac --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,2 @@ +Describe the resulting behavior and relevant verification. Identify Core, DSP and +plugin changes, compatibility requirements, and any migration steps. diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..bb8352f --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,44 @@ +name: Platform checks +on: + pull_request: + push: + branches: [main] + workflow_dispatch: +concurrency: + group: platform-checks-${{ github.ref }} + cancel-in-progress: true +permissions: + contents: read +jobs: + platform: + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 22 + cache: npm + cache-dependency-path: | + core/dashboard/package-lock.json + dsp/dashboard/package-lock.json + dsp/tooling/frontend/package-lock.json + - uses: ./.github/actions/prepare-test-host + - run: npm run bootstrap -- "$RUNNER_TEMP/platform" + - run: npm run check -- "$RUNNER_TEMP/platform" + - run: npm run build -- "$RUNNER_TEMP/platform" + - run: npm test -- "$RUNNER_TEMP/platform" + - run: npm run test:integration -- "$RUNNER_TEMP/platform" + - name: Verify updates in browser + run: | + cd "$RUNNER_TEMP/platform/core/dashboard" + npx playwright install --with-deps chromium + npx playwright test --config playwright.updates.config.cjs + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + if: always() + with: + name: verification + path: /tmp/dispatch-updates-browser + retention-days: 7 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..156c3ea --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,62 @@ +name: Publish platform release +on: + workflow_dispatch: + inputs: + version: + description: Owner-selected X.Y.Z platform version + required: true + type: string + expected_main: + description: Tested main commit (40 characters) + required: true + type: string + changes: + description: Reviewed JSON changelog with core, dsp and plugins strings + required: true + type: string +concurrency: + group: platform-publication + cancel-in-progress: false +permissions: + contents: read +jobs: + publish: + if: github.ref == 'refs/heads/main' && inputs.expected_main == github.sha + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: write + actions: read + id-token: write + attestations: write + env: + RELEASE_VERSION: ${{ inputs.version }} + RELEASE_COMMIT: ${{ inputs.expected_main }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ inputs.expected_main }} + fetch-depth: 0 + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 22 + - name: Verify main and unused version + env: + GH_TOKEN: ${{ github.token }} + run: node tooling/release.cjs guard + - run: npm run bootstrap -- "$RUNNER_TEMP/platform" + - run: npm run check -- "$RUNNER_TEMP/platform" + - run: npm run build -- "$RUNNER_TEMP/platform" + - name: Package platform release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_CHANGES: ${{ inputs.changes }} + run: node tooling/release.cjs package "$RUNNER_TEMP/platform" "$RUNNER_TEMP/publication" + - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 + with: + subject-path: ${{ runner.temp }}/publication/* + - name: Publish verified release + env: + GH_TOKEN: ${{ github.token }} + run: node tooling/release.cjs publish "$RUNNER_TEMP/publication" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c838f3d --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +node_modules/ +__pycache__/ +*.pyc +.env +.env.* +playwright-report/ +test-results/ +core/dashboard/public/assets/frontend.js +core/dashboard/public/assets/styles.css +dsp/dashboard/public/assets/frontend.js +dsp/dashboard/public/assets/styles.css diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..c75f223 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,21 @@ +# Dispatch Platform + +Public repository: dillonlille/dispatch-platform. Keep credentials, DSP data and +host configuration outside source. Develop in isolated feature branches; PRs may +be created and merged autonomously after review and passing checks. + +- core/: Platform Owner dashboard, shared API/services, host isolation, updater + and SDK source. Existing internal service modules remain under core/core/. +- dsp/: DSP runtime and DSP-owned dashboard entry point/pages. +- plugins/: optional plugin source, including Paycom; released with DSP updates. +- shared/dashboard/: shared UI source compiled separately into each dashboard. +- tooling/: monorepo assembly, checks, publication and migration helpers. + +Read DEVELOPMENT.md and RELEASES.md. Build/test in an external synthetic workspace. +SDK/support packages are real versioned copies. Dashboard assets are central per +release; each DSP selects its approved version and owns its installed plugins and +private state. Core builds must not contain DSP-owned dashboard pages. + +One user-requested vX.Y.Z release contains Core/DSP packages and three changelog +sections. Ask for a version unless supplied. Publishing never installs. Update Core +and Update Dev → Rollout Update are independent owner-controlled installation paths. diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..d65ec5f --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,35 @@ +# Development + +Use Node 22/24, npm, Python 3 and Linux. Build in a new directory outside source: + +```sh +npm run bootstrap -- /absolute/build/workspace +npm run check -- /absolute/build/workspace +npm run build -- /absolute/build/workspace +npm test -- /absolute/build/workspace +npm run test:integration -- /absolute/build/workspace +``` + +Bootstrap assembles portable Core and DSP source trees, copies top-level plugins +into the DSP build workspace, merges shared UI source with each product's owned +entry points, and installs locked compiler dependencies plus versioned SDK copies. +These assembled trees are disposable, not additional source repositories. After +editing source, bootstrap a fresh workspace before final verification. + +DSP browser collector tests need trusted Chrome, Xvfb, setpriv and X11 libraries; +the hosted checks prepare these on an isolated runner. Tests use synthetic data. + +Use feature branches and PRs. The owner authorizes autonomous PR creation and +merging after review and successful checks. Publication and installation remain +separate. See RELEASES.md. Never develop in live/ or installed DSP directories. + +After building both dashboards, run plugin previews from the assembled Core tree: + +```sh +/absolute/build/workspace/core/bin/dispatch create plugin sample-notes +/absolute/build/workspace/core/bin/dispatch plugin dev /absolute/plugin/source +``` + +The preview serves each synthetic DSP's DSP dashboard and uses independent test +accounts/state. An external DSP dashboard can be supplied through the explicit +DISPATCH_DSP_DASHBOARD development setting. Production always uses release receipts. diff --git a/README.md b/README.md index 048edcc..a0818b7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,18 @@ # Dispatch Platform -Core services, DSP dashboard/runtime, and optional plugins in one repository. -Core installation and Dev-first DSP rollout remain independent. +One repository and one platform release, with independent Core installation and +Dev-first DSP rollout. + +- [core/](core/): Platform Owner dashboard, API, SDK and shared services. +- [dsp/](dsp/): DSP dashboard and runtime. +- [plugins/](plugins/): Paycom and future optional plugins, released with DSP updates. +- [shared/](shared/): reusable dashboard source compiled separately per product. +- [tooling/](tooling/): development, verification, publication and migration. + +The Updates page groups Core, DSP and Plugins changes. **Update Core** installs +Core. **Update Dev → Rollout Update** installs the complete DSP experience, +including installed plugins and the approved catalog. Dashboard assets are stored +centrally per release; DSP credentials, settings and databases remain independent. + +See [DEVELOPMENT.md](DEVELOPMENT.md), [RELEASES.md](RELEASES.md) and [AGENTS.md](AGENTS.md). +No deployment credentials or DSP private data belong in this public repository. diff --git a/RELEASES.md b/RELEASES.md new file mode 100644 index 0000000..6092b3c --- /dev/null +++ b/RELEASES.md @@ -0,0 +1,35 @@ +# Platform releases + +One public repository and one platform version: vX.Y.Z. Prepare only on request. +Report the last published release, installed versions and changes under Core, +DSP and Plugins, then ask for a clickable patch/minor/major version unless supplied. + +The manual release.yml workflow accepts version, expected_main and changes (a JSON +object with core/dsp/plugins strings). It requires successful main checks, builds +immutable artifacts, verifies provenance and uploaded bytes, and publishes one +release. No deployment credentials are used by GitHub. Existing tags are immutable. + +platform-release.json binds the Core and DSP component digests. Unchanged components +reference the last release containing those exact packages, so plugin-only changes +never create a Core update. Shared source changes rebuild affected products. Core +and DSP manifests/archives have distinct asset names. SDK and plugin package +versions must change when their installed bytes change. + +The single owner Updates page has Core, DSP and Plugins sections. Update Core +installs only the selected release's Core package. Update Dev installs the DSP +package only on permanent Dev: dashboard, runtime, installed plugins and catalog. +Plugin-only changes are DSP updates. Other DSPs keep their current dashboard and +catalog. The owner tests Dev and clicks Rollout Update for sequential activation. +New candidates require another Dev test; active rollouts keep their pinned digest. +Failures pause rollout. Credentials/settings/databases remain independent. + +Core APIs must support installed DSP protocol versions; incompatible activation +is rejected. A Core update never rewrites DSP dependency copies. DSP dashboards +are cached centrally once per release and selected through authenticated sessions. +New DSPs use the completed production rollout and only builtin Cortex by default. + +Migration retains legacy manifests and freezes the currently deployed dashboard +for DSP releases that predate dashboard packaging. Do not remove old repositories +or release assets while installed/rollback references still use them. The initial +bootstrap requires verified release assets, an idle updater, dashboard snapshots, +and a reversible Core activation. Publishing alone does not perform this cutover. diff --git a/core/.containerignore b/core/.containerignore new file mode 100644 index 0000000..f6f7d3c --- /dev/null +++ b/core/.containerignore @@ -0,0 +1,22 @@ +** +!.containerignore +!runtime/ +!runtime/** +!shared/ +!shared/** +**/tests +**/test +**/examples +**/integration +**/scripts +**/*.md +**/node_modules +**/state +**/data +**/secrets +**/references +**/Containerfile +# Provider health commands are runtime entrypoints; other scripts are development tools. +!compatibility/providers/*/scripts +compatibility/providers/*/scripts/* +!compatibility/providers/*/scripts/health diff --git a/core/.gitignore b/core/.gitignore new file mode 100644 index 0000000..7b911c9 --- /dev/null +++ b/core/.gitignore @@ -0,0 +1,17 @@ +node_modules/ +__pycache__/ +*.pyc +.env +.env.* +!.env.example +*.sqlite +*.sqlite3 +*.sqlite3-* +*.db +*.db-* +*.log +*.har +playwright-report/ +test-results/ +coverage/ +.DS_Store diff --git a/core/AGENTS.md b/core/AGENTS.md new file mode 100644 index 0000000..f62512f --- /dev/null +++ b/core/AGENTS.md @@ -0,0 +1,4 @@ +# Core component + +Read ../AGENTS.md, ../DEVELOPMENT.md and ../RELEASES.md. This directory is part +of the dispatch-platform monorepo. Root tooling assembles portable product builds. diff --git a/core/DEVELOPMENT.md b/core/DEVELOPMENT.md new file mode 100644 index 0000000..78b418d --- /dev/null +++ b/core/DEVELOPMENT.md @@ -0,0 +1,3 @@ +# Development + +Use the monorepo commands in [../DEVELOPMENT.md](../DEVELOPMENT.md). diff --git a/core/README.md b/core/README.md new file mode 100644 index 0000000..e274dcb --- /dev/null +++ b/core/README.md @@ -0,0 +1,20 @@ +# Dispatch Core + +Shared services and dashboard for Dispatch. This project owns the complete web +application, platform-owner features, API, SDK source, authentication/browser +coordination, host management and update coordination. + +DSP runtime and plugin source belong to the separate `dispatch-dsp` project. +Each product builds independently, with explicit versioned dependency packages. +Every DSP retains its own installed code and private state. + +- [Directory map](AGENTS.md) +- [Local development](DEVELOPMENT.md) +- [PR and release workflow](RELEASES.md) +- [Independent update operations](core/updates/README.md) +- [Architecture and storage](docs/architecture.md) +- [Security policy](SECURITY.md) + +Core and DSP use separate public repositories and release versions. Publishing a +GitHub release makes it available; installing it is a separate owner action. See +the update operations guide for initial deployment and recovery prerequisites. diff --git a/core/RELEASES.md b/core/RELEASES.md new file mode 100644 index 0000000..3bde600 --- /dev/null +++ b/core/RELEASES.md @@ -0,0 +1,3 @@ +# Releases + +Use the platform workflow in [../RELEASES.md](../RELEASES.md). diff --git a/core/SECURITY.md b/core/SECURITY.md new file mode 100644 index 0000000..f5be48e --- /dev/null +++ b/core/SECURITY.md @@ -0,0 +1,40 @@ +--- +title: Repository security policy +status: current +last_verified: 2026-09-02 +--- + +# Security policy + +## Sensitive data boundary + +This repository must never contain: + +- passwords, PINs, recovery answers, API tokens, cookies, or authorization headers; +- Auth Broker master keys or credential databases; +- browser profiles, login databases, session storage, or authenticated caches; +- Paycom, CDF, Collection Manager, Access Control, Provisioner, or other operational databases and backups; +- employee, customer, provider, or account data copied from a live system; +- sockets, service records, logs, staging candidates, or runtime state. + +Use synthetic fixtures for tests and documentation. Local operational state belongs in an external owner-private directory configured through `DISPATCH_LOCAL_ROOT` or explicit absolute runtime roots. + +Managed installation tests must use temporary synthetic organization/runtime identities and credential-free provider/publication evidence. Do not copy a live installation manifest, outbox/job record, Auth Broker profile, collection batch, publication target, or readiness snapshot into Git. Installation lifecycle repair uses the durable reconciler or closed activation retry—never direct SQL or filesystem edits. + +## Reporting a vulnerability + +Do not include secret values, personal data, browser artifacts, or live provider responses in a public issue. Contact the repository owner privately or use GitHub private vulnerability reporting when it is enabled. + +Include only the minimum sanitized reproduction details needed to understand the problem. + +## Accidental exposure + +If a credential, key, cookie, browser profile, or private database is pushed, treat it as compromised even if the commit is later deleted: + +1. Revoke or rotate the affected credential or key. +2. Invalidate related browser sessions. +3. Preserve a private incident record without copying secret values into Git. +4. Remove the data from Git history and verify the rewritten remote. +5. Review forks, clones, pull-request diffs, workflow artifacts, and caches for continued exposure. + +See [runtime/auth-broker/SECURITY.md](runtime/auth-broker/SECURITY.md) for the component trust model and storage requirements. diff --git a/core/bin/dispatch b/core/bin/dispatch new file mode 100755 index 0000000..44cb117 --- /dev/null +++ b/core/bin/dispatch @@ -0,0 +1,12 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +process.umask(0o077); +const controller = new AbortController(); +process.once('SIGINT', () => controller.abort()); +if (['create', 'plugin'].includes(process.argv[2])) { + require('../tooling/plugin-cli').main(process.argv.slice(2)).then(result => console.log(JSON.stringify(result))).catch(error => { + console.error(error.message); process.exitCode = 1; + }); +} else { + process.stdout.write('Dispatch Core developer tools: create plugin ID | plugin generate/check/dev PATH\nDSP runtime commands belong to dispatch-dsp/bin/dispatch.\n'); +} diff --git a/core/bin/dispatch-access-admin b/core/bin/dispatch-access-admin new file mode 100755 index 0000000..81a8032 --- /dev/null +++ b/core/bin/dispatch-access-admin @@ -0,0 +1,95 @@ +#!/usr/bin/node --no-warnings +'use strict'; + +process.umask(0o077); +const fs = require('node:fs'); +const path = require('node:path'); +const { resolveAccessPaths } = require('../shared/paths/access-paths'); +const { AccessStore, AccessControlService } = require('../core/accounts/src'); + +function fail(message) { process.stderr.write(`dispatch-access-admin: ${message}\n`); process.exit(2); } +function option(argv, name, fallback = null) { + const index = argv.indexOf(name); + if (index < 0) return fallback; + if (index + 1 >= argv.length || argv[index + 1].startsWith('--')) fail(`${name} requires a value`); + return argv[index + 1]; +} +function safeOutputPath(file, root) { + const selected = path.resolve(file); + const allowed = path.resolve(root); + const relative = path.relative(allowed, selected); + if (!relative || relative.startsWith('..') || path.isAbsolute(relative) || path.dirname(selected) !== allowed) fail('output must be a direct child of the access-control secrets directory'); + fs.mkdirSync(allowed, { recursive: true, mode: 0o700 }); + const info = fs.lstatSync(allowed); + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== process.geteuid() || (info.mode & 0o777) !== 0o700) fail('unsafe secrets directory'); + return selected; +} +function safeOutput(file, root) { + const selected = safeOutputPath(file, root); + if (fs.existsSync(selected)) fail('output file already exists'); + return selected; +} + +const argv = process.argv.slice(2); +const action = argv[0]; +if (['rollout-status', 'rollout-start', 'rollout-pause', 'rollout-resume'].includes(action)) { + if (process.env.DISPATCH_PLATFORM_CONFIG !== undefined) fail('legacy rollouts are unavailable in directory mode'); + require('../core/accounts/src/rollout-admin-cli').main(argv).then(code => { process.exitCode = code; }); + return; +} +if (['owner-create', 'owner-recover', 'owner-list'].includes(action)) { + require('../core/accounts/src/owner-admin-cli').main(argv).then(code => { process.exitCode = code; }); + return; +} +if (!['status', 'bootstrap', 'revoke-bootstrap'].includes(action)) fail('usage: dispatch-access-admin rollout-status|rollout-start|rollout-pause|rollout-resume --help | owner-create | owner-recover | owner-list | status | bootstrap --email EMAIL [--organization ORGANIZATION_ID] [--output-file FILE] | revoke-bootstrap [--output-file FILE]'); +const paths = resolveAccessPaths(); +if (process.env.DISPATCH_PLATFORM_CONFIG !== undefined) { + require('../host/controller/operations').privateDirectory(path.dirname(paths.accessControl.databaseRoot)); +} +const store = new AccessStore(paths.accessControl); +const service = new AccessControlService(store); +try { + if (action === 'status') { + process.stdout.write(`${JSON.stringify({ ok: true, status: 'found', data: service.bootstrapStatus() })}\n`); + } else if (action === 'bootstrap') { + const email = option(argv, '--email'); + if (!email) fail('--email is required'); + const organizationId = option(argv, '--organization', null); + const secretsDirectory = path.join(paths.secretsRoot, 'access-control'); + const output = safeOutput(option(argv, '--output-file', path.join(secretsDirectory, 'platform-bootstrap.json')), secretsDirectory); + const result = service.createPlatformBootstrap({ email, organizationId }); + const payload = Buffer.from(`${JSON.stringify({ + invitationPath: `/#/invitation/${result.token}`, + invitedEmail: result.invitation.email, + organizationId: result.invitation.organizationId, + expiresAt: result.invitation.expiresAt, + }, null, 2)}\n`); + try { fs.writeFileSync(output, payload, { flag: 'wx', mode: 0o600 }); } + catch (error) { + try { store.revokeInvitation(result.invitation.id); } catch {} + throw error; + } + process.stdout.write(`${JSON.stringify({ ok: true, status: 'invitation_created', output, expiresAt: result.invitation.expiresAt })}\n`); + } else { + const secretsDirectory = path.join(paths.secretsRoot, 'access-control'); + const output = safeOutputPath(option(argv, '--output-file', path.join(secretsDirectory, 'platform-bootstrap.json')), secretsDirectory); + const invitation = service.revokePlatformBootstrap(); + let outputRemoved = false; + try { + fs.unlinkSync(output); + outputRemoved = true; + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + } + process.stdout.write(`${JSON.stringify({ + ok: true, + status: invitation ? 'invitation_revoked' : 'no_pending_invitation', + outputRemoved, + })}\n`); + } +} catch (error) { + process.stderr.write(`${JSON.stringify({ ok: false, status: error?.code || error?.message || 'access_admin_failed' })}\n`); + process.exitCode = 1; +} finally { + store.close(); +} diff --git a/core/bin/dispatch-api b/core/bin/dispatch-api new file mode 100755 index 0000000..fd3f6ca --- /dev/null +++ b/core/bin/dispatch-api @@ -0,0 +1,6 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +process.umask(0o077); +require('../core/api/main').main().then(code => { process.exitCode = code; }).catch(() => { + process.stderr.write('{"ok":false,"status":"api_startup_failed"}\n'); process.exitCode = 1; +}); diff --git a/core/bin/dispatch-dashboard b/core/bin/dispatch-dashboard new file mode 100755 index 0000000..ef77ee2 --- /dev/null +++ b/core/bin/dispatch-dashboard @@ -0,0 +1,6 @@ +#!/usr/bin/node --no-warnings +'use strict'; + +process.umask(0o077); +const { main } = require('../dashboard/server/main'); +main().then(code => { process.exitCode = code; }).catch(() => { process.exitCode = 1; }); diff --git a/core/bin/dispatch-local-backup b/core/bin/dispatch-local-backup new file mode 100755 index 0000000..d2b1ff6 --- /dev/null +++ b/core/bin/dispatch-local-backup @@ -0,0 +1,5 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +process.umask(0o077); +require('../host/storage/manual-backup-cli').main(process.argv.slice(2)) + .then(code => { process.exitCode = code; }); diff --git a/core/bin/dispatch-local-controller b/core/bin/dispatch-local-controller new file mode 100755 index 0000000..6853c89 --- /dev/null +++ b/core/bin/dispatch-local-controller @@ -0,0 +1,18 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +process.umask(0o077); +const { loadPlatformPaths } = require('../shared/paths/platform-paths'); +const { loadInstallation } = require('../host/services/installation'); +const { runController } = require('../host/controller/controller'); + +async function main() { + if (process.argv.length !== 2) throw new Error('directory_request_invalid'); + const paths = loadPlatformPaths(); + await runController({ paths, installation: loadInstallation(paths) }); +} +main().catch(error => { + const status = /^directory_[a-z_]+$/.test(error.code || error.message || '') + ? (error.code || error.message) : 'directory_controller_unavailable'; + process.stderr.write(JSON.stringify({ ok: false, status }) + '\n'); + process.exitCode = 1; +}); diff --git a/core/bin/dispatch-local-dsp b/core/bin/dispatch-local-dsp new file mode 100755 index 0000000..b358ede --- /dev/null +++ b/core/bin/dispatch-local-dsp @@ -0,0 +1,17 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +process.umask(0o077); +const { loadPlatformPaths } = require('../shared/paths/platform-paths'); +const { createDsp, inspectDsp } = require('../host/storage/storage'); +try { + const [action, ...args] = process.argv.slice(2); + const paths = loadPlatformPaths(); + let result; + if (action === 'create' && args.length === 0) result = createDsp(paths); + else if (action === 'inspect' && args.length === 1) result = inspectDsp(paths, args[0]); + else throw new Error('usage: dispatch-local-dsp create | inspect '); + process.stdout.write(JSON.stringify({ ok: true, ...result }) + '\n'); +} catch (error) { + const status = error.message?.startsWith('usage:') ? error.message : 'directory_dsp_unavailable'; + process.stderr.write(JSON.stringify({ ok: false, status }) + '\n'); process.exitCode = 1; +} diff --git a/core/bin/dispatch-local-install b/core/bin/dispatch-local-install new file mode 100755 index 0000000..bbfbc4d --- /dev/null +++ b/core/bin/dispatch-local-install @@ -0,0 +1,31 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +process.umask(0o077); +const path = require('node:path'); +const { loadPlatformPaths } = require('../shared/paths/platform-paths'); +const { privateJson } = require('../core/installations/src/release-delivery-files'); +const { installRuntime, activateRuntime } = require('../host/services/installation'); +const { preflight, prepareStartup } = require('../host/services/startup'); + +async function main() { + const args = process.argv.slice(2), paths = loadPlatformPaths(); + let result; + if (args.length === 1 && args[0] === 'check') result = await preflight(paths); + else if (args.length === 1 && ['install', 'prepare-upgrade'].includes(args[0])) { + const input = privateJson(path.join(paths.local, 'config/runtime-installation-source.json'), process.geteuid()); + if (!['browserSource,nodeRoot', 'browserSource,nodeSource,tiniSource'].includes(Object.keys(input).sort().join(','))) { + throw new Error('directory_installation_invalid'); + } + result = await installRuntime(paths, input, { prepare: args[0] === 'prepare-upgrade' }); + } else if (args.length === 1 && args[0] === 'activate-upgrade') { + result = await activateRuntime(paths); + } else if (args.length >= 1 && args.length <= 2 && args[0] === 'prepare-startup') { + result = await prepareStartup(paths, args.length === 2 ? { port: Number(args[1]) } : {}); + } else throw new Error('directory_request_invalid'); + process.stdout.write(JSON.stringify(result) + '\n'); +} +main().catch(error => { + const selected = error.code || error.message; + process.stderr.write(JSON.stringify({ ok: false, status: /^directory_[a-z_]+$/.test(selected || '') + ? selected : 'directory_installation_failed' }) + '\n'); process.exitCode = 1; +}); diff --git a/core/bin/dispatch-plugin-collector b/core/bin/dispatch-plugin-collector new file mode 100755 index 0000000..e911b1a --- /dev/null +++ b/core/bin/dispatch-plugin-collector @@ -0,0 +1,15 @@ +#!/usr/bin/env node +'use strict'; +// Stable framework command recorded in collection definitions. Directory +// runtimes dispatch directly through dispatch-sdk; this shim supports tooling. +let bytes = 0, chunks = []; +process.stdin.on('data', chunk => { bytes += chunk.length; if (bytes > 65536) process.exit(1); chunks.push(chunk); }); +process.stdin.on('end', async () => { + try { + const request = JSON.parse(Buffer.concat(chunks)); chunks = []; + const plugin = require('../shared/plugin-sdk/catalog').catalog().find(item => item.collectors.includes(request.source?.collector)); + if (!plugin) throw new Error(); + const value = await require('../sdk/node/framework').createFrameworkClient().request('plugin.collect', { pluginId: plugin.id, request }); + process.stdout.write(JSON.stringify(value) + '\n'); + } catch { process.exitCode = 1; } +}); diff --git a/core/bin/dispatch-updates b/core/bin/dispatch-updates new file mode 100755 index 0000000..d0ced09 --- /dev/null +++ b/core/bin/dispatch-updates @@ -0,0 +1,41 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +process.umask(0o077); +async function main() { + const [action, ...args] = process.argv.slice(2); + const { loadWorkerPaths, configure, adopt } = require('../host/releases/setup'); + const paths = loadWorkerPaths(); + if (action === 'worker' && !args.length) { + const entry = require('../host/releases/setup').workerEntrypoint(paths, __filename); + if (entry) { + const child = require('node:child_process').spawn(process.execPath, ['--no-warnings', entry, 'worker'], { stdio: 'inherit', env: process.env }); + process.once('SIGTERM', () => child.kill('SIGTERM')); process.once('SIGINT', () => child.kill('SIGINT')); + await new Promise((resolve, reject) => { child.once('error', reject); child.once('exit', code => code === 0 ? resolve() : reject(new Error('release_worker_failed'))); }); + return { status: 'stopped' }; + } + const app = await require('../core/updates/worker').startWorker(paths); + const startedDigest = app.worker.releases.state().active.core; + const monitor = setInterval(() => { + const state = app.worker.releases.state(); + if (!state.operation && state.active.core !== startedDigest) close(); + }, 1000); + const close = () => { clearInterval(monitor); return app.close().catch(() => { process.exitCode = 1; }); }; + process.once('SIGINT', close); process.once('SIGTERM', close); + return { status: 'ready' }; + } + if (action === 'configure' && args.length === 2) return configure(paths, args[0], Number(args[1])); + if (action === 'adopt' && args.length === 1) return adopt(paths, args[0]); + if (action === 'recover' && args.length === 1) { + const { authorizeOwner } = require('../core/updates/worker'); authorizeOwner(paths, args[0]); + const { UpdateCommands } = require('../core/updates/commands'); + const { rootFor } = require('../core/updates/configuration'); + const job = new UpdateCommands(rootFor(paths)).request(args[0], { action: 'recover', product: 'core', + digest: null, idempotencyKey: `recovery:${require('node:crypto').randomUUID()}` }); + return { status: 'queued', id: job.id }; + } + throw new Error('release_command_invalid'); +} +main().then(value => process.stdout.write(JSON.stringify({ ok: true, ...value }) + '\n')).catch(error => { + process.stderr.write(JSON.stringify({ ok: false, status: /^release_[a-z_]+$/.test(error.message) ? error.message : 'release_operation_failed' }) + '\n'); + process.exitCode = 1; +}); diff --git a/core/compatibility/provisioner/bin/dispatch-installation-lifecycle b/core/compatibility/provisioner/bin/dispatch-installation-lifecycle new file mode 100755 index 0000000..bc71ea1 --- /dev/null +++ b/core/compatibility/provisioner/bin/dispatch-installation-lifecycle @@ -0,0 +1,152 @@ +#!/usr/bin/env node +'use strict'; + +const crypto = require('node:crypto'); +const path = require('node:path'); +const { installationFailure } = require('../../../shared/contracts/src'); +const { PROJECT_ROOT } = require('../../../shared/paths/runtime-paths'); +const { AccessStore } = require('../../../core/accounts/src/store'); +const { createAccessInstallationLifecycleAuthority } = require('../../../core/accounts/src/installation-lifecycle'); +const { createSystemdUserSupervisor } = require('../../../core/installations/src/systemd-user'); +const { createManagedInstallationLifecycle } = require('../src/lifecycle'); +const { loadPrivateReleaseCatalog, loadPrivateOciReleaseCatalog } = require('../../../core/installations/src/release-catalog'); + +const { createProtectedOciHostClient } = require('../../../core/installations/src/oci-protected-client'); +const { createOciContainerAdapter } = require('../../../core/installations/src/oci-adapter'); +const { createOciRuntimeAgentCredentialPort } = require('../../../core/installations/src/oci-runtime-agent-credential'); +const { createOciInstallationLifecycle } = require('../../../core/installations/src/oci-lifecycle'); +const { createOciRuntimeLifecyclePort } = require('../../../core/installations/src/oci-runtime-lifecycle-port'); +const { createRuntimeAgentDispatchClient, runtimeAgentControlInvoke } = require('../../../core/agents/src'); + +function fail(code = 'invalid_input') { throw Object.assign(new Error(code), { code }); } +function absoluteEnvironment(name) { + const value = process.env[name]; + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) fail('runtime_boundary_violation'); + return value; +} +function parse(argv) { + const [command, organizationId, argument, extra] = argv; + if (!['status', 'backups', 'backup', 'restore', 'upgrade', 'suspend', 'resume', 'decommission', 'destroy'] + .includes(command) || !/^[a-z][a-z0-9_-]{2,95}$/.test(organizationId || '')) fail(); + if (['status', 'backups', 'backup', 'suspend', 'resume', 'decommission'].includes(command) + && (argument !== undefined || extra !== undefined)) fail(); + if (command === 'restore' && (!/^[a-z][a-z0-9_-]{2,95}$/.test(argument || '') || extra !== undefined)) fail(); + if (command === 'upgrade' && (!/^[a-z][a-z0-9_.-]{2,95}$/.test(argument || '') || extra !== undefined)) fail(); + if (command === 'destroy' && (argument !== '--approve-permanent-destruction' || extra !== undefined)) fail(); + return { command, organizationId, argument }; +} + +function idempotencyKey(input, revision) { + const digest = crypto.createHash('sha256') + .update(`${input.organizationId}\0${input.command}\0${input.argument || ''}\0${revision}`) + .digest('hex').slice(0, 32); + return `lifecycle:${input.command}:${digest}`; +} +function activeRequest(row) { + let receipts; + let request; + try { + receipts = JSON.parse(row.stage_receipts_json); + request = JSON.parse(receipts.__request); + } catch { fail('installation_operation_failed'); } + if (!request || typeof request !== 'object' || Array.isArray(request) + || request.operation !== row.operation) fail('installation_operation_failed'); + return request; +} + +async function main() { + const input = parse(process.argv.slice(2)); + const accessRoot = absoluteEnvironment('DISPATCH_ACCESS_CONTROL_DATABASE_ROOT'); + const installationsRoot = absoluteEnvironment('DISPATCH_INSTALLATIONS_ROOT'); + const unitRoot = absoluteEnvironment('DISPATCH_SYSTEMD_UNIT_ROOT'); + const runtimeAgentHubSocket = absoluteEnvironment('DISPATCH_RUNTIME_AGENT_HUB_SOCKET'); + const store = new AccessStore({ + databaseRoot: accessRoot, + database: path.join(accessRoot, 'access-control.sqlite3'), + }); + try { + const control = store.installationControl(input.organizationId); + if (!control) fail('installation_not_found'); + const oci = store.installationBackend(input.organizationId) === 'oci_container_v1'; + const catalog = oci ? loadPrivateOciReleaseCatalog(absoluteEnvironment('DISPATCH_OCI_RELEASE_CATALOG_FILE')) + : loadPrivateReleaseCatalog(process.env.DISPATCH_RELEASE_CATALOG_FILE); + + const authority = createAccessInstallationLifecycleAuthority({ + store, + organizationId: input.organizationId, + authorityScope: 'platform_lifecycle', + releaseCatalog: Object.keys(catalog), + destructionEnabled: input.command === 'destroy', + }); + if (input.command === 'status') { + const active = store.activeLifecycleJob(input.organizationId); + process.stdout.write(`${JSON.stringify({ ok: true, status: control.status, + revision: control.revision, manifestRevision: control.manifestRevision, + releaseId: control.releaseId, + lifecycle: active ? authority.inspect(active.id) : null })}\n`); + return; + } + if (input.command === 'backups') { + process.stdout.write(`${JSON.stringify({ ok: true, status: 'found', items: authority.backups() })}\n`); + return; + } + let lifecycle; + if (oci) { + const socket = absoluteEnvironment('DISPATCH_RUNTIME_AGENT_CONTROL_SOCKET'); + const credentials = createOciRuntimeAgentCredentialPort({ + credentialRoot: absoluteEnvironment('DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT'), + }); + const host = createProtectedOciHostClient({ dispatchRequest: (request, dispatch) => authority.dispatchHostRequest(request, dispatch) }); + const adapter = createOciContainerAdapter({ hostRegistry: host.hostRegistry, hostExecutor: host.hostExecutor, + credentialPort: credentials, releaseResolver: id => { if (!catalog[id]) fail('release_not_found'); return catalog[id]; } }); + lifecycle = createOciInstallationLifecycle({ authority, adapter, hostExecutor: host.hostExecutor, + backupManagerFactory: (plan, claim) => host.createBackupManager(plan, claim), + runtimeFactory: plan => createOciRuntimeLifecyclePort({ client: createRuntimeAgentDispatchClient({ + runtimeKey: plan.runtimeKey, hub: { invoke: (key, action, value) => runtimeAgentControlInvoke(socket, key, action, value) }, + }) }), + }); + } else { + lifecycle = createManagedInstallationLifecycle({ authority, installationsRoot, unitRoot, + supervisor: createSystemdUserSupervisor(), projectRoot: PROJECT_ROOT, releaseCatalog: catalog, runtimeAgentHubSocket }); + } + const active = store.activeLifecycleJob(input.organizationId); + if (active && active.status === 'running' && active.attempt >= active.max_attempts + && active.lease_expires_at <= Date.now()) { + authority.retryExhausted(active.id); + } + let jobId; + if (active) { + if (active.operation !== input.command) fail('installation_operation_in_progress'); + const request = activeRequest(active); + if (input.command === 'restore' && request.backupId !== input.argument + || input.command === 'upgrade' && request.releaseId !== input.argument) { + fail('idempotency_conflict'); + } + jobId = active.id; + } else { + const operation = { + operation: input.command, + idempotencyKey: idempotencyKey(input, control.revision), + expectedRevision: control.revision, + }; + if (input.command === 'restore') operation.backupId = input.argument; + if (input.command === 'upgrade') operation.releaseId = input.argument; + jobId = authority.request(operation).id; + } + const result = await lifecycle.run(jobId, `worker_${crypto.randomUUID().replaceAll('-', '')}`); + process.stdout.write(`${JSON.stringify({ ok: result.status === 'succeeded', status: result.status, + operation: result.operation, installationState: result.installationState, + installationRevision: result.installationRevision, manifestRevision: result.manifestRevision, + failure: result.failure, backups: authority.backups() })}\n`); + if (result.status !== 'succeeded') process.exitCode = 1; + } finally { store.close(); } +} + +main().catch(error => { + const selected = error?.code === 'invalid_input' + ? { code: 'invalid_input', category: 'request', recoverable: false } + : installationFailure(error); + process.stdout.write(`${JSON.stringify({ ok: false, status: selected.code, failure: selected })}\n`); + process.exitCode = 1; +}); diff --git a/core/compatibility/provisioner/bin/dispatch-installation-reconcile b/core/compatibility/provisioner/bin/dispatch-installation-reconcile new file mode 100755 index 0000000..566b63e --- /dev/null +++ b/core/compatibility/provisioner/bin/dispatch-installation-reconcile @@ -0,0 +1,198 @@ +#!/usr/bin/env node +'use strict'; + +const crypto = require('node:crypto'); +const path = require('node:path'); +const { installationFailure } = require('../../../shared/contracts/src'); +const { PROJECT_ROOT } = require('../../../shared/paths/runtime-paths'); +const { AccessStore } = require('../../../core/accounts/src/store'); +const { + createAccessControlLiveAuthorityResolver, + createInstallationProvisioningReconciler, +} = require('../../../core/accounts/src/installation-provisioning'); +const { createAccessInstallationLifecycleAuthority } = require('../../../core/accounts/src/installation-lifecycle'); +const { createDurableInstallationProvisioner } = require('../../../core/installations/src/jobs'); +const { createSystemdUserSupervisor } = require('../../../core/installations/src/systemd-user'); +const { createManagedInstallationLifecycle } = require('../src/lifecycle'); +const { createInstallationLifecycleReconciler } = require('../../../core/installations/src/lifecycle-reconcile'); +const { loadPrivateReleaseCatalog, loadPrivateOciReleaseCatalog } = require('../../../core/installations/src/release-catalog'); +const { createRuntimeAgentCredentialManager } = require('../../../core/installations/src/runtime-agent-credential'); +const { createOciRuntimeAgentCredentialPort } = require('../../../core/installations/src/oci-runtime-agent-credential'); +const { createProtectedOciHostClient } = require('../../../core/installations/src/oci-protected-client'); +const { createOciContainerAdapter } = require('../../../core/installations/src/oci-adapter'); +const { createOciInstallationLifecycle } = require('../../../core/installations/src/oci-lifecycle'); +const { createOciRuntimeLifecyclePort } = require('../../../core/installations/src/oci-runtime-lifecycle-port'); +const { createRuntimeAgentDispatchClient, runtimeAgentControlInvoke } = require('../../../core/agents/src'); + +function absoluteEnvironment(name) { + const value = process.env[name]; + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); + return value; +} + +function parse(argv) { + if (argv.length > 1) throw Object.assign(new Error('invalid_input'), { code: 'invalid_input' }); + if (argv.length === 0) return 20; + const match = /^--limit=([1-9]|[1-4][0-9]|50)$/.exec(argv[0]); + if (!match) throw Object.assign(new Error('invalid_input'), { code: 'invalid_input' }); + return Number.parseInt(match[1], 10); +} + +async function main() { + const limit = parse(process.argv.slice(2)); + const accessRoot = absoluteEnvironment('DISPATCH_ACCESS_CONTROL_DATABASE_ROOT'); + const stateRoot = absoluteEnvironment('DISPATCH_PROVISIONER_STATE_ROOT'); + const installationsRoot = absoluteEnvironment('DISPATCH_INSTALLATIONS_ROOT'); + const unitRoot = absoluteEnvironment('DISPATCH_SYSTEMD_UNIT_ROOT'); + const runtimeAgentHubSocket = absoluteEnvironment('DISPATCH_RUNTIME_AGENT_HUB_SOCKET'); + const releaseCatalog = loadPrivateReleaseCatalog(process.env.DISPATCH_RELEASE_CATALOG_FILE); + const ociReleaseCatalog = loadPrivateOciReleaseCatalog(process.env.DISPATCH_OCI_RELEASE_CATALOG_FILE); + const ociEnabled = process.env.DISPATCH_OCI_RELEASE_CATALOG_FILE !== undefined + || process.env.DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT !== undefined; + if (ociEnabled && (process.env.DISPATCH_OCI_RELEASE_CATALOG_FILE === undefined + || process.env.DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT === undefined)) { + throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); + } + const runtimeAgentCredentials = createRuntimeAgentCredentialManager({ installationsRoot }); + let ociRuntimeAgentCredentials = null; + let ociAdapter = null; + let ociHost = null; + let runtimeAgentControlSocket = null; + let provisioner; + let activeLifecycleAuthority = null; + if (ociEnabled) { + ociRuntimeAgentCredentials = createOciRuntimeAgentCredentialPort({ + credentialRoot: absoluteEnvironment('DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT'), + }); + runtimeAgentControlSocket = absoluteEnvironment('DISPATCH_RUNTIME_AGENT_CONTROL_SOCKET'); + ociHost = createProtectedOciHostClient({ dispatchRequest: (request, dispatch) => { + const authority = Object.hasOwn(request.claim, 'generation') ? provisioner : activeLifecycleAuthority; + if (!authority) throw new Error('runtime_boundary_violation'); + return authority.dispatchHostRequest(request, dispatch); + } }); + ociAdapter = createOciContainerAdapter({ + hostRegistry: ociHost.hostRegistry, + hostExecutor: ociHost.hostExecutor, + releaseResolver: (releaseId, fixture) => { + const selected = ociReleaseCatalog[releaseId]; + if (fixture || !selected) { + throw Object.assign(new Error('release_not_found'), { code: 'release_not_found' }); + } + return selected; + }, + credentialPort: ociRuntimeAgentCredentials, + }); + } + const store = new AccessStore({ + databaseRoot: accessRoot, + database: path.join(accessRoot, 'access-control.sqlite3'), + }); + provisioner = createDurableInstallationProvisioner({ + stateRoot, + installationsRoot, + unitRoot, + projectRoot: PROJECT_ROOT, + liveAuthorityResolver: createAccessControlLiveAuthorityResolver({ store }), + runtimeAgentHubSocket, + ...(ociAdapter === null ? {} : { ociAdapter }), + }); + try { + const platformReleases = require('../../../core/installations/src/platform-release-catalog') + .loadPlatformReleaseCatalog(process.env.DISPATCH_PLATFORM_RELEASE_CATALOG_FILE, ociReleaseCatalog); + const rolloutCoordinator = require('../../../core/accounts/src/platform-updates').createPlatformUpdates({ + store, releases: ociReleaseCatalog, platformReleases, enabled: true, + }); + rolloutCoordinator.tick(); + const corePending = store.db.prepare(`SELECT 1 FROM platform_rollouts r + LEFT JOIN platform_rollout_core c ON c.rollout_id=r.id + WHERE r.status!='completed' AND (c.status IS NULL OR c.status!='succeeded') LIMIT 1`).get(); + if (corePending) { + process.stdout.write(`${JSON.stringify({ ok: true, status: 'waiting_for_core', pending: 1 })}\n`); + return; + } + const workerId = `worker_${crypto.randomUUID().replaceAll('-', '')}`; + const result = createInstallationProvisioningReconciler({ + store, + provisioner, + runtimeAgentCredentialFactory: backend => backend === 'oci_container_v1' + ? ociRuntimeAgentCredentials : runtimeAgentCredentials, + }) + .runPending(workerId, limit); + require('../../../core/accounts/src/organization-profile').applyOrganizationProfiles(store); + const supervisor = createSystemdUserSupervisor(); + const authorityFactory = (organizationId, authorityScope) => + createAccessInstallationLifecycleAuthority({ + store, + organizationId, + authorityScope, + releaseCatalog: [...new Set([...Object.keys(releaseCatalog), ...Object.keys(ociReleaseCatalog)])], + }); + const lifecycle = await createInstallationLifecycleReconciler({ + store, + authorityFactory, + runtimeFactory: (organizationId, authority) => { + activeLifecycleAuthority = authority; + const backend = store.installationBackend(organizationId); + if (backend !== 'oci_container_v1') return createManagedInstallationLifecycle({ + authority, + installationsRoot, + unitRoot, + supervisor, + projectRoot: PROJECT_ROOT, + releaseCatalog, + runtimeAgentHubSocket, + }); + if (!ociAdapter || !ociHost || !runtimeAgentControlSocket) { + throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); + } + return createOciInstallationLifecycle({ + authority, + adapter: ociAdapter, + hostExecutor: ociHost.hostExecutor, + backupManagerFactory: (plan, claim) => ociHost.createBackupManager(plan, claim), + runtimeFactory: plan => createOciRuntimeLifecyclePort({ + client: createRuntimeAgentDispatchClient({ + runtimeKey: plan.runtimeKey, + hub: { + invoke: (runtimeKey, action, input) => + runtimeAgentControlInvoke(runtimeAgentControlSocket, runtimeKey, action, input), + }, + }), + }), + }); + }, + }).runPending(`lifecycle_${crypto.randomUUID().replaceAll('-', '')}`, limit); + rolloutCoordinator.tick(); + const credentialsRetired = ociRuntimeAgentCredentials + ? require('../../../core/installations/src/retire-oci-credentials').retireOciCredentials({ store, credentialPort: ociRuntimeAgentCredentials }) : 0; + const onboarding = runtimeAgentControlSocket ? await require('../../../core/installations/src/owner-onboarding').createOwnerOnboardingWorker({ + store, projectRoot: PROJECT_ROOT, + invoke: (runtimeKey, action, input) => runtimeAgentControlInvoke(runtimeAgentControlSocket, runtimeKey, action, input), + }).runPending(`onboard_${crypto.randomUUID().replaceAll('-', '')}`, limit) : { processed: 0, completed: 0, failed: 0 }; + const failed = result.failed + lifecycle.failed + onboarding.failed; + const rollout = rolloutCoordinator.view().rollout; + const pending = result.pending + (lifecycle.pending ? 1 : 0) + (rollout?.status === 'running' ? 1 : 0); + process.stdout.write(`${JSON.stringify({ + ok: true, + status: pending ? 'pending' : failed ? 'failed' : 'settled', + ...result, + failed, + pending, + lifecycle, + rollout: rollout ? { status: rollout.status, total: rollout.total, updated: rollout.updated } : null, + onboarding, + credentialsRetired, + })}\n`); + if (failed || pending) process.exitCode = 1; + } finally { + provisioner.close(); + store.close(); + } +} + +main().catch(error => { + const selected = installationFailure(error); + process.stdout.write(`${JSON.stringify({ ok: false, status: selected.code, failure: selected })}\n`); + process.exitCode = 1; +}); diff --git a/core/compatibility/provisioner/bin/dispatch-managed-activation b/core/compatibility/provisioner/bin/dispatch-managed-activation new file mode 100755 index 0000000..9bf8029 --- /dev/null +++ b/core/compatibility/provisioner/bin/dispatch-managed-activation @@ -0,0 +1,129 @@ +#!/usr/bin/env node +'use strict'; + +const crypto = require('node:crypto'); +const path = require('node:path'); +const { failure, installationFailure } = require('../../../shared/contracts/src'); +const { PROJECT_ROOT } = require('../../../shared/paths/runtime-paths'); +const { AccessStore } = require('../../../core/accounts/src/store'); +const { createAccessInstallationActivationAuthority } = require('../../../core/accounts/src/installation-activation'); +const { managedInstallationContext } = require('../../../core/accounts/src/installation-authority'); +const { createSystemdUserSupervisor } = require('../../../core/installations/src/systemd-user'); +const { createManagedPaycomActivationComposition } = require('../src/managed-activation-runtime'); +const { createManagedPaycomAuthSetup } = require('../src/managed-auth-setup'); +const { runManagedPaycomActivation } = require('../src/activation'); + +function absoluteEnvironment(name) { + const value = process.env[name]; + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); + return value; +} + +function parse(argv) { + if (argv.length < 2 || argv.length > 3) throw Object.assign(new Error('invalid_input'), { code: 'invalid_input' }); + const [command, organizationId, credentialAction] = argv; + if (!['status', 'prepare-auth', 'setup-auth', 'activate', 'retry-activation'].includes(command) + || !/^[a-z][a-z0-9_-]{2,95}$/.test(organizationId) + || command === 'setup-auth' && !['create', 'replace'].includes(credentialAction) + || command !== 'setup-auth' && credentialAction !== undefined) { + throw Object.assign(new Error('invalid_input'), { code: 'invalid_input' }); + } + return { command, organizationId, credentialAction }; +} + +function activationKey(organizationId, manifestRevision, installationRevision) { + const digest = crypto.createHash('sha256') + .update(`${organizationId}\0${manifestRevision}\0${installationRevision}`) + .digest('hex').slice(0, 32); + return `activation:${digest}`; +} + +async function main() { + const input = parse(process.argv.slice(2)); + const accessRoot = absoluteEnvironment('DISPATCH_ACCESS_CONTROL_DATABASE_ROOT'); + const installationsRoot = absoluteEnvironment('DISPATCH_INSTALLATIONS_ROOT'); + const unitRoot = absoluteEnvironment('DISPATCH_SYSTEMD_UNIT_ROOT'); + const runtimeAgentHubSocket = absoluteEnvironment('DISPATCH_RUNTIME_AGENT_HUB_SOCKET'); + const store = new AccessStore({ + databaseRoot: accessRoot, + database: path.join(accessRoot, 'access-control.sqlite3'), + }); + try { + const context = managedInstallationContext(store, input.organizationId); + if (input.command === 'status') { + process.stdout.write(`${JSON.stringify({ + ok: true, + status: context.installation.status, + revision: context.installation.revision, + manifestRevision: context.installation.manifestRevision, + ownerReady: context.ownerActive, + })}\n`); + return; + } + const authority = createAccessInstallationActivationAuthority({ + store, + organizationId: input.organizationId, + authorityScope: 'platform_activation', + idempotencyKey: activationKey( + input.organizationId, + context.installation.manifestRevision, + context.installation.revision, + ), + workerId: `worker_${crypto.randomUUID().replaceAll('-', '')}`, + releaseId: context.manifest.runtime.releaseId, + }); + if (input.command === 'retry-activation') { + const retry = authority.retry(); + process.stdout.write(`${JSON.stringify({ + ok: true, + status: retry.installation.state, + revision: retry.installation.revision, + })}\n`); + return; + } + const supervisor = createSystemdUserSupervisor(); + if (input.command === 'prepare-auth' || input.command === 'setup-auth') { + const setup = createManagedPaycomAuthSetup({ + manifest: context.manifest, + manifestAuthority: context.manifestAuthority, + authority, + installationsRoot, + unitRoot, + supervisor, + projectRoot: PROJECT_ROOT, + runtimeAgentHubSocket, + }); + const result = input.command === 'prepare-auth' + ? await setup.prepare() + : await setup.run({ credentialAction: input.credentialAction }); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.ok) process.exitCode = 1; + return; + } + const runtime = createManagedPaycomActivationComposition({ + manifest: context.manifest, + manifestAuthority: context.manifestAuthority, + installationsRoot, + unitRoot, + supervisor, + projectRoot: PROJECT_ROOT, + runtimeAgentHubSocket, + }); + const result = await runManagedPaycomActivation({ authority, runtime, projectRoot: PROJECT_ROOT }); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.ok) process.exitCode = 1; + } finally { + store.close(); + } +} + +main().catch(error => { + if (error?.code === 'invalid_input') { + process.stdout.write(`${JSON.stringify(failure('invalid_input'))}\n`); + } else { + const selected = installationFailure(error); + process.stdout.write(`${JSON.stringify({ ok: false, status: selected.code, failure: selected })}\n`); + } + process.exitCode = 1; +}); diff --git a/core/compatibility/provisioner/src/activation.js b/core/compatibility/provisioner/src/activation.js new file mode 100644 index 0000000..0688855 --- /dev/null +++ b/core/compatibility/provisioner/src/activation.js @@ -0,0 +1,9 @@ +'use strict'; +const { runManagedPaycomActivation: runActivation } = require('../../../core/installations/src/activation.js'); +const { managedPaycomDefinition } = require('./managed-paycom'); +function runManagedPaycomActivation(options) { + return runActivation({ ...options, definitionFactory: context => managedPaycomDefinition(context.manifest, context.manifestAuthority, { + ...(options.projectRoot === undefined ? {} : { projectRoot: options.projectRoot }), container: options.container === true, + }) }); +} +module.exports = { ...require('../../../core/installations/src/activation.js'), runManagedPaycomActivation }; diff --git a/core/compatibility/provisioner/src/lifecycle.js b/core/compatibility/provisioner/src/lifecycle.js new file mode 100644 index 0000000..b369ca8 --- /dev/null +++ b/core/compatibility/provisioner/src/lifecycle.js @@ -0,0 +1,11 @@ +'use strict'; +const { createManagedInstallationLifecycle: createLifecycle } = require('../../../core/installations/src/lifecycle.js'); +const { createManagedPaycomActivationComposition } = require('./managed-activation-runtime'); +function createManagedInstallationLifecycle(options) { + return createLifecycle({ ...options, activationRuntimeFactory: options.activationRuntimeFactory || (context => createManagedPaycomActivationComposition({ + manifest: context.manifest, manifestAuthority: context.manifestAuthority, installationsRoot: options.installationsRoot, + unitRoot: options.unitRoot, supervisor: options.supervisor, projectRoot: context.projectRoot, + ...(options.runtimeAgentHubSocket === undefined ? {} : { runtimeAgentHubSocket: options.runtimeAgentHubSocket }), + })) }); +} +module.exports = { createManagedInstallationLifecycle }; diff --git a/core/compatibility/provisioner/src/managed-activation-evidence.js b/core/compatibility/provisioner/src/managed-activation-evidence.js new file mode 100644 index 0000000..5a3a719 --- /dev/null +++ b/core/compatibility/provisioner/src/managed-activation-evidence.js @@ -0,0 +1,11 @@ +'use strict'; + +const { + createLocalPaycomActivationEvidencePort, +} = require('../../../plugins/paycom/backend/adapters/activation-evidence'); + +function createManagedPaycomActivationEvidenceVerifier(options) { + return createLocalPaycomActivationEvidencePort(options); +} + +module.exports = { createManagedPaycomActivationEvidenceVerifier }; diff --git a/core/compatibility/provisioner/src/managed-activation-runtime.js b/core/compatibility/provisioner/src/managed-activation-runtime.js new file mode 100644 index 0000000..97f9cba --- /dev/null +++ b/core/compatibility/provisioner/src/managed-activation-runtime.js @@ -0,0 +1,62 @@ +'use strict'; +const { PROJECT_ROOT, resolveManagedInstallationRuntimePaths, managedInstallationRuntimeEnvironment } = require('dispatch-protocol/paths/runtime-paths'); +const path = require('node:path'); +const { serverInstallationManifest } = require('dispatch-protocol/contracts/src'); +const { LocalCollectionAdminPort } = require('../../../runtime/adapters/local/collection-admin-port'); +const { createRuntimeGatewayDispatchClient } = require('../../../runtime/gateway/src'); +const { createManagedRuntimeDispatchClient } = require('../../../runtime/gateway/src/managed-runtime'); +const { createManagedPaycomActivationEvidenceVerifier } = require('./managed-activation-evidence'); +const shared = require('../../../plugins/paycom/backend/runtime/activation'); +const { createManagedPaycomActivationRuntime } = shared; +function fail(code) { throw Object.assign(new Error(code), { code }); } +function plain(value) { return Boolean(value) && typeof value === 'object' && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype; } +function createManagedPaycomActivationComposition(options) { + const optionFields = [ + 'manifest', 'manifestAuthority', 'installationsRoot', 'unitRoot', 'supervisor', 'projectRoot', + 'clock', 'delay', 'publicationTimeoutMs', 'publicationPollMs', 'runtimeAgentHubSocket', + ]; + if (!plain(options) || Object.keys(options).some(key => !optionFields.includes(key)) + || !['manifest', 'manifestAuthority', 'installationsRoot', 'unitRoot', 'supervisor'] + .every(key => Object.hasOwn(options, key))) fail('runtime_boundary_violation'); + const projectRoot = options.projectRoot === undefined ? PROJECT_ROOT : options.projectRoot; + const manifest = serverInstallationManifest(options.manifest, options.manifestAuthority); + const { createInstallationLayoutManager } = require('../../../core/installations/src/layout.js'); + const { createInstallationServiceManager } = require('../../../core/installations/src/services.js'); + const layout = createInstallationLayoutManager({ installationsRoot: options.installationsRoot, projectRoot }); + const selectedLayout = layout.derive(manifest, options.manifestAuthority); + const serviceManager = createInstallationServiceManager({ + unitRoot: options.unitRoot, + projectRoot, + ...(options.runtimeAgentHubSocket === undefined ? {} : { + runtimeAgentHubSocket: options.runtimeAgentHubSocket, + }), + }); + const paths = resolveManagedInstallationRuntimePaths(selectedLayout); + const environment = managedInstallationRuntimeEnvironment(selectedLayout); + const client = createManagedRuntimeDispatchClient({ paths }); + const collectionAdmin = new LocalCollectionAdminPort({ paths: paths.collection }); + const clock = options.clock === undefined ? Date.now : options.clock; + const evidenceVerifier = createManagedPaycomActivationEvidenceVerifier({ environment }); + const gateway = createRuntimeGatewayDispatchClient({ + socketPath: path.join(paths.runtimeRoot, 'runtime-gateway.sock'), + runtimeKey: manifest.runtime.key, + }); + return createManagedPaycomActivationRuntime({ + manifest, + manifestAuthority: options.manifestAuthority, + layout, + serviceManager, + supervisor: options.supervisor, + client, + collectionAdmin, + gateway, + evidenceVerifier, + projectRoot, + clock, + ...(options.delay === undefined ? {} : { delay: options.delay }), + ...(options.publicationTimeoutMs === undefined ? {} : { publicationTimeoutMs: options.publicationTimeoutMs }), + ...(options.publicationPollMs === undefined ? {} : { publicationPollMs: options.publicationPollMs }), + }); +} + +module.exports = { ...shared, createManagedPaycomActivationComposition }; diff --git a/core/compatibility/provisioner/src/managed-auth-setup.js b/core/compatibility/provisioner/src/managed-auth-setup.js new file mode 100644 index 0000000..ab50c0a --- /dev/null +++ b/core/compatibility/provisioner/src/managed-auth-setup.js @@ -0,0 +1,177 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { failure } = require('dispatch-protocol/contracts/src'); +const { + PROJECT_ROOT, + managedInstallationRuntimeEnvironment, + resolveManagedInstallationRuntimePaths, +} = require('dispatch-protocol/paths/runtime-paths'); +const { prepareAuthSetup } = require('../../../runtime/application/auth/prepare-auth-setup'); +const { runSetupAuth, RecordingEventSink } = require('../../../runtime/application/auth/setup-auth'); +const { AuthClient } = require('../../../runtime/sdk/src/auth-client'); +const { LocalAuthBrokerPort } = require('../../../runtime/adapters/local/auth-broker-port'); +const { LocalAuthSetupPort } = require('../../../runtime/adapters/local/auth-setup-port'); +const { LocalCredentialIngress } = require('../../../runtime/adapters/local/credential-ingress'); +const { createInstallationLayoutManager } = require('../../../core/installations/src/layout.js'); +const { createInstallationServiceManager } = require('../../../core/installations/src/services.js'); + +function fail(code) { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +class ManagedRuntimeServicePort { + constructor({ plan, supervisor, authority }) { + if (!plan || !supervisor || !['snapshot', 'start', 'stop', 'health'].every(method => typeof supervisor[method] === 'function') + || !authority || typeof authority.guard !== 'function') fail('runtime_boundary_violation'); + this.plan = plan; + this.supervisor = supervisor; + this.authority = authority; + } + + state() { + const values = this.supervisor.snapshot(this.plan); + if (!Array.isArray(values) || values.length !== this.plan.units.length) fail('broker_state_unknown'); + const active = values.filter(value => value.active).length; + if (active === values.length) return 'ready'; + if (active === 0) return 'stopped'; + fail('broker_state_unknown'); + } + + async status() { + const status = this.state(); + if (status === 'ready') this.supervisor.health(this.plan); + return { status, managed: true }; + } + + async stop() { + if (this.state() === 'stopped') return { status: 'stopped', managed: true, stopped: false }; + this.supervisor.stop(this.plan, mutation => this.authority.guard(mutation)); + if (this.state() !== 'stopped') fail('auth_broker_stop_failed'); + return { status: 'stopped', managed: true, stopped: true }; + } + + async start() { + if (this.state() === 'ready') { + this.supervisor.health(this.plan); + return { status: 'ready', managed: true, started: false }; + } + this.supervisor.start(this.plan, mutation => this.authority.guard(mutation)); + this.supervisor.health(this.plan); + if (this.state() !== 'ready') fail('auth_broker_start_failed'); + return { status: 'ready', managed: true, started: true }; + } +} + +function createManagedPaycomAuthSetup(options) { + const fields = [ + 'manifest', 'manifestAuthority', 'authority', 'installationsRoot', 'unitRoot', 'supervisor', + 'projectRoot', 'events', 'runtimeAgentHubSocket', + ]; + if (!plain(options) || Object.keys(options).some(key => !fields.includes(key)) + || !['manifest', 'manifestAuthority', 'authority', 'installationsRoot', 'unitRoot', 'supervisor'] + .every(key => Object.hasOwn(options, key))) fail('runtime_boundary_violation'); + if (!['peek', 'beginSetup', 'endSetup', 'guard'] + .every(method => typeof options.authority[method] === 'function')) { + fail('runtime_boundary_violation'); + } + const boundContext = context => { + if (!plain(context) || context.installation?.state !== 'waiting_for_provider_auth' + || JSON.stringify(context.manifest) !== JSON.stringify(options.manifest) + || JSON.stringify(context.manifestAuthority) !== JSON.stringify(options.manifestAuthority)) { + fail('runtime_identity_mismatch'); + } + return context; + }; + const initialContext = options.authority.peek(); + if (initialContext && typeof initialContext.then === 'function') fail('runtime_boundary_violation'); + boundContext(initialContext); + const projectRoot = options.projectRoot === undefined ? PROJECT_ROOT : options.projectRoot; + const layout = createInstallationLayoutManager({ installationsRoot: options.installationsRoot, projectRoot }); + const selectedLayout = layout.derive(options.manifest, options.manifestAuthority); + const paths = resolveManagedInstallationRuntimePaths(selectedLayout); + const serviceManager = createInstallationServiceManager({ + unitRoot: options.unitRoot, + projectRoot, + ...(options.runtimeAgentHubSocket === undefined ? {} : { + runtimeAgentHubSocket: options.runtimeAgentHubSocket, + }), + }); + const plan = serviceManager.plan(options.manifest, options.manifestAuthority, selectedLayout); + serviceManager.inspectInstalled(plan); + const environment = managedInstallationRuntimeEnvironment(selectedLayout); + const runOptions = { environment }; + const setup = new LocalAuthSetupPort({ paths: paths.auth, runOptions }); + const ingress = new LocalCredentialIngress({ runOptions }); + const guardedSetup = Object.freeze({ + inspect: profile => setup.inspect(profile), + initialize: () => options.authority.guard(() => setup.initialize()), + remove: profile => options.authority.guard(() => setup.remove(profile)), + }); + const guardedIngress = Object.freeze({ + available: () => ingress.available(), + capture: input => options.authority.guard(() => ingress.capture(input)), + }); + const service = new ManagedRuntimeServicePort({ plan, supervisor: options.supervisor, authority: options.authority }); + const authentication = new AuthClient({ port: new LocalAuthBrokerPort({ socketPath: paths.auth.socket }) }); + const events = options.events === undefined ? new RecordingEventSink() : options.events; + if (!events || typeof events.emit !== 'function') fail('runtime_boundary_violation'); + + async function prepare() { + try { + boundContext(await options.authority.peek()); + return prepareAuthSetup({ setup, service, ingress }, { provider: 'paycom', profile: 'paycom-main' }); + } catch (error) { + return failure(error?.code === 'installation_operation_in_progress' + ? 'installation_operation_in_progress' : 'installation_not_ready', { recoverable: true }); + } + } + + async function run(input) { + if (!plain(input) || Object.keys(input).sort().join(',') !== 'credentialAction' + || !['create', 'replace'].includes(input.credentialAction)) { + return failure('invalid_input'); + } + let claimed = false; + let result; + try { + boundContext(await options.authority.peek()); + boundContext(await options.authority.beginSetup()); + claimed = true; + result = await runSetupAuth({ + setup: guardedSetup, ingress: guardedIngress, service, authentication, events, + }, { + provider: 'paycom', + profile: 'paycom-main', + credentialAction: input.credentialAction === 'create' ? 'enroll' : 'replace', + startBroker: true, + testAuthentication: false, + operationId: `setup_auth_${crypto.randomUUID().replaceAll('-', '')}`, + }); + if (result.ok) boundContext(await options.authority.peek()); + } catch (error) { + const code = ['installation_operation_in_progress', 'installation_not_ready', 'runtime_identity_mismatch'] + .includes(error?.code) + ? error.code : 'setup_auth_failed'; + result = failure(code, { recoverable: code !== 'setup_auth_failed' }); + } finally { + if (claimed) { + try { boundContext(await options.authority.endSetup()); } + catch { result = failure('installation_operation_in_progress', { recoverable: true }); } + } + } + return result; + } + + return Object.freeze({ prepare, run }); +} + +module.exports = { + ManagedRuntimeServicePort, + createManagedPaycomAuthSetup, +}; diff --git a/core/compatibility/provisioner/src/managed-paycom.js b/core/compatibility/provisioner/src/managed-paycom.js new file mode 100644 index 0000000..d630ac3 --- /dev/null +++ b/core/compatibility/provisioner/src/managed-paycom.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('../../../plugins/paycom/backend/runtime/definition'); diff --git a/core/config/dispatch.env.example b/core/config/dispatch.env.example new file mode 100644 index 0000000..42c590f --- /dev/null +++ b/core/config/dispatch.env.example @@ -0,0 +1,19 @@ +# Path configuration only. Never put credentials or tokens here. +# Set this to an absolute directory outside the Git worktree. +# DISPATCH_LOCAL_ROOT= +# Managed installation commands use separate owner-private absolute roots; these +# are server/operator settings, never tenant/browser values. Reconciliation uses +# all four. Provider activation uses Access Control, installations, and systemd +# roots but does not open the Provisioner job database. Keep the service-unit +# root outside runtime/control roots. +# DISPATCH_ACCESS_CONTROL_DATABASE_ROOT= +# DISPATCH_PROVISIONER_STATE_ROOT= +# DISPATCH_INSTALLATIONS_ROOT= +# DISPATCH_SYSTEMD_UNIT_ROOT= +# Shared owner-private central Core socket used by the Dashboard hub and managed Runtime Agents. +# This remains same-user only until the per-DSP isolation transport is selected. +# DISPATCH_RUNTIME_AGENT_HUB_SOCKET= +# Optional absolute overrides when trusted executables are not discoverable on PATH. +# DISPATCH_CHROME_EXECUTABLE= +# DISPATCH_BROWSER_LAUNCHER= +# DISPATCH_STTY_EXECUTABLE= diff --git a/core/core/OVERVIEW.md b/core/core/OVERVIEW.md new file mode 100644 index 0000000..aa3c292 --- /dev/null +++ b/core/core/OVERVIEW.md @@ -0,0 +1,19 @@ +# Dispatch Core + +Core owns platform administration, shared login, DSP membership, provisioning, +removal, and Core-first release rollouts. The shared dashboard lives in +`dashboard/`. + +DSP services and provider implementations live in `runtime/`. Core sends +validated messages over the authenticated runtime-agent connection instead of +loading those implementations. The `shared/` package owns the messages, +transport helpers and runtime layout agreement. Both release artifacts carry +their own copy; neither application reads the other application's source tree. + +`compatibility/` contains opt-in tools for the older same-user runtime setup. +Those tools are excluded from the Core and DSP release artifacts. + +Run `./tooling/verify` from the repository root for source, behavior and package +boundary checks. Run `./runtime/tooling/verify` for the native package and live +two-account fixture. See [native DSPs](installations/NATIVE-DSPS.md) and +[backup and recovery](installations/RECOVERY.md). diff --git a/core/core/accounts/OVERVIEW.md b/core/core/accounts/OVERVIEW.md new file mode 100644 index 0000000..6cf6bb7 --- /dev/null +++ b/core/core/accounts/OVERVIEW.md @@ -0,0 +1,59 @@ +--- +title: Access Control overview +status: current +last_verified: 2026-09-03 +--- + +# Access Control + +This component is the human identity and DSP authorization control plane for Dispatch. + +## Owns + +- website user identities, password hashes, and authenticated password rotation; +- opaque server-side sessions and CSRF tokens; +- DSP organizations and station metadata; +- explicit user-to-DSP memberships; +- protected system roles and constrained custom roles; +- platform and DSP invitation lifecycles; +- authoritative organization-to-runtime installation lifecycle, revision, and current-job binding; +- durable provisioning outbox requests, activation fencing, lifecycle jobs, release authority, and backup inventory; +- bounded access audit events. + +## Does not own + +- Paycom or Amazon provider credentials; +- provider browser sessions or cookies; +- provider login automation; +- workforce, timecard, or CDF data; +- collection execution or scheduling; +- arbitrary runtime endpoints or commands. + +Those boundaries remain with the Auth Broker, plugins, Collection Manager, and SDK respectively. + +## Source + +- `src/store.js` — private SQLite schema and persistence +- `src/service.js` — identity, invitation, membership, role, permission, and session policy +- `src/installation-authority.js` — server-owned managed manifest projection +- `src/installation-provisioning.js` — Access Control outbox, Provisioner acknowledgement, and terminal reconciliation +- `src/installation-activation.js` — provider-activation lease/fence and atomic `ready` commit +- `src/installation-lifecycle.js` — durable backup/restore, upgrade, suspension, decommission, and destruction authority +- `src/passwords.js` — versioned scrypt hashing and constant-work fallback +- `src/validation.js` — closed bounded inputs +- `src/permissions.js` — platform and tenant permission catalogs +- `tests/access-control.test.js` — focused security and isolation acceptance + +## Runtime + +Version `0.4.0` owns schema version `6`. The database is `/access-control/access-control.sqlite3`, with an owner-only directory and file. Schema `2 -> 6` preserves the conservative lifecycle adoption, browser controls, and exact `local-dsp -> local` compatibility state while adding durable release/lifecycle state and per-installation Runtime Agent authority digests, generations, and revocation. Raw Agent tokens remain only in private per-runtime files. New DSPs receive pending installation records; no browser-provided address is accepted. + +Only `platform.installations.manage` may create a target-free provision/retry outbox request. The Provisioner job remains unclaimable until Access Control records its exact server-generated job ID and the Provisioner durably acknowledges that record. Owner acceptance advances `waiting_for_owner -> waiting_for_provider_auth` in the same transaction as membership creation. Protected setup acquires a durable worker/fence/expiry lease on the installation; concurrent setup and activation are denied while it is current, every credential or service mutation renews it, and stale workers cannot mutate after reclaim. Managed activation stores profile/provider metadata, a positive-test timestamp, job/manifest revisions, leases/fences, and a closed first-publication evidence bundle—never a provider secret or session. Only the current unexpired worker/fence may heartbeat, fail, or commit. The sole `ready` write validates the current activation job, all nine server-built gates, and the fresh evidence digest, then persists job success/evidence, installation readiness, and organization activation in one Access Control transaction with exact read-back. Public projections omit setup leases, evidence, and publication identities. + +The private Dashboard now exposes only fixed Access Control console capabilities. Its platform list returns an expiring opaque control reference and sanitized organization/installation state, not organization/runtime/job identity. CSRF-protected idempotent create, invitation, status, provision, and infrastructure-retry requests resolve that reference server-side. The setup status is target-free. For OCI installations, owners submit a fixed Paycom credential form into their runtime's vault through the private Agent, then the reconciler verifies and activates the DSP. Access Control schema 9 persists a credential-free onboarding ledger with expiring leases and fences. Platform removal and separate permanent deletion queue lifecycle jobs after exact-name confirmation. Runtime roots, keys, units, commands, endpoints and provider profile selectors remain unavailable to HTTP callers; credentials are accepted only by the owner setup input and never returned. + +The separate server-only lifecycle controller uses Access Control's current lease/fence around each fixed host mutation. Backup/restore, upgrade rollback, runtime suspension/resumption, retained removal and restoration, and permanent destruction use fixed lifecycle jobs. Private Dashboard routes can request removal/restoration and password-confirmed deletion; host mutations remain absent from the HTTP process, public SDK, and Runtime Gateway. + +See [`SECURITY.md`](SECURITY.md) for the authorization and storage boundary. + +Email-first creation atomically adds a pending business profile and provisioning request with the owner invitation. Submitted profiles are applied once infrastructure is prepared. Durable platform rollout records coordinate sequential lifecycle upgrades, block on unready DSPs or failures, and retain restart and retry state. See platform administration. diff --git a/core/core/accounts/SECURITY.md b/core/core/accounts/SECURITY.md new file mode 100644 index 0000000..f3fcbc9 --- /dev/null +++ b/core/core/accounts/SECURITY.md @@ -0,0 +1,87 @@ +--- +title: Access Control security contract +status: current +last_verified: 2026-09-03 +--- + +# Security contract + +## Trust boundary + +Human authentication and tenant authorization are separate from provider authentication. This component must never receive Paycom/Amazon credentials, browser cookies, CDP endpoints, vault records, or collector leases. The Auth Broker must never become the human account database. + +## Identity and secret storage + +- Email addresses are normalized for identity matching. +- Passwords are never normalized or trimmed and must contain 12–128 characters. +- Passwords use scrypt-v1 with `N=32768`, `r=8`, `p=1`, a random 24-byte salt, and a 64-byte output. +- Unknown-account authentication performs equivalent scrypt work. +- Session, invitation, and browser platform-control values contain 256 random bits and are stored only as SHA-256 hashes. +- The access database and directory must be owner-only regular paths outside the source tree. +- Browser storage never receives session or invitation tokens; the session token remains in an `HttpOnly` cookie. + +## Tenant invariants + +1. Account creation alone grants no organization access. +2. DSP-owner and member API paths contain no organization identifier; every organization operation derives the DSP from the session's active membership. +3. Platform ownership does not imply DSP workforce access. +4. Organization switching sends a membership ID, and the server derives the organization from that existing active membership. +5. Runtime selection comes from the server-owned installation registry. +6. Pending installations cannot call operational SDK endpoints. +7. A DSP abbreviation, station code, URL slug, request body, or opaque record ID is never authorization. +8. Authorization errors do not return workforce identities or runtime routing metadata. + +## Installation authority invariants + +1. Access Control schema `6` is authoritative for installation lifecycle, release and manifest revisions, current job, organization/runtime binding, fixed-stage lifecycle jobs, backup inventory, hash-only Runtime Agent authorities, hash-only browser platform-control references, and platform-mutation idempotency; the Provisioner database is an executor journal. +2. Only `platform.installations.manage` plus the separately enabled installation-operator composition may create a closed provision/retry request, and tenant roles cannot receive that permission. +3. One transaction checks lifecycle/revision/idempotency and writes the outbox request plus `provisioning`; browser organization/status/invitation commands use a separate actor/action/key ledger. Direct database edits are not an operational interface. +4. A Provisioner live job is unclaimable until Access Control records its exact server-generated ID and the Provisioner acknowledges that binding. +5. Every live host mutation revalidates the same Access Control organization/manifest/runtime/current-job authority while holding the authority transaction. +6. Owner acceptance moves `waiting_for_owner -> waiting_for_provider_auth` in the membership transaction. Provider credentials and sessions never enter Access Control. Protected credential setup owns a durable installation-local worker/fence/expiry lease; a current setup lease blocks both activation and another setup worker, and every service/vault/ingress mutation revalidates and renews it before execution. +7. Activation jobs are leased and fenced. Only the current unexpired worker/fence may heartbeat, fail, commit, or initiate a deadline cancellation; process loss reclaims with a larger fence. Long manager polling renews the lease. A stale worker exits without cancelling the shared idempotent run or batch, allowing its replacement to resume. +8. `ready` requires all nine gates for the current manifest/runtime/job plus a closed, digest-verified, fresh evidence bundle that independently binds the exact manager batch to active Paycom publications. Job success, private evidence, installation ready, and organization active commit and read back in one transaction. +9. Failed/partial publication, stale authority, unverified legacy managed ready, and unknown errors remain non-ready. Only exact `local-dsp -> local` may preserve legacy ready during schema migration. +10. Browser-safe platform/setup projections exclude organization/runtime/job/invitation identities, infrastructure details, provider responses, credentials, sessions, publication identifiers, evidence bundles, and workforce data. Platform mutations resolve only an expiring session/user/organization/purpose-bound random control reference in a JSON body. The private activation row may retain only the closed evidence contract needed to prove readiness; it is never a dashboard or public SDK DTO. +11. Lifecycle jobs are authority-scope/idempotency-key bound, one-active-job constrained, leased, and fenced. Their fixed stages and bounded aggregate receipts never contain a path, command, credential, provider response, or business record. +12. Restore accepts only an available backup already bound to the same organization/runtime and only while suspended. Upgrade accepts only a different release from the server-owned catalog and advances release/manifest authority only after backup, target health, publication continuity, and journal commit. +13. Resume rechecks infrastructure and exact publication continuity without provider authentication, collection, or publication. Decommission retains data and a final backup; permanent deletion is a separate decommissioned-only server-local operation with literal approval. + +## Invitation invariants + +- The organization, email, and role are fixed before token generation. +- Tokens are one-time, revocable, hashed at rest, and expire. +- Initial platform-owner invitations can be revoked idempotently from the owner-private bootstrap command before acceptance. +- Pending-owner and per-DSP/per-email uniqueness is checked inside the write transaction and enforced by partial unique indexes. +- Local invitation links keep the token in the URL fragment, inspect it through a JSON body, and clear the fragment after acceptance so reverse-proxy request logs and referrers do not receive it. +- Existing accounts must sign in with the exact invited email. +- Initial ownership uses a platform-issued invitation; additional Owners can be invited or assigned through team management. +- Only an authenticated platform owner may create a DSP shell or replacement initial-owner invitation. +- Without outbound email, invitation paths are returned only once to the authorized creator or written by the bootstrap command to an owner-only secret file. + +## Role invariants + +- Every DSP has exactly four fixed system roles: Owner, Manager, Dispatcher, and Driver. +- All four currently share the complete DSP permission catalog, including `organization.owner`, but never platform authority. +- Custom role creation and all role edits/deletions are rejected by the service; `roles.manage` is not granted. +- An actor cannot grant permissions they do not possess. +- System roles cannot be edited or deleted. +- All four roles can be invited and assigned. The last active Owner cannot be demoted or removed. +- Schema 13 migrates Administrator to Manager, Viewer to Driver, and custom roles to a matching standard name or Dispatcher. Existing canonical IDs, memberships, and invitation tokens/statuses are preserved; retired role references are repointed transactionally. +- Self-role change and self-removal are rejected. +- Role and membership changes are checked on every request rather than trusted from stale browser state. + +## HTTP invariants + +- Platform target references are random, hashed at rest, bound to the issuing session/user/organization/purpose, expire after 15 minutes, never enter a URL, and are not authorization without a fresh permission check. +- Platform mutations use fixed routes and exact bodies. Organization/status/invitation commands use durable actor/action/idempotency-key request matching; provision/retry uses the installation/authority-scope/key/canonical-operation outbox record. Replay never reissues a one-time invitation value. +- Protected routes require an opaque session cookie. +- Mutations require JSON, same-site browser context, and the session's CSRF token. +- Login attempts are bounded in memory by address/account and across accounts per address; invitation inspection/redemption is also address-bounded. Failures remain generic. +- Public cookies use `__Host-dispatch_session` with `Secure`, `HttpOnly`, `SameSite=Strict`, `Path=/`, and no `Domain` attribute. +- Responses containing identity, invitation, membership, or workforce information are `no-store`. +- Static content keeps the restrictive dashboard CSP, frame denial, referrer denial, and permissions policy. + +## Public deployment boundary + +The invitation-only Dashboard is public only through the exact `https://dispatch.example.test` Cloudflare Tunnel route to its loopback listener. Public mode requires exact Host, Cloudflare HTTPS proof, exact mutation Origin, secure host-only cookies, managed WAF/DDoS protection, and no-store application responses. Bot Fight Mode is disabled during testing because its zone-wide policy challenged scripted/headless acceptance; review sibling subdomains and automated clients before production enablement. Do not expose an origin port, ordinary reverse proxy, Funnel, wildcard/per-DSP route, Runtime Gateway, Provisioner, or lifecycle executor. Verified outbound email/recovery, administrator MFA/passkeys, stronger distributed rate limiting/alerting, backup/incident review, and final independent review remain hardening responsibilities. diff --git a/core/core/accounts/package.json b/core/core/accounts/package.json new file mode 100644 index 0000000..5d09f29 --- /dev/null +++ b/core/core/accounts/package.json @@ -0,0 +1,15 @@ +{ + "name": "dispatch-access-control", + "version": "0.4.0", + "private": true, + "description": "Human identity, tenant authorization, and installation lifecycle authority for Dispatch", + "type": "commonjs", + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "./scripts/build", + "test": "./scripts/test", + "verify": "./scripts/verify" + } +} diff --git a/core/core/accounts/scripts/build b/core/core/accounts/scripts/build new file mode 100755 index 0000000..c1ec584 --- /dev/null +++ b/core/core/accounts/scripts/build @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +for file in "$ROOT"/src/*.js "$ROOT"/tests/*.js; do + node --no-warnings --check "$file" +done +node --no-warnings -e "require('$ROOT/src')" +printf '%s\n' '{"ok":true,"status":"built"}' diff --git a/core/core/accounts/scripts/test b/core/core/accounts/scripts/test new file mode 100755 index 0000000..f6e53ac --- /dev/null +++ b/core/core/accounts/scripts/test @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +cd "$ROOT" +node --no-warnings --test tests/*.test.js diff --git a/core/core/accounts/scripts/verify b/core/core/accounts/scripts/verify new file mode 100755 index 0000000..2a3e236 --- /dev/null +++ b/core/core/accounts/scripts/verify @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +"$ROOT/tooling/build" +"$ROOT/tooling/test" diff --git a/core/core/accounts/src/backup-categories.js b/core/core/accounts/src/backup-categories.js new file mode 100644 index 0000000..512ccd1 --- /dev/null +++ b/core/core/accounts/src/backup-categories.js @@ -0,0 +1,43 @@ +'use strict'; +const crypto = require('node:crypto'); +const categories = new Set(['scheduled', 'manual', 'pre_update']); +function recordCategory(db, backupId, category) { + if (!categories.has(category)) throw Error('backup_category_invalid'); + db.prepare('INSERT OR IGNORE INTO backup_categories VALUES(?,?)').run(backupId, category); +} +function categoryForRequest(request) { + const input = JSON.parse(request?.input_json || '{}'); + return input.category || (request?.idempotency_key?.startsWith('scheduled:') ? 'scheduled' : 'manual'); +} +function classifyExisting(db) { + db.exec(`INSERT OR IGNORE INTO backup_categories + SELECT r.id,CASE + WHEN json_extract(q.input_json,'$.category') IN ('scheduled','manual','pre_update') THEN json_extract(q.input_json,'$.category') + WHEN b.purpose='upgrade' THEN 'pre_update' + WHEN q.idempotency_key LIKE 'scheduled:%' THEN 'scheduled' + ELSE 'manual' END + FROM platform_backup_records r LEFT JOIN installation_backups b ON b.id=r.id + LEFT JOIN installation_lifecycle_jobs j ON j.id=b.lifecycle_job_id + LEFT JOIN platform_backup_requests q ON q.id=r.id OR q.job_id=j.id;`); +} +// Replacement is upload-first. Deletion uses the existing root-side, resumable +// remote deletion path; a failed upload never removes an older recovery point. +function replacePreUpdateBackups(store, archive, now) { + const db = store.db; + classifyExisting(db); + const rows = db.prepare(`SELECT r.* FROM platform_backup_records r JOIN backup_categories c ON c.backup_id=r.id + WHERE c.category='pre_update' AND r.deleted_at IS NULL ORDER BY r.created_at DESC,r.rowid DESC`).all(); + const current = new Map(); + for (const row of rows) { + const scope = row.organization_id || 'core'; + const proof = archive.backups?.[row.id]; + if (!current.has(scope)) { + if (proof?.status === 'verified' && proof.metadataDigest === crypto.createHash('sha256').update(row.metadata_json).digest('hex')) current.set(scope, row.id); + continue; + } + if (db.prepare("SELECT 1 FROM backup_deletions WHERE backup_id=? AND status IN ('queued','completed')").get(row.id)) continue; + db.prepare("INSERT INTO backup_deletions VALUES(?,?,?,'queued',?,NULL)") + .run(`bdel_${crypto.randomBytes(16).toString('hex')}`, row.id, row.organization_id, now); + } +} +module.exports = { recordCategory, categoryForRequest, classifyExisting, replacePreUpdateBackups }; diff --git a/core/core/accounts/src/backup-metadata.js b/core/core/accounts/src/backup-metadata.js new file mode 100644 index 0000000..bb57e58 --- /dev/null +++ b/core/core/accounts/src/backup-metadata.js @@ -0,0 +1,179 @@ +'use strict'; +const { AccessError } = require('./validation'); +const { managedInstallationContext } = require('./installation-authority'); +const fail = () => { + throw new AccessError('backup_identity_conflict', 409); +}; +function recordDspBackup(store, backupId, organizationId, now) { + const request = store.db + .prepare( + "SELECT input_json,idempotency_key FROM platform_backup_requests WHERE organization_id=? AND status='running' ORDER BY created_at DESC LIMIT 1", + ) + .get(organizationId); + const settings = store.db + .prepare('SELECT settings_json FROM platform_backup_settings WHERE id=1') + .get(); + const days = request + ? JSON.parse(request.input_json).retentionDays + : settings + ? JSON.parse(settings.settings_json).retentionDays + : null; + store.db + .prepare("INSERT OR IGNORE INTO platform_backup_records VALUES(?,?,'dsp',?,?,?,?,NULL)") + .run( + backupId, + organizationId, + JSON.stringify(captureDspMetadata(store, organizationId)), + days, + now, + days === null ? null : now + days * 86400000, + ); + const backup = store.installationBackup(backupId); + require('./backup-categories').recordCategory(store.db, backupId, request + ? require('./backup-categories').categoryForRequest(request) + : backup?.purpose === 'upgrade' ? 'pre_update' : 'manual'); +} +function captureDspMetadata(store, organizationId) { + const db = store.db, + context = managedInstallationContext(store, organizationId); + const rows = (table) => + db.prepare(`SELECT * FROM ${table} WHERE organization_id=?`).all(organizationId); + return { + schemaVersion: 1, + schedule: db.prepare('SELECT * FROM backup_scope_settings WHERE scope=?').get(organizationId) || null, + recovery: { + provisioning: db.prepare("SELECT * FROM installation_provisioning_requests WHERE organization_id=? AND status='completed' ORDER BY updated_at DESC,rowid DESC LIMIT 1").get(organizationId) || null, + installation: db.prepare('SELECT * FROM installations WHERE organization_id=?').get(organizationId), + authority: db.prepare('SELECT * FROM runtime_agent_authorities WHERE organization_id=?').get(organizationId) || null, + activation: db.prepare("SELECT * FROM installation_activation_jobs WHERE organization_id=? AND status='succeeded' ORDER BY updated_at DESC,rowid DESC LIMIT 1").get(organizationId) || null, + lifecycle: ['resume', 'upgrade', 'suspend'].map(operation => db.prepare("SELECT * FROM installation_lifecycle_jobs WHERE organization_id=? AND operation=? AND status='succeeded' ORDER BY updated_at DESC,rowid DESC LIMIT 1").get(organizationId, operation)).filter(Boolean), + }, + organizationId, + manifest: context.manifest, + organization: db.prepare('SELECT * FROM organizations WHERE id=?').get(organizationId), + stations: rows('stations'), + roles: rows('roles'), + memberships: rows('memberships'), + permissions: db + .prepare( + 'SELECT p.* FROM role_permissions p JOIN roles r ON r.id=p.role_id WHERE r.organization_id=?', + ) + .all(organizationId), + users: db + .prepare( + 'SELECT u.* FROM users u JOIN memberships m ON m.user_id=u.id WHERE m.organization_id=?', + ) + .all(organizationId), + profile: + db + .prepare('SELECT * FROM organization_profiles WHERE organization_id=?') + .get(organizationId) || null, + }; +} +function checkDspMetadata(store, organizationId, value) { + if ( + !value || + value.schemaVersion !== 1 || + value.organizationId !== organizationId || + value.organization?.id !== organizationId || + value.manifest?.organization?.id !== organizationId + ) + fail(); + const current = managedInstallationContext(store, organizationId); + // Restore data under the current pinned runtime, then verify that runtime + // before reopening access. Never replace its host identity or registration. + if ( + JSON.stringify(current.manifest.organization) !== JSON.stringify(value.manifest.organization) || + current.manifest.runtime.key !== value.manifest.runtime.key || + current.manifest.runtime.templateId !== value.manifest.runtime.templateId + ) + fail(); + for (const key of ['stations', 'roles', 'memberships']) { + if (!Array.isArray(value[key]) || value[key].some((r) => r.organization_id !== organizationId)) + fail(); + } + if (!Array.isArray(value.users) || !Array.isArray(value.permissions)) fail(); + if (value.profile && value.profile.organization_id !== organizationId) fail(); + for (const user of value.users) { + if (user.platform_role !== null) fail(); + const sameId = store.userById(user.id), + sameEmail = store.userByEmail(user.email); + if ( + (sameId && sameId.email.toLowerCase() !== user.email.toLowerCase()) || + (sameEmail && sameEmail.id !== user.id) + ) + fail(); + } + const roles = new Set(value.roles.map((r) => r.id)), + users = new Set(value.users.map((u) => u.id)); + if ( + value.memberships.some((m) => !roles.has(m.role_id) || !users.has(m.user_id)) || + value.permissions.some((p) => !roles.has(p.role_id)) + ) + fail(); + return value; +} +function restoreDspMetadata(store, organizationId, value, now = Date.now()) { + checkDspMetadata(store, organizationId, value); + return store.transaction(() => { + const db = store.db; + // Sign-in credentials are global identities. Keep current passwords and + // account disablement; restore missing DSP users without ever granting a + // platform role or reviving old sessions/invitations. + const insert = (table, row) => { + const columns = Object.keys(row); + const allowed = new Set( + db + .prepare(`PRAGMA table_info(${table})`) + .all() + .map((c) => c.name), + ); + if (columns.some((c) => !allowed.has(c))) fail(); + db.prepare( + `INSERT INTO ${table} (${columns.join(',')}) VALUES (${columns.map(() => '?').join(',')})`, + ).run(...Object.values(row)); + }; + for (const user of value.users) + if (!store.userById(user.id)) + insert('users', { + ...user, + platform_role: null, + auth_version: user.auth_version + 1, + updated_at: now, + }); + db.prepare('DELETE FROM sessions WHERE active_organization_id=?').run(organizationId); + db.prepare('DELETE FROM invitations WHERE organization_id=?').run(organizationId); + db.prepare('DELETE FROM memberships WHERE organization_id=?').run(organizationId); + db.prepare('DELETE FROM roles WHERE organization_id=?').run(organizationId); + for (const row of value.roles) + insert('roles', { + ...row, + created_by: row.created_by && store.userById(row.created_by) ? row.created_by : null, + }); + for (const row of value.permissions) insert('role_permissions', row); + for (const row of value.memberships) + insert('memberships', { + ...row, + created_by: row.created_by && store.userById(row.created_by) ? row.created_by : null, + }); + db.prepare('DELETE FROM stations WHERE organization_id=?').run(organizationId); + for (const row of value.stations) insert('stations', row); + { + if(value.schedule && value.schedule.scope !== organizationId) fail(); + const savedSettings = value.schedule?.settings_json || JSON.stringify(require('./backup-schedule').DEFAULT_BACKUP_SETTINGS); + require('./backup-schedule').backupSettings(JSON.parse(savedSettings)); + const prior=db.prepare('SELECT revision FROM backup_scope_settings WHERE scope=?').get(organizationId); + db.prepare('INSERT INTO backup_scope_settings VALUES(?,?,?,?) ON CONFLICT(scope) DO UPDATE SET revision=excluded.revision,settings_json=excluded.settings_json,updated_at=excluded.updated_at') + .run(organizationId,(prior?.revision || 0)+1,savedSettings,now); + } + const o = value.organization; + db.prepare( + 'UPDATE organizations SET name=?,abbreviation=?,timezone=?,updated_at=? WHERE id=?', + ).run(o.name, o.abbreviation, o.timezone, now, organizationId); + if (value.profile) { + db.prepare('DELETE FROM organization_profiles WHERE organization_id=?').run(organizationId); + insert('organization_profiles', value.profile); + } + }); +} +module.exports = { captureDspMetadata, checkDspMetadata, restoreDspMetadata, recordDspBackup }; diff --git a/core/core/accounts/src/backup-schedule.js b/core/core/accounts/src/backup-schedule.js new file mode 100644 index 0000000..9d79b81 --- /dev/null +++ b/core/core/accounts/src/backup-schedule.js @@ -0,0 +1,64 @@ +'use strict'; +const { AccessError, exact, timezone } = require('./validation'); +const DEFAULT_BACKUP_SETTINGS = Object.freeze({ + enabled: false, + frequency: 'daily', + time: '02:00', + timezone: 'America/Los_Angeles', + weekday: 0, + retentionDays: null, +}); +function backupSettings(value) { + exact(value, Object.keys(DEFAULT_BACKUP_SETTINGS)); + if ( + Object.keys(value).length !== Object.keys(DEFAULT_BACKUP_SETTINGS).length || + typeof value.enabled !== 'boolean' || + !['hourly', 'daily', 'weekly'].includes(value.frequency) || + !/^([01][0-9]|2[0-3]):[0-5][0-9]$/.test(value.time) || + !Number.isInteger(value.weekday) || + value.weekday < 0 || + value.weekday > 6 || + ![null, 7, 30, 90, 365].includes(value.retentionDays) + ) + throw new AccessError('invalid_input'); + return { ...value, timezone: timezone(value.timezone) }; +} +function scheduleClock(settings) { + const formatter = new Intl.DateTimeFormat('en-US', { + timeZone: settings.timezone, + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + hourCycle: 'h23', + weekday: 'short', + }); + return (now) => Object.fromEntries(formatter.formatToParts(now).map((p) => [p.type, p.value])); +} +function scheduledSlot(settings, now, parts = scheduleClock(settings)(now)) { + if (!settings.enabled) return null; + const date = `${parts.year}-${parts.month}-${parts.day}`; + // A repeated DST hour runs once. A skipped daily clock time runs at the first + // available minute after it. No backlog of obsolete snapshots is generated. + if (settings.frequency === 'hourly') return `${date}T${parts.hour}`; + if (`${parts.hour}:${parts.minute}` < settings.time) return null; + if ( + settings.frequency === 'weekly' && + ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'].indexOf(parts.weekday) !== settings.weekday + ) + return null; + return date; +} +function nextScheduledAt(settings, now) { + if (!settings.enabled) return null; + const parts = scheduleClock(settings), + current = scheduledSlot(settings, now, parts(now)); + const start = Math.floor(now / 60000) * 60000 + 60000; + for (let at = start; at < start + 8 * 86400000; at += 60000) { + const slot = scheduledSlot(settings, at, parts(at)); + if (slot && slot !== current) return new Date(at).toISOString(); + } + return null; +} +module.exports = { DEFAULT_BACKUP_SETTINGS, backupSettings, scheduledSlot, nextScheduledAt }; diff --git a/core/core/accounts/src/backup-schema.js b/core/core/accounts/src/backup-schema.js new file mode 100644 index 0000000..8b4f282 --- /dev/null +++ b/core/core/accounts/src/backup-schema.js @@ -0,0 +1,61 @@ +'use strict'; +// Additive tables: old Core versions can still open the access database during +// recovery. Existing tables, constraints and SCHEMA_VERSION are unchanged. +function initializeBackupSchema(db) { + db.exec(` + CREATE TABLE IF NOT EXISTS backup_categories ( + backup_id TEXT PRIMARY KEY, + category TEXT NOT NULL CHECK(category IN ('scheduled','manual','pre_update')) + ) STRICT; + CREATE TABLE IF NOT EXISTS platform_rollout_backups ( + rollout_id TEXT PRIMARY KEY, set_id TEXT NOT NULL UNIQUE + ) STRICT; + CREATE TABLE IF NOT EXISTS backup_scope_settings ( + scope TEXT PRIMARY KEY, revision INTEGER NOT NULL, + settings_json TEXT NOT NULL, updated_at INTEGER NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS backup_scope_slots ( + scope TEXT NOT NULL, revision INTEGER NOT NULL, slot TEXT NOT NULL, created_at INTEGER NOT NULL, + PRIMARY KEY(scope,revision,slot) + ) STRICT; + CREATE TABLE IF NOT EXISTS backup_sets ( + id TEXT PRIMARY KEY, created_at INTEGER NOT NULL, members_json TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('pending','verified','incomplete','deleting','deleted')) + ) STRICT; + CREATE TABLE IF NOT EXISTS backup_set_settings ( + set_id TEXT PRIMARY KEY, settings_json TEXT NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS backup_deletions ( + id TEXT PRIMARY KEY, backup_id TEXT NOT NULL, organization_id TEXT, + status TEXT NOT NULL CHECK(status IN ('queued','completed','failed')), + created_at INTEGER NOT NULL, failure_code TEXT + ) STRICT; + CREATE TABLE IF NOT EXISTS platform_backup_settings ( + id INTEGER PRIMARY KEY CHECK(id=1), revision INTEGER NOT NULL, + settings_json TEXT NOT NULL, updated_at INTEGER NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS platform_backup_requests ( + id TEXT PRIMARY KEY, organization_id TEXT, kind TEXT NOT NULL CHECK(kind IN ('backup','core','restore')), + status TEXT NOT NULL CHECK(status IN ('queued','running','completed','failed')), + phase TEXT NOT NULL, job_id TEXT, input_json TEXT NOT NULL, actor_user_id TEXT, + idempotency_key TEXT NOT NULL UNIQUE, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, + failure_code TEXT + ) STRICT; + CREATE UNIQUE INDEX IF NOT EXISTS one_platform_backup_operation ON platform_backup_requests(organization_id) + WHERE status IN ('queued','running') AND organization_id IS NOT NULL; + CREATE TABLE IF NOT EXISTS platform_backup_records ( + id TEXT PRIMARY KEY, organization_id TEXT, kind TEXT NOT NULL CHECK(kind IN ('dsp','core')), + metadata_json TEXT NOT NULL, retention_days INTEGER, created_at INTEGER NOT NULL, + expires_at INTEGER, deleted_at INTEGER + ) STRICT; + CREATE TABLE IF NOT EXISTS platform_backup_commands ( + actor_user_id TEXT NOT NULL, idempotency_key TEXT NOT NULL, input_json TEXT NOT NULL, + created_at INTEGER NOT NULL, PRIMARY KEY(actor_user_id,idempotency_key) + ) STRICT; + CREATE TABLE IF NOT EXISTS platform_backup_schedule_slots ( + revision INTEGER NOT NULL, slot TEXT NOT NULL, created_at INTEGER NOT NULL, + PRIMARY KEY(revision,slot) + ) STRICT; + `); +} +module.exports = { initializeBackupSchema }; diff --git a/core/core/accounts/src/core-backup.js b/core/core/accounts/src/core-backup.js new file mode 100644 index 0000000..5acc547 --- /dev/null +++ b/core/core/accounts/src/core-backup.js @@ -0,0 +1,104 @@ +'use strict'; +// Core snapshots contain only platform-owned records. Coordination, tenant +// identities, sessions and archive catalogs always remain live during restore. +const { DatabaseSync } = require('node:sqlite'); +const fail = () => { + throw Error('core_backup_invalid'); +}; +function sanitizeCoreDatabase(file) { + const db = new DatabaseSync(file); + try { + db.exec('PRAGMA foreign_keys=OFF; PRAGMA secure_delete=ON; BEGIN IMMEDIATE'); + for (const { name } of db + .prepare("SELECT name FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'sqlite_%'") + .all()) { + if (!/^[a-z_]+$/.test(name)) fail(); + if (name === 'users') db.exec('DELETE FROM users WHERE platform_role IS NULL'); + else if (name === 'backup_scope_settings') + db.exec("DELETE FROM backup_scope_settings WHERE scope!='core'"); + else db.exec(`DELETE FROM ${name}`); + } + db.exec('COMMIT; VACUUM; PRAGMA foreign_keys=ON'); + verifyCoreDatabase(db); + } finally { + db.close(); + } +} +function verifyCoreDatabase(db) { + if ( + db.prepare('PRAGMA quick_check').get().quick_check !== 'ok' || + db.prepare('PRAGMA foreign_key_check').all().length + ) + fail(); + if ( + db.prepare('SELECT 1 FROM users WHERE platform_role IS NULL').get() || + !db.prepare("SELECT 1 FROM users WHERE platform_role='owner' AND status='active'").get() + ) + fail(); + for (const { name } of db + .prepare("SELECT name FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'sqlite_%'") + .all()) { + if (!/^[a-z_]+$/.test(name)) fail(); + if ( + !['users', 'backup_scope_settings'].includes(name) && + db.prepare(`SELECT 1 FROM ${name} LIMIT 1`).get() + ) + fail(); + } + if (db.prepare("SELECT 1 FROM backup_scope_settings WHERE scope!='core'").get()) fail(); +} +function restoreCoreDatabase(store, file, now = Date.now(), { removeOwnerIds = [] } = {}) { + const saved = new DatabaseSync(file, { readOnly: true }); + try { + verifyCoreDatabase(saved); + return store.transaction(() => { + const db = store.db, + owners = saved.prepare('SELECT * FROM users').all(); + for (const id of removeOwnerIds) { + if (owners.some(row => row.id === id)) fail(); + db.prepare("DELETE FROM users WHERE id=? AND platform_role='owner'").run(id); + } + for (const row of owners) { + const prior = store.userById(row.id), + email = store.userByEmail(row.email); + if ((prior && prior.platform_role !== 'owner') || (email && email.id !== row.id)) fail(); + // Keep current passwords and revocation versions. Recovery must not + // revive old passwords or elevate an existing tenant account. + if (prior) + db.prepare('UPDATE users SET first_name=?,last_name=?,updated_at=? WHERE id=?').run( + row.first_name, + row.last_name, + now, + row.id, + ); + else + db.prepare('INSERT INTO users VALUES(?,?,?,?,?,?,?,?,?,?)').run( + row.id, + row.email, + row.first_name, + row.last_name, + row.password_hash, + row.status, + 'owner', + row.auth_version + 1, + row.created_at, + now, + ); + } + db.prepare( + "DELETE FROM sessions WHERE user_id IN (SELECT id FROM users WHERE platform_role='owner')", + ).run(); + for (const row of saved.prepare('SELECT * FROM backup_scope_settings').all()) { + const live = db + .prepare('SELECT revision FROM backup_scope_settings WHERE scope=?') + .get(row.scope); + db.prepare( + 'INSERT INTO backup_scope_settings VALUES(?,?,?,?) ON CONFLICT(scope) DO UPDATE SET revision=excluded.revision,settings_json=excluded.settings_json,updated_at=excluded.updated_at', + ).run(row.scope, (live?.revision || 0) + 1, row.settings_json, now); + } + }); + } finally { + saved.close(); + } +} +module.exports = { sanitizeCoreDatabase, verifyCoreDatabase, restoreCoreDatabase }; diff --git a/core/core/accounts/src/directory-lifecycle.js b/core/core/accounts/src/directory-lifecycle.js new file mode 100644 index 0000000..8cb5065 --- /dev/null +++ b/core/core/accounts/src/directory-lifecycle.js @@ -0,0 +1,127 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { AccessError, idempotencyKey } = require('./validation'); +const { platformInstallationReceipt, installationFailure } = require('../../../shared/contracts/src'); +const BACKEND = 'directory_service_v1'; +const ACTIVE = ['queued', 'running']; +function fail(code = 'installation_operation_not_allowed') { throw new AccessError(code, 409); } + +function initializeDirectoryLifecycleSchema(db) { + db.exec(`CREATE TABLE IF NOT EXISTS directory_lifecycle_requests ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL REFERENCES organizations(id), + runtime_key TEXT NOT NULL, + actor_user_id TEXT REFERENCES users(id), + idempotency_key TEXT NOT NULL, + action TEXT NOT NULL CHECK(action IN ('suspend','resume','restart','decommission','restore_dsp')), + expected_revision INTEGER NOT NULL, + installation_revision INTEGER NOT NULL, + starting_state TEXT NOT NULL, + starting_organization_status TEXT NOT NULL, + target_state TEXT NOT NULL, + target_organization_status TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('queued','running','succeeded','failed')), + failure_code TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(organization_id,idempotency_key) + ) STRICT; + CREATE UNIQUE INDEX IF NOT EXISTS one_active_directory_lifecycle ON directory_lifecycle_requests(organization_id) + WHERE status IN ('queued','running');`); +} + +function createDirectoryLifecycle({ store, clock = Date.now }) { + const db = store.db; + const latest = id => db.prepare('SELECT * FROM directory_lifecycle_requests WHERE organization_id=? ORDER BY created_at DESC,rowid DESC LIMIT 1').get(id); + const active = id => db.prepare("SELECT * FROM directory_lifecycle_requests WHERE organization_id=? AND status IN ('queued','running')").get(id); + const removal = id => db.prepare('SELECT * FROM dsp_removals WHERE organization_id=?').get(id); + + function receipt(row, replayed) { + const current = store.installationControl(row.organization_id); + return platformInstallationReceipt({ action: row.action, status: replayed ? 'replayed' : 'accepted', + installationState: current.status, installationRevision: current.revision, replayed }); + } + + function request({ organizationId, actorUserId = null, action, expectedRevision, requestId }) { + idempotencyKey(requestId); + if (!['suspend', 'resume', 'restart', 'decommission', 'restore_dsp'].includes(action)) fail(); + return store.transaction(() => { + const control = store.installationControl(organizationId), organization = store.organization(organizationId); + if (action === 'decommission' && control?.runtimeKey === store.permanentDevId) fail('directory_dev_protected'); + if (store.releaseBlocked?.(organizationId)) fail('release_busy'); + if (store.directoryDeletion?.get(organizationId)) fail('installation_operation_not_allowed'); + if (!control || store.installationBackend(organizationId) !== BACKEND) fail(); + const prior = db.prepare('SELECT * FROM directory_lifecycle_requests WHERE organization_id=? AND idempotency_key=?').get(organizationId, requestId); + if (prior) { + if (prior.action !== action || prior.expected_revision !== expectedRevision || prior.actor_user_id !== actorUserId) fail('idempotency_conflict'); + return receipt(prior, true); + } + if (expectedRevision !== control.revision) fail('installation_revision_conflict'); + if (active(organizationId) || store.activeLifecycleJob(organizationId) || store.runningActivationJob(organizationId) + || db.prepare("SELECT 1 FROM installation_onboarding_requests WHERE organization_id=? AND status IN ('enrolling','queued','running')").get(organizationId)) fail('installation_operation_in_progress'); + const removed = removal(organizationId), previous = latest(organizationId); + const retry = previous?.status === 'failed' && previous.action === action && control.currentJobId === previous.id; + if (!retry && (removed ? action !== 'restore_dsp' || control.status !== 'decommissioned' + : action === 'suspend' ? !['ready', 'waiting_for_owner', 'waiting_for_provider_auth'].includes(control.status) + : action === 'resume' ? control.status !== 'suspended' + : action === 'restart' ? control.status !== 'ready' || organization.status !== 'active' + : action === 'decommission' ? !['ready', 'waiting_for_owner', 'waiting_for_provider_auth', 'suspended', 'failed'].includes(control.status) + : true)) fail(); + const timestamp = clock(), id = `dop_${crypto.randomBytes(16).toString('hex')}`; + const startingState = retry ? previous.starting_state : control.status; + const startingOrganization = retry ? previous.starting_organization_status : organization.status; + let targetState, targetOrganization; + if (retry) { targetState = previous.target_state; targetOrganization = previous.target_organization_status; } + else if (action === 'suspend') { targetState = 'suspended'; targetOrganization = 'suspended'; } + else if (action === 'decommission') { targetState = 'decommissioned'; targetOrganization = 'suspended'; } + else if (action === 'restore_dsp') { targetState = removed.installation_state; targetOrganization = removed.organization_status; } + else if (action === 'resume') { + const suspended = db.prepare("SELECT * FROM directory_lifecycle_requests WHERE organization_id=? AND action='suspend' AND status='succeeded' ORDER BY created_at DESC,rowid DESC LIMIT 1").get(organizationId); + if (!suspended) fail(); + targetState = suspended.starting_state; targetOrganization = suspended.starting_organization_status; + } else { targetState = 'ready'; targetOrganization = 'active'; } + if (action === 'decommission' && !removed) { + db.prepare('INSERT INTO dsp_removals(organization_id,installation_state,organization_status,removed_at,actor_user_id) VALUES(?,?,?,?,?)') + .run(organizationId, startingState, startingOrganization, timestamp, actorUserId); + } + db.prepare(`INSERT INTO directory_lifecycle_requests VALUES(?,?,?,?,?,?,?,?,?,?,?,?,'queued',NULL,?,?)`) + .run(id, organizationId, control.runtimeKey, actorUserId, requestId, action, expectedRevision, expectedRevision + 1, + startingState, startingOrganization, targetState, targetOrganization, timestamp, timestamp); + const state = action === 'decommission' ? 'decommissioning' : action === 'suspend' ? 'suspended' : 'verifying'; + store.updateInstallationControl({ organizationId, expectedStatus: control.status, expectedRevision, + status: state, revision: expectedRevision + 1, currentJobId: id, timestamp }); + // Gate user access immediately; the worker then makes the runtime match. + store.updateOrganizationStatus(organizationId, ['suspend', 'decommission'].includes(action) || targetState === 'suspended' + ? 'suspended' : targetOrganization, timestamp); + store.createAudit({ id: `aud_${id}`, actorUserId, organizationId, action: `installation.${action}.requested`, + targetType: 'organization', targetId: organizationId, result: 'succeeded', timestamp }); + require('./worker-wakeup').afterCommit(store, ['reconcile']); + return receipt(latest(organizationId), false); + }); + } + + function projection(organizationId, fallback, enabled) { + const row = latest(organizationId), control = store.installationControl(organizationId), removed = removal(organizationId); + if (!row && !enabled) return fallback; + const availableActions = [...fallback.availableActions]; + if (enabled && !active(organizationId)) { + if (row?.status === 'failed' && control.currentJobId === row.id) availableActions.push(row.action); + else if (removed) { if (control.status === 'decommissioned') availableActions.push('restore_dsp'); } + else { + if (control.status === 'ready') availableActions.push('restart'); + if (['ready', 'waiting_for_owner', 'waiting_for_provider_auth'].includes(control.status)) availableActions.push('suspend'); + if (control.status === 'suspended') availableActions.push('resume'); + if (['ready', 'waiting_for_owner', 'waiting_for_provider_auth', 'suspended', 'failed'].includes(control.status)) availableActions.push('decommission'); + } + } + return { ...fallback, availableActions: [...new Set(availableActions)], + ...(row && (control.currentJobId === row.id || ACTIVE.includes(row.status)) ? { + operation: { kind: row.action, status: row.status }, + failure: row.status === 'failed' ? installationFailure(row.failure_code) : null, + } : {}) }; + } + return { request, projection, latest, active, removal }; +} + +module.exports = { initializeDirectoryLifecycleSchema, createDirectoryLifecycle }; diff --git a/core/core/accounts/src/erase-dsp.js b/core/core/accounts/src/erase-dsp.js new file mode 100644 index 0000000..121f600 --- /dev/null +++ b/core/core/accounts/src/erase-dsp.js @@ -0,0 +1,82 @@ +'use strict'; + +// Shared by the live account store and retained Core snapshots. Discover foreign +// keys so newly added tenant tables cannot silently escape deletion. +function eraseDsp(db, { organizationId, runtimeKey }) { + if (!/^org_[a-zA-Z0-9_]+$/.test(organizationId) || !/^dsp_[a-f0-9]{32}$/.test(runtimeKey)) throw Error('invalid_erasure_identity'); + const tables = db.prepare("SELECT name FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'sqlite_%'").all().map(row => row.name); + if (tables.some(name => !/^[a-z_]+$/.test(name))) throw Error('erasure_schema_unsupported'); + const columns = new Map(tables.map(name => [name, db.prepare(`PRAGMA table_info("${name}")`).all()])); + const links = []; + for (const child of tables) { + const groups = new Map(); + for (const fk of db.prepare(`PRAGMA foreign_key_list("${child}")`).all()) { + if (!groups.has(fk.id)) groups.set(fk.id, []); + groups.get(fk.id).push(fk); + } + for (const group of groups.values()) { + const parent = group[0].table; + if (!columns.has(parent)) throw Error('erasure_schema_unsupported'); + const keys = columns.get(parent).filter(c => c.pk).sort((a, b) => a.pk - b.pk); + const join = group.sort((a, b) => a.seq - b.seq).map((fk, i) => { + const to = fk.to || keys[i]?.name; + if (![fk.from, to].every(name => /^[a-z_]+$/.test(name))) throw Error('erasure_schema_unsupported'); + return `c."${fk.from}"=p."${to}"`; + }).join(' AND '); + links.push({ child, parent, join }); + } + } + db.exec('PRAGMA secure_delete=ON; BEGIN IMMEDIATE; PRAGMA defer_foreign_keys=ON; CREATE TEMP TABLE erasure_rows(table_name TEXT,row_id INTEGER,PRIMARY KEY(table_name,row_id));'); + try { + const seed = (table, condition, ...args) => db.prepare(`INSERT OR IGNORE INTO erasure_rows SELECT ?,rowid FROM "${table}" WHERE ${condition}`).run(table, ...args); + for (const table of tables) { + const names = columns.get(table).map(c => c.name); + for (const key of ['organization_id', 'active_organization_id']) if (names.includes(key)) seed(table, `"${key}"=?`, organizationId); + if (names.includes('runtime_key')) seed(table, 'runtime_key=?', runtimeKey); + if (names.includes('scope')) seed(table, 'scope IN (?,?)', organizationId, `dsp:${organizationId}`); + if (table === 'audit_events') seed(table, 'target_id IN (?,?)', organizationId, runtimeKey); + } + seed('organizations', 'id=?', organizationId); + const expand = () => { + let changed; + do { + changed = 0; + for (const { child, parent, join } of links) changed += db.prepare(`INSERT OR IGNORE INTO erasure_rows + SELECT ?,c.rowid FROM "${child}" c JOIN "${parent}" p ON ${join} + JOIN erasure_rows e ON e.table_name=? AND e.row_id=p.rowid`).run(child, parent).changes; + } while (changed); + }; + expand(); + // Platform users and accounts referenced by another DSP remain independent + // identities. Remove accounts that existed only for the erased DSP. + const candidates = db.prepare(`SELECT DISTINCT u.* FROM users u JOIN memberships m ON m.user_id=u.id + WHERE m.organization_id=? AND u.platform_role IS NULL`).all(organizationId); + const privateChildren = new Set(['sessions', 'password_reset_tokens', 'release_popup_dismissals', 'audit_events', 'platform_target_refs', 'platform_mutation_requests']); + const removedUsers = []; + for (const user of candidates) { + const userRow = db.prepare('SELECT rowid FROM users WHERE id=?').get(user.id).rowid; + const shared = links.filter(link => link.parent === 'users').some(({ child, join }) => { + const names = columns.get(child).map(c => c.name); + if (privateChildren.has(child) && !names.includes('organization_id')) return false; + const scope = privateChildren.has(child) ? ' AND c.organization_id IS NOT NULL AND c.organization_id<>?' : ''; + return Boolean(db.prepare(`SELECT 1 FROM "${child}" c JOIN users p ON ${join} WHERE p.rowid=? + AND NOT EXISTS(SELECT 1 FROM erasure_rows e WHERE e.table_name=? AND e.row_id=c.rowid)${scope} LIMIT 1`) + .get(userRow, child, ...(scope ? [organizationId] : []))); + }); + if (!shared) { seed('users', 'id=?', user.id); removedUsers.push(user.id); } + } + expand(); + for (const table of tables) db.prepare(`DELETE FROM "${table}" WHERE rowid IN (SELECT row_id FROM erasure_rows WHERE table_name=?)`).run(table); + if (db.prepare('PRAGMA foreign_key_check').all().length) throw Error('erasure_integrity_failed'); + db.exec('DROP TABLE erasure_rows; COMMIT'); + return { removedUsers }; + } catch (error) { db.exec('ROLLBACK'); throw error; } +} + +function compactErasedDatabase(db) { + db.exec('PRAGMA secure_delete=ON; VACUUM'); + const checkpoint = db.prepare('PRAGMA wal_checkpoint(TRUNCATE)').get(); + if (checkpoint.busy) throw Error('erasure_checkpoint_busy'); + if (db.prepare('PRAGMA quick_check').get().quick_check !== 'ok') throw Error('erasure_integrity_failed'); +} +module.exports = { eraseDsp, compactErasedDatabase }; diff --git a/core/core/accounts/src/fixed-roles-migration.js b/core/core/accounts/src/fixed-roles-migration.js new file mode 100644 index 0000000..ab47d7c --- /dev/null +++ b/core/core/accounts/src/fixed-roles-migration.js @@ -0,0 +1,50 @@ +'use strict'; + +const { randomUUID } = require('node:crypto'); +const { SYSTEM_ROLES } = require('./permissions'); + +// Called inside the schema transaction. Keep canonical role IDs and repoint all +// memberships and invitations before deleting retired roles (including custom +// roles whose names collide with the new catalog). +function migrateFixedRoles(db) { + const timestamp = Date.now(); + for (const organization of db.prepare('SELECT id FROM organizations').all()) { + const existing = db.prepare('SELECT * FROM roles WHERE organization_id=?').all(organization.id); + const targets = new Map(); + for (const definition of SYSTEM_ROLES) { + let role = existing.find(candidate => candidate.key === definition.key); + if (!role) { + const id = `role_${randomUUID().replaceAll('-', '')}`; + db.prepare(`INSERT INTO roles VALUES(?,?,?,?,?,1,NULL,?,?)`).run( + id, organization.id, definition.key, id, definition.description, timestamp, timestamp, + ); + role = { id }; + } + targets.set(definition.key, role.id); + } + for (const role of existing) { + if (targets.get(role.key) === role.id) continue; + const key = role.key === 'administrator' ? 'manager' + : role.key === 'viewer' ? 'driver' + : SYSTEM_ROLES.find(definition => definition.name.toLowerCase() === role.name.trim().toLowerCase())?.key || 'dispatcher'; + const target = targets.get(key); + db.prepare('UPDATE memberships SET role_id=?,updated_at=? WHERE organization_id=? AND role_id=?') + .run(target, timestamp, organization.id, role.id); + // Preserve tokens, expiry, status, and acceptance history. + db.prepare('UPDATE invitations SET role_id=? WHERE organization_id=? AND role_id=?') + .run(target, organization.id, role.id); + db.prepare('DELETE FROM roles WHERE id=?').run(role.id); + } + for (const definition of SYSTEM_ROLES) { + const roleId = targets.get(definition.key); + db.prepare('UPDATE roles SET name=?,description=?,is_system=1,updated_at=? WHERE id=?') + .run(definition.name, definition.description, timestamp, roleId); + db.prepare('DELETE FROM role_permissions WHERE role_id=?').run(roleId); + for (const permission of definition.permissions) { + db.prepare('INSERT INTO role_permissions VALUES(?,?)').run(roleId, permission); + } + } + } +} + +module.exports = { migrateFixedRoles }; diff --git a/core/core/accounts/src/index.js b/core/core/accounts/src/index.js new file mode 100644 index 0000000..93158e2 --- /dev/null +++ b/core/core/accounts/src/index.js @@ -0,0 +1,25 @@ +'use strict'; + +const { AccessStore } = require('./store'); +const { AccessControlService } = require('./service'); +const installationAuthority = require('./installation-authority'); +const installationProvisioning = require('./installation-provisioning'); +const installationActivation = require('./installation-activation'); +const installationLifecycle = require('./installation-lifecycle'); +const runtimeAgentAuthority = require('./runtime-agent-authority'); +const validation = require('./validation'); +const permissions = require('./permissions'); +const passwords = require('./passwords'); + +module.exports = { + AccessStore, + AccessControlService, + ...installationAuthority, + ...installationProvisioning, + ...installationActivation, + ...installationLifecycle, + ...runtimeAgentAuthority, + ...validation, + ...permissions, + ...passwords, +}; diff --git a/core/core/accounts/src/installation-activation.js b/core/core/accounts/src/installation-activation.js new file mode 100644 index 0000000..9b60650 --- /dev/null +++ b/core/core/accounts/src/installation-activation.js @@ -0,0 +1,522 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { + IDEMPOTENCY_RE, + installationFailure, + installationJob, + installationActivationEvidence, + installationTransition, + serverInstallationActivation, +} = require('../../../shared/contracts/src'); +const { AccessError, identifier } = require('./validation'); +const { + DEFAULT_MANAGED_TEMPLATE_ID, + DEFAULT_MANAGED_RELEASE_ID, + managedInstallationContext, +} = require('./installation-authority'); + +const DEFAULT_ACTIVATION_LEASE_MS = 5 * 60 * 1000; +const DEFAULT_SETUP_LEASE_MS = 20 * 60 * 1000; +const MAX_PROVIDER_EVIDENCE_AGE_MS = 15 * 60 * 1000; +const MAX_ACTIVATION_EVIDENCE_AGE_MS = 15 * 60 * 1000; + + +function fail(code, statusCode = 409) { + throw new AccessError(code, statusCode); +} + +function activationJobView(row, replayed = false) { + if (!row) fail('installation_operation_not_found'); + return installationJob({ + id: row.id, + operation: row.operation, + status: row.status, + installationState: row.installation_state, + revision: row.installation_revision, + replayed, + failure: row.failure_code === null ? null : installationFailure(row.failure_code), + }); +} + +function providerEvidence(value, now) { + if (!value || typeof value !== 'object' || Array.isArray(value) + || Object.keys(value).sort().join(',') !== 'profileId,provider,status,testedAt' + || value.profileId !== 'paycom-main' || value.provider !== 'paycom' + || value.status !== 'authenticated' || typeof value.testedAt !== 'string') { + fail('provider_auth_required'); + } + const testedAt = Date.parse(value.testedAt); + if (!Number.isSafeInteger(testedAt) || testedAt > now + 60_000 || now - testedAt > MAX_PROVIDER_EVIDENCE_AGE_MS) { + fail('provider_auth_required'); + } + return testedAt; +} + +function createAccessInstallationActivationAuthority(options) { + if (!options || typeof options !== 'object' || Array.isArray(options) + || Object.keys(options).some(key => ![ + 'store', 'organizationId', 'authorityScope', 'idempotencyKey', 'workerId', 'clock', + 'leaseMs', 'setupLeaseMs', 'jobFactory', 'templateId', 'releaseId', + ].includes(key))) fail('runtime_boundary_violation', 500); + const store = options.store; + if (!store || !['transaction', 'installationControl', 'installationSetup', 'activationJob'] + .every(method => typeof store[method] === 'function')) fail('runtime_boundary_violation', 500); + const organizationId = identifier(options.organizationId); + const authorityScope = identifier(options.authorityScope); + const workerId = identifier(options.workerId); + const idempotencyKey = options.idempotencyKey; + if (typeof idempotencyKey !== 'string' || idempotencyKey.length < 16 || idempotencyKey.length > 128 + || !IDEMPOTENCY_RE.test(idempotencyKey)) fail('invalid_input', 400); + const clock = options.clock === undefined ? Date.now : options.clock; + const jobFactory = options.jobFactory === undefined + ? () => `act_${crypto.randomUUID().replaceAll('-', '')}` : options.jobFactory; + const leaseMs = options.leaseMs === undefined ? DEFAULT_ACTIVATION_LEASE_MS : options.leaseMs; + const setupLeaseMs = options.setupLeaseMs === undefined ? DEFAULT_SETUP_LEASE_MS : options.setupLeaseMs; + const templateId = options.templateId === undefined ? DEFAULT_MANAGED_TEMPLATE_ID : options.templateId; + const releaseId = options.releaseId === undefined ? DEFAULT_MANAGED_RELEASE_ID : options.releaseId; + if (typeof clock !== 'function' || typeof jobFactory !== 'function' + || !Number.isSafeInteger(leaseMs) || leaseMs < 10_000 || leaseMs > 2 * 60 * 60 * 1000 + || !Number.isSafeInteger(setupLeaseMs) || setupLeaseMs < 10_000 || setupLeaseMs > 2 * 60 * 60 * 1000 + || !/^[a-z][a-z0-9_.-]{2,95}$/.test(templateId) + || !/^[a-z][a-z0-9_.-]{2,95}$/.test(releaseId)) fail('runtime_boundary_violation', 500); + let claim = null; + let setupClaim = null; + + function now() { + const value = clock(); + if (!Number.isSafeInteger(value) || value < 0) fail('installation_operation_failed', 500); + return value; + } + + function manifestContext() { + return managedInstallationContext(store, organizationId, { templateId, releaseId }); + } + + function requireUsableOrganization(context) { + if (!context.ownerActive || !['setup_required', 'active'].includes(context.organization.status)) { + fail('installation_not_ready'); + } + } + + function claimRow(row, timestamp) { + if (!row || row.status !== 'running') fail('installation_not_ready'); + if (row.worker_id === workerId && row.lease_expires_at > timestamp) { + row = store.renewActivationJob(row.id, workerId, row.fence, timestamp + leaseMs, timestamp); + } else if (row.lease_expires_at <= timestamp) { + row = store.claimActivationJob(row.id, workerId, row.fence, timestamp + leaseMs, timestamp); + } else { + fail('installation_operation_in_progress'); + } + claim = Object.freeze({ id: row.id, fence: row.fence }); + return row; + } + + function contextView(context, row = null, replayed = false) { + return Object.freeze({ + manifest: context.manifest, + manifestAuthority: Object.freeze(context.manifestAuthority), + installation: Object.freeze({ + state: context.installation.status, + revision: context.installation.revision, + currentJobId: context.installation.currentJobId, + }), + job: row ? activationJobView(row, replayed) : null, + owner: Object.freeze({ active: context.ownerActive }), + }); + } + + function completedRow(context) { + if (context.installation.status !== 'ready' || !context.installation.currentJobId) { + fail('installation_not_ready'); + } + const row = store.activationJob(context.installation.currentJobId); + if (!row || row.status !== 'succeeded' || row.installation_state !== 'ready' + || row.installation_revision !== context.installation.revision + || row.manifest_revision !== context.installation.manifestRevision + || row.runtime_key !== context.installation.runtimeKey || row.evidence_json === null + || row.evidence_digest === null) fail('installation_not_ready'); + let evidence; + try { evidence = installationActivationEvidence(JSON.parse(row.evidence_json)); } + catch { fail('installation_not_ready'); } + if (evidence.evidenceDigest !== row.evidence_digest || evidence.jobId !== row.id) { + fail('installation_not_ready'); + } + return row; + } + + function peek() { + return store.transaction(() => { + const context = manifestContext(); + requireUsableOrganization(context); + if (context.installation.status === 'waiting_for_provider_auth') { + if (context.installation.currentJobId !== null) fail('runtime_boundary_violation', 500); + return contextView(context); + } + if (context.installation.status === 'ready') return contextView(context, completedRow(context), true); + if (context.installation.status !== 'verifying' || !context.installation.currentJobId) { + fail('installation_not_ready'); + } + const row = store.activationJob(context.installation.currentJobId); + if (!row || row.status !== 'running') fail('installation_not_ready'); + return contextView(context, row, row.authority_scope === authorityScope + && row.idempotency_key === idempotencyKey); + }); + } + + function inspect() { + return store.transaction(() => { + const timestamp = now(); + const context = manifestContext(); + requireUsableOrganization(context); + if (context.installation.status === 'waiting_for_provider_auth') { + if (context.installation.currentJobId !== null) fail('runtime_boundary_violation', 500); + claim = null; + return contextView(context); + } + if (context.installation.status === 'ready') { + claim = null; + return contextView(context, completedRow(context), true); + } + if (context.installation.status !== 'verifying' || !context.installation.currentJobId) { + fail('installation_not_ready'); + } + const row = claimRow(store.activationJob(context.installation.currentJobId), timestamp); + return contextView(manifestContext(), row, row.authority_scope === authorityScope + && row.idempotency_key === idempotencyKey); + }); + } + + function begin(evidence) { + return store.transaction(() => { + const timestamp = now(); + const testedAt = providerEvidence(evidence, timestamp); + let context = manifestContext(); + requireUsableOrganization(context); + const prior = store.activationJobByRequest(organizationId, authorityScope, idempotencyKey); + if (prior) { + if (prior.runtime_key !== context.installation.runtimeKey + || prior.manifest_revision !== context.installation.manifestRevision + || prior.provider !== 'paycom' || prior.profile_id !== 'paycom-main') { + fail('idempotency_conflict'); + } + if (context.installation.status !== 'verifying' || context.installation.currentJobId !== prior.id) { + fail('installation_operation_not_allowed'); + } + const row = claimRow(prior, timestamp); + return contextView(manifestContext(), row, true); + } + if (context.installation.status !== 'waiting_for_provider_auth' + || context.installation.currentJobId !== null) fail('installation_operation_not_allowed'); + const setup = store.installationSetup(organizationId); + if (setup?.workerId !== null && setup?.leaseExpiresAt > timestamp) { + fail('installation_operation_in_progress'); + } + if (store.runningActivationJob(organizationId)) fail('installation_operation_in_progress'); + installationTransition('waiting_for_provider_auth', 'verifying'); + const jobId = identifier(jobFactory()); + const nextRevision = context.installation.revision + 1; + store.updateInstallationControl({ + organizationId, + expectedStatus: 'waiting_for_provider_auth', + expectedRevision: context.installation.revision, + status: 'verifying', + revision: nextRevision, + currentJobId: jobId, + timestamp, + }); + const row = store.createActivationJob({ + id: jobId, + organizationId, + installationRevision: nextRevision, + manifestRevision: context.installation.manifestRevision, + runtimeKey: context.installation.runtimeKey, + authorityScope, + idempotencyKey, + workerId, + leaseExpiresAt: timestamp + leaseMs, + providerTestedAt: testedAt, + timestamp, + }); + claim = Object.freeze({ id: row.id, fence: row.fence }); + store.createAudit({ + id: `aud_${crypto.randomUUID().replaceAll('-', '')}`, + actorUserId: null, + organizationId, + action: 'installation.activation.begin', + targetType: 'installation_job', + targetId: row.id, + result: 'succeeded', + timestamp, + }); + context = manifestContext(); + return contextView(context, row); + }); + } + + function claimedRow(jobId, timestamp) { + if (!claim || claim.id !== jobId) fail('installation_operation_in_progress'); + const row = store.activationJob(jobId); + if (!row || row.status !== 'running' || row.worker_id !== workerId || row.fence !== claim.fence + || row.lease_expires_at <= timestamp) fail('installation_operation_in_progress'); + const control = store.installationControl(organizationId); + if (!control || control.status !== 'verifying' || control.currentJobId !== jobId + || control.runtimeKey !== row.runtime_key || control.manifestRevision !== row.manifest_revision + || control.revision !== row.installation_revision) fail('installation_operation_in_progress'); + return { row, control }; + } + + function heartbeat() { + return store.transaction(() => { + const timestamp = now(); + if (!claim) fail('installation_operation_in_progress'); + const { row } = claimedRow(claim.id, timestamp); + const renewed = store.renewActivationJob( + row.id, workerId, row.fence, timestamp + leaseMs, timestamp, + ); + claim = Object.freeze({ id: renewed.id, fence: renewed.fence }); + return contextView(manifestContext(), renewed, true); + }); + } + + function commit(activation, activationAuthorityValue) { + return store.transaction(() => { + const timestamp = now(); + const jobId = activationAuthorityValue?.jobId; + const { row, control } = claimedRow(jobId, timestamp); + const context = manifestContext(); + requireUsableOrganization(context); + const expectedJob = activationJobView(row); + const suppliedJob = installationJob({ ...activation?.job, replayed: false }); + if (JSON.stringify(suppliedJob) !== JSON.stringify(expectedJob)) fail('installation_not_ready'); + const selected = serverInstallationActivation({ + manifest: activation?.manifest, + job: expectedJob, + readiness: activation?.readiness, + evidence: activation?.evidence, + }, { + manifestAuthority: context.manifestAuthority, + jobId: row.id, + }); + const evidenceCapturedAt = Date.parse(selected.evidence.capturedAt); + if (!Number.isSafeInteger(evidenceCapturedAt) || evidenceCapturedAt > timestamp + 60_000 + || timestamp - evidenceCapturedAt > MAX_ACTIVATION_EVIDENCE_AGE_MS) fail('installation_not_ready'); + if (activationAuthorityValue.jobId !== row.id + || JSON.stringify(activationAuthorityValue.manifestAuthority) !== JSON.stringify(context.manifestAuthority)) { + fail('installation_not_ready'); + } + installationTransition('verifying', 'ready', { + activation: { + manifest: selected.manifest, + job: expectedJob, + readiness: selected.readiness, + evidence: selected.evidence, + }, + authority: { manifestAuthority: context.manifestAuthority, jobId: row.id }, + }); + const nextRevision = control.revision + 1; + store.finishActivationJob( + row.id, workerId, row.fence, 'succeeded', 'ready', nextRevision, null, + selected.evidence, timestamp, + ); + store.updateInstallationControl({ + organizationId, + expectedStatus: 'verifying', + expectedRevision: control.revision, + status: 'ready', + revision: nextRevision, + currentJobId: row.id, + timestamp, + }); + if (context.organization.status === 'setup_required') { + store.updateOrganizationStatus(organizationId, 'active', timestamp); + } + store.createAudit({ + id: `aud_${crypto.randomUUID().replaceAll('-', '')}`, + actorUserId: null, + organizationId, + action: 'installation.activation.ready', + targetType: 'installation_job', + targetId: row.id, + result: 'succeeded', + timestamp, + }); + const installed = store.installationControl(organizationId); + const finished = store.activationJob(row.id); + const organization = store.organization(organizationId); + if (installed?.status !== 'ready' || installed.revision !== nextRevision + || installed.currentJobId !== row.id || finished?.status !== 'succeeded' + || finished.installation_state !== 'ready' || finished.installation_revision !== nextRevision + || finished.evidence_digest !== selected.evidence.evidenceDigest + || organization?.status !== 'active') fail('installation_not_ready'); + claim = null; + return Object.freeze({ state: installed.status, revision: installed.revision }); + }); + } + + function failActivation(jobIdValue, failureValue) { + return store.transaction(() => { + const timestamp = now(); + const jobId = identifier(jobIdValue); + const failure = installationFailure(failureValue); + const { row, control } = claimedRow(jobId, timestamp); + installationTransition('verifying', 'failed'); + const nextRevision = control.revision + 1; + store.finishActivationJob( + row.id, workerId, row.fence, 'failed', 'failed', nextRevision, failure.code, null, timestamp, + ); + store.updateInstallationControl({ + organizationId, + expectedStatus: 'verifying', + expectedRevision: control.revision, + status: 'failed', + revision: nextRevision, + currentJobId: row.id, + timestamp, + }); + store.createAudit({ + id: `aud_${crypto.randomUUID().replaceAll('-', '')}`, + actorUserId: null, + organizationId, + action: 'installation.activation.fail', + targetType: 'installation_job', + targetId: row.id, + result: 'succeeded', + timestamp, + }); + claim = null; + return activationJobView(store.activationJob(row.id)); + }); + } + + function retryFailure() { + return store.transaction(() => { + const timestamp = now(); + const context = manifestContext(); + requireUsableOrganization(context); + if (context.installation.status !== 'failed' || !context.installation.currentJobId) { + fail('installation_operation_not_allowed'); + } + const failedJob = store.activationJob(context.installation.currentJobId); + if (!failedJob || failedJob.status !== 'failed' || failedJob.installation_state !== 'failed' + || failedJob.runtime_key !== context.installation.runtimeKey + || failedJob.manifest_revision !== context.installation.manifestRevision) { + fail('installation_operation_not_allowed'); + } + installationTransition('failed', 'provisioning'); + installationTransition('provisioning', 'waiting_for_provider_auth'); + const selected = store.updateInstallationControl({ + organizationId, + expectedStatus: 'failed', + expectedRevision: context.installation.revision, + status: 'waiting_for_provider_auth', + revision: context.installation.revision + 2, + currentJobId: null, + timestamp, + }); + store.createAudit({ + id: `aud_${crypto.randomUUID().replaceAll('-', '')}`, + actorUserId: null, + organizationId, + action: 'installation.activation.retry', + targetType: 'installation_job', + targetId: failedJob.id, + result: 'succeeded', + timestamp, + }); + claim = null; + return contextView({ ...manifestContext(), installation: selected }); + }); + } + + function guardSetupMutation(mutation) { + if (typeof mutation !== 'function') fail('runtime_boundary_violation', 500); + const renew = () => store.transaction(() => { + const timestamp = now(); + if (!setupClaim) fail('installation_operation_in_progress'); + const context = manifestContext(); + requireUsableOrganization(context); + const control = context.installation; + const setup = store.installationSetup(organizationId); + if (control.status !== 'waiting_for_provider_auth' || control.currentJobId !== null + || setup?.workerId !== workerId || setup?.fence !== setupClaim.fence + || setup?.leaseExpiresAt <= timestamp) fail('installation_operation_in_progress'); + store.renewInstallationSetup( + organizationId, workerId, setupClaim.fence, timestamp + setupLeaseMs, timestamp, + ); + const renewed = store.installationSetup(organizationId); + setupClaim = Object.freeze({ fence: renewed.fence }); + return control; + }); + const before = renew(); + const result = mutation(); + if (result && typeof result.then === 'function') fail('runtime_boundary_violation', 500); + const after = renew(); + if (after.revision !== before.revision || after.manifestRevision !== before.manifestRevision + || after.runtimeKey !== before.runtimeKey) fail('installation_operation_in_progress'); + return result; + } + + function beginSetup() { + return store.transaction(() => { + const timestamp = now(); + const context = manifestContext(); + requireUsableOrganization(context); + const control = context.installation; + const setup = store.installationSetup(organizationId); + if (control.status !== 'waiting_for_provider_auth' || control.currentJobId !== null) { + fail('installation_operation_not_allowed'); + } + let selected; + if (setup.workerId === workerId && setup.leaseExpiresAt > timestamp) { + selected = store.renewInstallationSetup( + organizationId, workerId, setup.fence, timestamp + setupLeaseMs, timestamp, + ); + } else { + if (setup.workerId !== null && setup.leaseExpiresAt > timestamp) { + fail('installation_operation_in_progress'); + } + selected = store.claimInstallationSetup( + organizationId, workerId, setup.fence, timestamp + setupLeaseMs, timestamp, + ); + } + setupClaim = Object.freeze({ fence: store.installationSetup(organizationId).fence }); + return contextView({ ...context, installation: selected }); + }); + } + + function endSetup() { + return store.transaction(() => { + const timestamp = now(); + if (!setupClaim) fail('installation_operation_in_progress'); + const selected = store.releaseInstallationSetup( + organizationId, workerId, setupClaim.fence, timestamp, + ); + setupClaim = null; + return contextView({ ...manifestContext(), installation: selected }); + }); + } + + return Object.freeze({ + peek, + inspect, + begin, + heartbeat, + commit, + fail: failActivation, + retry: retryFailure, + beginSetup, + endSetup, + guard: guardSetupMutation, + }); +} + +module.exports = { + DEFAULT_ACTIVATION_LEASE_MS, + DEFAULT_SETUP_LEASE_MS, + MAX_PROVIDER_EVIDENCE_AGE_MS, + DEFAULT_MANAGED_TEMPLATE_ID, + DEFAULT_MANAGED_RELEASE_ID, + activationJobView, + createAccessInstallationActivationAuthority, +}; diff --git a/core/core/accounts/src/installation-authority.js b/core/core/accounts/src/installation-authority.js new file mode 100644 index 0000000..6f9155c --- /dev/null +++ b/core/core/accounts/src/installation-authority.js @@ -0,0 +1,71 @@ +'use strict'; + +const { serverInstallationManifest } = require('../../../shared/contracts/src'); +const { AccessError, identifier } = require('./validation'); +const { runtimeBackend } = require('../../runtime-deployment'); + +const DEFAULT_MANAGED_TEMPLATE_ID = 'isolated_dsp_v1'; +const DEFAULT_MANAGED_RELEASE_ID = 'dispatch_current_1'; +const CATALOG_IDENTIFIER_RE = /^[a-z][a-z0-9_.-]{2,95}$/; + +function fail(code, statusCode = 409) { + throw new AccessError(code, statusCode); +} + +function managedInstallationContext(store, organizationIdValue, options = {}) { + if (!store || typeof store.organization !== 'function' || typeof store.installationControl !== 'function' + || typeof store.installationBackend !== 'function' + || !options || typeof options !== 'object' || Array.isArray(options) + || Object.keys(options).some(key => !['templateId', 'releaseId', 'backend'].includes(key))) { + fail('runtime_boundary_violation', 500); + } + const organizationId = identifier(organizationIdValue); + const templateId = options.templateId === undefined ? DEFAULT_MANAGED_TEMPLATE_ID : options.templateId; + const organization = store.organization(organizationId); + const installation = store.installationControl(organizationId); + if (!organization || !installation) fail('installation_not_found', 404); + const backend = runtimeBackend(store.installationBackend(organizationId)); + if (options.backend !== undefined && backend !== runtimeBackend(options.backend)) { + fail('runtime_identity_mismatch'); + } + if (installation.runtimeKey === 'local') fail('installation_operation_not_allowed'); + const releaseId = options.releaseId === undefined ? installation.releaseId : options.releaseId; + if (!CATALOG_IDENTIFIER_RE.test(templateId) || !CATALOG_IDENTIFIER_RE.test(releaseId)) { + fail('runtime_boundary_violation', 500); + } + const primary = organization.stations.find(station => station.primary); + if (!primary) fail('runtime_boundary_violation', 500); + const manifest = { + manifestVersion: 1, + revision: installation.manifestRevision, + organization: { + id: organization.id, + stationCode: primary.code, + timezone: organization.timezone, + }, + runtime: { + key: installation.runtimeKey, + templateId, + releaseId, + }, + }; + const manifestAuthority = { + revision: manifest.revision, + organization: { ...manifest.organization }, + runtime: { ...manifest.runtime }, + }; + return Object.freeze({ + organization, + installation, + backend, + manifest: serverInstallationManifest(manifest, manifestAuthority), + manifestAuthority: Object.freeze(manifestAuthority), + ownerActive: store.activeOwnerCount(organizationId) > 0, + }); +} + +module.exports = { + DEFAULT_MANAGED_TEMPLATE_ID, + DEFAULT_MANAGED_RELEASE_ID, + managedInstallationContext, +}; diff --git a/core/core/accounts/src/installation-lifecycle.js b/core/core/accounts/src/installation-lifecycle.js new file mode 100644 index 0000000..9e0a90f --- /dev/null +++ b/core/core/accounts/src/installation-lifecycle.js @@ -0,0 +1,877 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { workspaceWithoutPaycom } = require('./workspace-readiness'); +const { publicationBaseline } = require('../../../shared/contracts/src/publication-baseline'); +const { + INSTALLATION_LIFECYCLE_OPERATIONS, + INSTALLATION_READINESS_GATES, + assertInstallationOperationAllowed, + installationActivationEvidence, + installationFailure, + installationJob, + installationOperation, + installationPublicationContinuity, + installationTransition, +} = require('../../../shared/contracts/src'); +const { AccessError, identifier } = require('./validation'); +const { managedInstallationContext } = require('./installation-authority'); + +const DEFAULT_LIFECYCLE_LEASE_MS = 5 * 60 * 1000; +const MAX_LIFECYCLE_RECEIPT_BYTES = 32 * 1024; +const LIFECYCLE_STAGES = Object.freeze({ + backup: Object.freeze([ + 'inspect_schedule', 'quiesce_schedule', 'stop_if_running', 'snapshot', + 'restart_if_needed', 'restore_schedule', 'verify_runtime', + ]), + restore: Object.freeze(['verify_stopped', 'safety_snapshot', 'restore_snapshot', 'verify_restored']), + upgrade: Object.freeze([ + 'inspect_schedule', 'quiesce_schedule', 'stop_runtime', 'upgrade_backup', + 'install_release', 'start_release', 'verify_release', 'verify_release_publication', + 'restore_schedule', 'commit_release', + ]), + suspend: Object.freeze(['inspect_schedule', 'quiesce_schedule', 'stop_runtime', 'verify_stopped']), + resume: Object.freeze([ + 'start_runtime', 'verify_infrastructure', 'verify_publication', 'restore_schedule', + ]), + decommission: Object.freeze([ + 'inspect_schedule', 'quiesce_schedule', 'stop_runtime', + 'disable_runtime', 'verify_retained', + ]), + destroy: Object.freeze(['destroy_runtime', 'verify_destroyed']), +}); +function lifecycleStages(operation, backend, startingState = 'ready', removal = null, withoutPaycom = false) { + if (operation === 'resume' && removal && removal.installation_state !== 'ready') { + return removal.installation_state === 'pending' ? ['verify_unallocated'] : [...(removal.legacy_services ? ['restore_services'] : []), 'start_runtime', 'verify_infrastructure']; + } + const stages = [...LIFECYCLE_STAGES[operation]]; + if (['oci_container_v1', 'native_service_v1'].includes(backend) && ['upgrade', 'resume'].includes(operation)) { + stages.splice(operation === 'upgrade' ? stages.indexOf('upgrade_backup') : 0, 0, 'capture_publication'); + } + if (backend === 'native_service_v1' && operation === 'upgrade') { + if (startingState === 'suspended') { + stages.splice(stages.indexOf('start_release'), 1); + stages[stages.indexOf('verify_release')] = 'verify_stopped_release'; + } else if (['waiting_for_owner', 'waiting_for_provider_auth'].includes(startingState)) { + stages.splice(stages.indexOf('capture_publication'), 1); + stages.splice(stages.indexOf('verify_release_publication'), 1); + } + } + if (operation === 'resume' && removal?.legacy_services) stages.unshift('restore_services'); + return withoutPaycom ? stages.filter(stage => !['capture_publication', 'verify_release_publication', 'verify_publication'].includes(stage)) : stages; +} +const RECEIPT_STATUSES = Object.freeze([ + 'stopped', 'snapshot', 'started', 'healthy', 'inactive', 'restored', 'installed', + 'verified', 'committed', 'disabled', 'removed', 'retained', 'destroyed', 'absent', +]); +const STAGE_RECEIPT_STATUSES = Object.freeze({ + restore_services: Object.freeze(['installed']), + verify_unallocated: Object.freeze(['absent']), + capture_publication: Object.freeze(['verified']), + inspect_schedule: Object.freeze(['verified']), + quiesce_schedule: Object.freeze(['stopped']), + stop_if_running: Object.freeze(['stopped', 'inactive']), + snapshot: Object.freeze(['snapshot']), + restart_if_needed: Object.freeze(['started', 'inactive']), + restore_schedule: Object.freeze(['started']), + verify_runtime: Object.freeze(['healthy', 'inactive']), + verify_stopped: Object.freeze(['inactive']), + safety_snapshot: Object.freeze(['snapshot']), + restore_snapshot: Object.freeze(['restored']), + verify_restored: Object.freeze(['verified']), + stop_runtime: Object.freeze(['stopped', 'absent']), + upgrade_backup: Object.freeze(['snapshot']), + install_release: Object.freeze(['installed']), + start_release: Object.freeze(['started']), + verify_release: Object.freeze(['verified']), + verify_stopped_release: Object.freeze(['verified']), + verify_release_publication: Object.freeze(['verified']), + commit_release: Object.freeze(['committed']), + start_runtime: Object.freeze(['started']), + verify_infrastructure: Object.freeze(['verified']), + verify_publication: Object.freeze(['verified']), + final_backup: Object.freeze(['snapshot', 'absent']), + disable_runtime: Object.freeze(['disabled', 'absent']), + remove_services: Object.freeze(['removed', 'absent']), + verify_retained: Object.freeze(['retained', 'absent']), + destroy_runtime: Object.freeze(['destroyed']), + verify_destroyed: Object.freeze(['absent']), +}); + +function fail(code, statusCode = 409) { throw new AccessError(code, statusCode); } +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} +function exact(value, allowed, required, code = 'installation_operation_failed') { + if (!plain(value)) fail(code, 500); + const keys = Object.keys(value); + if (keys.some(key => !allowed.includes(key)) || required.some(key => !Object.hasOwn(value, key))) { + fail(code, 500); + } +} +function timestamp(clock) { + const value = clock(); + if (!Number.isSafeInteger(value) || value < 0) fail('installation_operation_failed', 500); + return value; +} +function defaultJobId() { return `life_${crypto.randomUUID().replaceAll('-', '')}`; } +function defaultBackupId() { return `backup_${crypto.randomUUID().replaceAll('-', '')}`; } +function parseJson(value) { + try { return JSON.parse(value); } catch { fail('installation_operation_failed', 500); } +} +function manifestAuthority(manifest) { + return Object.freeze({ + revision: manifest.revision, + organization: Object.freeze({ ...manifest.organization }), + runtime: Object.freeze({ ...manifest.runtime }), + }); +} +function lifecycleJobView(row, replayed = false) { + if (!row) fail('installation_operation_not_found', 404); + const failure = row.failure_code === null ? null : installationFailure(row.failure_code); + return Object.freeze({ + id: row.id, + operation: row.operation, + status: row.status, + installationState: row.installation_state, + installationRevision: row.installation_revision, + manifestRevision: row.manifest_revision, + attempt: row.attempt, + maxAttempts: row.max_attempts, + completedStages: row.next_stage, + totalStages: parseJson(row.stages_json).length, + replayed, + failure, + }); +} +function closedReceipt(value) { + exact(value, [ + 'status', 'changed', 'serviceCount', 'fileCount', 'totalBytes', 'treeDigest', + 'releaseId', 'activationEvidence', 'syncWasRunning', 'publicationBaseline', 'publicationBaselineDigest', + ], ['status']); + if (!RECEIPT_STATUSES.includes(value.status)) fail('installation_operation_failed', 500); + const result = { status: value.status }; + if (Object.hasOwn(value, 'changed')) { + if (typeof value.changed !== 'boolean') fail('installation_operation_failed', 500); + result.changed = value.changed; + } + if (Object.hasOwn(value, 'syncWasRunning')) { + if (typeof value.syncWasRunning !== 'boolean') fail('installation_operation_failed', 500); + result.syncWasRunning = value.syncWasRunning; + } + for (const key of ['serviceCount', 'fileCount', 'totalBytes']) { + if (!Object.hasOwn(value, key)) continue; + if (!Number.isSafeInteger(value[key]) || value[key] < 0) fail('installation_operation_failed', 500); + result[key] = value[key]; + } + if (Object.hasOwn(value, 'treeDigest')) { + if (typeof value.treeDigest !== 'string' || !/^[a-f0-9]{64}$/.test(value.treeDigest)) { + fail('installation_operation_failed', 500); + } + result.treeDigest = value.treeDigest; + } + if (Object.hasOwn(value, 'releaseId')) { + if (typeof value.releaseId !== 'string' || !/^[a-z][a-z0-9_.-]{2,95}$/.test(value.releaseId)) { + fail('installation_operation_failed', 500); + } + result.releaseId = value.releaseId; + } + if (Object.hasOwn(value, 'publicationBaseline')) result.publicationBaseline = publicationBaseline(value.publicationBaseline); + if (Object.hasOwn(value, 'publicationBaselineDigest')) { + if (typeof value.publicationBaselineDigest !== 'string' || !/^[a-f0-9]{64}$/.test(value.publicationBaselineDigest)) fail('first_publication_failed'); + result.publicationBaselineDigest = value.publicationBaselineDigest; + } + if (Object.hasOwn(value, 'activationEvidence')) { + result.activationEvidence = installationActivationEvidence(value.activationEvidence); + } + if (Buffer.byteLength(JSON.stringify(result), 'utf8') > MAX_LIFECYCLE_RECEIPT_BYTES) { + fail('installation_operation_failed', 500); + } + return Object.freeze(result); +} +function checkedStageReceipt(row, stage, receipt) { + const statuses = STAGE_RECEIPT_STATUSES[stage]; + if (!statuses || !statuses.includes(receipt.status)) fail('installation_operation_failed', 500); + if (stage === 'capture_publication' && !receipt.publicationBaseline) fail('first_publication_failed'); + if (stage === 'inspect_schedule' && typeof receipt.syncWasRunning !== 'boolean') { + fail('installation_operation_failed', 500); + } + if (['snapshot', 'safety_snapshot', 'upgrade_backup', 'final_backup'].includes(stage) + && receipt.status === 'snapshot' + && (!Number.isSafeInteger(receipt.fileCount) || !Number.isSafeInteger(receipt.totalBytes) + || typeof receipt.treeDigest !== 'string')) fail('backup_failed', 500); + if (['verify_release_publication', 'verify_publication'].includes(stage) + && !receipt.activationEvidence) fail('installation_not_ready', 500); + if (['verify_release', 'verify_stopped_release', 'commit_release'].includes(stage) + && receipt.releaseId !== row.target_release_id) fail('upgrade_failed', 500); + if (stage === 'restore_schedule' && typeof receipt.syncWasRunning !== 'boolean') { + fail('installation_operation_failed', 500); + } + return receipt; +} +function backupPurpose(operation, safety = false) { + if (safety) return 'restore_safety'; + return Object.freeze({ backup: 'manual', upgrade: 'upgrade' })[operation] || null; +} +function workState(operation, startingState) { + if (operation === 'decommission') return 'decommissioning'; + if (['resume', 'upgrade'].includes(operation) + || (operation === 'backup' && startingState === 'ready')) return 'verifying'; + if (operation === 'suspend') return 'suspended'; + return startingState; +} +function successState(operation, startingState) { + if (operation === 'suspend' || operation === 'restore') return 'suspended'; + if (operation === 'resume') return 'ready'; + if (operation === 'upgrade') return startingState; + if (operation === 'decommission' || operation === 'destroy') return 'decommissioned'; + return startingState; +} +function failureState(operation, startingState, error) { + const code = installationFailure(error).code; + if (['lifecycle_compensation_failed', 'upgrade_rollback_required'].includes(code)) return 'failed'; + if (operation === 'decommission') return 'failed'; + if (operation === 'resume') return 'suspended'; + if (operation === 'destroy') return 'failed'; + return startingState; +} +function failureForOperation(operation, error) { + const expected = Object.freeze({ + backup: 'backup_failed', restore: 'restore_failed', upgrade: 'upgrade_failed', + suspend: 'installation_operation_failed', resume: 'installation_not_ready', + decommission: 'decommission_failed', destroy: 'destruction_failed', + })[operation]; + const selected = installationFailure(error); + if (selected.code === 'installation_operation_in_progress' + || ['lifecycle_compensation_failed', 'upgrade_rollback_required'].includes(selected.code)) { + return selected.code; + } + return expected; +} + +function createAccessInstallationLifecycleAuthority(options) { + exact(options, [ + 'store', 'organizationId', 'authorityScope', 'actorUserId', 'clock', 'jobFactory', + 'backupFactory', 'leaseMs', 'releaseCatalog', 'destructionEnabled', + ], ['store', 'organizationId', 'authorityScope']); + const store = options.store; + if (!store || typeof store.transaction !== 'function') fail('runtime_boundary_violation', 500); + const organizationId = identifier(options.organizationId); + const authorityScope = identifier(options.authorityScope); + const actorUserId = options.actorUserId === undefined || options.actorUserId === null + ? null : identifier(options.actorUserId); + const clock = options.clock === undefined ? Date.now : options.clock; + const jobFactory = options.jobFactory === undefined ? defaultJobId : options.jobFactory; + const backupFactory = options.backupFactory === undefined ? defaultBackupId : options.backupFactory; + const leaseMs = options.leaseMs === undefined ? DEFAULT_LIFECYCLE_LEASE_MS : options.leaseMs; + const releaseCatalog = options.releaseCatalog === undefined ? [] : options.releaseCatalog; + const destructionEnabled = options.destructionEnabled === true; + if (typeof clock !== 'function' || typeof jobFactory !== 'function' || typeof backupFactory !== 'function' + || !Number.isSafeInteger(leaseMs) || leaseMs < 1_000 || leaseMs > 10 * 60 * 1000 + || !Array.isArray(releaseCatalog) + || releaseCatalog.some(value => typeof value !== 'string' || !/^[a-z][a-z0-9_.-]{2,95}$/.test(value))) { + fail('runtime_boundary_violation', 500); + } + const releases = new Set(releaseCatalog); + + function request(operationValue) { + const operation = installationOperation(operationValue); + if (!INSTALLATION_LIFECYCLE_OPERATIONS.includes(operation.operation)) { + fail('installation_operation_not_allowed'); + } + if (operation.operation === 'destroy' && !destructionEnabled) fail('installation_operation_not_allowed'); + return store.transaction(() => { + const at = timestamp(clock); + let context = managedInstallationContext(store, organizationId); + if (context.backend === 'directory_service_v1') fail('installation_operation_not_allowed'); + const removal = store.db.prepare('SELECT * FROM dsp_removals WHERE organization_id=?').get(organizationId); + const restoring = operation.operation === 'resume' && Boolean(removal); + if (restoring && store.lifecycleJob(context.installation.currentJobId)?.operation === 'destroy') fail('installation_operation_not_allowed'); + const prior = store.lifecycleJobByRequest(organizationId, authorityScope, operation.idempotencyKey); + if (prior) { + const expectedRequest = JSON.stringify(operation); + if (prior.result_json === '__request_conflict__' || prior.idempotency_key !== operation.idempotencyKey + || prior.operation !== operation.operation + || prior.installation_revision < operation.expectedRevision + 1) fail('idempotency_conflict'); + const storedRequest = parseJson(prior.stage_receipts_json).__request; + if (storedRequest !== expectedRequest) fail('idempotency_conflict'); + return lifecycleJobView(prior, true); + } + if (operation.operation === 'destroy' && store.lifecycleJob(context.installation.currentJobId)?.operation === 'destroy' && store.lifecycleJob(context.installation.currentJobId)?.status === 'succeeded') fail('installation_operation_not_allowed'); + if (removal && !['decommission', 'destroy', 'resume'].includes(operation.operation)) fail('installation_operation_not_allowed'); + if (operation.operation === 'destroy' && context.installation.status !== 'decommissioned' + && !(store.lifecycleJob(context.installation.currentJobId)?.operation === 'destroy' && (removal || store.lifecycleJob(context.installation.currentJobId)?.starting_state === 'decommissioned'))) fail('installation_operation_not_allowed'); + let interruptedSync = null; + // Cancel backup work under the same database lock that fences host mutations. + if (operation.operation === 'decommission') { + const activeBackup = store.activeLifecycleJob(organizationId); + if (activeBackup?.operation === 'backup') { + const intent = parseJson(activeBackup.stage_receipts_json).inspect_schedule?.syncWasRunning; + interruptedSync = typeof intent === 'boolean' ? Number(intent) : null; + store.db.prepare("UPDATE installation_lifecycle_jobs SET status='failed',failure_code='backup_failed',lease_expires_at=NULL,fence=fence+1,finished_at=?,updated_at=? WHERE id=?").run(at, at, activeBackup.id); + store.db.prepare('UPDATE installations SET status=? WHERE organization_id=?').run(activeBackup.starting_state, organizationId); + context = managedInstallationContext(store, organizationId); + } + store.db.prepare("UPDATE platform_backup_requests SET status='failed',phase='cancelled',failure_code='installation_operation_not_allowed',updated_at=? WHERE organization_id=? AND kind='backup' AND status IN ('queued','running')").run(at, organizationId); + } + if (!['pending_owner', 'setup_required', 'active', 'suspended'].includes(context.organization.status) + || ['provisioning', 'verifying', 'decommissioning'].includes(context.installation.status) + || context.installation.revision !== operation.expectedRevision) { + fail(context.installation.revision !== operation.expectedRevision + ? 'installation_revision_conflict' : 'installation_operation_in_progress'); + } + if (authorityScope !== 'platform_backups' && store.db.prepare("SELECT 1 FROM platform_backup_requests WHERE organization_id=? AND status IN ('queued','running')").get(organizationId)) { + fail('installation_operation_in_progress'); + } + const setup = store.installationSetup(organizationId); + if (setup?.workerId && setup.leaseExpiresAt > at) fail('installation_operation_in_progress'); + // Keep teardown/backup out of the short ready-to-scheduled-update window. + if (store.db.prepare("SELECT 1 FROM installation_onboarding_requests WHERE organization_id=? AND (status='queued' OR status IN ('enrolling','running') AND lease_expires_at>?) AND (status='running' OR EXISTS (SELECT 1 FROM installations i WHERE i.organization_id=installation_onboarding_requests.organization_id AND i.status='ready'))").get(organizationId, at)) { + fail('installation_operation_in_progress'); + } + if (operation.operation === 'backup' && !['ready', 'suspended'].includes(context.installation.status) + && context.backend !== 'native_service_v1') fail('installation_operation_not_allowed'); + if (!restoring) assertInstallationOperationAllowed(context.installation.status, operation.operation); + else if (!['decommissioned', 'failed', 'suspended'].includes(context.installation.status)) fail('installation_operation_not_allowed'); + if (operation.operation === 'upgrade' && context.installation.status !== 'ready' && context.backend !== 'native_service_v1') fail('installation_operation_not_allowed'); + if (operation.operation === 'upgrade' && store.db.prepare("SELECT 1 FROM installation_onboarding_requests WHERE organization_id=? AND status IN ('enrolling','queued','running')").get(organizationId)) fail('installation_operation_in_progress'); + if ((operation.operation === 'resume' && !restoring || operation.operation === 'upgrade' && context.installation.status === 'ready' + || (operation.operation === 'backup' && context.installation.status === 'ready')) + && context.organization.status !== 'active') { + fail('installation_operation_not_allowed'); + } + if (operation.operation === 'upgrade' + && (!releases.has(operation.releaseId) || operation.releaseId === context.installation.releaseId)) { + fail('installation_operation_not_allowed'); + } + let sourceBackup = null; + if (operation.operation === 'restore') { + sourceBackup = store.installationBackup(operation.backupId); + if (!sourceBackup || sourceBackup.organization_id !== organizationId + || sourceBackup.runtime_key !== context.installation.runtimeKey + || sourceBackup.status !== 'available' + || authorityScope !== 'platform_backups' && (sourceBackup.manifest_revision !== context.installation.manifestRevision + || sourceBackup.release_id !== context.installation.releaseId)) { + fail('installation_operation_not_allowed'); + } + if (authorityScope === 'platform_backups') { + const archived = store.db.prepare('SELECT metadata_json FROM platform_backup_records WHERE id=?').get(sourceBackup.id); + if (!archived) fail('installation_operation_not_allowed'); + require('./backup-metadata').checkDspMetadata(store, organizationId, JSON.parse(archived.metadata_json)); + } + } + const active = store.activeLifecycleJob(organizationId); + if (active) fail('installation_operation_in_progress'); + let preUpdateBackup = null; + if (operation.operation === 'upgrade' && authorityScope === 'platform_rollout') { + const rolloutId = operation.idempotencyKey.split(':')[0]; + const progress = require('./rollout-backups').rolloutBackupProgress(store.db, rolloutId); + if (progress) { + const member = progress.members.find(m => m.organizationId === organizationId); + preUpdateBackup = member?.backupId ? store.installationBackup(member.backupId) : null; + if (progress.status !== 'completed' || !preUpdateBackup || preUpdateBackup.status !== 'available' + || preUpdateBackup.runtime_key !== context.installation.runtimeKey + || preUpdateBackup.release_id !== context.installation.releaseId + || preUpdateBackup.manifest_revision !== context.installation.manifestRevision) fail('backup_failed'); + } + } + const id = identifier(jobFactory()); + const purpose = backupPurpose(operation.operation); + const backupId = preUpdateBackup?.id || (purpose && !(operation.operation === 'decommission' && context.installation.status === 'pending') + ? identifier(backupFactory()) : null); + const safetyBackupId = operation.operation === 'restore' ? identifier(backupFactory()) : null; + const startingState = context.installation.status; + const selectedWorkState = workState(operation.operation, startingState); + if (operation.operation === 'resume' && !restoring) installationTransition('suspended', 'verifying'); + else if (operation.operation === 'decommission') installationTransition(startingState, 'decommissioning'); + const nextRevision = context.installation.revision + 1; + store.updateInstallationControl({ + organizationId, + expectedStatus: startingState, + expectedRevision: context.installation.revision, + status: selectedWorkState, + revision: nextRevision, + currentJobId: id, + timestamp: at, + }); + if (operation.operation === 'decommission') { + const priorState = startingState === 'suspended' && (store.latestReadyEvidence(organizationId) || workspaceWithoutPaycom(store, organizationId)) ? 'ready' : startingState; + store.db.prepare('INSERT OR IGNORE INTO dsp_removals (organization_id,installation_state,organization_status,sync_running,removed_at,actor_user_id) VALUES(?,?,?,?,?,?)').run(organizationId, priorState, context.organization.status, + startingState === 'suspended' ? Number(store.latestSuspensionResult(organizationId)?.resumeSync === true) : interruptedSync, at, actorUserId); + } + if (restoring && removal.legacy_services) { + store.db.prepare("UPDATE runtime_agent_authorities SET status='active',revoked_at=NULL,updated_at=? WHERE organization_id=?").run(at, organizationId); + } + if (['decommission', 'destroy'].includes(operation.operation)) { + store.db.prepare('DELETE FROM sessions WHERE user_id IN (SELECT user_id FROM memberships WHERE organization_id=?) AND user_id IN (SELECT id FROM users WHERE platform_role IS NULL)').run(organizationId); + store.updateOrganizationStatus(organizationId, 'suspended', at); + } + const row = store.createLifecycleJob({ + id, + organizationId, + operation: operation.operation, + startingState, + installationState: selectedWorkState, + installationRevision: nextRevision, + manifestRevision: context.installation.manifestRevision, + runtimeKey: context.installation.runtimeKey, + releaseId: context.installation.releaseId, + targetReleaseId: operation.releaseId || null, + backupId, + safetyBackupId, + authorityScope, + idempotencyKey: operation.idempotencyKey, + stages: lifecycleStages(operation.operation, context.backend, context.installation.status, removal, workspaceWithoutPaycom(store, organizationId)), + timestamp: at, + }); + const receipts = { __withoutPaycom: workspaceWithoutPaycom(store, organizationId), __request: JSON.stringify(operation), ...(preUpdateBackup ? { __preUpdateBackup: preUpdateBackup.id } : {}) }; + store.db.prepare('UPDATE installation_lifecycle_jobs SET stage_receipts_json=? WHERE id=?') + .run(JSON.stringify(receipts), id); + if (backupId && !preUpdateBackup) store.reserveInstallationBackup({ + id: backupId, organizationId, runtimeKey: context.installation.runtimeKey, + manifestRevision: context.installation.manifestRevision, releaseId: context.installation.releaseId, + purpose, lifecycleJobId: id, timestamp: at, + }); + if (safetyBackupId) store.reserveInstallationBackup({ + id: safetyBackupId, organizationId, runtimeKey: context.installation.runtimeKey, + manifestRevision: context.installation.manifestRevision, releaseId: context.installation.releaseId, + purpose: backupPurpose(operation.operation, true), lifecycleJobId: id, timestamp: at, + }); + for (const selected of [preUpdateBackup ? null : backupId, safetyBackupId].filter(Boolean)) { + require('./backup-metadata').recordDspBackup(store, selected, organizationId, at); + } + store.createAudit({ + id: `aud_${crypto.randomUUID().replaceAll('-', '')}`, + actorUserId, + organizationId, + action: `installation.${operation.operation}.request`, + targetType: 'installation_lifecycle_job', + targetId: id, + result: 'succeeded', + timestamp: at, + }); + return lifecycleJobView(row); + }); + } + + function claim(jobIdValue, workerIdValue) { + const jobId = identifier(jobIdValue); + const workerId = identifier(workerIdValue); + return store.transaction(() => { + const at = timestamp(clock); + const row = store.claimLifecycleJob(jobId, workerId, at + leaseMs, at); + if (row.organization_id !== organizationId || row.authority_scope !== authorityScope) { + fail('installation_operation_not_allowed'); + } + const removal = store.db.prepare('SELECT * FROM dsp_removals WHERE organization_id=?').get(organizationId); + const storedStages = parseJson(row.stages_json); + const legacyRemoval = row.operation === 'decommission' && JSON.stringify(storedStages) === JSON.stringify(['inspect_schedule', 'quiesce_schedule', 'stop_runtime', 'final_backup', 'disable_runtime', 'remove_services', 'verify_retained']); + if (!legacyRemoval && JSON.stringify(storedStages) !== JSON.stringify(lifecycleStages(row.operation, store.installationBackend(organizationId), row.starting_state, removal, parseJson(row.stage_receipts_json).__withoutPaycom === true))) { + fail('runtime_boundary_violation', 500); + } + const context = managedInstallationContext(store, organizationId); + const manifest = context.manifest; + const receipts = parseJson(row.stage_receipts_json); + const withoutPaycom = receipts.__withoutPaycom === true; + if (withoutPaycom && !workspaceWithoutPaycom(store, organizationId)) fail('installation_not_ready'); + const targetManifest = row.operation === 'upgrade' ? Object.freeze({ + ...manifest, + revision: manifest.revision + 1, + runtime: Object.freeze({ ...manifest.runtime, releaseId: row.target_release_id }), + }) : manifest; + const backup = row.backup_id === null ? null : store.installationBackup(row.backup_id); + const sourceBackup = row.operation === 'restore' + ? store.installationBackup(parseJson(receipts.__request).backupId) : null; + const safetyBackup = row.safety_backup_id === null ? null : store.installationBackup(row.safety_backup_id); + const priorEvidence = ['resume', 'upgrade'].includes(row.operation) + ? store.latestReadyEvidence(organizationId) : null; + if (['resume', 'upgrade'].includes(row.operation) && !withoutPaycom && !priorEvidence && !(row.operation === 'resume' && removal && removal.installation_state !== 'ready') + && !(context.backend === 'native_service_v1' && row.operation === 'upgrade' && ['waiting_for_owner', 'waiting_for_provider_auth'].includes(row.starting_state))) fail('installation_not_ready'); + const suspension = row.operation === 'resume' ? store.latestSuspensionResult(organizationId) : null; + return Object.freeze({ + job: lifecycleJobView(row), + claim: Object.freeze({ jobId: row.id, workerId, fence: row.fence }), + operation: row.operation, + legacyRemoval, + withoutPaycom, + requireOffsiteSafety: authorityScope === 'platform_backups', + startingState: row.starting_state, + stages: Object.freeze([...storedStages]), + nextStage: row.next_stage, + stageReceipts: { ...receipts }, + manifest, + manifestAuthority: context.manifestAuthority, + backend: context.backend, + targetManifest, + targetManifestAuthority: manifestAuthority(targetManifest), + backup: backup ? Object.freeze({ + id: backup.id, purpose: backup.purpose, manifestRevision: backup.manifest_revision, + releaseId: backup.release_id, status: backup.status, + ...(receipts.__preUpdateBackup ? { treeDigest: backup.tree_digest, fileCount: backup.file_count, totalBytes: backup.total_bytes } : {}), + }) : null, + sourceBackup: sourceBackup ? Object.freeze({ + id: sourceBackup.id, purpose: sourceBackup.purpose, manifestRevision: sourceBackup.manifest_revision, + releaseId: sourceBackup.release_id, status: sourceBackup.status, treeDigest: sourceBackup.tree_digest, + fileCount: sourceBackup.file_count, totalBytes: sourceBackup.total_bytes, + }) : null, + safetyBackup: safetyBackup ? Object.freeze({ + id: safetyBackup.id, purpose: safetyBackup.purpose, manifestRevision: safetyBackup.manifest_revision, + releaseId: safetyBackup.release_id, status: safetyBackup.status, + }) : null, + priorEvidence, + removal: removal ? { ...removal } : null, + resumeSync: row.operation === 'resume' && removal ? removal.sync_running === 1 : suspension?.resumeSync === true, + }); + }); + } + + function checkedClaim(claimValue, at) { + const selected = store.lifecycleClaim( + identifier(claimValue.jobId), identifier(claimValue.workerId), claimValue.fence, at, + ); + if (selected.row.organization_id !== organizationId + || selected.row.authority_scope !== authorityScope) { + fail('installation_operation_not_found', 404); + } + return selected; + } + + function renew(claimValue) { + exact(claimValue, ['jobId', 'workerId', 'fence'], ['jobId', 'workerId', 'fence']); + return store.transaction(() => { + const at = timestamp(clock); + checkedClaim(claimValue, at); + store.renewLifecycleJob( + identifier(claimValue.jobId), identifier(claimValue.workerId), claimValue.fence, at + leaseMs, at, + ); + return true; + }); + } + + function desiredRuntimeState(claimValue) { + return store.transaction(() => { + const at = timestamp(clock); + const { row } = checkedClaim(claimValue, at); + const organization = store.organization(organizationId); + if (!organization) fail('installation_not_found', 404); + return row.operation === 'resume' && store.db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(organizationId) || organization.status === 'active' || store.installationBackend(organizationId) === 'native_service_v1' + && row.operation === 'upgrade' && ['waiting_for_owner', 'waiting_for_provider_auth'].includes(row.starting_state) && organization.status !== 'suspended' ? 'active' : 'suspended'; + }); + } + + let hostMutationDepth = 0; + function mutate(claimValue, mutation) { + if (typeof mutation !== 'function') fail('runtime_boundary_violation', 500); + return store.transaction(() => { + const at = timestamp(clock); + checkedClaim(claimValue, at); + let result; + hostMutationDepth += 1; + try { result = mutation(); } finally { hostMutationDepth -= 1; } + checkedClaim(claimValue, timestamp(clock)); + return result; + }); + } + + function beginCompensation(claimValue, error) { + return mutate(claimValue, () => { + const { row } = checkedClaim(claimValue, timestamp(clock)); + const receipts = parseJson(row.stage_receipts_json); + if (!receipts.__compensating) receipts.__compensationFailure = failureForOperation(row.operation, error); + receipts.__compensating = true; + store.db.prepare('UPDATE installation_lifecycle_jobs SET stage_receipts_json=? WHERE id=?') + .run(JSON.stringify(receipts), row.id); + }); + } + + // Persist before the restored runtime (including its background sync) starts. + // A resumed compensation must never rewind data accepted after this point. + function checkpointCompensationRestore(claimValue) { + return mutate(claimValue, () => { + const { row } = checkedClaim(claimValue, timestamp(clock)); + const receipts = parseJson(row.stage_receipts_json); + if (row.operation !== 'upgrade' || !receipts.__compensating) fail('runtime_boundary_violation'); + receipts.__compensationRestored = true; + store.db.prepare('UPDATE installation_lifecycle_jobs SET stage_receipts_json=? WHERE id=?') + .run(JSON.stringify(receipts), row.id); + }); + } + + function completeCompensation(claimValue) { + return mutate(claimValue, () => { + const { row } = checkedClaim(claimValue, timestamp(clock)); + const receipts = parseJson(row.stage_receipts_json); + if (!receipts.__compensating) fail('runtime_boundary_violation'); + receipts.__compensated = true; + store.db.prepare('UPDATE installation_lifecycle_jobs SET stage_receipts_json=? WHERE id=?') + .run(JSON.stringify(receipts), row.id); + }); + } + + function dispatchHostRequest(request, dispatch) { + const { authorizeHostRequest } = require('../../installations/src/oci-host-permissions'); + const authorize = () => { + const { row } = checkedClaim(request.claim, timestamp(clock)); + const context = managedInstallationContext(store, organizationId); + const stages = parseJson(row.stages_json); + const receipts = parseJson(row.stage_receipts_json); + const lease = authorizeHostRequest({ kind: 'lifecycle', claim: request.claim, + manifest: context.manifest, backend: store.installationBackend(organizationId), + stage: stages[row.next_stage] || null, compensation: receipts.__compensating === true, + canSettle: row.next_stage === 0 && !receipts.__compensating && (['ready', 'suspended'].includes(row.starting_state) + || store.installationBackend(organizationId) === 'native_service_v1' && ['waiting_for_owner', 'waiting_for_provider_auth'].includes(row.starting_state)), + operation: row.operation, targetReleaseId: row.target_release_id, + installationRevision: row.installation_revision, expiresAt: row.lease_expires_at }, request); + return dispatch(lease); + }; + return hostMutationDepth ? authorize() : mutate(request.claim, authorize); + } + + function startStage(claimValue, stage) { + const { row } = checkedClaim(claimValue, timestamp(clock)); + if (parseJson(row.stages_json)[row.next_stage] !== stage) fail('runtime_boundary_violation'); + return require('../../installations/src/operation-timing').start(store.db, { jobId: row.id, attempt: row.attempt, stage }, clock); + } + + function checkpoint(claimValue, stage, receiptValue) { + const receipt = closedReceipt(receiptValue); + return store.transaction(() => { + const at = timestamp(clock); + const { row } = checkedClaim(claimValue, at); + checkedStageReceipt(row, stage, receipt); + if (stage === 'inspect_schedule' && row.operation === 'decommission') { + store.db.prepare('UPDATE dsp_removals SET sync_running=COALESCE(sync_running,?) WHERE organization_id=?').run(Number(receipt.syncWasRunning), organizationId); + } + const snapshotBackupId = Object.freeze({ + snapshot: row.backup_id, + upgrade_backup: row.backup_id, + final_backup: row.backup_id, + safety_snapshot: row.safety_backup_id, + })[stage] || null; + if (snapshotBackupId) { + store.afterCommit?.(() => require('../../installations/src/worker-notify').exportReady()); + if (stage === 'final_backup' && receipt.status === 'absent') { + store.discardReservedInstallationBackup(snapshotBackupId, row.id); + } else { + if (receipt.status !== 'snapshot' || !Number.isSafeInteger(receipt.fileCount) + || !Number.isSafeInteger(receipt.totalBytes) || !receipt.treeDigest) fail('backup_failed', 500); + if (stage === 'upgrade_backup' && parseJson(row.stage_receipts_json).__preUpdateBackup === snapshotBackupId) { + const saved = store.installationBackup(snapshotBackupId); + if (!saved || saved.status !== 'available' || saved.tree_digest !== receipt.treeDigest + || saved.file_count !== receipt.fileCount || saved.total_bytes !== receipt.totalBytes) fail('backup_failed'); + } else store.completeInstallationBackup(snapshotBackupId, row.id, receipt, at); + if (!parseJson(row.stage_receipts_json).__preUpdateBackup) require('./backup-metadata').recordDspBackup(store, snapshotBackupId, organizationId, at); + } + } + return lifecycleJobView(store.completeLifecycleStage( + row.id, claimValue.workerId, claimValue.fence, stage, receipt, at, + )); + }); + } + + function retryExhausted(jobIdValue) { + const jobId = identifier(jobIdValue); + return store.transaction(() => { + const at = timestamp(clock); + const row = store.lifecycleJob(jobId); + if (!row || row.organization_id !== organizationId || row.authority_scope !== authorityScope) { + fail('installation_operation_not_found', 404); + } + const reopened = store.reopenExhaustedLifecycleJob(jobId, at); + store.createAudit({ + id: `aud_${crypto.randomUUID().replaceAll('-', '')}`, + actorUserId, + organizationId, + action: 'installation.lifecycle.retry_exhausted', + targetType: 'installation_lifecycle_job', + targetId: jobId, + result: 'succeeded', + timestamp: at, + }); + return lifecycleJobView(reopened, true); + }); + } + + function succeed(claimValue) { + return store.transaction(() => { + const at = timestamp(clock); + const { row, control } = checkedClaim(claimValue, at); + const receipts = parseJson(row.stage_receipts_json); + const stages = parseJson(row.stages_json); + for (const stage of stages) { + if (!plain(receipts[stage])) fail('installation_operation_failed', 500); + checkedStageReceipt(row, stage, closedReceipt(receipts[stage])); + } + const removal = store.db.prepare('SELECT * FROM dsp_removals WHERE organization_id=?').get(organizationId); + const restoring = row.operation === 'resume' && Boolean(removal); + const withoutPaycom = receipts.__withoutPaycom === true; + if (withoutPaycom && !workspaceWithoutPaycom(store, organizationId)) fail('installation_not_ready'); + const setupRestore = restoring && removal.installation_state !== 'ready'; + const setupUpgrade = store.installationBackend(organizationId) === 'native_service_v1' && row.operation === 'upgrade' && ['waiting_for_owner', 'waiting_for_provider_auth'].includes(row.starting_state); + if (!withoutPaycom && !setupUpgrade && !setupRestore && ['oci_container_v1', 'native_service_v1'].includes(store.installationBackend(organizationId)) && ['upgrade', 'resume'].includes(row.operation)) { + const baseline = receipts.capture_publication?.publicationBaseline; + const proof = receipts[row.operation === 'upgrade' ? 'verify_release_publication' : 'verify_publication']; + if (!baseline || publicationBaseline(baseline).digest !== proof?.publicationBaselineDigest) fail('first_publication_failed'); + } + const destination = setupRestore ? removal.installation_state : successState(row.operation, row.starting_state); + const organization = store.organization(organizationId); + if ((row.operation === 'backup' && row.starting_state === 'ready' + || row.operation === 'resume' && !restoring || row.operation === 'upgrade' && row.starting_state === 'ready') + && organization?.status !== 'active') fail('installation_operation_not_allowed'); + const result = { status: destination, operation: row.operation }; + if (receipts.capture_publication?.publicationBaseline) result.publicationBaseline = receipts.capture_publication.publicationBaseline; + const finishOptions = {}; + if (row.operation === 'suspend') { + if (typeof receipts.inspect_schedule?.syncWasRunning !== 'boolean') fail('suspension_failed', 500); + result.resumeSync = receipts.inspect_schedule.syncWasRunning; + } + if (row.operation === 'upgrade') { + if (receipts.commit_release?.status !== 'committed' + || receipts.commit_release.releaseId !== row.target_release_id) fail('upgrade_failed', 500); + if (!setupUpgrade && !withoutPaycom) { + const currentEvidence = receipts.verify_release_publication?.activationEvidence; + const priorEvidence = store.latestReadyEvidence(organizationId); + if (!currentEvidence || !priorEvidence || currentEvidence.jobId !== row.id + || currentEvidence.runtimeKey !== row.runtime_key + || currentEvidence.manifestRevision !== control.manifestRevision + 1) { + fail('installation_not_ready'); + } + installationPublicationContinuity(priorEvidence, currentEvidence, { + allowNextManifestRevision: true, + }); + result.activationEvidence = currentEvidence; + } + finishOptions.manifestRevision = control.manifestRevision + 1; + finishOptions.releaseId = row.target_release_id; + result.releaseId = row.target_release_id; + } + if (row.operation === 'resume' && !setupRestore && !withoutPaycom) { + const priorEvidence = store.latestReadyEvidence(organizationId); + const currentEvidence = receipts.verify_publication?.activationEvidence; + if (!priorEvidence || !currentEvidence) fail('installation_not_ready'); + const manifest = managedInstallationContext(store, organizationId).manifest; + const publicJob = installationJob({ + id: row.id, + operation: 'resume', + status: 'running', + installationState: 'verifying', + revision: control.revision, + replayed: false, + failure: null, + }); + const readiness = { + manifestRevision: manifest.revision, + jobId: row.id, + runtimeKey: manifest.runtime.key, + gates: Object.fromEntries(INSTALLATION_READINESS_GATES.map(gate => [gate, 'passed'])), + }; + const activation = { manifest, job: publicJob, readiness, evidence: currentEvidence }; + const authority = { manifestAuthority: manifestAuthority(manifest), jobId: row.id }; + installationTransition('suspended', 'ready', { + resume: { activation, priorEvidence }, + authority, + }); + result.activationEvidence = currentEvidence; + } else if (row.operation === 'suspend') installationTransition('ready', 'suspended'); + else if (row.operation === 'decommission') { + if (!['retained', 'absent'].includes(receipts.verify_retained.status)) fail('decommission_failed', 500); + installationTransition('decommissioning', 'decommissioned'); + } else if (row.operation === 'destroy' + && (receipts.destroy_runtime.status !== 'destroyed' + || receipts.verify_destroyed.status !== 'absent')) fail('destruction_failed', 500); + const finished = store.finishLifecycleJob( + row.id, claimValue.workerId, claimValue.fence, destination, result, at, finishOptions, + ); + if (restoring) { + const restoredStatus = destination === 'ready' ? 'active' : removal.organization_status === 'suspended' ? 'setup_required' : removal.organization_status; + store.updateOrganizationStatus(organizationId, restoredStatus, at); + store.db.prepare('DELETE FROM dsp_removals WHERE organization_id=?').run(organizationId); + } + if (row.operation === 'destroy') { + store.destroyInstallationBackups(organizationId, at); + // Native DSPs are fully erased after the owning worker has also removed + // their Core-side registration credential. Keep membership ownership + // until that step so exclusive user accounts can be identified. + if (store.installationBackend(organizationId) !== 'native_service_v1') store.destroyOrganizationAccess(organizationId); + } + if (row.operation === 'destroy') { + const agent = store.runtimeAgentAuthority(row.runtime_key); + if (agent?.status === 'active') store.revokeRuntimeAgentAuthority({ + organizationId, runtimeKey: row.runtime_key, expectedGeneration: agent.generation, timestamp: at, + }); + store.db.prepare("UPDATE installation_onboarding_requests SET status='failed',failure_code='installation_not_ready',lease_expires_at=NULL,updated_at=? WHERE organization_id=? AND status IN ('enrolling','queued','running')").run(at, organizationId); + } + store.createAudit({ + id: `aud_${crypto.randomUUID().replaceAll('-', '')}`, + actorUserId, + organizationId, + action: `installation.${row.operation}.complete`, + targetType: 'installation_lifecycle_job', + targetId: row.id, + result: 'succeeded', + timestamp: at, + }); + return lifecycleJobView(finished); + }); + } + + function failed(claimValue, error) { + return store.transaction(() => { + const at = timestamp(clock); + const { row } = checkedClaim(claimValue, at); + const destination = row.operation === 'resume' && store.db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(organizationId) + ? 'decommissioned' : failureState(row.operation, row.starting_state, error); + const failureCode = failureForOperation(row.operation, error); + const finished = store.failLifecycleJob( + row.id, claimValue.workerId, claimValue.fence, destination, failureCode, at, + ); + store.createAudit({ + id: `aud_${crypto.randomUUID().replaceAll('-', '')}`, + actorUserId, + organizationId, + action: `installation.${row.operation}.complete`, + targetType: 'installation_lifecycle_job', + targetId: row.id, + result: 'denied', + timestamp: at, + }); + return lifecycleJobView(finished); + }); + } + + function inspect(jobIdValue) { + const row = store.lifecycleJob(identifier(jobIdValue)); + if (!row || row.organization_id !== organizationId || row.authority_scope !== authorityScope) { + fail('installation_operation_not_found', 404); + } + return lifecycleJobView(row); + } + + function backups() { + return Object.freeze(store.installationBackups(organizationId).map(row => Object.freeze({ + id: row.id, + purpose: row.purpose, + manifestRevision: row.manifest_revision, + releaseId: row.release_id, + fileCount: row.file_count, + totalBytes: row.total_bytes, + treeDigest: row.tree_digest, + createdAt: new Date(row.created_at).toISOString(), + }))); + } + + return Object.freeze({ + request, claim, renew, desiredRuntimeState, mutate, checkpoint, succeed, failed, retryExhausted, startStage, + beginCompensation, checkpointCompensationRestore, completeCompensation, dispatchHostRequest, + inspect, backups, + }); +} + +module.exports = { + DEFAULT_LIFECYCLE_LEASE_MS, + LIFECYCLE_STAGES, + lifecycleStages, + lifecycleJobView, + createAccessInstallationLifecycleAuthority, +}; diff --git a/core/core/accounts/src/installation-provisioning.js b/core/core/accounts/src/installation-provisioning.js new file mode 100644 index 0000000..afec73a --- /dev/null +++ b/core/core/accounts/src/installation-provisioning.js @@ -0,0 +1,361 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { + INSTALLATION_IDENTIFIER_RE, + installationFailure, + installationJob, + installationOperation, + installationProvisioningRequest, +} = require('../../../shared/contracts/src'); +const { AccessError, identifier } = require('./validation'); +const { + DEFAULT_MANAGED_TEMPLATE_ID, + managedInstallationContext, +} = require('./installation-authority'); + +function fail(code, statusCode = 409) { + throw new AccessError(code, statusCode); +} + +function timestamp(clock) { + const value = clock(); + if (!Number.isSafeInteger(value) || value < 0) fail('installation_operation_failed', 500); + return value; +} + +function requestView(row, replayed = false) { + if (!row) fail('installation_operation_not_found', 404); + return installationProvisioningRequest({ + id: row.id, + status: row.status, + installationRevision: row.installation_revision, + manifestRevision: row.manifest_revision, + jobId: row.provisioner_job_id, + replayed, + failure: row.failure_code === null ? null : installationFailure(row.failure_code), + }); +} + +function createAccessInstallationProvisioningAuthority(options) { + if (!options || typeof options !== 'object' || Array.isArray(options) + || Object.keys(options).some(key => ![ + 'store', 'organizationId', 'authorityScope', 'actorUserId', 'clock', 'requestFactory', 'backend', + ].includes(key)) + || !options.store || typeof options.store.transaction !== 'function') { + fail('runtime_boundary_violation', 500); + } + const store = options.store; + const organizationId = identifier(options.organizationId); + const authorityScope = identifier(options.authorityScope); + const actorUserId = identifier(options.actorUserId); + const clock = options.clock === undefined ? Date.now : options.clock; + const requestFactory = options.requestFactory === undefined + ? () => `prq_${crypto.randomUUID().replaceAll('-', '')}` : options.requestFactory; + if (typeof clock !== 'function' || typeof requestFactory !== 'function') fail('runtime_boundary_violation', 500); + const contextOptions = options.backend === undefined ? {} : { backend: options.backend }; + + function request(operationValue) { + const operation = installationOperation(operationValue); + if (!['provision', 'retry'].includes(operation.operation)) fail('installation_operation_not_allowed'); + return store.transaction(() => { + const at = timestamp(clock); + const context = managedInstallationContext(store, organizationId, contextOptions); + if (!['pending_owner', 'setup_required', 'active'].includes(context.organization.status)) { + fail('installation_operation_not_allowed'); + } + const id = identifier(requestFactory()); + const selected = store.createProvisioningRequest({ + id, + organizationId, + authorityScope, + operation, + timestamp: at, + }); + store.createAudit({ + id: `aud_${crypto.randomUUID().replaceAll('-', '')}`, + actorUserId, + organizationId, + action: `installation.${operation.operation}.request`, + targetType: 'installation_request', + targetId: selected.row.id, + result: 'succeeded', + timestamp: at, + }); + return requestView(selected.row, selected.replayed); + }); + } + + function inspect(requestIdValue) { + const requestId = identifier(requestIdValue); + const row = store.provisioningRequest(requestId); + if (!row || row.organization_id !== organizationId || row.authority_scope !== authorityScope) { + fail('installation_operation_not_found', 404); + } + return requestView(row); + } + + return Object.freeze({ request, inspect }); +} + +function createAccessControlLiveAuthorityResolver(options) { + if (!options || typeof options !== 'object' || Array.isArray(options) + || Object.keys(options).some(key => !['store', 'templateId', 'releaseId', 'backend'].includes(key)) + || !options.store) fail('runtime_boundary_violation', 500); + const store = options.store; + const templateId = options.templateId === undefined ? DEFAULT_MANAGED_TEMPLATE_ID : options.templateId; + const releaseId = options.releaseId; + const backend = options.backend; + return function resolve(request, mutation = null) { + if (!request || typeof request !== 'object' || Array.isArray(request) + || Object.keys(request).sort().join(',') !== 'installationRevision,jobId,manifestRevision,organizationId,runtimeKey' + || typeof request.organizationId !== 'string' || typeof request.runtimeKey !== 'string' + || typeof request.jobId !== 'string' || !INSTALLATION_IDENTIFIER_RE.test(request.organizationId) + || !INSTALLATION_IDENTIFIER_RE.test(request.runtimeKey) || !INSTALLATION_IDENTIFIER_RE.test(request.jobId) + || !Number.isSafeInteger(request.manifestRevision) || request.manifestRevision < 1 + || !Number.isSafeInteger(request.installationRevision) || request.installationRevision < 1 + || mutation !== null && typeof mutation !== 'function') { + fail('runtime_boundary_violation', 500); + } + return store.transaction(() => { + const read = () => { + const context = managedInstallationContext(store, request.organizationId, { + templateId, ...(releaseId === undefined ? {} : { releaseId }), ...(backend === undefined ? {} : { backend }), + }); + if (!['pending_owner', 'setup_required', 'active'].includes(context.organization.status) + || context.installation.status !== 'provisioning' + || context.installation.currentJobId !== request.jobId + || context.installation.runtimeKey !== request.runtimeKey + || context.installation.revision !== request.installationRevision + || context.installation.manifestRevision !== request.manifestRevision) { + fail('installation_operation_in_progress'); + } + return context; + }; + const before = read(); + if (mutation !== null) mutation(); + const after = read(); + if (after.installation.revision !== before.installation.revision + || JSON.stringify(after.manifest) !== JSON.stringify(before.manifest)) { + fail('installation_operation_in_progress'); + } + return Object.freeze({ + manifestAuthority: after.manifestAuthority, + backend: after.backend, + organizationStatus: after.organization.status, + installationState: after.installation.status, + installationRevision: after.installation.revision, + currentJobId: after.installation.currentJobId, + }); + }); + }; +} + +function createInstallationProvisioningReconciler(options) { + if (!options || typeof options !== 'object' || Array.isArray(options) + || Object.keys(options).some(key => ![ + 'store', 'provisioner', 'provisionerFactory', 'clock', 'templateId', 'releaseId', + 'runtimeAgentCredentials', 'runtimeAgentCredentialFactory', 'backend', + ].includes(key)) + || !options.store || (options.provisioner === undefined) === (options.provisionerFactory === undefined)) { + fail('runtime_boundary_violation', 500); + } + const store = options.store; + const provisioner = options.provisioner === undefined ? null : options.provisioner; + const provisionerFactory = options.provisionerFactory === undefined ? null : options.provisionerFactory; + const required = ['registerLive', 'request', 'authorizeLive', 'inspect', 'runNext']; + if ((provisioner !== null && required.some(method => typeof provisioner[method] !== 'function')) + || (provisionerFactory !== null && typeof provisionerFactory !== 'function')) { + fail('runtime_boundary_violation', 500); + } + const clock = options.clock === undefined ? Date.now : options.clock; + const templateId = options.templateId === undefined ? DEFAULT_MANAGED_TEMPLATE_ID : options.templateId; + const releaseId = options.releaseId; + const backend = options.backend; + const runtimeAgentCredentials = options.runtimeAgentCredentials === undefined + ? null : options.runtimeAgentCredentials; + const runtimeAgentCredentialFactory = options.runtimeAgentCredentialFactory === undefined + ? null : options.runtimeAgentCredentialFactory; + if (runtimeAgentCredentials !== null && runtimeAgentCredentialFactory !== null) { + fail('runtime_boundary_violation', 500); + } + if (typeof clock !== 'function' || (runtimeAgentCredentialFactory !== null + && typeof runtimeAgentCredentialFactory !== 'function') || (runtimeAgentCredentials !== null + && (typeof runtimeAgentCredentials.issue !== 'function' + || typeof runtimeAgentCredentials.revoke !== 'function'))) fail('runtime_boundary_violation', 500); + + function requestRow(requestIdValue) { + const requestId = identifier(requestIdValue); + const row = store.provisioningRequest(requestId); + if (!row) fail('installation_operation_not_found', 404); + return row; + } + + function contextFor(row) { + const context = managedInstallationContext(store, row.organization_id, { + templateId, ...(releaseId === undefined ? {} : { releaseId }), ...(backend === undefined ? {} : { backend }), + }); + if (context.backend === 'directory_service_v1') fail('installation_operation_not_allowed'); + if (context.installation.runtimeKey !== row.runtime_key + || context.installation.manifestRevision !== row.manifest_revision) fail('runtime_identity_mismatch'); + return context; + } + + function provisionerFor(context) { + const selected = provisioner === null ? provisionerFactory(context.backend) : provisioner; + if (!selected || required.some(method => typeof selected[method] !== 'function')) { + fail('runtime_boundary_violation', 500); + } + return selected; + } + + function credentialsFor(context) { + const selected = runtimeAgentCredentialFactory === null + ? runtimeAgentCredentials : runtimeAgentCredentialFactory(context.backend); + if (selected !== null && (!selected || typeof selected.issue !== 'function' + || typeof selected.revoke !== 'function')) fail('runtime_boundary_violation', 500); + return selected; + } + + function mutationAuthority(row) { + return Object.freeze({ + scope: row.authority_scope, + permission: 'platform.installations.manage', + operatorEnabled: true, + }); + } + + function dispatch(requestIdValue) { + let row = requestRow(requestIdValue); + if (row.status === 'completed' || row.status === 'failed') return requestView(row); + let context = contextFor(row); + let selectedProvisioner = provisionerFor(context); + const operation = installationOperation(JSON.parse(row.request_json)); + const expectedStartingState = operation.operation === 'provision' ? 'pending' + : operation.operation === 'retry' ? 'failed' : null; + if (expectedStartingState === null || row.starting_state !== expectedStartingState + || row.installation_revision !== operation.expectedRevision + 1) fail('runtime_boundary_violation', 500); + const credentials = credentialsFor(context); + if (credentials !== null) { + let credential = null; + try { + store.transaction(() => { + credential = credentials.issue(context.installation.runtimeKey); + if (!credential || credential.runtimeKey !== context.installation.runtimeKey + || !/^[a-f0-9]{64}$/.test(credential.tokenHash) + || typeof credential.tokenChanged !== 'boolean') fail('runtime_boundary_violation', 500); + store.recordRuntimeAgentAuthority({ + organizationId: row.organization_id, + runtimeKey: row.runtime_key, + tokenHash: credential.tokenHash, + timestamp: timestamp(clock), + }); + }); + } catch (error) { + if (credential?.tokenChanged) { + try { credentials.revoke(row.runtime_key, credential.tokenHash); } catch {} + } + throw error; + } + } + if (operation.operation === 'provision') { + selectedProvisioner.registerLive(context.manifest, context.manifestAuthority, { + source: 'access_control', + installationState: 'pending', + organizationStatus: context.organization.status, + retainedData: false, + }); + } + const job = installationJob(selectedProvisioner.request( + context.manifest, + context.manifestAuthority, + operation, + mutationAuthority(row), + context.backend, + )); + store.transaction(() => { + store.acknowledgeProvisioningRequest(row.id, job, timestamp(clock)); + }); + row = requestRow(row.id); + context = contextFor(row); + selectedProvisioner = provisionerFor(context); + selectedProvisioner.authorizeLive( + context.manifest, + context.manifestAuthority, + mutationAuthority(row), + job.id, + { + source: 'access_control', + installationState: 'provisioning', + currentJobId: job.id, + organizationStatus: context.organization.status, + }, + ); + return requestView(requestRow(row.id)); + } + + function reconcile(requestIdValue) { + let row = requestRow(requestIdValue); + if (row.status === 'pending') dispatch(row.id); + row = requestRow(row.id); + if (row.status !== 'dispatched') return requestView(row); + const context = contextFor(row); + const job = installationJob(provisionerFor(context).inspect( + context.manifest, + context.manifestAuthority, + { scope: row.authority_scope, permission: 'platform.installations.read' }, + )); + if (!['succeeded', 'failed'].includes(job.status)) return requestView(row); + store.transaction(() => { + store.finishProvisioningRequest(row.id, job, timestamp(clock)); + require('./organization-profile').applyOrganizationProfiles(store, clock); + store.createAudit({ + id: `aud_${crypto.randomUUID().replaceAll('-', '')}`, + actorUserId: null, + organizationId: row.organization_id, + action: 'installation.provision.reconcile', + targetType: 'installation_request', + targetId: row.id, + result: job.status === 'succeeded' ? 'succeeded' : 'denied', + timestamp: timestamp(clock), + }); + }); + return requestView(requestRow(row.id)); + } + + function runNext(requestIdValue, workerIdValue) { + const row = requestRow(requestIdValue); + dispatch(row.id); + provisionerFor(contextFor(requestRow(row.id))).runNext(identifier(workerIdValue)); + return reconcile(row.id); + } + + function runPending(workerIdValue, limit = 20) { + const workerId = identifier(workerIdValue); + const rows = store.pendingProvisioningRequests(limit, + require('../../runtime-deployment').RUNTIME_BACKENDS.filter(value => value !== 'directory_service_v1')); + let processed = 0; + for (const row of rows) { + dispatch(row.id); + const current = requestRow(row.id); + const result = provisionerFor(contextFor(current)).runNext(workerId); + if (result?.status !== 'idle') processed += 1; + } + const results = rows.map(row => reconcile(row.id)); + return Object.freeze({ + processed, + completed: results.filter(result => result.status === 'completed').length, + failed: results.filter(result => result.status === 'failed').length, + pending: results.filter(result => ['pending', 'dispatched'].includes(result.status)).length, + }); + } + + return Object.freeze({ dispatch, reconcile, runNext, runPending }); +} + +module.exports = { + requestView, + createAccessInstallationProvisioningAuthority, + createAccessControlLiveAuthorityResolver, + createInstallationProvisioningReconciler, +}; diff --git a/core/core/accounts/src/onboarding-store.js b/core/core/accounts/src/onboarding-store.js new file mode 100644 index 0000000..3cc48a6 --- /dev/null +++ b/core/core/accounts/src/onboarding-store.js @@ -0,0 +1,66 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { AccessError, idempotencyKey } = require('./validation'); +const LEASE_MS = 360_000; +function fail(code = 'installation_operation_in_progress') { throw new AccessError(code, 409); } +function createOnboardingStore(store, clock = Date.now) { + const db = store.db; + const get = id => db.prepare('SELECT * FROM installation_onboarding_requests WHERE id=?').get(id) || null; + const latest = org => db.prepare('SELECT * FROM installation_onboarding_requests WHERE organization_id=? ORDER BY created_at DESC,rowid DESC LIMIT 1').get(org) || null; + const prior = (org, actor, key) => db.prepare('SELECT * FROM installation_onboarding_requests WHERE organization_id=? AND actor_user_id=? AND idempotency_key=?').get(org, actor, idempotencyKey(key)) || null; + function begin(org, actor, key, intent, revision) { + return store.transaction(() => { + const existing = prior(org, actor, key); + if (existing) { if (existing.intent !== intent) fail('idempotency_conflict'); return existing; } + if (db.prepare("SELECT 1 FROM installation_onboarding_requests WHERE organization_id=? AND status IN ('enrolling','queued','running')").get(org)) fail(); + const id = `setup_${crypto.randomBytes(16).toString('hex')}`; + db.prepare(`INSERT INTO installation_onboarding_requests(id,organization_id,actor_user_id,idempotency_key,intent,manifest_revision,status,lease_expires_at,created_at,updated_at) + VALUES(?,?,?,?,?,?,'enrolling',?,?,?)`).run(id, org, actor, key, intent, revision, clock() + LEASE_MS, clock(), clock()); + return get(id); + }); + } + function enrolled(id) { + if (db.prepare("UPDATE installation_onboarding_requests SET status='queued',lease_expires_at=NULL,updated_at=? WHERE id=? AND status='enrolling'").run(clock(), id).changes !== 1) fail(); + require('./worker-wakeup').afterCommit(store, ['reconcile']); + return get(id); + } + function enrollmentFailed(id, code) { + db.prepare("UPDATE installation_onboarding_requests SET status='failed',failure_code=?,lease_expires_at=NULL,updated_at=? WHERE id=? AND status='enrolling'").run(code, clock(), id); + } + function candidates(limit = 20, backends = null) { + if (!Number.isInteger(limit) || limit < 1 || limit > 100) fail('invalid_input'); + if (backends !== null && (!Array.isArray(backends) || backends.length === 0)) fail('invalid_input'); + const selected = backends?.map(require('../../runtime-deployment').runtimeBackend) || []; + const filter = selected.length ? ` AND organization_id IN (SELECT organization_id FROM installations WHERE backend IN (${selected.map(() => '?').join(',')}))` : ''; + db.prepare("UPDATE installation_onboarding_requests SET status='failed',failure_code='setup_interrupted',lease_expires_at=NULL,updated_at=? WHERE status='enrolling' AND lease_expires_at<=?" + filter).run(clock(), clock(), ...selected); + db.prepare("UPDATE installation_onboarding_requests SET status='failed',failure_code='setup_interrupted',lease_expires_at=NULL,updated_at=? WHERE status='running' AND attempt>=3 AND lease_expires_at<=?" + filter).run(clock(), clock(), ...selected); + return db.prepare("SELECT id FROM installation_onboarding_requests WHERE (status='queued' OR (status='running' AND lease_expires_at<=? AND attempt<3)) AND NOT EXISTS (SELECT 1 FROM dsp_removals d WHERE d.organization_id=installation_onboarding_requests.organization_id)" + filter + ' ORDER BY created_at,id LIMIT ?').all(clock(), ...selected, limit); + } + function requeue(id) { + if (db.prepare("UPDATE installation_onboarding_requests SET status='queued',failure_code=NULL,attempt=0,worker_id=NULL,lease_expires_at=NULL,updated_at=? WHERE id=? AND status='failed'").run(clock(), id).changes !== 1) fail(); + require('./worker-wakeup').afterCommit(store, ['reconcile']); + } + function claim(id, worker) { + return store.transaction(() => { + if (db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(get(id)?.organization_id || '')) fail(); + if (db.prepare(`UPDATE installation_onboarding_requests SET status='running',worker_id=?,fence=fence+1,attempt=attempt+1,lease_expires_at=?,updated_at=? + WHERE id=? AND attempt<3 AND (status='queued' OR (status='running' AND lease_expires_at<=?))`).run(worker, clock() + LEASE_MS, clock(), id, clock()).changes !== 1) fail(); + return get(id); + }); + } + function renew(row) { + if (db.prepare("UPDATE installation_onboarding_requests SET lease_expires_at=?,updated_at=? WHERE id=? AND status='running' AND worker_id=? AND fence=? AND lease_expires_at>?") + .run(clock() + LEASE_MS, clock(), row.id, row.worker_id, row.fence, clock()).changes !== 1) fail(); + } + function finish(row, code = null) { + if (db.prepare("UPDATE installation_onboarding_requests SET status=?,failure_code=?,lease_expires_at=NULL,updated_at=? WHERE id=? AND status='running' AND worker_id=? AND fence=? AND lease_expires_at>?") + .run(code ? 'failed' : 'succeeded', code, clock(), row.id, row.worker_id, row.fence, clock()).changes !== 1) fail(); + } + function defer(row) { + if (db.prepare("UPDATE installation_onboarding_requests SET status='queued',attempt=attempt-1,worker_id=NULL,lease_expires_at=NULL,updated_at=? WHERE id=? AND status='running' AND worker_id=? AND fence=? AND lease_expires_at>?") + .run(clock(), row.id, row.worker_id, row.fence, clock()).changes !== 1) fail(); + } + return { get, latest, prior, begin, enrolled, enrollmentFailed, candidates, claim, renew, finish, requeue, defer }; +} +module.exports = { createOnboardingStore, LEASE_MS }; diff --git a/core/core/accounts/src/optional-paycom-activation.js b/core/core/accounts/src/optional-paycom-activation.js new file mode 100644 index 0000000..aa01f21 --- /dev/null +++ b/core/core/accounts/src/optional-paycom-activation.js @@ -0,0 +1,112 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { installationFailure, serverInstallationActivation } = require('../../../shared/contracts/src'); +const { managedInstallationContext } = require('./installation-authority'); +const { activationJobView, MAX_PROVIDER_EVIDENCE_AGE_MS } = require('./installation-activation'); +const { AccessError } = require('./validation'); + +// The existing publication pipeline still verifies Paycom's data, but its job +// cannot change a usable DSP's installation or organization status. +function createOptionalPaycomActivation({ store, row, requests, clock = Date.now }) { + const scope = 'optional_paycom'; + const leaseMs = 300_000; + let claim = null; + const fail = (code = 'installation_operation_in_progress') => { throw new AccessError(code, 409); }; + function context() { + requests.renew(row); + const value = managedInstallationContext(store, row.organization_id); + if (value.installation.status !== 'ready' || value.organization.status !== 'active' + || !value.ownerActive || value.manifest.revision !== row.manifest_revision + || store.activeLifecycleJob(row.organization_id) + || store.db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(row.organization_id)) fail(); + return value; + } + function latest() { + return store.db.prepare(`SELECT * FROM installation_activation_jobs WHERE organization_id=? + AND authority_scope=? AND substr(idempotency_key,1,?)=? ORDER BY rowid DESC LIMIT 1`) + .get(row.organization_id, scope, row.id.length + 1, `${row.id}:`) || null; + } + function view(value, job) { + return { manifest: value.manifest, manifestAuthority: value.manifestAuthority, + installation: { state: job?.status === 'succeeded' ? 'ready' : job?.status === 'running' ? 'verifying' : 'waiting_for_provider_auth', + revision: value.installation.revision, currentJobId: job?.id || null }, + owner: { active: value.ownerActive }, job: job ? activationJobView(job) : null }; + } + function peek() { return store.transaction(() => view(context(), latest())); } + function inspect() { + return store.transaction(() => { + const value = context(); + let job = latest(); + if (job?.status === 'running') { + const at = clock(); + if (job.worker_id === row.worker_id && job.lease_expires_at > at) { + job = store.renewActivationJob(job.id, row.worker_id, job.fence, at + leaseMs, at); + } else if (job.lease_expires_at <= at) { + job = store.claimActivationJob(job.id, row.worker_id, job.fence, at + leaseMs, at); + } else fail(); + claim = job; + } + return view(value, job?.status === 'failed' ? null : job); + }); + } + function begin(evidence) { + return store.transaction(() => { + const value = context(); + const at = clock(); + const testedAt = Date.parse(evidence?.testedAt); + if (evidence?.provider !== 'paycom' || evidence?.profileId !== 'paycom-main' + || evidence?.status !== 'authenticated' || !Number.isSafeInteger(testedAt) + || testedAt > at + 60_000 || at - testedAt > MAX_PROVIDER_EVIDENCE_AGE_MS) fail('provider_auth_required'); + if (store.runningActivationJob(row.organization_id)) fail(); + claim = store.createActivationJob({ id: `act_${crypto.randomBytes(16).toString('hex')}`, + organizationId: row.organization_id, installationRevision: value.installation.revision, + manifestRevision: value.manifest.revision, runtimeKey: value.manifest.runtime.key, + authorityScope: scope, idempotencyKey: `${row.id}:${row.fence}`, workerId: row.worker_id, + leaseExpiresAt: at + leaseMs, providerTestedAt: testedAt, timestamp: at }); + return view(value, claim); + }); + } + function checked() { + const value = context(); + const job = claim && store.activationJob(claim.id); + if (!job || job.status !== 'running' || job.worker_id !== row.worker_id || job.fence !== claim.fence + || job.lease_expires_at <= clock() || job.installation_revision !== value.installation.revision + || job.manifest_revision !== value.manifest.revision) fail(); + return { value, job }; + } + function heartbeat() { + return store.transaction(() => { + const { value, job } = checked(); + claim = store.renewActivationJob(job.id, row.worker_id, job.fence, clock() + leaseMs, clock()); + return view(value, claim); + }); + } + function commit(activation, authority) { + return store.transaction(() => { + const { value, job } = checked(); + if (authority?.jobId !== job.id || JSON.stringify(authority.manifestAuthority) !== JSON.stringify(value.manifestAuthority)) fail(); + const verified = serverInstallationActivation({ ...activation, job: activationJobView(job) }, + { manifestAuthority: value.manifestAuthority, jobId: job.id }); + const at = clock(); + const captured = Date.parse(verified.evidence.capturedAt); + if (!Number.isSafeInteger(captured) || captured > at + 60_000 || at - captured > 900_000) fail('installation_not_ready'); + store.finishActivationJob(job.id, row.worker_id, job.fence, 'succeeded', 'ready', + value.installation.revision, null, verified.evidence, at); + claim = null; + return { state: 'ready', revision: value.installation.revision }; + }); + } + function failed(jobId, error) { + return store.transaction(() => { + const { value, job } = checked(); + if (job.id !== jobId) fail(); + store.finishActivationJob(job.id, row.worker_id, job.fence, 'failed', 'failed', + value.installation.revision, installationFailure(error).code, null, clock()); + claim = null; + }); + } + return { peek, inspect, begin, heartbeat, commit, fail: failed }; +} + +module.exports = { createOptionalPaycomActivation }; diff --git a/core/core/accounts/src/organization-profile.js b/core/core/accounts/src/organization-profile.js new file mode 100644 index 0000000..da31d45 --- /dev/null +++ b/core/core/accounts/src/organization-profile.js @@ -0,0 +1,23 @@ +'use strict'; + +// Apply business details only after infrastructure provisioning releases its authority. +// The organization's ID, runtime key, host allocation and data paths never change. +function applyOrganizationProfiles(store, clock = Date.now) { + return store.transaction(() => { + const rows = store.db.prepare(`SELECT p.* FROM organization_profiles p JOIN installations i ON i.organization_id=p.organization_id + JOIN organizations o ON o.id=p.organization_id WHERE p.details_json IS NOT NULL AND p.applied_at IS NULL + AND i.status IN ('waiting_for_owner','waiting_for_provider_auth') AND o.status!='suspended' + AND i.setup_worker_id IS NULL`).all(); + for (const row of rows) { + const details = JSON.parse(row.details_json); + store.db.prepare('UPDATE organizations SET name=?,abbreviation=?,timezone=?,updated_at=? WHERE id=?') + .run(details.name, details.abbreviation, details.timezone, clock(), row.organization_id); + store.db.prepare('DELETE FROM stations WHERE organization_id=?').run(row.organization_id); + store.insertStation(row.organization_id, details.stationCode, true, clock()); + store.db.prepare('UPDATE organization_profiles SET applied_at=? WHERE organization_id=?').run(clock(), row.organization_id); + } + require('./workspace-readiness').completeWorkspaceSetup(store, clock); + return rows.length; + }); +} +module.exports = { applyOrganizationProfiles }; diff --git a/core/core/accounts/src/owner-admin-cli.js b/core/core/accounts/src/owner-admin-cli.js new file mode 100644 index 0000000..fd5495d --- /dev/null +++ b/core/core/accounts/src/owner-admin-cli.js @@ -0,0 +1,87 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { resolveAccessPaths } = require('../../../shared/paths/access-paths'); +const { AccessStore } = require('./store'); +const { administerOwner, listOwners } = require('./owner-admin'); + +function collectOwnerInput(action) { + let fd; + try { fd = fs.openSync('/dev/tty', fs.constants.O_RDWR | fs.constants.O_NOCTTY | fs.constants.O_NOFOLLOW); } + catch { throw new Error('tty_required'); } + let savedMode; + const stty = args => { + const result = spawnSync('/usr/bin/stty', args, { stdio: [fd, 'pipe', 'pipe'], encoding: 'utf8' }); + if (result.status !== 0 || result.error) throw new Error('tty_required'); + return result.stdout.trim(); + }; + const restore = () => { if (savedMode) { try { stty([savedMode]); } catch {} } }; + const interrupt = () => { restore(); process.exit(130); }; + const terminate = () => { restore(); process.exit(143); }; + const input = {}; + try { + if (!fs.fstatSync(fd).isCharacterDevice()) throw new Error('tty_required'); + savedMode = stty(['-g']); + process.once('exit', restore); process.once('SIGINT', interrupt); process.once('SIGTERM', terminate); + stty(['-echo', '-echonl']); + fs.writeSync(fd, 'Enter platform login details privately. Input is hidden.\n'); + if (action === 'owner-recover') fs.writeSync(fd, 'Recovery replaces the login credentials and signs out all sessions.\n'); + const fields = action === 'owner-create' + ? [['email', 'Owner email', 254], ['firstName', 'First name', 80], ['lastName', 'Last name', 80]] + : [['email', 'Current owner email', 254], ['newEmail', 'New owner email (Enter to keep current)', 254]]; + fields.push(['password', 'New password (12–128 characters)', 128], ['confirmPassword', 'Confirm new password', 128]); + for (const [key, label, maximum] of fields) { + fs.writeSync(fd, `${label}: `); + const bytes = Buffer.alloc(maximum * 4 + 1); + let length = 0; + try { + while (true) { + if (length >= bytes.length || fs.readSync(fd, bytes, length, 1, null) !== 1) throw new Error('input_cancelled'); + if (bytes[length] === 10 || bytes[length] === 13) break; + length += 1; + } + input[key] = bytes.subarray(0, length).toString('utf8'); + } finally { bytes.fill(0); fs.writeSync(fd, '\n'); } + } + return input; + } finally { + restore(); + process.removeListener('exit', restore); process.removeListener('SIGINT', interrupt); process.removeListener('SIGTERM', terminate); + fs.closeSync(fd); + } +} + +async function main(argv, { paths = resolveAccessPaths(), collect = collectOwnerInput, + write = value => process.stdout.write(value) } = {}) { + const [action] = argv; + if (argv.length !== 1 || !['owner-create', 'owner-recover', 'owner-list'].includes(action)) { + write('Invalid arguments. Use owner-create, owner-recover, or owner-list without credential arguments.\n'); + return 2; + } + let store; + let input; + try { + // Recovery/listing must never silently initialize a different database. + if (action !== 'owner-create' && !fs.existsSync(paths.accessControl.database)) throw new Error('access_not_initialized'); + if (action !== 'owner-list') input = collect(action); + if (action === 'owner-create') fs.mkdirSync(path.dirname(paths.accessControl.databaseRoot), { recursive: true, mode: 0o700 }); + store = new AccessStore(paths.accessControl); + const data = action === 'owner-list' ? { owners: listOwners(store) } : await administerOwner(store, action, input); + write(`${JSON.stringify({ ok: true, ...data })}\n`); + return 0; + } catch (error) { + const safe = new Set(['tty_required', 'input_cancelled', 'access_not_initialized', 'invalid_input', 'password_policy_failed', + 'password_confirmation_mismatch', 'platform_owner_exists', 'platform_owner_not_found', 'email_in_use', 'account_changed', + 'unsafe_access_storage', 'access_schema_incompatible']); + const code = error?.code || error?.message; + write(`${JSON.stringify({ ok: false, status: safe.has(code) ? code : 'owner_admin_failed' })}\n`); + return 1; + } finally { + if (input) for (const key of Object.keys(input)) input[key] = ''; + store?.close(); + } +} + +module.exports = { main, collectOwnerInput }; diff --git a/core/core/accounts/src/owner-admin.js b/core/core/accounts/src/owner-admin.js new file mode 100644 index 0000000..53f4a7f --- /dev/null +++ b/core/core/accounts/src/owner-admin.js @@ -0,0 +1,58 @@ +'use strict'; + +// Server-local capability. Deliberately not part of the HTTP service or SDK. +const crypto = require('node:crypto'); +const { AccessError, exact, email, text, password } = require('./validation'); +const { hashPassword } = require('./passwords'); + +function listOwners(store) { + return store.db.prepare(`SELECT id,email,first_name AS firstName,last_name AS lastName,status + FROM users WHERE platform_role='owner' ORDER BY created_at,id`).all(); +} + +async function administerOwner(store, action, input, now = Date.now) { + if (!['owner-create', 'owner-recover'].includes(action)) throw new AccessError('invalid_input'); + exact(input, action === 'owner-create' + ? ['email', 'firstName', 'lastName', 'password', 'confirmPassword'] + : ['email', 'newEmail', 'password', 'confirmPassword']); + const selectedEmail = email(input.email); + const newEmail = action === 'owner-recover' && input.newEmail ? email(input.newEmail) : selectedEmail; + password(input.password); + if (input.password !== input.confirmPassword) throw new AccessError('password_confirmation_mismatch'); + const firstName = action === 'owner-create' ? text(input.firstName, 'firstName', { maximum: 80 }) : null; + const lastName = action === 'owner-create' ? text(input.lastName, 'lastName', { maximum: 80 }) : null; + const previous = store.userByEmail(selectedEmail); + if (action === 'owner-recover' && previous?.platform_role !== 'owner') throw new AccessError('platform_owner_not_found', 404); + const passwordHash = await hashPassword(input.password); + return store.transaction(() => { + const timestamp = now(); + let userId; + if (action === 'owner-create') { + if (listOwners(store).length) throw new AccessError('platform_owner_exists', 409); + if (store.userByEmail(selectedEmail)) throw new AccessError('email_in_use', 409); + userId = `usr_${crypto.randomUUID().replaceAll('-', '')}`; + store.insertUser({ id: userId, email: selectedEmail, firstName, lastName, passwordHash, platformRole: 'owner', timestamp }); + // An earlier bootstrap link must not remain able to create another owner. + store.db.prepare("UPDATE invitations SET status='revoked' WHERE kind='platform_owner' AND status='pending'").run(); + } else { + const current = store.userById(previous.id); + if (current?.platform_role !== 'owner' || current.email !== selectedEmail || current.auth_version !== previous.auth_version) { + throw new AccessError('account_changed', 409); + } + const conflict = store.userByEmail(newEmail); + if (conflict && conflict.id !== current.id) throw new AccessError('email_in_use', 409); + userId = current.id; + store.db.prepare(`UPDATE users SET email=?,password_hash=?,status='active',auth_version=auth_version+1,updated_at=? WHERE id=?`) + .run(newEmail, passwordHash, timestamp, userId); + store.deleteUserSessions(userId); + store.db.prepare("UPDATE invitations SET status='revoked' WHERE status='pending' AND email IN (?,?)") + .run(selectedEmail, newEmail); + } + store.createAudit({ id: `aud_${crypto.randomUUID().replaceAll('-', '')}`, actorUserId: null, organizationId: null, + action: action === 'owner-create' ? 'platform.owner.create_cli' : 'platform.owner.recover_cli', + targetType: 'user', targetId: userId, result: 'succeeded', timestamp }); + return { status: action === 'owner-create' ? 'platform_owner_created' : 'platform_owner_recovered' }; + }); +} + +module.exports = { administerOwner, listOwners }; diff --git a/core/core/accounts/src/owner-connections.js b/core/core/accounts/src/owner-connections.js new file mode 100644 index 0000000..6c61def --- /dev/null +++ b/core/core/accounts/src/owner-connections.js @@ -0,0 +1,110 @@ +'use strict'; + +const { AccessError } = require('./validation'); +const { managedInstallationContext } = require('./installation-authority'); +const { SERVICES, service, credentialsFor, verificationInput, connectionView, connectionList, REASONS } = require('../../../shared/contracts/src/connections'); + +function createOwnerConnections({ store, access, invoke, clock = Date.now, paycomSetup = null }) { + const onboarding = require('./onboarding-store').createOnboardingStore(store, clock); + function context(session) { + const { organization } = access.requireDspOwner(session); + const selected = managedInstallationContext(store, organization.id); + if (organization.status !== 'active' || !['ready', 'waiting_for_provider_auth'].includes(selected.installation.status) + || store.activeLifecycleJob(organization.id)) { + throw new AccessError('installation_not_ready', 409); + } + return { ...selected, organization }; + } + function reauthorize(session, previous) { + const current = context(session); + if (current.organization.id !== previous.organization.id || current.manifest.runtime.key !== previous.manifest.runtime.key + || current.installation.revision !== previous.installation.revision) throw new AccessError('installation_not_ready', 409); + } + async function call(session, input) { + const selected = context(session); + let result; + try { result = await invoke(selected.manifest.runtime.key, 'connections.manage', input); } + catch { throw new AccessError('auth_unavailable', 503); } + reauthorize(session, selected); + if (!result?.ok) { + const code = REASONS.includes(result?.status) || result?.status === 'profile_not_configured' ? result.status : 'auth_unavailable'; + throw new AccessError(code, code === 'auth_unavailable' ? 503 : 409); + } + try { + if (input.command === 'list') { + if (result.status !== 'found') throw new Error(); + const listed = connectionList(result.data); + return { items: listed.items.filter(item => { + const owner = require('../../../shared/plugin-sdk/catalog').catalog().find(plugin => plugin.services.includes(item.service)); + return !owner || require('./plugins').available(store, selected.organization.id, owner.id); + }).map(item => { + // A confirmed save can outlive a lost check acknowledgement. Its + // durable onboarding request still owns the pending verification. + if (item.service === 'paycom' && item.state === 'not_verified') { + const pending = onboarding.latest(selected.organization.id); + if (['queued', 'running'].includes(pending?.status)) return { ...item, state: 'checking', reason: null }; + if (pending?.status === 'failed') return { ...item, state: 'temporarily_unavailable', reason: 'auth_unavailable' }; + } + return item; + }) }; + } + if (result.status !== 'accepted') throw new Error(); + const view = connectionView(result.data); + if (view.service !== input.service) throw new Error(); + access.audit({ actorUserId: session.user.id, organizationId: selected.organization.id, + action: `connection.${input.command}`, targetType: 'connection', targetId: input.service }); + return view; + } catch { throw new AccessError('auth_unavailable', 503); } + } + return { + list: session => call(session, { command: 'list' }), + async change(session, id, command, body) { + const selected = context(session); + if (store.runningActivationJob(selected.organization.id)) throw new AccessError('session_busy', 409); + try { + service(id); + const owner = require('../../../shared/plugin-sdk/catalog').catalog().find(plugin => plugin.services.includes(id)); + if (owner) require('./plugins').requirePlugin(access, session, owner.id); + const keys = command === 'save' ? ['credentials'] : command === 'verify' ? ['code', 'verificationId'] : []; + if (!body || Object.getPrototypeOf(body) !== Object.prototype + || Object.keys(body).sort().join(',') !== keys.join(',') || !['save', 'test', 'disconnect', 'verify'].includes(command)) { + throw new AccessError('invalid_input', 400); + } + const input = { command, service: id, ...(command === 'save' + ? { credentials: credentialsFor(id, body.credentials), expiresAt: clock() + 30_000 } + : command === 'verify' ? { ...verificationInput(body), expiresAt: clock() + 30_000 } : {}) }; + if (command === 'verify' && id !== 'cortex') throw new AccessError('invalid_input', 400); + if (id === 'paycom' && paycomSetup && ['save', 'test'].includes(command)) { + const setup = await paycomSetup.status(session); + if (command === 'save' && setup.canSubmit) { + const current = await call(session, { command: 'list' }); + const configured = current.items.find(item => item.service === id).configured; + await paycomSetup.submit(session, { + idempotencyKey: `connections:${require('node:crypto').randomUUID()}`, + intent: configured ? 'replace' : 'create', credentials: input.credentials, + }); + const listed = await call(session, { command: 'list' }); + access.audit({ actorUserId: session.user.id, organizationId: selected.organization.id, + action: 'connection.save', targetType: 'connection', targetId: id }); + return listed.items.find(item => item.service === id); + } + if (command === 'test' && setup.canRetry) { + await paycomSetup.retry(session, {}); + access.audit({ actorUserId: session.user.id, organizationId: selected.organization.id, + action: 'connection.test', targetType: 'connection', targetId: id }); + return (await call(session, { command: 'list' })).items.find(item => item.service === id); + } + } + return await call(session, input); + } catch (error) { + if (error instanceof AccessError) throw error; + if (error?.code === 'invalid_input') throw new AccessError('invalid_input', 400); + // Transport or audit failures can occur after persistence. Do not claim + // the user's input was rejected when the save outcome is uncertain. + throw new AccessError('auth_unavailable', 503); + } + }, + services: Object.values(SERVICES).map(({ id, name, fields }) => ({ id, name, fields })), + }; +} +module.exports = { createOwnerConnections }; diff --git a/core/core/accounts/src/owner-paycom-setup.js b/core/core/accounts/src/owner-paycom-setup.js new file mode 100644 index 0000000..a7ae12e --- /dev/null +++ b/core/core/accounts/src/owner-paycom-setup.js @@ -0,0 +1,173 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { AccessError, exact, idempotencyKey } = require('./validation'); +const { paycomCredentials, setupFailure, paycomReadiness } = require('../../../shared/contracts/src/paycom-setup'); +const { managedInstallationContext } = require('./installation-authority'); +const { createAccessInstallationActivationAuthority } = require('./installation-activation'); +const { createOnboardingStore } = require('./onboarding-store'); +function fail(code, status = 409) { throw new AccessError(code, status); } +function createOwnerPaycomSetup({ store, access, invoke, clock = Date.now, + beginVerification = null, readReadiness = null, + enroll = (key, input) => invoke(key, 'paycom.setup', input) }) { + const requests = createOnboardingStore(store, clock); + function context(session) { + require('./plugins').requirePlugin(access, session, 'paycom'); + const { organization } = access.requireDspOwner(session); + const context = managedInstallationContext(store, organization.id); + const profile = store.db.prepare('SELECT applied_at FROM organization_profiles WHERE organization_id=?').get(organization.id); + if (profile && profile.applied_at === null) fail('organization_details_required'); + if (!['oci_container_v1', 'native_service_v1', 'directory_service_v1'].includes(context.backend)) fail('installation_operation_not_allowed'); + return context; + } + function view(context) { + const row = requests.latest(context.organization.id); + const idle = !store.activeLifecycleJob(context.organization.id); + const workforceAvailable = Boolean(store.latestReadyEvidence(context.organization.id)); + const connected = workforceAvailable || row?.status === 'succeeded'; + return { + installationState: context.installation.status, workforceAvailable, + status: row?.status || (connected ? 'succeeded' : 'not_started'), failureCode: row?.failure_code || null, + canSubmit: idle && !connected && ['ready', 'waiting_for_provider_auth'].includes(context.installation.status) + && (!row || ['succeeded', 'failed'].includes(row.status)), + canRetry: false, retryState: null, retryAt: null, + }; + } + async function status(session) { + const selected = context(session); + const row = requests.latest(selected.organization.id); + const result = view(selected); + if (row?.status !== 'failed' || store.activeLifecycleJob(selected.organization.id) + || !['failed', 'ready', 'verifying', 'waiting_for_provider_auth'].includes(selected.installation.status)) return result; + let readiness; + try { + const response = readReadiness ? await readReadiness(selected.manifest.runtime.key) + : await invoke(selected.manifest.runtime.key, 'paycom.setup', { + command: 'status', requestId: row.id, step: 'readiness', manifest: selected.manifest, + manifestAuthority: selected.manifestAuthority, parameters: {}, + }); + if (!response?.ok || response.status !== 'succeeded') throw new Error('readiness_unavailable'); + readiness = paycomReadiness(response.data); + } catch { readiness = { state: 'unavailable', retryAllowed: false, retryAt: null }; } + // Reauthorize after transport; a replacement or lifecycle operation may have won. + const current = context(session); + const latest = requests.latest(current.organization.id); + if (latest?.id !== row.id || latest?.fence !== row.fence || latest?.status !== 'failed' + || current.installation.revision !== selected.installation.revision + || current.installation.status !== selected.installation.status + || JSON.stringify(current.manifest) !== JSON.stringify(selected.manifest) + || store.activeLifecycleJob(current.organization.id)) return view(current); + return { ...result, canRetry: readiness.retryAllowed, retryState: readiness.state, retryAt: readiness.retryAt }; + } + async function submit(session, input) { + access.requireDspOwner(session); + exact(input, ['idempotencyKey', 'intent', 'credentials']); + idempotencyKey(input.idempotencyKey); + if (!['create', 'replace'].includes(input.intent)) fail('invalid_input', 400); + let credentials; + try { credentials = paycomCredentials(input.credentials); } + catch { fail('paycom_credentials_invalid', 400); } + let selected = context(session); + const prior = requests.prior(selected.organization.id, session.user.id, input.idempotencyKey); + if (prior) { + if (prior.intent !== input.intent) fail('idempotency_conflict'); + return { ...view(selected), replayed: true }; + } + if (!['ready', 'waiting_for_provider_auth'].includes(selected.installation.status)) fail('installation_not_ready'); + if (store.latestReadyEvidence(selected.organization.id) || requests.latest(selected.organization.id)?.status === 'succeeded') fail('installation_operation_not_allowed'); + if (selected.installation.status === 'ready') { + // Changing an established connection requires a separate reconnect workflow. + // Failed initial login verification can retry. + let row; + const guard = () => { + const current = context(session); + if (current.installation.status !== 'ready' || current.organization.status !== 'active' + || current.installation.revision !== selected.installation.revision + || store.activeLifecycleJob(selected.organization.id) + || store.runningActivationJob(selected.organization.id) + || store.db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(selected.organization.id)) fail('installation_operation_in_progress'); + }; + try { + store.transaction(() => { + guard(); + row = requests.begin(selected.organization.id, session.user.id, input.idempotencyKey, input.intent, selected.manifest.revision); + }); + const result = await enroll(selected.manifest.runtime.key, { + command: 'enroll', requestId: row.id, expiresAt: clock() + 30_000, credentials, intent: input.intent, + }); + if (!result?.ok || result.status !== 'succeeded' || result.data?.configured !== true) fail(setupFailure(result?.status)); + store.transaction(() => { guard(); requests.enrolled(row.id); }); + return { ...view(context(session)), replayed: false }; + } catch (error) { + if (row) requests.enrollmentFailed(row.id, setupFailure(error?.code)); + if (error instanceof AccessError) throw error; + fail('provider_setup_failed'); + } + } + const authority = createAccessInstallationActivationAuthority({ + store, organizationId: selected.organization.id, authorityScope: 'owner_onboarding', + workerId: `owner_${crypto.randomBytes(16).toString('hex')}`, idempotencyKey: input.idempotencyKey, + clock, setupLeaseMs: 180_000, releaseId: selected.manifest.runtime.releaseId, + }); + authority.beginSetup(); + let row; + try { + row = requests.begin(selected.organization.id, session.user.id, input.idempotencyKey, input.intent, selected.manifest.revision); + authority.guard(() => true); + const result = await enroll(selected.manifest.runtime.key, { + command: 'enroll', requestId: row.id, expiresAt: clock() + 30_000, credentials, intent: input.intent, + }); + authority.guard(() => true); + if (!result?.ok || result.status !== 'succeeded' || result.data?.configured !== true) fail(setupFailure(result?.status)); + // Reauthorize after transport completion before making the queued work visible. + selected = context(session); + requests.enrolled(row.id); + return { ...view(selected), replayed: false }; + } catch (error) { + if (row) requests.enrollmentFailed(row.id, setupFailure(error?.code)); + if (error instanceof AccessError) throw error; + fail('provider_setup_failed'); + } finally { authority.endSetup(); } + } + async function retry(session, input) { + access.requireDspOwner(session); + exact(input, []); + const selected = context(session); + const row = requests.latest(selected.organization.id); + const readiness = await status(session); + if (!readiness.canRetry) fail('installation_operation_not_allowed'); + const guard = () => { + const current = context(session); + const latest = requests.latest(current.organization.id); + if (latest?.id !== row?.id || latest?.fence !== row?.fence || latest?.status !== 'failed' + || current.installation.revision !== selected.installation.revision + || current.installation.status !== selected.installation.status + || JSON.stringify(current.manifest) !== JSON.stringify(selected.manifest) + || store.activeLifecycleJob(current.organization.id)) fail('installation_operation_in_progress'); + }; + guard(); + // An owner retry starts a fresh check before the worker sees the request. + if (beginVerification) { + try { await beginVerification(selected.manifest.runtime.key); } + catch { fail('auth_unavailable', 503); } + guard(); + } + if (['ready', 'verifying', 'waiting_for_provider_auth'].includes(selected.installation.status)) { + store.transaction(() => { + const row = requests.latest(selected.organization.id); + requests.requeue(row.id); + if (selected.installation.status === 'ready') store.db.prepare('UPDATE installation_onboarding_requests SET manifest_revision=? WHERE id=?').run(selected.manifest.revision, row.id); + }); + return view(selected); + } + const authority = createAccessInstallationActivationAuthority({ + store, organizationId: selected.organization.id, authorityScope: 'owner_onboarding', + workerId: `owner_${crypto.randomBytes(16).toString('hex')}`, idempotencyKey: requests.latest(selected.organization.id).id, + clock, releaseId: selected.manifest.runtime.releaseId, + }); + authority.retry(); + return view(context(session)); + } + return { status, submit, retry }; +} +module.exports = { createOwnerPaycomSetup }; diff --git a/core/core/accounts/src/password-recovery.js b/core/core/accounts/src/password-recovery.js new file mode 100644 index 0000000..064cfbb --- /dev/null +++ b/core/core/accounts/src/password-recovery.js @@ -0,0 +1,76 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { AccessError, exact, email, password } = require('./validation'); +const { hashPassword } = require('./passwords'); +const RESET_TTL_MS = 30 * 60 * 1000; +const digest = value => crypto.createHash('sha256').update(value).digest('hex'); + +// Persist all buckets, including unknown emails, across processes/restarts. +// Never evict live buckets: reaching the storage bound fails closed. +function consumeRecoveryLimits(store, limits, now) { + return store.transaction(() => { + const db = store.db; + db.prepare('DELETE FROM password_recovery_limits WHERE reset_at<=?').run(now); + const entries = limits.map(({ key, ...limit }) => { + const hash = digest(key); + return { ...limit, hash, row: db.prepare('SELECT * FROM password_recovery_limits WHERE key_hash=?').get(hash) }; + }); + if (entries.some(({ row, count, cooldown = 0 }) => row && (row.count >= count || now - row.last_at < cooldown))) return false; + const size = db.prepare('SELECT count(*) AS size FROM password_recovery_limits').get().size; + if (size + entries.filter(entry => !entry.row).length > 10000) return false; + for (const { hash, row, window } of entries) { + db.prepare(`INSERT INTO password_recovery_limits VALUES(?,?,?,?) + ON CONFLICT(key_hash) DO UPDATE SET count=excluded.count,last_at=excluded.last_at`) + .run(hash, (row?.count || 0) + 1, row?.reset_at ?? now + window, now); + } + return true; + }); +} + +// Internal return value is for the email worker only, never an HTTP response. +function requestPasswordReset(input) { + exact(input, ['email']); + const selectedEmail = email(input.email); + const now = this.now(); + return this.store.transaction(() => { + if (!consumeRecoveryLimits(this.store, [{ key: `email:${selectedEmail}`, count: 5, + window: 60 * 60 * 1000, cooldown: 60 * 1000 }], now)) return null; + this.store.db.prepare('DELETE FROM password_reset_tokens WHERE expires_at<=?').run(now); + const user = this.store.userByEmail(selectedEmail); + if (!user || user.status !== 'active') return null; + const token = crypto.randomBytes(32).toString('base64url'); + this.store.db.prepare('INSERT INTO password_reset_tokens VALUES(?,?,?,?,?,?)') + .run(digest(token), user.id, user.auth_version, user.email, now + RESET_TTL_MS, now); + this.audit({ action: 'account.password.reset.request', targetType: 'user', targetId: user.id }); + return { token, email: user.email, userId: user.id, expiresAt: new Date(now + RESET_TTL_MS).toISOString() }; + }); +} + +async function resetPassword(input) { + exact(input, ['token', 'newPassword', 'confirmPassword']); + const invalid = () => { throw new AccessError('password_reset_invalid', 400); }; + if (typeof input.token !== 'string' || !/^[A-Za-z0-9_-]{43}$/.test(input.token)) invalid(); + const hash = digest(input.token); + const read = () => this.store.db.prepare(`SELECT r.* FROM password_reset_tokens r JOIN users u ON u.id=r.user_id + WHERE r.token_hash=? AND r.expires_at>? AND u.status='active' + AND u.auth_version=r.auth_version AND u.email=r.email`).get(hash, this.now()); + const initial = read(); + if (!initial) invalid(); + password(input.newPassword); + if (input.newPassword !== input.confirmPassword) throw new AccessError('password_confirmation_mismatch'); + const passwordHash = await hashPassword(input.newPassword); + // Hashing yields. Recheck the token, expiry and account inside the write lock, + // so competing resets, CLI recovery and password changes have just one winner. + return this.store.transaction(() => { + const current = read(); + if (!current || current.user_id !== initial.user_id || current.auth_version !== initial.auth_version) invalid(); + this.store.updatePassword(current.user_id, passwordHash, this.now()); + // The users trigger removes ALL reset tokens, including tokens from other requests. + this.store.deleteUserSessions(current.user_id); + this.audit({ action: 'account.password.reset.complete', targetType: 'user', targetId: current.user_id }); + return { email: current.email, userId: current.user_id }; + }); +} + +module.exports = { RESET_TTL_MS, consumeRecoveryLimits, requestPasswordReset, resetPassword }; diff --git a/core/core/accounts/src/passwords.js b/core/core/accounts/src/passwords.js new file mode 100644 index 0000000..a6def97 --- /dev/null +++ b/core/core/accounts/src/passwords.js @@ -0,0 +1,43 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { promisify } = require('node:util'); +const { AccessError, password } = require('./validation'); + +const scrypt = promisify(crypto.scrypt); +const PARAMETERS = Object.freeze({ N: 32768, r: 8, p: 1, maxmem: 64 * 1024 * 1024 }); +const KEY_BYTES = 64; + +async function hashPassword(value, salt = crypto.randomBytes(24)) { + password(value); + const derived = await scrypt(value, salt, KEY_BYTES, PARAMETERS); + return `scrypt-v1$${PARAMETERS.N}$${PARAMETERS.r}$${PARAMETERS.p}$${salt.toString('base64url')}$${Buffer.from(derived).toString('base64url')}`; +} + +function parse(encoded) { + if (typeof encoded !== 'string') throw new AccessError('credential_record_invalid', 500); + const parts = encoded.split('$'); + if (parts.length !== 6 || parts[0] !== 'scrypt-v1') throw new AccessError('credential_record_invalid', 500); + const [N, r, p] = parts.slice(1, 4).map(Number); + const salt = Buffer.from(parts[4], 'base64url'); + const expected = Buffer.from(parts[5], 'base64url'); + if (N !== PARAMETERS.N || r !== PARAMETERS.r || p !== PARAMETERS.p || salt.length !== 24 || expected.length !== KEY_BYTES) { + throw new AccessError('credential_record_invalid', 500); + } + return { salt, expected }; +} + +async function verifyPassword(value, encoded) { + const { salt, expected } = parse(encoded); + let derived; + try { derived = Buffer.from(await scrypt(value, salt, expected.length, PARAMETERS)); } + catch { return false; } + return crypto.timingSafeEqual(derived, expected); +} + +async function consumeEquivalentPasswordWork(value) { + const selected = typeof value === 'string' ? value : ''; + await scrypt(selected, Buffer.alloc(24, 0x5a), KEY_BYTES, PARAMETERS); +} + +module.exports = { PARAMETERS, hashPassword, verifyPassword, consumeEquivalentPasswordWork }; diff --git a/core/core/accounts/src/permissions.js b/core/core/accounts/src/permissions.js new file mode 100644 index 0000000..787fe49 --- /dev/null +++ b/core/core/accounts/src/permissions.js @@ -0,0 +1,40 @@ +'use strict'; + +const TENANT_PERMISSIONS = Object.freeze([ + 'dashboard.view', + 'workforce.read', + 'integrations.read', + 'sync.run', + 'members.read', + 'members.invite', + 'members.manage', + 'roles.read', + 'organization.settings.manage', + 'audit.read', + 'organization.owner', +]); + +// Role labels are fixed; their permissions can be separated here later. +const SYSTEM_ROLES = Object.freeze([ + ['owner', 'Owner', 'DSP ownership'], + ['manager', 'Manager', 'DSP team management'], + ['dispatcher', 'Dispatcher', 'Dispatch operations'], + ['driver', 'Driver', 'Delivery operations'], +].map(([key, name, description]) => Object.freeze({ + key, name, description, permissions: TENANT_PERMISSIONS, +}))); + +const PLATFORM_PERMISSIONS = Object.freeze([ + 'platform.organizations.read', + 'platform.organizations.create', + 'platform.organizations.suspend', + 'platform.invitations.manage', + 'platform.installations.read', + 'platform.installations.manage', +]); + +module.exports = { + TENANT_PERMISSIONS, + SYSTEM_ROLES, + PLATFORM_PERMISSIONS, +}; diff --git a/core/core/accounts/src/platform-backups.js b/core/core/accounts/src/platform-backups.js new file mode 100644 index 0000000..9b3f4d8 --- /dev/null +++ b/core/core/accounts/src/platform-backups.js @@ -0,0 +1,641 @@ +'use strict'; +const crypto = require('node:crypto'); +const { AccessError, exact, identifier, idempotencyKey } = require('./validation'); +const { + DEFAULT_BACKUP_SETTINGS, + backupSettings, + scheduledSlot, + nextScheduledAt, +} = require('./backup-schedule'); +const { checkDspMetadata } = require('./backup-metadata'); +const fail = (code) => { + throw new AccessError(code, 409); +}; +const newId = () => `breq_${crypto.randomBytes(16).toString('hex')}`; +const iso = (n) => (n === null ? null : new Date(n).toISOString()); +function createPlatformBackups({ + store, + enabled = false, + clock = Date.now, + archive = () => ({ status: 'unavailable', backups: {} }), +}) { + const db = store.db; + db.prepare('INSERT OR IGNORE INTO platform_backup_settings VALUES(1,1,?,?)').run( + JSON.stringify(DEFAULT_BACKUP_SETTINGS), + clock(), + ); + function settingsRow(scope = 'system') { + db.prepare('INSERT OR IGNORE INTO backup_scope_settings VALUES(?,1,?,?)').run( + scope, + JSON.stringify(DEFAULT_BACKUP_SETTINGS), + clock(), + ); + return db.prepare('SELECT * FROM backup_scope_settings WHERE scope=?').get(scope); + } + const settings = (scope = 'system') => JSON.parse(settingsRow(scope).settings_json); + function scopeOf(input) { + if (input.scope === 'core' || input.scope === 'system') { + if (input.organizationId || input.organizationIds) fail('invalid_input'); + return input.scope; + } + if (input.organizationId) { + const id = identifier(input.organizationId); + if (!fleet().some((o) => o.id === id)) fail('backup_dsp_unavailable'); + return id; + } + if (input.scope !== undefined) fail('invalid_input'); + return 'system'; + } + const fleet = () => + db + .prepare( + `SELECT o.id,o.name,o.status AS organization_status,i.status,i.backend,i.runtime_key + FROM organizations o JOIN installations i ON i.organization_id=o.id WHERE i.backend IN ('oci_container_v1','native_service_v1') AND NOT EXISTS (SELECT 1 FROM dsp_removals d WHERE d.organization_id=o.id) AND NOT EXISTS (SELECT 1 FROM installation_lifecycle_jobs j WHERE j.organization_id=o.id AND j.operation='destroy' AND j.status='succeeded') ORDER BY o.name,o.id`, + ) + .all(); + const active = (org) => + db + .prepare( + "SELECT * FROM platform_backup_requests WHERE organization_id=? AND status IN ('queued','running')", + ) + .get(org); + function enqueue(kind, organizationId, key, actorId = null, extra = {}) { + const previous = db + .prepare('SELECT * FROM platform_backup_requests WHERE idempotency_key=?') + .get(key); + if (previous) return previous.id; + if ( + !organizationId && + db + .prepare( + "SELECT 1 FROM platform_backup_requests WHERE kind='core' AND status IN ('queued','running')", + ) + .get() + ) + fail('backup_operation_in_progress'); + if (organizationId && active(organizationId)) fail('backup_operation_in_progress'); + if (organizationId) { + const target = fleet().find((o) => o.id === organizationId); + if ( + !target || + !['ready', 'suspended'].includes(target.status) || + !['active', 'suspended'].includes(target.organization_status) || + store.activeLifecycleJob(organizationId) + ) + fail('backup_dsp_unavailable'); + } + const id = newId(), + input = { category: key.startsWith('scheduled:') ? 'scheduled' : 'manual', retentionDays: settings(organizationId || 'core').retentionDays, ...extra }; + db.prepare( + "INSERT INTO platform_backup_requests VALUES(?,?,?,'queued','queued',NULL,?,?,?,?,?,NULL)", + ).run(id, organizationId, kind, JSON.stringify(input), actorId, key, clock(), clock()); + require('./worker-wakeup').afterCommit(store, ['reconcile']); + return id; + } + function restoreEligibility(row, remote) { + if ( + row.deleted_at || + remote?.status === 'expired' || + (!remote?.retained && + (remote?.expiresAt ?? row.expires_at) !== null && + (remote?.expiresAt ?? row.expires_at) <= clock()) + ) + return 'This backup has reached its retention date.'; + if (!remote || remote.status !== 'verified') + return 'Waiting for the encrypted backup upload to Cloudflare R2.'; + if ( + remote.metadataDigest !== + crypto.createHash('sha256').update(row.metadata_json).digest('hex') || + remote.format !== 2 + ) + return 'This older snapshot does not contain a complete DSP backup.'; + const target = fleet().find((o) => o.id === row.organization_id); + if (!target || !['ready', 'suspended'].includes(target.status)) + return 'This DSP must be available or suspended before restoring.'; + try { + checkDspMetadata(store, row.organization_id, JSON.parse(row.metadata_json)); + } catch { + return 'This backup requires its original DSP configuration and a compatible release.'; + } + if (active(row.organization_id) || store.activeLifecycleJob(row.organization_id)) + return 'Another operation is already running for this DSP.'; + return null; + } + function coreRestoreBlocked(row, proof) { + if (JSON.parse(row.metadata_json).scope !== 'core') + return 'This older backup is not isolated to Core.'; + if ( + !proof || + proof.status !== 'verified' || + proof.metadataDigest !== crypto.createHash('sha256').update(row.metadata_json).digest('hex') + ) + return 'Waiting for a verified Core backup.'; + if (row.deleted_at || (proof.expiresAt != null && proof.expiresAt <= clock())) + return 'This backup has expired.'; + if ( + db + .prepare( + "SELECT 1 FROM platform_backup_requests WHERE kind='core' AND status IN ('queued','running')", + ) + .get() + ) + return 'Another Core operation is running.'; + return null; + } + function command(session, input) { + if ( + !session?.user || + (session.user.platformRole !== 'owner' && session.user.platform_role !== 'owner') + ) + throw new AccessError('permission_denied', 403); + if (!enabled) fail('installation_operator_disabled'); + exact(input, [ + 'action', + 'idempotencyKey', + 'settings', + 'revision', + 'organizationId', + 'organizationIds', + 'scope', + 'backupId', + 'confirmation', + 'setId', + ]); + const fields = { + settings: ['action', 'idempotencyKey', 'settings', 'revision', 'scope', 'organizationId'], + backup: ['action', 'idempotencyKey', 'scope', 'organizationId', 'organizationIds'], + restore: [ + 'action', + 'idempotencyKey', + 'scope', + 'organizationId', + 'backupId', + 'setId', + 'confirmation', + ], + delete: [ + 'action', + 'idempotencyKey', + 'scope', + 'organizationId', + 'backupId', + 'setId', + 'confirmation', + ], + }[input.action]; + if (!fields) fail('invalid_input'); + exact(input, fields); + if (input.scope !== undefined && !['core', 'system', 'dsps'].includes(input.scope)) + fail('invalid_input'); + const key = idempotencyKey(input.idempotencyKey), + encoded = JSON.stringify(input); + return store.transaction(() => { + const prior = db + .prepare( + 'SELECT input_json FROM platform_backup_commands WHERE actor_user_id=? AND idempotency_key=?', + ) + .get(session.user.id, key); + if (prior) { + if (prior.input_json !== encoded) fail('idempotency_conflict'); + return; + } + if (input.action === 'settings') { + const scope = scopeOf(input), + value = backupSettings(input.settings), + row = settingsRow(scope); + if (input.revision !== row.revision) fail('backup_settings_conflict'); + db.prepare( + 'UPDATE backup_scope_settings SET revision=revision+1,settings_json=?,updated_at=? WHERE scope=?', + ).run(JSON.stringify(value), clock(), scope); + } else if (input.action === 'backup') { + if (db.prepare("SELECT 1 FROM platform_rollouts WHERE status!='completed'").get()) + fail('backup_operation_in_progress'); + if (input.scope !== undefined && !['dsps', 'core', 'system'].includes(input.scope)) + fail('invalid_input'); + if ( + input.organizationIds !== undefined && + (!Array.isArray(input.organizationIds) || + !input.organizationIds.length || + input.organizationIds.length > 1000 || + input.organizationId || + input.scope !== 'dsps') + ) + fail('invalid_input'); + if ( + ['core', 'system'].includes(input.scope) && + (input.organizationId || input.organizationIds) + ) + fail('invalid_input'); + if (input.scope === 'dsps' && !input.organizationId && !input.organizationIds) + fail('invalid_input'); + const targets = + input.scope === 'core' + ? [] + : input.organizationIds + ? [...new Set(input.organizationIds.map(identifier))] + : input.organizationId + ? [identifier(input.organizationId)] + : fleet() + .filter((o) => ['ready', 'suspended'].includes(o.status)) + .map((o) => o.id); + const system = input.scope === 'system' || (!input.scope && !input.organizationId); + const members = []; + const extra = system + ? { setId: newId(), retentionDays: settings('system').retentionDays } + : {}; + if (system && targets.length !== fleet().length) fail('backup_dsp_unavailable'); + if (input.scope === 'core' || system) + members.push({ + organizationId: null, + requestId: enqueue('core', null, `${key}:core`, session.user.id, extra), + }); + for (const id of targets) + members.push({ + organizationId: id, + requestId: enqueue('backup', id, `${key}:${id}`, session.user.id, extra), + }); + if (system) { + db.prepare("INSERT INTO backup_sets VALUES(?,?,?,'pending')").run( + extra.setId, + clock(), + JSON.stringify(members), + ); + db.prepare('INSERT INTO backup_set_settings VALUES(?,?)').run( + extra.setId, + JSON.stringify(settings('system')), + ); + } + } else if (['restore', 'delete'].includes(input.action)) { + if (db.prepare("SELECT 1 FROM platform_rollouts WHERE status!='completed'").get()) + fail('backup_operation_in_progress'); + let selected; + if (input.scope === 'system') { + if (input.organizationId || input.backupId) fail('invalid_input'); + const set = db + .prepare('SELECT * FROM backup_sets WHERE id=?') + .get(identifier(input.setId)); + if ( + !set || + (input.action === 'restore' && + (set.status !== 'verified' || + archive().sets?.[set.id]?.setDigest !== + crypto.createHash('sha256').update(JSON.stringify(set)).digest('hex'))) + ) + fail('backup_restore_unavailable'); + const members = JSON.parse(set.members_json); + if (members.some(m => m.requestId && db.prepare("SELECT 1 FROM platform_backup_requests WHERE id=? AND status IN ('queued','running')").get(m.requestId))) + fail('backup_operation_in_progress'); + if (input.action === 'restore') { + const current = fleet() + .map((o) => o.id) + .sort(), + captured = members + .map((m) => m.organizationId) + .filter(Boolean) + .sort(); + if (JSON.stringify(current) !== JSON.stringify(captured)) + fail('backup_identity_conflict'); + } + selected = members + .map((m) => + db + .prepare('SELECT * FROM platform_backup_records WHERE id=? AND deleted_at IS NULL') + .get(m.backupId), + ) + .filter(Boolean); + if (input.action === 'restore' && selected.length !== members.length) + fail('backup_restore_unavailable'); + } else { + const row = db + .prepare('SELECT * FROM platform_backup_records WHERE id=? AND deleted_at IS NULL') + .get(identifier(input.backupId)); + if ( + !row || + (input.scope === 'core' + ? row.kind !== 'core' || input.organizationId + : row.organization_id !== input.organizationId || row.kind !== 'dsp') + ) + fail('backup_not_found'); + selected = [row]; + } + const label = + input.scope === 'system' + ? 'Full system' + : input.scope === 'core' + ? 'Platform Core' + : fleet().find((o) => o.id === input.organizationId)?.name; + if (input.confirmation !== label) fail('backup_confirmation_required'); + selected.sort((a, b) => Number(b.kind === 'core') - Number(a.kind === 'core')); + for (const [position, row] of selected.entries()) { + if ( + active(row.organization_id) || + (row.organization_id && store.activeLifecycleJob(row.organization_id)) || + db + .prepare("SELECT 1 FROM backup_deletions WHERE backup_id=? AND status='queued'") + .get(row.id) + ) + fail('backup_operation_in_progress'); + if (input.action === 'restore') { + const proof = archive().backups?.[row.id]; + if ( + row.kind === 'dsp' ? restoreEligibility(row, proof) : coreRestoreBlocked(row, proof) + ) + fail('backup_restore_unavailable'); + enqueue( + row.kind === 'core' ? 'core' : 'restore', + row.organization_id, + `${key}:${row.id}`, + session.user.id, + { + ...(input.scope === 'system' + ? { + restoreSet: key, + setId: input.setId, + position, + systemSchedule: JSON.parse( + db + .prepare('SELECT settings_json FROM backup_set_settings WHERE set_id=?') + .get(input.setId)?.settings_json || + JSON.stringify(DEFAULT_BACKUP_SETTINGS), + ), + } + : {}), + backupId: row.id, + action: 'restore', + wasRunning: fleet().find((o) => o.id === row.organization_id)?.status === 'ready', + }, + ); + } else { + if ( + db + .prepare( + "SELECT 1 FROM platform_backup_requests WHERE status IN ('queued','running') AND json_extract(input_json,'$.backupId')=?", + ) + .get(row.id) + ) + fail('backup_operation_in_progress'); + db.prepare("INSERT INTO backup_deletions VALUES(?,?,?,'queued',?,NULL)").run( + newId(), + row.id, + row.organization_id, + clock(), + ); + } + } + if (input.action === 'delete' && input.scope === 'system') + db.prepare('UPDATE backup_sets SET status=? WHERE id=?').run(selected.length ? 'incomplete' : 'deleting', input.setId); + } else fail('invalid_input'); + db.prepare('INSERT INTO platform_backup_commands VALUES(?,?,?,?)').run( + session.user.id, + key, + encoded, + clock(), + ); + store.createAudit({ + id: `aud_${crypto.randomBytes(16).toString('hex')}`, + actorUserId: session.user.id, + organizationId: input.organizationId || null, + action: `platform.backup.${input.action}`, + targetType: 'platform_backup', + targetId: input.backupId || null, + result: 'succeeded', + timestamp: clock(), + }); + }); + } + function schedule() { + return store.transaction(() => { + if (!enabled || db.prepare("SELECT 1 FROM platform_rollouts WHERE status!='completed'").get()) + return; + for (const scope of ['system', 'core', ...fleet().map((o) => o.id)]) { + const row = settingsRow(scope), + value = settings(scope), + slot = scheduledSlot(value, clock()); + if ( + !slot || + db + .prepare('SELECT 1 FROM backup_scope_slots WHERE scope=? AND revision=? AND slot=?') + .get(scope, row.revision, slot) + ) + continue; + const targets = scope === 'system' ? fleet() : fleet().filter((o) => o.id === scope); + // A full-system set never silently skips a busy DSP. + if ( + targets.some( + (o) => + !['ready', 'suspended'].includes(o.status) || + active(o.id) || + store.activeLifecycleJob(o.id), + ) + ) + continue; + if ( + ['system', 'core'].includes(scope) && + db + .prepare( + "SELECT 1 FROM platform_backup_requests WHERE kind='core' AND status IN ('queued','running')", + ) + .get() + ) + continue; + const key = `scheduled:${scope}:${row.revision}:${slot}`, + setId = scope === 'system' ? newId() : null; + const extra = { retentionDays: value.retentionDays, ...(setId ? { setId } : {}) }, + members = []; + if (['system', 'core'].includes(scope)) + members.push({ + organizationId: null, + requestId: enqueue('core', null, `${key}:core`, null, extra), + }); + for (const org of targets) + members.push({ + organizationId: org.id, + requestId: enqueue('backup', org.id, `${key}:${org.id}`, null, extra), + }); + if (setId) { + db.prepare("INSERT INTO backup_sets VALUES(?,?,?,'pending')").run( + setId, + clock(), + JSON.stringify(members), + ); + db.prepare('INSERT INTO backup_set_settings VALUES(?,?)').run( + setId, + JSON.stringify(value), + ); + } + db.prepare('INSERT INTO backup_scope_slots VALUES(?,?,?,?)').run( + scope, + row.revision, + slot, + clock(), + ); + } + }); + } + function latestSetRestore(setId) { + const latest = db.prepare("SELECT json_extract(input_json,'$.restoreSet') AS restore_set FROM platform_backup_requests WHERE json_extract(input_json,'$.setId')=? AND json_extract(input_json,'$.restoreSet') IS NOT NULL ORDER BY created_at DESC,rowid DESC LIMIT 1").get(setId); + if (!latest) return null; + const members = db.prepare("SELECT status FROM platform_backup_requests WHERE json_extract(input_json,'$.restoreSet')=?").all(latest.restore_set); + return {status: members.some(m => m.status === 'failed') ? 'failed' : members.every(m => m.status === 'completed') ? 'completed' : 'running'}; + } + function storageUsage(remote) { + const measured = remote.usage; + if (!measured || !['ready','stale'].includes(measured.status)) return {status:'unavailable', checkedAt:null}; + const organizations = db.prepare(`SELECT o.id,o.name,EXISTS(SELECT 1 FROM dsp_removals d WHERE d.organization_id=o.id) AS removed + FROM organizations o JOIN installations i ON i.organization_id=o.id ORDER BY o.name,o.id`).all(); + const byId = new Map(measured.dsps.map(row => [row.organizationId,row])); + const scopes = [{scope:'core', name:'Platform Core', removed:false, ...measured.core}, + ...organizations.map(org => ({scope:org.id,name:org.name,removed:!!org.removed,bytes:byId.get(org.id)?.bytes || 0,backupCount:byId.get(org.id)?.backupCount || 0}))]; + const other = {...measured.other}; + for (const row of measured.dsps) if (!organizations.some(org => org.id === row.organizationId)) { + other.bytes += row.bytes; other.backupCount += row.backupCount; + } + return { + status:measured.status === 'stale' || remote.status !== 'connected' || clock()-measured.checkedAt >= 300000 ? 'stale' : 'ready', + checkedAt:iso(measured.checkedAt), bytes:measured.bytes, backupCount:measured.backupCount, + retainedBytes:scopes.filter(s => s.removed).reduce((n,s) => n+s.bytes,0), + manifestBytes:measured.manifestBytes, legacyBytes:measured.legacyBytes, artifactBytes:measured.artifactBytes || 0, other, scopes, sets:measured.sets, + }; + } + function view() { + require('./backup-categories').classifyExisting(db); + const remote = archive(), + value = settings(), + row = settingsRow(); + const records = db + .prepare( + 'SELECT * FROM platform_backup_records WHERE deleted_at IS NULL ORDER BY created_at DESC LIMIT 1000', + ) + .all(); + const backups = records.map((r) => { + const proof = remote.backups?.[r.id], + metadata = JSON.parse(r.metadata_json); + return { + id: r.id, + organizationId: r.organization_id, + kind: r.kind, + name: metadata.organization?.name || 'Platform Core', + createdAt: iso(r.created_at), + expiresAt: proof?.retained ? null : iso(proof?.expiresAt ?? r.expires_at), + retentionDays: r.retention_days, + size: proof?.size ?? null, + status: + proof?.status === 'expired' || + (!proof?.retained && + (proof?.expiresAt ?? r.expires_at) !== null && + (proof?.expiresAt ?? r.expires_at) <= clock()) + ? 'expired' + : proof?.status === 'verified' + ? 'verified' + : 'pending', + verifiedAt: iso(proof?.verifiedAt ?? null), + trigger: db.prepare('SELECT category FROM backup_categories WHERE backup_id=?').get(r.id)?.category || proof?.trigger || 'manual', + category: db.prepare('SELECT category FROM backup_categories WHERE backup_id=?').get(r.id)?.category || 'manual', + verification: proof?.verification || 'restore', + restoreBlocked: + r.kind === 'core' ? coreRestoreBlocked(r, proof) : restoreEligibility(r, proof), + }; + }); + const operations = db + .prepare( + `SELECT * FROM (SELECT *, + ROW_NUMBER() OVER (PARTITION BY organization_id ORDER BY created_at DESC,id DESC) AS scope_rank, + ROW_NUMBER() OVER (ORDER BY created_at DESC,id DESC) AS recent_rank + FROM platform_backup_requests) + WHERE scope_rank=1 OR recent_rank<=50 OR status IN ('queued','running') + ORDER BY created_at DESC,id DESC`, + ) + .all() + .map((r) => { + const input = JSON.parse(r.input_json); + const job = r.job_id ? store.lifecycleJob(r.job_id) : null; + const restoreJob = + r.kind === 'restore' + ? store.lifecycleJobByRequest( + r.organization_id, + 'platform_backups', + `${r.id}:restoring`, + ) + : null; + return { + id: r.id, + organizationId: r.organization_id, + kind: input.action === 'restore' ? 'restore' : r.kind, + setId: input.setId || null, + category: require('./backup-categories').categoryForRequest(r), + restoreSet: input.restoreSet || null, + status: r.status, + phase: r.phase, + createdAt: iso(r.created_at), + updatedAt: iso(r.updated_at), + backupId: + r.kind === 'restore' || input.action === 'restore' + ? input.backupId + : job?.backup_id || (r.kind === 'core' ? r.id : null), + safetyBackupId: restoreJob?.safety_backup_id || null, + failureCode: r.failure_code, + }; + }); + const rolloutActive = !!db + .prepare("SELECT 1 FROM platform_rollouts WHERE status!='completed'") + .get(); + return { + enabled, + canBackupCore: + enabled && + !rolloutActive && + !operations.some((o) => o.organizationId === null && ['queued', 'running'].includes(o.status)), + operationBlocked: rolloutActive + ? 'Backups and restores are unavailable while a platform rollout is in progress.' + : null, + schedules: ['system', 'core', ...fleet().map((o) => o.id)].map((scope) => ({ + scope, + settings: settings(scope), + revision: settingsRow(scope).revision, + nextBackupAt: nextScheduledAt(settings(scope), clock()), + })), + sets: db + .prepare('SELECT * FROM backup_sets ORDER BY created_at DESC') + .all() + .map((set) => ({ + id: set.id, + createdAt: iso(set.created_at), + restore: latestSetRestore(set.id), + status: + set.status === 'verified' && + remote.sets?.[set.id]?.setDigest !== + crypto.createHash('sha256').update(JSON.stringify(set)).digest('hex') + ? 'pending' + : set.status, + members: JSON.parse(set.members_json), + busy: JSON.parse(set.members_json).some(m => m.requestId && db.prepare("SELECT 1 FROM platform_backup_requests WHERE id=? AND status IN ('queued','running')").get(m.requestId)), + })), + settings: value, + revision: row.revision, + nextBackupAt: nextScheduledAt(value, clock()), + storage: { status: remote.status, checkedAt: remote.checkedAt || null }, + storageUsage: storageUsage(remote), + organizations: fleet().map((o) => ({ + id: o.id, + name: o.name, + status: o.status, + canBackup: + enabled && + !rolloutActive && + ['ready', 'suspended'].includes(o.status) && + ['active', 'suspended'].includes(o.organization_status) && + !active(o.id) && + !store.activeLifecycleJob(o.id), + })), + backups, + deletions: db + .prepare( + 'SELECT id,backup_id AS backupId,organization_id AS organizationId,status,failure_code AS failureCode FROM backup_deletions ORDER BY created_at DESC', + ) + .all(), + operations, + }; + } + return { view, command, schedule, settings, enqueue }; +} +module.exports = { createPlatformBackups }; diff --git a/core/core/accounts/src/platform-updates.js b/core/core/accounts/src/platform-updates.js new file mode 100644 index 0000000..c439685 --- /dev/null +++ b/core/core/accounts/src/platform-updates.js @@ -0,0 +1,310 @@ +'use strict'; + +const { compareVersions } = require('../../../shared/release-version'); +const crypto = require('node:crypto'); +const { queueRolloutBackups, rolloutBackupProgress, retryRolloutBackups } = require('./rollout-backups'); +const { AccessError, exact, idempotencyKey } = require('./validation'); +const { createAccessInstallationLifecycleAuthority } = require('./installation-lifecycle'); +const fail = code => { throw new AccessError(code, 409); }; + +function createPlatformUpdates({ store, releases = {}, platformReleases = {}, enabled = false, clock = Date.now, loadCatalogs = null, delivery = null, + canaryVerifier = null, + cleanupReady = require('../../installations/src/release-retention-status').cleanupReady }) { + const db = store.db; + function refreshCatalogs() { + if (!loadCatalogs) return; + const current = loadCatalogs(); + releases = current.releases; platformReleases = current.platformReleases; + } + const core = id => db.prepare('SELECT * FROM platform_rollout_core WHERE rollout_id=?').get(id); + const available = id => Object.hasOwn(releases, id) && Object.hasOwn(platformReleases, id); + const settled = (member, releaseId) => member.release_id === releaseId && (member.installation_status === 'ready' + || member.backend === 'native_service_v1' && ['pending', 'waiting_for_owner', 'waiting_for_provider_auth', 'suspended'].includes(member.installation_status)); + const latest = () => db.prepare('SELECT * FROM platform_rollouts ORDER BY created_at DESC,rowid DESC LIMIT 1').get(); + const members = id => db.prepare(`SELECT m.*,o.name,i.status AS installation_status,i.release_id,i.backend,o.status AS organization_status + FROM platform_rollout_members m JOIN organizations o ON o.id=m.organization_id JOIN installations i ON i.organization_id=o.id + WHERE m.rollout_id=? ORDER BY m.position,m.organization_id`).all(id); + function includeFleet(id) { + db.prepare(`INSERT OR IGNORE INTO platform_rollout_members(rollout_id,organization_id,position,status) + SELECT ?,i.organization_id,(SELECT count(*) FROM platform_rollout_members WHERE rollout_id=?)+row_number() OVER (ORDER BY o.created_at,o.id),'queued' + FROM installations i JOIN organizations o ON o.id=i.organization_id WHERE i.status NOT IN ('decommissioning','decommissioned') + AND NOT EXISTS (SELECT 1 FROM dsp_removals d WHERE d.organization_id=i.organization_id) + AND NOT EXISTS (SELECT 1 FROM installation_lifecycle_jobs j WHERE j.id=i.current_job_id AND j.operation IN ('decommission','destroy'))`).run(id, id); + } + function command(session, input) { + if (input?.action === 'retry_download') { + exact(input, ['action']); + if (!enabled || !delivery) fail('installation_operator_disabled'); + delivery.retry(); return; + } + if (input?.action !== 'pause') refreshCatalogs(); + exact(input, ['action', 'idempotencyKey', 'releaseId', 'canaryOrganizationId']); + if (!enabled) fail('installation_operator_disabled'); + if (input.canaryOrganizationId !== undefined && input.action !== 'start') fail('invalid_input'); + if (input.action === 'start') { + const key = idempotencyKey(input.idempotencyKey); + if (!available(input.releaseId)) fail('update_unavailable'); + store.transaction(() => { + const replay = db.prepare('SELECT * FROM platform_rollouts WHERE actor_user_id=? AND idempotency_key=?').get(session.user.id, key); + if (replay) { + const selected = members(replay.id).find(member => member.position === 0)?.organization_id; + if (replay.release_id !== input.releaseId || selected !== input.canaryOrganizationId) fail('idempotency_conflict'); + return; + } + if (latest() && latest().status !== 'completed') fail('rollout_in_progress'); + if (view().releases[0]?.id !== input.releaseId) fail('update_unavailable'); + if (db.prepare("SELECT 1 FROM platform_backup_requests WHERE status IN ('queued','running')").get()) fail('installation_operation_in_progress'); + // Native recovery requires a native fleet. Reject before queuing Core + // so an incompatible DSP cannot strand the update in backup preparation. + if (releases[input.releaseId].backend === 'native_service_v1' + && db.prepare("SELECT 1 FROM installations WHERE backend<>'native_service_v1' AND status<>'decommissioned'").get()) { + fail('native_migration_required'); + } + if (db.prepare("SELECT 1 FROM installation_lifecycle_jobs WHERE status IN ('queued','running')").get() + || db.prepare("SELECT 1 FROM installation_provisioning_requests p JOIN installations i ON i.organization_id=p.organization_id WHERE p.status IN ('pending','dispatched') AND i.status='provisioning'").get() + || db.prepare("SELECT 1 FROM installation_onboarding_requests q JOIN installations i ON i.organization_id=q.organization_id WHERE q.status IN ('enrolling','queued','running') AND i.status='provisioning'").get()) fail('installation_operation_in_progress'); + const id = `rollout_${crypto.randomBytes(16).toString('hex')}`; + db.prepare("INSERT INTO platform_rollouts VALUES(?,?,?,?,'running',?,?)").run(id, input.releaseId, session.user.id, key, clock(), clock()); + includeFleet(id); + if (input.canaryOrganizationId !== undefined) { + const canary = members(id).find(member => member.organization_id === input.canaryOrganizationId); + if (!canary || canary.backend !== 'native_service_v1' || canary.installation_status !== 'ready' + || canary.organization_status !== 'active') fail('rollout_canary_unavailable'); + db.prepare('UPDATE platform_rollout_members SET position=0 WHERE rollout_id=? AND organization_id=?').run(id, canary.organization_id); + } + db.prepare("INSERT INTO platform_rollout_core VALUES(?,'queued',?,0,NULL,?)").run(id, JSON.stringify(platformReleases[input.releaseId]), clock()); + queueRolloutBackups(store, { id, actor_user_id: session.user.id }, clock()); + audit(session, 'platform.rollout.start', id); + require('./worker-wakeup').afterCommit(store, ['reconcile']); + }); + } else if (['pause', 'resume'].includes(input.action)) { + store.transaction(() => { + const row = latest(); + if (!row || row.status === 'completed') fail('rollout_not_active'); + if (input.action === 'resume' && !available(row.release_id)) fail('update_unavailable'); + if (input.action === 'resume') { + retryRolloutBackups(store, row.id, clock()); + db.prepare("UPDATE platform_rollout_core SET status='queued',failure_code=NULL,updated_at=? WHERE rollout_id=? AND status='failed'").run(clock(), row.id); + for (const member of members(row.id).filter(m => m.status === 'blocked')) { + const job = member.job_id ? store.lifecycleJob(member.job_id) : null; + if (job?.status === 'running' && job.attempt >= job.max_attempts && job.lease_expires_at <= clock()) { + createAccessInstallationLifecycleAuthority({ store, organizationId: member.organization_id, + authorityScope: 'platform_rollout', actorUserId: session.user.id, releaseCatalog: Object.keys(releases), clock }).retryExhausted(job.id); + db.prepare("UPDATE platform_rollout_members SET status='updating',message=NULL WHERE rollout_id=? AND organization_id=?").run(row.id, member.organization_id); + } else { + db.prepare("UPDATE platform_rollout_members SET status='queued',job_id=NULL,attempt=attempt+1,message=NULL WHERE rollout_id=? AND organization_id=?").run(row.id, member.organization_id); + } + } + } + db.prepare('UPDATE platform_rollouts SET status=?,updated_at=? WHERE id=?').run(input.action === 'pause' ? 'paused' : 'running', clock(), row.id); + audit(session, `platform.rollout.${input.action}`, row.id); + if (input.action === 'resume') require('./worker-wakeup').afterCommit(store, ['reconcile', 'core']); + }); + } else fail('invalid_input'); + } + function audit(session, action, id) { + store.createAudit({ id: `aud_${crypto.randomBytes(16).toString('hex')}`, actorUserId: session.user.id, + organizationId: null, action, targetType: 'platform_rollout', targetId: id, result: 'succeeded', timestamp: clock() }); + } + function block(row, member, message) { + store.transaction(() => { + db.prepare("UPDATE platform_rollout_members SET status='blocked',message=? WHERE rollout_id=? AND organization_id=?").run(message, row.id, member.organization_id); + db.prepare("UPDATE platform_rollouts SET status='paused',updated_at=? WHERE id=?").run(clock(), row.id); + }); + } + const canaryChecks = new Set(); + function canaryReady(row) { + const canary = members(row.id).find(member => member.position === 0); + if (!canary) return true; + if (canary.status === 'removed') { block(row, canary, 'The test DSP was removed. Restore it before resuming this rollout.'); return false; } + if (canary.status !== 'updated') return false; + const verificationId = `canary_${crypto.createHash('sha256').update(`${row.id}:${canary.attempt}`).digest('hex').slice(0, 32)}`; + const event = action => db.prepare('SELECT * FROM audit_events WHERE target_id=? AND action=? ORDER BY created_at DESC LIMIT 1') + .get(verificationId, `platform.rollout.canary.${action}`); + if (event('succeeded')) return true; + if (canaryChecks.has(verificationId)) return false; + if (!canaryVerifier) { block(row, canary, 'Test DSP verification is unavailable. Resolve it before resuming.'); return false; } + const record = (action, result) => store.createAudit({ id: `aud_${crypto.randomBytes(16).toString('hex')}`, + actorUserId: row.actor_user_id, organizationId: canary.organization_id, action: `platform.rollout.canary.${action}`, + targetType: 'platform_rollout_canary', targetId: verificationId, result, timestamp: clock() }); + if (!event('started')) record('started', 'succeeded'); + const startedAt = event('started').created_at; + const runtimeKey = db.prepare('SELECT runtime_key FROM installations WHERE organization_id=?').get(canary.organization_id).runtime_key; + db.prepare('UPDATE platform_rollout_members SET message=? WHERE rollout_id=? AND organization_id=?').run('Verifying a fresh collection on the test DSP.', row.id, canary.organization_id); + canaryChecks.add(verificationId); + store.afterCommit(() => { + Promise.resolve().then(() => canaryVerifier({ runtimeKey, verificationId, startedAt })).then(result => { + if (result !== true) throw Error('canary_collection_failed'); + if (!store.db) return; + store.transaction(() => { + const current = members(row.id).find(member => member.organization_id === canary.organization_id); + if (current?.attempt !== canary.attempt || current.status !== 'updated') return; + if (!event('succeeded')) record('succeeded', 'succeeded'); + db.prepare('UPDATE platform_rollout_members SET message=? WHERE rollout_id=? AND organization_id=?').run('Test DSP collection verified.', row.id, canary.organization_id); + }); + }).catch(() => { + if (!store.db) return; + store.transaction(() => { + const current = members(row.id).find(member => member.organization_id === canary.organization_id); + if (current?.attempt !== canary.attempt || event('succeeded')) return; + record('failed', 'denied'); + block(row, canary, 'Test DSP collection verification failed. Resolve it and resume before updating other DSPs.'); + }); + }).finally(() => canaryChecks.delete(verificationId)).catch(() => { + // A failed database write leaves no success proof; the next tick + // rechecks the gate and cannot advance the fleet. + }); + }); + return false; + } + // This coordinator queues at most one lifecycle job. The existing private worker + // owns execution, fenced leases, backup, health verification and rollback. + function tick() { refreshCatalogs(); return store.transaction(advance); } + function advance() { + let row = latest(); + if (!row || row.status === 'completed') return; + // Older DSP-only rollouts must also pass the Core stage before proceeding. + if (!core(row.id)) { + if (!available(row.release_id)) { + db.prepare("UPDATE platform_rollouts SET status='paused',updated_at=? WHERE id=?").run(clock(), row.id); + return; + } + db.prepare("INSERT INTO platform_rollout_core VALUES(?,'queued',?,0,NULL,?)").run(row.id, JSON.stringify(platformReleases[row.release_id]), clock()); + } + const coreState = core(row.id); + if (!available(row.release_id) || JSON.stringify(platformReleases[row.release_id]) !== coreState.release_json) { + db.prepare("UPDATE platform_rollout_core SET status='failed',failure_code='release_bundle_changed',updated_at=? WHERE rollout_id=?").run(clock(), row.id); + db.prepare("UPDATE platform_rollouts SET status='paused',updated_at=? WHERE id=?").run(clock(), row.id); + return; + } + const backups = rolloutBackupProgress(db, row.id); + if (backups && backups.status !== 'completed') { + if (backups.status === 'failed') db.prepare("UPDATE platform_rollouts SET status='paused',updated_at=? WHERE id=?").run(clock(), row.id); + return; + } + if (coreState.status !== 'succeeded') return; + store.transaction(() => includeFleet(row.id)); + // Removal is explicit in history and no longer belongs to the current fleet. + db.prepare("UPDATE platform_rollout_members SET status='removed',message='Removed from the platform' WHERE rollout_id=? AND organization_id IN (SELECT organization_id FROM installations WHERE status='decommissioned')").run(row.id); + const active = members(row.id).find(m => m.status === 'updating'); + if (active) { + const key = `${row.id}:${active.organization_id}:${active.attempt}`; + const job = active.job_id ? store.lifecycleJob(active.job_id) : store.lifecycleJobByRequest(active.organization_id, 'platform_rollout', key); + if (job) { + db.prepare('UPDATE platform_rollout_members SET job_id=? WHERE rollout_id=? AND organization_id=?').run(job.id, row.id, active.organization_id); + if (job.status === 'running' && job.attempt >= job.max_attempts && job.lease_expires_at <= clock()) return block(row, active, 'The worker was interrupted repeatedly. Resume to retry the same update safely.'); + if (job.status === 'succeeded') { + if (!settled(active, row.release_id)) return block(row, active, 'Update verification is incomplete.'); + db.prepare("UPDATE platform_rollout_members SET status='updated',job_id=?,message=NULL WHERE rollout_id=? AND organization_id=?").run(job.id, row.id, active.organization_id); + } else if (job.status === 'failed') block(row, active, 'Update failed. Review this DSP and resume to retry.'); + return; + } + if (row.status === 'paused') return; + try { + const control = store.installationControl(active.organization_id); + const authority = createAccessInstallationLifecycleAuthority({ store, organizationId: active.organization_id, + authorityScope: 'platform_rollout', actorUserId: row.actor_user_id, releaseCatalog: Object.keys(releases), clock }); + const result = authority.request({ operation: 'upgrade', idempotencyKey: key, expectedRevision: control.revision, releaseId: row.release_id }); + db.prepare('UPDATE platform_rollout_members SET job_id=? WHERE rollout_id=? AND organization_id=?').run(result.id, row.id, active.organization_id); + } catch (error) { + if (error.code === 'installation_operation_in_progress') { + db.prepare("UPDATE platform_rollout_members SET status='queued',job_id=NULL WHERE rollout_id=? AND organization_id=?").run(row.id, active.organization_id); + return; + } + // Another coordinator may have persisted the same idempotent job. + if (!store.lifecycleJobByRequest(active.organization_id, 'platform_rollout', key)) block(row, active, 'This DSP is not ready to update. Resolve its setup or runtime issue, then resume.'); + } + return; + } + if (row.status === 'paused') return; + store.transaction(() => { + row = latest(); + if (row.status !== 'running') return; + const allMembers = members(row.id); + const selectedCanary = allMembers.find(member => member.position === 0); + if (selectedCanary && ['updated', 'removed'].includes(selectedCanary.status) && !canaryReady(row)) return; + const fleet = allMembers.filter(m => m.status !== 'removed'); + if (fleet.some(m => m.status === 'updating')) return; + // Recheck previously updated members before declaring fleet completion. + const next = fleet.find(m => m.status !== 'updated' || !settled(m, row.release_id)); + if (!next) { + if (releases[row.release_id]?.backend === 'native_service_v1' && !cleanupReady(row.id, row.release_id)) return; + db.prepare("UPDATE platform_rollouts SET status='completed',updated_at=? WHERE id=?").run(clock(), row.id); + return; + } + if (settled(next, row.release_id)) { + db.prepare("UPDATE platform_rollout_members SET status='updated',message=NULL WHERE rollout_id=? AND organization_id=?").run(row.id, next.organization_id); + } else if (store.activeLifecycleJob(next.organization_id) + || ['provisioning', 'verifying'].includes(next.installation_status) + || db.prepare("SELECT 1 FROM installation_onboarding_requests WHERE organization_id=? AND status IN ('enrolling','queued','running')").get(next.organization_id)) { + return; + } else if (next.backend === 'native_service_v1' && next.installation_status === 'pending') { + if (db.prepare("SELECT 1 FROM installation_provisioning_requests WHERE organization_id=? AND status IN ('pending','dispatched')").get(next.organization_id)) return; + // An invited DSP without a runtime needs only the current release + // assignment. Its eventual provisioning uses this updated manifest. + db.prepare('UPDATE installations SET release_id=?,revision=revision+1,manifest_revision=manifest_revision+1,updated_at=? WHERE organization_id=?') + .run(row.release_id, clock(), next.organization_id); + } else if (!(next.installation_status === 'ready' && next.organization_status === 'active' + || next.backend === 'native_service_v1' && ['suspended', 'waiting_for_owner', 'waiting_for_provider_auth'].includes(next.installation_status)) + || next.backend === 'local_reference' || !Object.hasOwn(releases, row.release_id)) { + db.prepare("UPDATE platform_rollout_members SET status='blocked',message=? WHERE rollout_id=? AND organization_id=?") + .run(next.backend === 'local_reference' ? 'This DSP must use an isolated runtime before it can receive runtime updates.' : 'Finish setup or resolve this DSP’s runtime status, then resume the rollout.', row.id, next.organization_id); + db.prepare("UPDATE platform_rollouts SET status='paused',updated_at=? WHERE id=?").run(clock(), row.id); + } else { + db.prepare("UPDATE platform_rollout_members SET status='updating',message=NULL WHERE rollout_id=? AND organization_id=?").run(row.id, next.organization_id); + } + }); + } + function view(selectedReleaseId = null) { + refreshCatalogs(); + const row = latest(); + const fleet = row ? members(row.id) : []; + const coreRow = row ? core(row.id) : null; + const release = coreRow ? JSON.parse(coreRow.release_json) : null; + const backups = row ? rolloutBackupProgress(db, row.id) : null; + const phase = backups && backups.status !== 'completed' ? 'backups' : coreRow?.status === 'failed' && coreRow.failure_code === 'core_verification_failed' ? 'verify_core' + : !coreRow || ['queued', 'updating', 'failed'].includes(coreRow.status) ? 'core' + : coreRow.status === 'verifying' ? 'verify_core' : row.status === 'completed' ? 'complete' : 'dsps'; + const completed = db.prepare("SELECT r.release_id,c.release_json FROM platform_rollouts r LEFT JOIN platform_rollout_core c ON c.rollout_id=r.id WHERE r.status='completed' ORDER BY r.updated_at DESC LIMIT 1").get(); + const installed = completed?.release_json ? JSON.parse(completed.release_json) : null; + const offered = Object.entries(platformReleases).filter(([id, item]) => available(id) && id !== completed?.release_id && (!installed || (compareVersions(item.version, installed.version) || item.publishedAt.localeCompare(installed.publishedAt)) > 0)) + .sort((a, b) => compareVersions(b[1].version, a[1].version) || b[1].publishedAt.localeCompare(a[1].publishedAt)) + .map(([id, item]) => ({ id, version: item.version, publishedAt: item.publishedAt, changelog: item.changelog })); + // Installation catalogs may be pruned; release history and rollout snapshots + // keep their human-readable notes available independently of package retention. + const known = { ...(delivery?.history?.() || {}), ...platformReleases }; + for (const snapshot of db.prepare('SELECT r.release_id,c.release_json FROM platform_rollouts r JOIN platform_rollout_core c ON c.rollout_id=r.id').all()) { + if (!known[snapshot.release_id]) known[snapshot.release_id] = JSON.parse(snapshot.release_json); + } + const defaultId = row && row.status !== 'completed' ? row.release_id : offered[0]?.id || completed?.release_id; + const selectedId = selectedReleaseId || defaultId; + if (selectedReleaseId && !Object.hasOwn(known, selectedReleaseId)) throw new AccessError('release_not_found', 404); + const state = id => row?.release_id === id && row.status !== 'completed' ? 'rolling_out' + : completed?.release_id === id ? 'installed' : offered[0]?.id === id ? 'available' : 'historical'; + const selected = selectedId && known[selectedId]; + const displayedRelease = selected ? { id: selectedId, version: selected.version, publishedAt: selected.publishedAt, + changelog: selected.changelog, state: state(selectedId), notes: delivery?.notes?.(selectedId, selected) || null } : null; + const releaseHistory = Object.entries(known).sort((a,b) => compareVersions(b[1].version,a[1].version) + || b[1].publishedAt.localeCompare(a[1].publishedAt)).map(([id,item]) => ({ id, version: item.version, publishedAt: item.publishedAt, state: state(id) })); + const activity = []; + if (coreRow?.status === 'succeeded') activity.push({ label: 'Core checks passed', at: new Date(coreRow.updated_at).toISOString() }); + for (const member of fleet) { + const job = member.job_id ? store.lifecycleJob(member.job_id) : null; + if (member.status === 'updated' && job?.finished_at) activity.push({ label: `${member.name} updated`, at: new Date(job.finished_at).toISOString() }); + else if (member.status === 'updating' && job?.started_at) activity.push({ label: `${member.name} update started`, at: new Date(job.started_at).toISOString() }); + } + return { enabled, releases: offered, displayedRelease, releaseHistory, ...(delivery ? { delivery: delivery.view() } : {}), + rollout: row ? { status: row.status, phase, release: row.release_id, version: release?.version || null, + backups: backups ? { status: backups.status, total: backups.total, completed: backups.completed, + members: backups.members.map(m => ({ name: m.name, status: m.status, phase: m.phase })) } : null, + core: { status: coreRow?.status || 'unverified', message: coreRow?.status === 'failed' + ? 'Core could not be updated or verified. Resolve the server issue, then resume.' : !coreRow ? 'A complete platform release is required to continue.' : null }, + total: fleet.filter(m => m.status !== 'removed').length, + updated: fleet.filter(m => m.status === 'updated').length, + members: fleet.map(m => ({ name: m.name, status: m.status, message: m.message })), + activity: activity.sort((a, b) => b.at.localeCompare(a.at)).slice(0, 3), + updatedAt: new Date(row.updated_at).toISOString() } : null }; + } + return { command, view, tick }; +} +module.exports = { createPlatformUpdates }; diff --git a/core/core/accounts/src/plugin-settings.js b/core/core/accounts/src/plugin-settings.js new file mode 100644 index 0000000..5cc6691 --- /dev/null +++ b/core/core/accounts/src/plugin-settings.js @@ -0,0 +1,117 @@ +"use strict"; +const { AccessError } = require("./validation"); +const { plugin } = require("../../../shared/plugin-sdk/catalog"); +function createPluginSettings({ store, access, port }) { + const fail = (code, status = 409) => { + throw new AccessError(code, status); + }; + function context(session, id, write) { + const view = access.organizationFor(session, "dashboard.view"); + const definition = store.pluginMetadataFor ? store.pluginMetadataFor(view.organization.id,id) : plugin(id); + if (!definition) fail("plugin_not_found", 404); + const selected = write + ? access.requireDspOwner(session) + : access.organizationFor( + session, + definition.pages[0]?.permission || "dashboard.view", + ); + const installation = store.installationControl(selected.organization.id); + const row = store.db + .prepare( + "SELECT * FROM dsp_plugins WHERE organization_id=? AND plugin_id=?", + ) + .get(selected.organization.id, id); + if ( + selected.organization.status !== "active" || + installation?.status !== "ready" || + !row || + row.desired_state !== "enabled" || + row.applied_state !== "enabled" || + row.revision !== row.applied_revision || + store.activeLifecycleJob(selected.organization.id) || + store.db + .prepare("SELECT 1 FROM dsp_removals WHERE organization_id=?") + .get(selected.organization.id) || + store.db + .prepare( + "SELECT 1 FROM directory_lifecycle_requests WHERE organization_id=? AND status IN ('queued','running')", + ) + .get(selected.organization.id) + ) + fail("plugin_unavailable"); + return { + organizationId: selected.organization.id, + runtimeKey: installation.runtimeKey, + installationRevision: installation.revision, + revision: row.revision, + }; + } + return async function settings(session, id, action, input) { + if (!["get", "options", "update", "history"].includes(action)) + fail("invalid_input", 400); + const write = action === "update", + before = context(session, id, write || action === "history"); + if (typeof port !== "function") fail("plugin_unavailable", 503); + let result; + try { + result = await port(before.runtimeKey, id, { + action, + ...(write + ? { input, actor: session.user.id } + : action === "history" + ? { input } + : {}), + }); + } catch (error) { + const code = error.code || error.message; + if (["invalid_request", "settings_invalid"].includes(code)) + fail("settings_invalid", 400); + if ( + [ + "settings_revision_conflict", + "idempotency_conflict", + "settings_migration_required", + "settings_incompatible", + ].includes(code) + ) + fail(code); + fail("settings_unavailable", 503); + } + const after = context(session, id, write || action === "history"); + if (JSON.stringify(before) !== JSON.stringify(after)) + fail("plugin_revision_conflict"); + if (write) + access.audit({ + actorUserId: session.user.id, + organizationId: before.organizationId, + action: "plugin.settings.update", + targetType: "plugin", + targetId: id, + }); + if (action === "history") { + const names = new Map(); + result = { + ...result, + items: result.items.map((item) => { + if (item.updatedBy && !names.has(item.updatedBy)) + names.set( + item.updatedBy, + store.db + .prepare( + "SELECT first_name || ' ' || last_name AS name FROM users WHERE id=?", + ) + .get(item.updatedBy)?.name || "Former user", + ); + return { + ...item, + actorName: item.updatedBy + ? names.get(item.updatedBy) + : "Plugin update", + }; + }), + }; + } + return result; + }; +} +module.exports = { createPluginSettings }; diff --git a/core/core/accounts/src/plugins.js b/core/core/accounts/src/plugins.js new file mode 100644 index 0000000..36eabe3 --- /dev/null +++ b/core/core/accounts/src/plugins.js @@ -0,0 +1,215 @@ +'use strict'; + +const { AccessError, exact, idempotencyKey } = require('./validation'); +const { catalog, plugin, publicPlugin } = require('../../../shared/plugin-sdk/catalog'); +const { pluginStatus } = require('../../../shared/plugin-sdk/contract'); +const STATES = { install: 'enabled', enable: 'enabled', disable: 'disabled', uninstall: 'uninstalled' }; +function fail(code, status = 409) { throw new AccessError(code, status); } +function rowFor(store, organizationId, id) { + return store.db.prepare('SELECT * FROM dsp_plugins WHERE organization_id=? AND plugin_id=?').get(organizationId, id); +} +function available(store, organizationId, id) { + const row = rowFor(store, organizationId, id); + return Boolean(row && row.desired_state === 'enabled' && row.applied_state === 'enabled' && row.applied_revision === row.revision); +} +function requirePlugin(access, session, id) { + const { organization } = access.organizationFor(session, 'dashboard.view'); + if (!(access.store.pluginMetadataFor ? access.store.pluginMetadataFor(organization.id, id) : plugin(id)) || !available(access.store, organization.id, id)) fail('plugin_disabled'); + return organization; +} +function projection(definition, row) { + return { ...publicPlugin(definition), latestVersion: plugin(definition.id)?.version || definition.version, + version: row?.version || definition.version, state: row?.desired_state || 'uninstalled', + appliedState: row?.applied_state || 'uninstalled', revision: row?.revision || 0, + pending: Boolean(row && row.revision !== row.applied_revision), failureCode: row?.failure_code || null, + available: Boolean(row && row.desired_state === 'enabled' && row.applied_state === 'enabled' && row.revision === row.applied_revision) }; +} +function listFor(store, organizationId) { return catalog().map(item => { + const row = rowFor(store, organizationId, item.id); + let installed; + try { installed = row && store.pluginMetadataFor?.(organizationId, item.id); } catch {} + return projection(installed || item, row); +}); } + +function createPluginService({ store, access, invoke, settingsPort = null, installationCoordinator = null, backends = ['directory_service_v1'], clock = Date.now }) { + let running = null; + function context(session) { + const selected = access.requireDspOwner(session); + const installation = store.installationControl(selected.organization.id); + if (store.releaseBlocked?.(selected.organization.id)) fail('release_busy'); + if (selected.organization.status !== 'active' || installation?.status !== 'ready' + || !backends.includes(store.installationBackend(installation.organizationId)) || store.activeLifecycleJob(selected.organization.id) + || store.db.prepare("SELECT 1 FROM directory_lifecycle_requests WHERE organization_id=? AND status IN ('queued','running')").get(selected.organization.id)) fail('installation_not_ready'); + return { ...selected, installation }; + } + // Internal session serialization and the Plugins endpoint must use the same + // release-scoped catalog. The organization id comes from the signed session. + function listForOrganization(organizationId) { + const key = store.installationControl(organizationId)?.runtimeKey; + return listFor(store, organizationId).flatMap(item => { + const approved = installationCoordinator?.latest?.(item.id, key); + if (installationCoordinator && !approved && item.state === 'uninstalled') return []; + return [{ ...item, latestVersion: approved?.version || item.version, automaticUpdates: true }]; + }); + } + function list(session) { + const { organization } = access.organizationFor(session, 'dashboard.view'); + return { items: listForOrganization(organization.id) }; + } + function change(session, id, input) { + const selected = context(session); + exact(input, ['action', 'expectedRevision', 'idempotencyKey']); idempotencyKey(input.idempotencyKey); + const definition = plugin(id); + if (!definition || !Object.hasOwn(STATES, input.action) || !Number.isSafeInteger(input.expectedRevision) + || input.expectedRevision < 0) fail('invalid_input', 400); + return store.transaction(() => { + context(session); + const previous = store.db.prepare(`SELECT * FROM dsp_plugin_requests + WHERE organization_id=? AND plugin_id=? AND idempotency_key=?`).get(selected.organization.id, id, input.idempotencyKey); + if (previous) { + if (previous.action !== input.action || previous.expected_revision !== input.expectedRevision + || previous.actor_user_id !== session.user.id) fail('idempotency_conflict'); + return projection(definition, rowFor(store, selected.organization.id, id)); + } + const current = rowFor(store, selected.organization.id, id); + if ((current?.revision || 0) !== input.expectedRevision) fail('plugin_revision_conflict'); + if (current && current.revision !== current.applied_revision) fail('plugin_busy'); + const before = current?.desired_state || 'uninstalled'; + if (input.action === 'install' && before !== 'uninstalled' || input.action === 'enable' && before !== 'disabled' + || input.action === 'disable' && before !== 'enabled' || input.action === 'uninstall' && before === 'uninstalled') fail('plugin_operation_not_allowed'); + if (store.db.prepare(`SELECT 1 FROM installation_onboarding_requests WHERE organization_id=? + AND status IN ('enrolling','queued','running')`).get(selected.organization.id) + || store.runningActivationJob(selected.organization.id)) fail('plugin_busy'); + const revision = (current?.revision || 0) + 1; + const version = ['install','enable'].includes(input.action) ? (installationCoordinator ? installationCoordinator.latest(id,selected.installation.runtimeKey)?.version : definition.version) : current.version; + if (!version) fail('plugin_package_not_approved'); + store.db.prepare(`INSERT INTO dsp_plugins(organization_id,plugin_id,version,desired_state,applied_state, + revision,applied_revision,failure_code,actor_user_id,updated_at) VALUES(?,?,?,?,'uninstalled',?,0,NULL,?,?) + ON CONFLICT(organization_id,plugin_id) DO UPDATE SET desired_state=excluded.desired_state,version=excluded.version, + revision=excluded.revision,failure_code=NULL,actor_user_id=excluded.actor_user_id,updated_at=excluded.updated_at`) + .run(selected.organization.id, id, version, STATES[input.action], revision, session.user.id, clock()); + store.db.prepare(`INSERT INTO dsp_plugin_requests(organization_id,plugin_id,idempotency_key,action, + expected_revision,actor_user_id) VALUES(?,?,?,?,?,?)`) + .run(selected.organization.id, id, input.idempotencyKey, input.action, input.expectedRevision, session.user.id); + access.audit({ actorUserId: session.user.id, organizationId: selected.organization.id, + action: `plugin.${input.action}`, targetType: 'plugin', targetId: id }); + store.wakeWorkers?.(['reconcile']); + return projection(definition, rowFor(store, selected.organization.id, id)); + }); + } + function ready(organizationId, runtimeKey = null) { + const installation = store.installationControl(organizationId); + return !store.releaseBlocked?.(organizationId) && installation?.status === 'ready' && backends.includes(store.installationBackend(installation.organizationId)) + && (!runtimeKey || installation.runtimeKey === runtimeKey) + && store.organization(organizationId)?.status === 'active' && !store.activeLifecycleJob(organizationId) + && !store.db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(organizationId) + && !store.db.prepare("SELECT 1 FROM directory_lifecycle_requests WHERE organization_id=? AND status IN ('queued','running')").get(organizationId); + } + async function reconcile(row) { + if (!ready(row.organization_id)) return; + const installation = store.installationControl(row.organization_id); + let receipt, error = null; + try { + const request = { command: 'apply', pluginId: row.plugin_id, revision: row.revision, state: row.desired_state, version: row.version }; + const authorize = () => { + const current = rowFor(store, row.organization_id, row.plugin_id); + return ready(row.organization_id, installation.runtimeKey) + && store.installationControl(row.organization_id).revision === installation.revision + && current?.revision === row.revision && current.desired_state === row.desired_state && current.version === row.version; + }; + const result = installationCoordinator + ? await installationCoordinator.apply({ runtimeKey: installation.runtimeKey, request, invoke, authorize }) + : await invoke(installation.runtimeKey, 'plugins.manage', request); + if (!result?.ok || result.status !== 'applied') throw new Error('plugin_unavailable'); + receipt = pluginStatus(result.data); + if (receipt.id !== row.plugin_id || receipt.version !== row.version || receipt.revision !== row.revision || receipt.state !== row.desired_state) throw new Error(); + } catch { error = 'plugin_unavailable'; } + store.transaction(() => { + const current = rowFor(store, row.organization_id, row.plugin_id); + if (!ready(row.organization_id, installation.runtimeKey) || store.installationControl(row.organization_id).revision !== installation.revision + || current?.revision !== row.revision) return; + store.db.prepare(`UPDATE dsp_plugins SET applied_state=?,applied_revision=?,failure_code=?,updated_at=? + WHERE organization_id=? AND plugin_id=? AND revision=?`) + .run(error ? current.applied_state : receipt.state, error ? current.applied_revision : receipt.revision, + error, clock(), row.organization_id, row.plugin_id, row.revision); + }); + if (!error) await resume(row); + } + async function resume(row) { + if (!installationCoordinator?.resume || !ready(row.organization_id)) return; + const current = rowFor(store, row.organization_id, row.plugin_id); + if (current?.revision !== row.revision || current.applied_revision !== row.revision) return; + let failure = null; + try { await installationCoordinator.resume(store.installationControl(row.organization_id).runtimeKey, row); } + catch { failure = 'plugin_runtime_unavailable'; } + store.db.prepare('UPDATE dsp_plugins SET failure_code=? WHERE organization_id=? AND plugin_id=? AND revision=? AND applied_revision=?') + .run(failure, row.organization_id, row.plugin_id, row.revision, row.revision); + } + async function discoverLegacy() { + const rows = store.db.prepare(`SELECT i.organization_id,i.runtime_key,i.revision FROM installations i + WHERE i.status='ready' AND NOT EXISTS(SELECT 1 FROM plugin_migration_checks m WHERE m.organization_id=i.organization_id) + ORDER BY i.created_at`).all().filter(row => ready(row.organization_id)).slice(0, 20); + for (const row of rows) { + try { + const result = await invoke(row.runtime_key, 'plugins.manage', { command: 'status' }); + if (!result?.ok || result.status !== 'found' || !Array.isArray(result.data?.items)) continue; + const items = result.data.items.map(pluginStatus); + if (items.length !== catalog().length || new Set(items.map(item => item.id)).size !== items.length) continue; + store.transaction(() => { + if (!ready(row.organization_id, row.runtime_key) || store.installationControl(row.organization_id).revision !== row.revision) return; + for (const item of items) { + if (item.state !== 'enabled' || item.revision !== 0 || rowFor(store, row.organization_id, item.id)) continue; + store.db.prepare(`INSERT INTO dsp_plugins(organization_id,plugin_id,version,desired_state,applied_state, + revision,applied_revision,failure_code,actor_user_id,updated_at) VALUES(?,?,?,'enabled','enabled',1,0,NULL,NULL,?)`) + .run(row.organization_id, item.id, item.version, clock()); + access.audit({ organizationId: row.organization_id, action: 'plugin.migrate', targetType: 'plugin', targetId: item.id }); + } + store.db.prepare('INSERT OR IGNORE INTO plugin_migration_checks(organization_id) VALUES(?)').run(row.organization_id); + }); + } catch { /* A disconnected DSP is retried without changing its enrollment. */ } + } + } + function runPending() { + if (running) return running; + running = (async () => { + await discoverLegacy(); + // Platform delivery selects the latest approved immutable package. DSPs + // keep their own copies and desired enabled/disabled state. + for (const row of store.db.prepare("SELECT * FROM dsp_plugins WHERE desired_state<>'uninstalled' AND revision=applied_revision").all()) { + const key=store.installationControl(row.organization_id)?.runtimeKey; + const version=installationCoordinator ? installationCoordinator.latest(row.plugin_id,key)?.version : plugin(row.plugin_id)?.version; + if(!version||version===row.version||!ready(row.organization_id))continue; + const a=version.split('.').map(Number),b=row.version.split('.').map(Number); + if((a[0]-b[0]||a[1]-b[1]||a[2]-b[2])<=0)continue; + store.transaction(()=>{ + if(!ready(row.organization_id))return; + const changed=store.db.prepare(`UPDATE dsp_plugins SET version=?,revision=revision+1,failure_code=NULL,actor_user_id=NULL,updated_at=? + WHERE organization_id=? AND plugin_id=? AND revision=? AND applied_revision=? AND desired_state<>'uninstalled'`) + .run(version,clock(),row.organization_id,row.plugin_id,row.revision,row.revision); + if(changed.changes)access.audit({organizationId:row.organization_id,action:'plugin.auto_update',targetType:'plugin',targetId:row.plugin_id}); + }); + } + if (installationCoordinator?.needsMigration) { + for (const row of store.db.prepare('SELECT * FROM dsp_plugins WHERE revision=applied_revision').all().filter(row => ready(row.organization_id))) { + try { + const installation = store.installationControl(row.organization_id); + if (!installationCoordinator.needsMigration(installation.runtimeKey, row)) continue; + store.transaction(() => { + if (!ready(row.organization_id, installation.runtimeKey)) return; + const changed = store.db.prepare(`UPDATE dsp_plugins SET revision=revision+1,failure_code=NULL,updated_at=? + WHERE organization_id=? AND plugin_id=? AND revision=? AND applied_revision=?`).run(clock(), row.organization_id, row.plugin_id, row.revision, row.revision); + if (changed.changes) access.audit({ organizationId: row.organization_id, action: 'plugin.migrate', targetType: 'plugin', targetId: row.plugin_id }); + }); + } catch { /* Unmounted DSPs are retried after their storage is prepared. */ } + } + } + for (const row of store.db.prepare('SELECT * FROM dsp_plugins WHERE revision<>applied_revision ORDER BY updated_at').all().filter(row => ready(row.organization_id)).slice(0, 20)) await reconcile(row); + for (const row of store.db.prepare("SELECT * FROM dsp_plugins WHERE revision=applied_revision AND failure_code='plugin_runtime_unavailable'").all().slice(0, 20)) await resume(row); + })().finally(() => { running = null; }); + return running; + } + return { list, listForOrganization, change, runPending, + settings: require('./plugin-settings').createPluginSettings({store,access,port:settingsPort}), + catalog: () => ({ items: catalog().map(publicPlugin) }) }; +} +module.exports = { createPluginService, available, requirePlugin, listFor }; diff --git a/core/core/accounts/src/release-popup.js b/core/core/accounts/src/release-popup.js new file mode 100644 index 0000000..7a34b2a --- /dev/null +++ b/core/core/accounts/src/release-popup.js @@ -0,0 +1,83 @@ +'use strict'; +const fs = require('node:fs'); +const { AccessError, exact } = require('./validation'); +const { VERSION } = require('../../../shared/release-version'); +const MAX_BYTES = 128 * 1024; + +// This file is generated inside the verified Core bundle, never served as a +// static asset. Older updaters can install it without a new sidecar contract. +function validatePopup(value, identity) { + exact(value, ['schemaVersion', 'releaseId', 'version', 'sourceCommit', 'changelog', 'afterUpdating']); + if (!identity || value.schemaVersion !== 1 || value.releaseId !== identity.releaseId + || value.version !== identity.version || value.sourceCommit !== identity.sourceCommit + || typeof value.version !== 'string' || !VERSION.test(value.version) + || !/^[a-z][a-z0-9_.-]{2,95}$/.test(value.releaseId) || !/^[a-f0-9]{40}$/.test(value.sourceCommit) + || !Array.isArray(value.changelog) || !value.changelog.length || value.changelog.length > 100 + || !Array.isArray(value.afterUpdating) || value.afterUpdating.length > 10 + || Buffer.byteLength(JSON.stringify(value)) > MAX_BYTES) throw Error('release_popup_invalid'); + function row(item, change) { + exact(item, [...(change ? ['kind'] : []), 'title', 'description', 'audience']); + if (!['platform', 'dsp'].includes(item.audience) + || change && !['added', 'improved', 'changed', 'fixed', 'removed'].includes(item.kind) + || typeof item.title !== 'string' || !item.title.trim() || item.title.length > 160 + || typeof item.description !== 'string' || item.description.length > 600 + || /[\x00-\x1f\x7f]/.test(item.title + item.description)) throw Error('release_popup_invalid'); + } + value.changelog.forEach(item => row(item, true)); + value.afterUpdating.forEach(item => row(item, false)); + return value; +} +function loadPopup(file, identity) { + try { + if (!identity || fs.statSync(file).size > MAX_BYTES) return null; + return validatePopup(JSON.parse(fs.readFileSync(file, 'utf8')), identity); + } catch { return null; } // Missing/invalid optional copy never blocks the dashboard. +} +function createReleasePopup({ store, release = null, clock = Date.now }) { + if (release) validatePopup(release, release); + const db = store.db; + function available(session) { + if (!release || session.dspView) return null; + const platform = session.platformPermissions.includes('platform.organizations.read'); + if (!platform) { + const membership = session.memberships.find(m => m.organizationId === session.activeOrganizationId + && m.organization.status === 'active' && m.permissions.includes('organization.owner')); + if (!membership) return null; + const installed = db.prepare('SELECT release_id,status FROM installations WHERE organization_id=?').get(membership.organizationId); + if (installed?.release_id !== release.releaseId || installed.status !== 'ready') return null; + } + // Gate on successful fleet rollout, not discovery/download or Core promotion. + // Also bind to the running Core identity so a newer prepared version cannot leak in. + const rollout = db.prepare(`SELECT c.release_json FROM platform_rollouts r + JOIN platform_rollout_core c ON c.rollout_id=r.id + WHERE r.release_id=? AND r.status='completed' AND c.status='succeeded' + ORDER BY r.updated_at DESC,r.rowid DESC LIMIT 1`).get(release.releaseId); + if (!rollout) return null; + let metadata; + try { metadata = JSON.parse(rollout.release_json); } catch { return null; } + if (metadata.version !== release.version || metadata.sourceCommit !== release.sourceCommit + || typeof metadata.publishedAt !== 'string' || !Number.isFinite(Date.parse(metadata.publishedAt))) return null; + const visible = items => items.filter(item => platform || item.audience === 'dsp') + .map(({ audience, ...item }) => item); + const changelog = visible(release.changelog); + if (!changelog.length) return null; + return { releaseId: release.releaseId, version: release.version, publishedAt: metadata.publishedAt, + changelog, afterUpdating: visible(release.afterUpdating) }; + } + return { + pending(session) { + const current = available(session); + return { release: current && !db.prepare('SELECT 1 FROM release_popup_dismissals WHERE user_id=? AND release_id=?') + .get(session.user.id, current.releaseId) ? current : null }; + }, + dismiss(session, input) { + exact(input, ['releaseId']); + const current = available(session); + if (!current || input.releaseId !== current.releaseId) throw new AccessError('release_popup_unavailable', 409); + db.prepare('INSERT OR IGNORE INTO release_popup_dismissals(user_id,release_id,dismissed_at) VALUES(?,?,?)') + .run(session.user.id, current.releaseId, clock()); + return { release: null }; + }, + }; +} +module.exports = { validatePopup, loadPopup, createReleasePopup }; diff --git a/core/core/accounts/src/rollout-admin-cli.js b/core/core/accounts/src/rollout-admin-cli.js new file mode 100644 index 0000000..349835b --- /dev/null +++ b/core/core/accounts/src/rollout-admin-cli.js @@ -0,0 +1,123 @@ +'use strict'; + +// Server-local operator capability, protected by the same filesystem ownership +// checks as owner-admin. No browser session or credentials are created. +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { resolveLocalRuntimePaths } = require('../../../shared/paths/runtime-paths'); +const { VERSION } = require('../../../shared/release-version'); +const { AccessStore } = require('./store'); +const { createPlatformUpdates } = require('./platform-updates'); +const { loadPrivateOciReleaseCatalog } = require('../../installations/src/release-catalog'); +const { loadPlatformReleaseCatalog } = require('../../installations/src/platform-release-catalog'); + +const usage = 'dispatch-access-admin rollout-status|rollout-start|rollout-pause|rollout-resume --local-root PATH --version VERSION --commit SHA [--canary-organization ID]'; +const fail = code => { throw Object.assign(new Error(code), { code }); }; +function parse(argv) { + const [action, ...args] = argv; + if (!['rollout-status', 'rollout-start', 'rollout-pause', 'rollout-resume'].includes(action)) fail('invalid_input'); + const input = { action }; + for (let i = 0; i < args.length; i += 2) { + const key = { '--canary-organization': 'canaryOrganizationId', '--local-root': 'localRoot', '--version': 'version', '--commit': 'commit' }[args[i]]; + if (!key || input[key] !== undefined || !args[i + 1] || args[i + 1].startsWith('--')) fail('invalid_input'); + input[key] = args[i + 1]; + } + if (!input.localRoot || !path.isAbsolute(input.localRoot) || path.resolve(input.localRoot) !== input.localRoot + || !input.version || input.version.length > 80 || !VERSION.test(input.version) || !/^[a-f0-9]{40}$/.test(input.commit || '')) fail('invalid_input'); + if (input.canaryOrganizationId !== undefined && (input.action !== 'rollout-start' || !/^[a-z][a-z0-9_-]{2,95}$/.test(input.canaryOrganizationId))) fail('invalid_input'); + return input; +} +function operate(store, updates, catalogs, input) { + // Bind commands to the requested identity, including resume/pause, inside the + // same transaction that changes state so another rollout cannot race the guard. + const operation = () => { + const row = store.db.prepare(`SELECT r.*,c.release_json FROM platform_rollouts r + JOIN platform_rollout_core c ON c.rollout_id=r.id ORDER BY r.created_at DESC,r.rowid DESC LIMIT 1`).get(); + const current = row && JSON.parse(row.release_json); + const matching = current?.version === input.version && current?.sourceCommit === input.commit; + const candidates = Object.entries(catalogs.platformReleases).filter(([, release]) => release.version === input.version); + if (candidates.some(([, release]) => release.sourceCommit !== input.commit)) fail('release_identity_mismatch'); + if (current?.version === input.version && !matching) fail('release_identity_mismatch'); + const prepared = candidates.find(([id, release]) => release.sourceCommit === input.commit && catalogs.releases[id]); + let status = matching ? row.status : prepared ? 'ready' : 'not_prepared'; + if (input.action !== 'rollout-status') { + const owner = store.db.prepare("SELECT id FROM users WHERE platform_role='owner' AND status='active' ORDER BY created_at,id LIMIT 1").get(); + if (!owner) fail('platform_owner_required'); + const session = { user: { id: owner.id } }; + if (input.action === 'rollout-start') { + if (matching && input.canaryOrganizationId) { + const selected = store.db.prepare('SELECT organization_id FROM platform_rollout_members WHERE rollout_id=? AND position=0').get(row.id); + if (selected?.organization_id !== input.canaryOrganizationId) fail('rollout_canary_mismatch'); + } + if (row && row.status !== 'completed' && !matching) fail('rollout_in_progress'); + // A repeated start observes the existing rollout, including a paused one. + // Only an explicit resume after diagnosis may retry failed work. + if (!matching) { + if (!prepared) fail('release_not_prepared'); + updates.command(session, { action: 'start', releaseId: prepared[0], + ...(input.canaryOrganizationId ? { canaryOrganizationId: input.canaryOrganizationId } : {}), + idempotencyKey: `operator:${crypto.createHash('sha256').update(`${input.version}:${input.commit}`).digest('hex')}` }); + } + } else { + if (!matching) fail('rollout_target_mismatch'); + if (row.status !== 'completed') { + if (!prepared && input.action === 'rollout-resume') fail('release_not_prepared'); + updates.command(session, { action: input.action.slice('rollout-'.length) }); + } + } + } + const view = updates.view(); + const target = view.rollout?.version === input.version && (matching || prepared) ? view.rollout : null; + if (target) { + status = target.status; + if (target.status === 'running' && target.phase === 'dsps' && target.core.status === 'succeeded' && target.updated === target.total) target.phase = 'cleanup'; + } + const timings = []; + if (target && store.db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name='operation_stage_timings'").get()) { + const active = store.db.prepare('SELECT id FROM platform_rollouts WHERE release_id=? ORDER BY created_at DESC LIMIT 1').get(target.release); + if (active) timings.push(...store.db.prepare(`SELECT job_id,attempt,stage,started_at,finished_at,duration_ms,status,failure_code FROM operation_stage_timings + WHERE job_id=? OR job_id IN (SELECT job_id FROM platform_rollout_members WHERE rollout_id=?) + OR job_id IN (SELECT id FROM platform_backup_requests WHERE json_extract(input_json,'$.rolloutId')=?) + OR job_id IN (SELECT job_id FROM platform_backup_requests WHERE json_extract(input_json,'$.rolloutId')=?) + ORDER BY started_at DESC LIMIT 200`).all(active.id, active.id, active.id, active.id).reverse()); + } + return { ok: true, version: input.version, sourceCommit: input.commit, status, + releaseId: target?.release || prepared?.[0] || null, + timings, rollout: target, activeReleaseId: view.rollout?.status !== 'completed' ? view.rollout?.release || null : null }; + }; + if (input.action !== 'rollout-status') return store.transaction(operation); + // A deferred read transaction gives one consistent WAL snapshot without + // competing with the backup and rollout workers for the writer lock. + store.db.exec('BEGIN'); + try { const result = operation(); store.db.exec('COMMIT'); return result; } + catch (error) { store.db.exec('ROLLBACK'); throw error; } +} +async function main(argv, { write = value => process.stdout.write(value), wake = require('./worker-wakeup').wake } = {}) { + if (argv.length === 2 && argv[1] === '--help') { write(`${usage}\n`); return 0; } + let store; + try { + const input = parse(argv); + const paths = resolveLocalRuntimePaths({ localRoot: input.localRoot }); + if (paths.accessControl.database !== path.join(input.localRoot, 'data/access-control/access-control.sqlite3')) fail('runtime_path_mismatch'); + if (!fs.existsSync(paths.accessControl.database)) fail('access_not_initialized'); + const releases = loadPrivateOciReleaseCatalog(path.join(input.localRoot, 'config/oci-releases.json')); + const platformReleases = loadPlatformReleaseCatalog(path.join(input.localRoot, 'config/platform-releases.json'), releases); + store = new AccessStore(paths.accessControl, { readOnly: input.action === 'rollout-status' }); + const updates = createPlatformUpdates({ store, releases, platformReleases, enabled: true }); + const result = operate(store, updates, { releases, platformReleases }, input); + if (input.action !== 'rollout-status' && result.status === 'running') { + wake(['reconcile', 'core'], { databaseRoot: paths.accessControl.databaseRoot, localRoot: input.localRoot }); + } + write(`${JSON.stringify(result)}\n`); + return 0; + } catch (error) { + const safe = new Set(['invalid_input', 'runtime_path_mismatch', 'access_not_initialized', 'unsafe_access_storage', + 'access_schema_incompatible', 'platform_owner_required', 'release_identity_mismatch', 'release_not_prepared', + 'rollout_in_progress', 'rollout_target_mismatch', 'update_unavailable', 'installation_operation_in_progress', + 'native_migration_required', 'rollout_canary_mismatch', 'rollout_canary_unavailable', 'platform_release_invalid', 'runtime_boundary_violation']); + write(`${JSON.stringify({ ok: false, status: safe.has(error.code) ? error.code : 'rollout_admin_failed' })}\n`); + return 1; + } finally { store?.close(); } +} +module.exports = { main, parse, operate }; diff --git a/core/core/accounts/src/rollout-backups.js b/core/core/accounts/src/rollout-backups.js new file mode 100644 index 0000000..cd8af9b --- /dev/null +++ b/core/core/accounts/src/rollout-backups.js @@ -0,0 +1,65 @@ +'use strict'; +const crypto = require('node:crypto'); +const id = () => `breq_${crypto.randomBytes(16).toString('hex')}`; + +// One durable set per rollout. Requests and membership are committed with the +// rollout, before the independently supervised Core updater can see it. +function queueRolloutBackups(store, rollout, now) { + const db = store.db; + const previous = db.prepare('SELECT set_id FROM platform_rollout_backups WHERE rollout_id=?').get(rollout.id); + if (previous) return previous.set_id; + const setId = id(); + const targets = [null, ...db.prepare(`SELECT m.organization_id FROM platform_rollout_members m + JOIN installations i ON i.organization_id=m.organization_id WHERE m.rollout_id=? + AND i.status!='pending' ORDER BY m.position`).all(rollout.id).map(r => r.organization_id)]; + const members = targets.map(organizationId => { + const requestId = id(); + db.prepare("INSERT INTO platform_backup_requests VALUES(?,?,?,'queued','queued',NULL,?,?,?,?,?,NULL)") + .run(requestId, organizationId, organizationId ? 'backup' : 'core', + JSON.stringify({ category: 'pre_update', rolloutId: rollout.id, setId, retentionDays: null }), + rollout.actor_user_id, `${rollout.id}:backup:${organizationId || 'core'}`, now, now); + return { organizationId, requestId }; + }); + db.prepare("INSERT INTO backup_sets VALUES(?,?,?,'pending')").run(setId, now, JSON.stringify(members)); + const schedule = db.prepare("SELECT settings_json FROM backup_scope_settings WHERE scope='system'").get(); + db.prepare('INSERT INTO backup_set_settings VALUES(?,?)').run(setId, schedule?.settings_json || JSON.stringify(require('./backup-schedule').DEFAULT_BACKUP_SETTINGS)); + db.prepare('INSERT INTO platform_rollout_backups VALUES(?,?)').run(rollout.id, setId); + return setId; +} +function rolloutBackupProgress(db, rolloutId) { + const set = db.prepare(`SELECT s.* FROM platform_rollout_backups r JOIN backup_sets s ON s.id=r.set_id WHERE r.rollout_id=?`).get(rolloutId); + if (!set) return null; // Existing in-flight rollouts retain their original protocol. + const members = JSON.parse(set.members_json).map(member => { + const request = db.prepare('SELECT * FROM platform_backup_requests WHERE id=?').get(member.requestId); + const job = request?.job_id ? db.prepare('SELECT backup_id FROM installation_lifecycle_jobs WHERE id=?').get(request.job_id) : null; + // A retry can replace the lifecycle job and its snapshot. The request owns + // that identity; persisted set membership is only a historical fallback. + return { ...member, backupId: request ? job?.backup_id || (request.kind === 'core' ? request.id : null) : member.backupId || null, + name: member.organizationId ? db.prepare('SELECT name FROM organizations WHERE id=?').get(member.organizationId)?.name || 'DSP' : 'Dispatch Core', + status: request?.status || 'failed', phase: request?.phase || 'failed', updatedAt: request?.updated_at || set.created_at }; + }); + return { setId: set.id, total: members.length, completed: members.filter(m => m.status === 'completed').length, + status: members.some(m => m.status === 'failed') ? 'failed' : members.every(m => m.status === 'completed') ? 'completed' : 'running', members }; +} +function retryRolloutBackups(store, rolloutId, now) { + const progress = rolloutBackupProgress(store.db, rolloutId); + if (!progress) return; + for (const member of progress.members.filter(m => m.status === 'failed')) { + const row = store.db.prepare('SELECT * FROM platform_backup_requests WHERE id=?').get(member.requestId); + // Retry the same snapshot/upload when it exists. A failed lifecycle needs a + // fresh idempotency key, so its request attempt is durable too. + const job = row.job_id ? store.lifecycleJob(row.job_id) : null; + if (job?.status === 'running' && job.attempt >= job.max_attempts && job.lease_expires_at <= now) { + require('./installation-lifecycle').createAccessInstallationLifecycleAuthority({ store, organizationId: row.organization_id, authorityScope: 'platform_backups', clock: () => now }).retryExhausted(job.id); + store.db.prepare("UPDATE platform_backup_requests SET status='running',phase='backing_up',failure_code=NULL,updated_at=? WHERE id=?").run(now, row.id); + continue; + } + const input = JSON.parse(row.input_json); + input.attempt = (input.attempt || 0) + 1; + const uploaded = job?.status === 'succeeded' || row.kind === 'core'; + store.db.prepare("UPDATE platform_backup_requests SET status='queued',phase=?,job_id=?,failure_code=NULL,input_json=?,updated_at=? WHERE id=?") + .run(uploaded && row.kind !== 'core' ? 'uploading' : 'queued', uploaded ? row.job_id : null, JSON.stringify(input), now, row.id); + } + store.db.prepare("UPDATE backup_sets SET status='pending' WHERE id=?").run(progress.setId); +} +module.exports = { queueRolloutBackups, rolloutBackupProgress, retryRolloutBackups }; diff --git a/core/core/accounts/src/rollout-canary.js b/core/core/accounts/src/rollout-canary.js new file mode 100644 index 0000000..50b77a6 --- /dev/null +++ b/core/core/accounts/src/rollout-canary.js @@ -0,0 +1,30 @@ +'use strict'; +const { setTimeout: delay } = require('node:timers/promises'); +const { PAYCOM_SYNC_ID } = require('../../../shared/paycom-activation'); +function instant(value) { return typeof value === 'number' ? value : Date.parse(value); } +function createCanaryVerifier(hub, { clock = Date.now, pollMs = 5000, timeoutMs = 30 * 60_000 } = {}) { + return async ({ runtimeKey, verificationId, startedAt }) => { + if (!hub) throw Error('canary_unavailable'); + if (clock() >= startedAt + timeoutMs) throw Error('canary_collection_timeout'); + const invoke = async (action, input) => { + const result = await hub.invoke(runtimeKey, action, input); + if (!result?.ok) throw Error('canary_collection_failed'); + return result.data; + }; + const initial = await invoke('sync.status', { id: PAYCOM_SYNC_ID }); + if (initial.desiredState !== 'running') throw Error('canary_sync_stopped'); + await invoke('sync.run_now', { id: PAYCOM_SYNC_ID, options: { idempotencyKey: verificationId } }); + while (clock() < startedAt + timeoutMs) { + const status = await invoke('sync.status', { id: PAYCOM_SYNC_ID }); + if (status.desiredState !== 'running' || (!status.activeRun && status.lastError + && instant(status.lastStartedAt) >= startedAt)) throw Error('canary_collection_failed'); + if (!status.lastError && status.lastSucceededAt !== null && instant(status.lastSucceededAt) >= startedAt) { + await invoke('collections.health', {}); + return true; + } + await delay(pollMs); + } + throw Error('canary_collection_timeout'); + }; +} +module.exports = { createCanaryVerifier }; diff --git a/core/core/accounts/src/runtime-agent-authority.js b/core/core/accounts/src/runtime-agent-authority.js new file mode 100644 index 0000000..8dee523 --- /dev/null +++ b/core/core/accounts/src/runtime-agent-authority.js @@ -0,0 +1,30 @@ +'use strict'; + +const { INSTALLATION_IDENTIFIER_RE } = require('../../../shared/contracts/src'); + +function fail(code = 'runtime_boundary_violation') { + throw Object.assign(new Error(code), { code }); +} + +function createAccessRuntimeAgentAuthorityCatalog({ store } = {}) { + if (!store || typeof store.activeRuntimeAgentAuthority !== 'function' + || typeof store.activeRuntimeAgentAuthorityCount !== 'function') fail(); + + return Object.freeze({ + resolve(runtimeKey) { + if (typeof runtimeKey !== 'string' || !INSTALLATION_IDENTIFIER_RE.test(runtimeKey)) return null; + const authority = store.activeRuntimeAgentAuthority(runtimeKey); + if (!authority || authority.runtimeKey !== runtimeKey || !/^[a-f0-9]{64}$/.test(authority.tokenHash) + || !Number.isSafeInteger(authority.generation) || authority.generation < 1) return null; + return Object.freeze({ digest: authority.tokenHash, generation: authority.generation }); + }, + + count() { + const value = store.activeRuntimeAgentAuthorityCount(); + if (!Number.isSafeInteger(value) || value < 0) fail(); + return value; + }, + }); +} + +module.exports = { createAccessRuntimeAgentAuthorityCatalog }; diff --git a/core/core/accounts/src/schema.js b/core/core/accounts/src/schema.js new file mode 100644 index 0000000..3e98ea9 --- /dev/null +++ b/core/core/accounts/src/schema.js @@ -0,0 +1,658 @@ +'use strict'; +const { AccessError } = require('./validation'); +const SCHEMA_VERSION = 17; +const REVIEWED_UPGRADE_SCHEMAS = Object.freeze([10, 11, 12, 13, 14, 15, 16, 17]); +function fail(code) { throw new AccessError(code, 500); } +const LEGACY_INSTALLATION_COLUMNS = Object.freeze([ + 'organization_id', 'runtime_key', 'status', 'created_at', 'updated_at', +]); +const CRITICAL_SCHEMA_COLUMNS = Object.freeze({ + dsp_plugins: Object.freeze(['organization_id','plugin_id','version','desired_state','applied_state','revision','applied_revision','failure_code','actor_user_id','updated_at']), + dsp_plugin_requests: Object.freeze(['organization_id','plugin_id','idempotency_key','action','expected_revision','actor_user_id']), + plugin_migration_checks: Object.freeze(['organization_id']), + directory_lifecycle_requests: Object.freeze(['id', 'organization_id', 'runtime_key', 'actor_user_id', 'idempotency_key', + 'action', 'expected_revision', 'installation_revision', 'starting_state', 'starting_organization_status', + 'target_state', 'target_organization_status', 'status', 'failure_code', 'created_at', 'updated_at']), + password_reset_tokens: Object.freeze(['token_hash', 'user_id', 'auth_version', 'email', 'expires_at', 'created_at']), + password_recovery_limits: Object.freeze(['key_hash', 'count', 'reset_at', 'last_at']), + diagnostic_dsps: Object.freeze(['organization_id', 'actor_user_id', 'idempotency_key', 'status', 'created_at']), + platform_rollout_core: Object.freeze(['rollout_id', 'status', 'release_json', 'attempt', 'failure_code', 'updated_at']), + organization_profiles: Object.freeze(['organization_id', 'owner_email', 'details_json', 'applied_at']), + platform_rollouts: Object.freeze(['id', 'release_id', 'actor_user_id', 'idempotency_key', 'status', 'created_at', 'updated_at']), + platform_rollout_members: Object.freeze(['rollout_id', 'organization_id', 'position', 'status', 'job_id', 'attempt', 'message']), + installation_onboarding_requests: Object.freeze([ + 'id', 'organization_id', 'actor_user_id', 'idempotency_key', 'intent', 'manifest_revision', + 'status', 'worker_id', 'fence', 'lease_expires_at', 'attempt', 'failure_code', 'created_at', 'updated_at', + ]), + installations: Object.freeze([ + 'organization_id', 'runtime_key', 'status', 'revision', 'manifest_revision', + 'current_job_id', 'setup_worker_id', 'setup_fence', 'setup_lease_expires_at', + 'created_at', 'updated_at', 'release_id', 'backend', + ]), + runtime_agent_authorities: Object.freeze([ + 'organization_id', 'runtime_key', 'token_hash', 'generation', 'status', + 'created_at', 'updated_at', 'revoked_at', + ]), + installation_provisioning_requests: Object.freeze([ + 'id', 'organization_id', 'authority_scope', 'idempotency_key', 'request_json', + 'starting_state', 'installation_revision', 'manifest_revision', 'runtime_key', + 'status', 'provisioner_job_id', 'failure_code', 'created_at', 'updated_at', + 'finished_at', + ]), + installation_activation_jobs: Object.freeze([ + 'id', 'organization_id', 'operation', 'status', 'installation_state', + 'installation_revision', 'manifest_revision', 'runtime_key', 'authority_scope', + 'idempotency_key', 'worker_id', 'fence', 'lease_expires_at', 'provider', + 'profile_id', 'provider_tested_at', 'evidence_json', 'evidence_digest', + 'failure_code', 'created_at', 'started_at', + 'finished_at', 'updated_at', + ]), + installation_lifecycle_jobs: Object.freeze([ + 'id', 'organization_id', 'operation', 'status', 'starting_state', 'installation_state', + 'installation_revision', 'manifest_revision', 'runtime_key', 'release_id', 'target_release_id', + 'backup_id', 'safety_backup_id', 'authority_scope', 'idempotency_key', 'stages_json', + 'next_stage', 'stage_receipts_json', 'worker_id', 'attempt', 'max_attempts', 'fence', + 'lease_expires_at', 'failure_code', 'result_json', + 'created_at', 'started_at', 'finished_at', 'updated_at', + ]), + installation_backups: Object.freeze([ + 'id', 'organization_id', 'runtime_key', 'manifest_revision', 'release_id', 'purpose', + 'status', 'tree_digest', 'file_count', 'total_bytes', 'lifecycle_job_id', + 'created_at', 'completed_at', 'destroyed_at', + ]), + platform_target_refs: Object.freeze([ + 'reference_hash', 'session_hash', 'user_id', 'organization_id', 'purpose', 'expires_at', 'created_at', + ]), + platform_mutation_requests: Object.freeze([ + 'id', 'actor_user_id', 'action', 'idempotency_key', 'request_digest', + 'organization_id', 'result_json', 'created_at', + ]), +}); + +function tableColumns(db, table) { + return db.prepare(`PRAGMA table_info(${table})`).all().map(row => row.name); +} +const CRITICAL_SCHEMA_INDEXES = Object.freeze({ + one_active_directory_lifecycle: Object.freeze({ table: 'directory_lifecycle_requests', + columns: Object.freeze(['organization_id']), unique: true, predicate: "WHERE status IN ('queued','running')" }), + one_active_platform_rollout: Object.freeze({ + table: 'platform_rollouts', columns: Object.freeze([null]), unique: true, + predicate: "WHERE status IN ('running','paused')", + }), + one_active_installation_onboarding: Object.freeze({ + table: 'installation_onboarding_requests', columns: Object.freeze(['organization_id']), unique: true, + predicate: "WHERE status IN ('enrolling','queued','running')", + }), + platform_mutation_idempotency: Object.freeze({ + table: 'platform_mutation_requests', columns: Object.freeze(['actor_user_id', 'action', 'idempotency_key']), unique: true, + }), + one_active_installation_lifecycle: Object.freeze({ + table: 'installation_lifecycle_jobs', columns: Object.freeze(['organization_id']), unique: true, + }), +}); +const CRITICAL_SCHEMA_TRIGGERS = Object.freeze({ + password_reset_invalidate: "AFTER UPDATE OF auth_version,email,status ON users BEGIN DELETE FROM password_reset_tokens WHERE user_id=NEW.id; END", + installations_backend_immutable: "BEFORE UPDATE OF backend ON installations FOR EACH ROW WHEN NEW.backend IS NOT OLD.backend BEGIN SELECT RAISE(ABORT, 'installation_backend_immutable'); END", +}); + +function requireColumns(db, table, expected) { + if (JSON.stringify(tableColumns(db, table)) !== JSON.stringify(expected)) fail('access_schema_incompatible'); +} +function requireIndex(db, name, expected) { + const index = db.prepare(`PRAGMA index_list(${expected.table})`).all().find(row => row.name === name); + const columns = index ? db.prepare(`PRAGMA index_info(${name})`).all() + .sort((left, right) => left.seqno - right.seqno).map(row => row.name) : []; + if (!index || Boolean(index.unique) !== expected.unique + || JSON.stringify(columns) !== JSON.stringify(expected.columns)) fail('access_schema_incompatible'); + if (expected.predicate) { + const sql = db.prepare("SELECT sql FROM sqlite_schema WHERE type='index' AND name=?").get(name)?.sql; + if (!sql?.replace(/\s+/g, ' ').includes(expected.predicate)) fail('access_schema_incompatible'); + } +} +function requireTrigger(db, name, expected) { + const row = db.prepare("SELECT sql FROM sqlite_schema WHERE type='trigger' AND name=?").get(name); + const normalized = row?.sql?.replace(/\s+/g, ' ').trim(); + if (!normalized || !normalized.includes(expected)) fail('access_schema_incompatible'); +} + +function initializeAccessSchema(db, initialVersion) { + if (![0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, SCHEMA_VERSION].includes(initialVersion)) fail('access_schema_incompatible'); + if (initialVersion === 2) { + db.exec(`BEGIN IMMEDIATE; + ALTER TABLE installations RENAME TO installations_legacy; + CREATE TABLE installations ( + organization_id TEXT PRIMARY KEY REFERENCES organizations(id) ON DELETE CASCADE, + runtime_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL CHECK(status IN ( + 'pending','provisioning','waiting_for_owner','waiting_for_provider_auth','verifying', + 'ready','failed','suspended','decommissioning','decommissioned' + )), + revision INTEGER NOT NULL CHECK(revision>=1), + manifest_revision INTEGER NOT NULL CHECK(manifest_revision>=1), + current_job_id TEXT, + setup_worker_id TEXT, + setup_fence INTEGER NOT NULL DEFAULT 0 CHECK(setup_fence>=0), + setup_lease_expires_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + release_id TEXT NOT NULL DEFAULT 'dispatch_current_1', + CHECK((setup_worker_id IS NULL)=(setup_lease_expires_at IS NULL)) + ) STRICT; + INSERT INTO installations( + organization_id,runtime_key,status,revision,manifest_revision,release_id,current_job_id, + setup_worker_id,setup_fence,setup_lease_expires_at,created_at,updated_at + ) SELECT organization_id,runtime_key, + CASE + WHEN status='ready' AND organization_id='local-dsp' AND runtime_key='local' THEN 'ready' + WHEN status='pending' THEN 'pending' + ELSE 'failed' + END, + 1,1,'dispatch_current_1',NULL,NULL,0,NULL,created_at,updated_at FROM installations_legacy; + DROP TABLE installations_legacy; + COMMIT;`); + } + if ([3, 4].includes(initialVersion) + && !tableColumns(db, 'installations').includes('release_id')) { + db.exec("ALTER TABLE installations ADD COLUMN release_id TEXT NOT NULL DEFAULT 'dispatch_current_1'"); + } + if (initialVersion !== 0 && initialVersion < 7 && !tableColumns(db, 'installations').includes('backend')) { + db.exec(`BEGIN IMMEDIATE; + ALTER TABLE installations ADD COLUMN backend TEXT NOT NULL DEFAULT 'systemd_user' + CHECK(backend IN ('local_reference','systemd_user','oci_container_v1')); + UPDATE installations SET backend='local_reference' + WHERE organization_id='local-dsp' AND runtime_key='local'; + COMMIT;`); + } + db.exec(` + CREATE TABLE IF NOT EXISTS users ( + id TEXT PRIMARY KEY, + email TEXT NOT NULL UNIQUE COLLATE NOCASE, + first_name TEXT NOT NULL, + last_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('active','disabled')), + platform_role TEXT CHECK(platform_role IS NULL OR platform_role='owner'), + auth_version INTEGER NOT NULL CHECK(auth_version>=1), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS password_reset_tokens ( + token_hash TEXT PRIMARY KEY CHECK(length(token_hash)=64), + user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + auth_version INTEGER NOT NULL, + email TEXT NOT NULL, + expires_at INTEGER NOT NULL, + created_at INTEGER NOT NULL + ) STRICT; + CREATE INDEX IF NOT EXISTS password_resets_by_user ON password_reset_tokens(user_id); + CREATE INDEX IF NOT EXISTS password_resets_by_expiry ON password_reset_tokens(expires_at); + CREATE TABLE IF NOT EXISTS password_recovery_limits ( + key_hash TEXT PRIMARY KEY CHECK(length(key_hash)=64), + count INTEGER NOT NULL CHECK(count>0), + reset_at INTEGER NOT NULL, + last_at INTEGER NOT NULL + ) STRICT; + CREATE TRIGGER IF NOT EXISTS password_reset_invalidate + AFTER UPDATE OF auth_version,email,status ON users BEGIN DELETE FROM password_reset_tokens WHERE user_id=NEW.id; END; + CREATE TABLE IF NOT EXISTS release_popup_dismissals ( + user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + release_id TEXT NOT NULL, + dismissed_at INTEGER NOT NULL, + PRIMARY KEY(user_id, release_id) + ) STRICT; + CREATE TABLE IF NOT EXISTS organizations ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + abbreviation TEXT, + timezone TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('pending_owner','setup_required','active','suspended')), + created_by TEXT REFERENCES users(id), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS stations ( + organization_id TEXT NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + code TEXT NOT NULL, + is_primary INTEGER NOT NULL CHECK(is_primary IN (0,1)), + created_at INTEGER NOT NULL, + PRIMARY KEY(organization_id,code) + ) STRICT; + CREATE UNIQUE INDEX IF NOT EXISTS stations_one_primary ON stations(organization_id) WHERE is_primary=1; + CREATE TABLE IF NOT EXISTS roles ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + key TEXT, + name TEXT NOT NULL, + description TEXT NOT NULL, + is_system INTEGER NOT NULL CHECK(is_system IN (0,1)), + created_by TEXT REFERENCES users(id), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(organization_id,key), + UNIQUE(organization_id,id) + ) STRICT; + CREATE UNIQUE INDEX IF NOT EXISTS roles_name ON roles(organization_id,lower(name)); + CREATE TABLE IF NOT EXISTS role_permissions ( + role_id TEXT NOT NULL REFERENCES roles(id) ON DELETE CASCADE, + permission TEXT NOT NULL, + PRIMARY KEY(role_id,permission) + ) STRICT; + CREATE TABLE IF NOT EXISTS memberships ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + role_id TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('active','suspended')), + created_by TEXT REFERENCES users(id), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(organization_id,user_id), + FOREIGN KEY(organization_id,role_id) REFERENCES roles(organization_id,id) + ) STRICT; + CREATE TABLE IF NOT EXISTS invitations ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL CHECK(kind IN ('platform_owner','organization_owner','organization_member')), + organization_id TEXT REFERENCES organizations(id) ON DELETE CASCADE, + role_id TEXT, + email TEXT NOT NULL COLLATE NOCASE, + token_hash TEXT NOT NULL UNIQUE, + status TEXT NOT NULL CHECK(status IN ('pending','accepted','revoked')), + expires_at INTEGER NOT NULL, + created_by TEXT REFERENCES users(id), + accepted_by TEXT REFERENCES users(id), + created_at INTEGER NOT NULL, + accepted_at INTEGER, + FOREIGN KEY(organization_id,role_id) REFERENCES roles(organization_id,id) + ) STRICT; + CREATE UNIQUE INDEX IF NOT EXISTS one_pending_platform_invitation ON invitations(kind) WHERE kind='platform_owner' AND status='pending'; + UPDATE invitations AS older SET status='revoked' + WHERE older.status='pending' AND older.organization_id IS NOT NULL AND EXISTS ( + SELECT 1 FROM invitations AS newer + WHERE newer.status='pending' AND newer.organization_id=older.organization_id AND newer.email=older.email + AND ( + CASE newer.kind WHEN 'platform_owner' THEN 3 WHEN 'organization_owner' THEN 2 ELSE 1 END + > CASE older.kind WHEN 'platform_owner' THEN 3 WHEN 'organization_owner' THEN 2 ELSE 1 END + OR ( + newer.kind=older.kind + AND (newer.created_at>older.created_at OR (newer.created_at=older.created_at AND newer.id>older.id)) + ) + ) + ); + UPDATE invitations AS older SET status='revoked' + WHERE older.status='pending' AND older.kind='organization_owner' AND EXISTS ( + SELECT 1 FROM invitations AS newer + WHERE newer.status='pending' AND newer.kind='organization_owner' + AND newer.organization_id=older.organization_id + AND (newer.created_at>older.created_at OR (newer.created_at=older.created_at AND newer.id>older.id)) + ); + CREATE UNIQUE INDEX IF NOT EXISTS one_pending_invitation_per_organization_email + ON invitations(organization_id,email) WHERE status='pending' AND organization_id IS NOT NULL; + CREATE UNIQUE INDEX IF NOT EXISTS one_pending_organization_owner_invitation + ON invitations(organization_id) WHERE kind='organization_owner' AND status='pending'; + CREATE TABLE IF NOT EXISTS installations ( + organization_id TEXT PRIMARY KEY REFERENCES organizations(id) ON DELETE CASCADE, + runtime_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL CHECK(status IN ( + 'pending','provisioning','waiting_for_owner','waiting_for_provider_auth','verifying', + 'ready','failed','suspended','decommissioning','decommissioned' + )), + revision INTEGER NOT NULL CHECK(revision>=1), + manifest_revision INTEGER NOT NULL CHECK(manifest_revision>=1), + current_job_id TEXT, + setup_worker_id TEXT, + setup_fence INTEGER NOT NULL DEFAULT 0 CHECK(setup_fence>=0), + setup_lease_expires_at INTEGER, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + release_id TEXT NOT NULL DEFAULT 'dispatch_current_1', + backend TEXT NOT NULL DEFAULT 'systemd_user' + CHECK(backend IN ('local_reference','systemd_user','oci_container_v1')), + CHECK((setup_worker_id IS NULL)=(setup_lease_expires_at IS NULL)) + ) STRICT; + CREATE TABLE IF NOT EXISTS runtime_agent_authorities ( + organization_id TEXT PRIMARY KEY REFERENCES installations(organization_id) ON DELETE CASCADE, + runtime_key TEXT NOT NULL UNIQUE, + token_hash TEXT NOT NULL UNIQUE, + generation INTEGER NOT NULL CHECK(generation>=1), + status TEXT NOT NULL CHECK(status IN ('active','revoked')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + revoked_at INTEGER, + CHECK((status='revoked')=(revoked_at IS NOT NULL)) + ) STRICT; + CREATE TABLE IF NOT EXISTS installation_provisioning_requests ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL REFERENCES installations(organization_id) ON DELETE CASCADE, + authority_scope TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + request_json TEXT NOT NULL, + starting_state TEXT NOT NULL CHECK(starting_state IN ('pending','failed')), + installation_revision INTEGER NOT NULL CHECK(installation_revision>=1), + manifest_revision INTEGER NOT NULL CHECK(manifest_revision>=1), + runtime_key TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('pending','dispatched','completed','failed')), + provisioner_job_id TEXT, + failure_code TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + finished_at INTEGER, + UNIQUE(organization_id,authority_scope,idempotency_key), + CHECK((status='pending')=(provisioner_job_id IS NULL)), + CHECK((status='failed')=(failure_code IS NOT NULL)), + CHECK((status IN ('completed','failed'))=(finished_at IS NOT NULL)) + ) STRICT; + CREATE UNIQUE INDEX IF NOT EXISTS one_active_installation_provisioning + ON installation_provisioning_requests(organization_id) WHERE status IN ('pending','dispatched'); + CREATE TABLE IF NOT EXISTS installation_activation_jobs ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL REFERENCES installations(organization_id) ON DELETE CASCADE, + operation TEXT NOT NULL CHECK(operation='resume'), + status TEXT NOT NULL CHECK(status IN ('running','succeeded','failed')), + installation_state TEXT NOT NULL CHECK(installation_state IN ('verifying','ready','failed')), + installation_revision INTEGER NOT NULL CHECK(installation_revision>=1), + manifest_revision INTEGER NOT NULL CHECK(manifest_revision>=1), + runtime_key TEXT NOT NULL, + authority_scope TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + worker_id TEXT NOT NULL, + fence INTEGER NOT NULL CHECK(fence>=1), + lease_expires_at INTEGER, + provider TEXT NOT NULL CHECK(provider='paycom'), + profile_id TEXT NOT NULL CHECK(profile_id='paycom-main'), + provider_tested_at INTEGER NOT NULL, + evidence_json TEXT, + evidence_digest TEXT, + failure_code TEXT, + created_at INTEGER NOT NULL, + started_at INTEGER NOT NULL, + finished_at INTEGER, + updated_at INTEGER NOT NULL, + UNIQUE(organization_id,authority_scope,idempotency_key), + CHECK((status='failed')=(failure_code IS NOT NULL)), + CHECK((status='succeeded')=(evidence_json IS NOT NULL)), + CHECK((status='succeeded')=(evidence_digest IS NOT NULL)), + CHECK((status='running')=(lease_expires_at IS NOT NULL)), + CHECK((status='running')=(finished_at IS NULL)) + ) STRICT; + CREATE UNIQUE INDEX IF NOT EXISTS one_running_installation_activation + ON installation_activation_jobs(organization_id) WHERE status='running'; + CREATE TABLE IF NOT EXISTS installation_lifecycle_jobs ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL REFERENCES installations(organization_id) ON DELETE CASCADE, + operation TEXT NOT NULL CHECK(operation IN ( + 'backup','restore','upgrade','suspend','resume','decommission','destroy' + )), + status TEXT NOT NULL CHECK(status IN ('queued','running','succeeded','failed')), + starting_state TEXT NOT NULL CHECK(starting_state IN ( + 'pending','provisioning','waiting_for_owner','waiting_for_provider_auth','verifying', + 'ready','failed','suspended','decommissioning','decommissioned' + )), + installation_state TEXT NOT NULL CHECK(installation_state IN ( + 'pending','provisioning','waiting_for_owner','waiting_for_provider_auth','verifying', + 'ready','failed','suspended','decommissioning','decommissioned' + )), + installation_revision INTEGER NOT NULL CHECK(installation_revision>=1), + manifest_revision INTEGER NOT NULL CHECK(manifest_revision>=1), + runtime_key TEXT NOT NULL, + release_id TEXT NOT NULL, + target_release_id TEXT, + backup_id TEXT, + safety_backup_id TEXT, + authority_scope TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + stages_json TEXT NOT NULL, + next_stage INTEGER NOT NULL DEFAULT 0 CHECK(next_stage>=0), + stage_receipts_json TEXT NOT NULL DEFAULT '{}', + worker_id TEXT, + attempt INTEGER NOT NULL DEFAULT 0 CHECK(attempt>=0 AND attempt<=max_attempts), + max_attempts INTEGER NOT NULL DEFAULT 3 CHECK(max_attempts BETWEEN 1 AND 5), + fence INTEGER NOT NULL DEFAULT 0 CHECK(fence>=0), + lease_expires_at INTEGER, + failure_code TEXT, + result_json TEXT, + created_at INTEGER NOT NULL, + started_at INTEGER, + finished_at INTEGER, + updated_at INTEGER NOT NULL, + UNIQUE(organization_id,authority_scope,idempotency_key), + CHECK((status='running')=(lease_expires_at IS NOT NULL)), + CHECK((status='failed')=(failure_code IS NOT NULL)), + CHECK((status IN ('succeeded','failed'))=(finished_at IS NOT NULL)), + CHECK((status='succeeded')=(result_json IS NOT NULL)) + ) STRICT; + CREATE UNIQUE INDEX IF NOT EXISTS one_active_installation_lifecycle + ON installation_lifecycle_jobs(organization_id) WHERE status IN ('queued','running'); + CREATE TABLE IF NOT EXISTS installation_backups ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL REFERENCES installations(organization_id) ON DELETE CASCADE, + runtime_key TEXT NOT NULL, + manifest_revision INTEGER NOT NULL CHECK(manifest_revision>=1), + release_id TEXT NOT NULL, + purpose TEXT NOT NULL CHECK(purpose IN ('manual','upgrade','restore_safety','decommission')), + status TEXT NOT NULL CHECK(status IN ('reserved','available','destroyed')), + tree_digest TEXT, + file_count INTEGER CHECK(file_count IS NULL OR file_count>=0), + total_bytes INTEGER CHECK(total_bytes IS NULL OR total_bytes>=0), + lifecycle_job_id TEXT NOT NULL REFERENCES installation_lifecycle_jobs(id) ON DELETE CASCADE, + created_at INTEGER NOT NULL, + completed_at INTEGER, + destroyed_at INTEGER, + CHECK((status='available')=(tree_digest IS NOT NULL)), + CHECK((status='available')=(file_count IS NOT NULL)), + CHECK((status='available')=(total_bytes IS NOT NULL)), + CHECK((status='available')=(completed_at IS NOT NULL)), + CHECK((status='destroyed')=(destroyed_at IS NOT NULL)) + ) STRICT; + CREATE INDEX IF NOT EXISTS installation_backups_by_organization + ON installation_backups(organization_id,created_at DESC,id); + CREATE TABLE IF NOT EXISTS sessions ( + token_hash TEXT PRIMARY KEY, + user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + csrf_token TEXT NOT NULL, + active_organization_id TEXT REFERENCES organizations(id), + auth_version INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL + ) STRICT; + CREATE INDEX IF NOT EXISTS sessions_by_user ON sessions(user_id,expires_at); + CREATE TABLE IF NOT EXISTS audit_events ( + id TEXT PRIMARY KEY, + actor_user_id TEXT REFERENCES users(id), + organization_id TEXT REFERENCES organizations(id), + action TEXT NOT NULL, + target_type TEXT NOT NULL, + target_id TEXT, + result TEXT NOT NULL CHECK(result IN ('succeeded','denied')), + created_at INTEGER NOT NULL + ) STRICT; + CREATE INDEX IF NOT EXISTS audit_by_org ON audit_events(organization_id,created_at DESC); + CREATE TABLE IF NOT EXISTS platform_target_refs ( + reference_hash TEXT PRIMARY KEY, + session_hash TEXT NOT NULL REFERENCES sessions(token_hash) ON DELETE CASCADE, + user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + organization_id TEXT NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + purpose TEXT NOT NULL CHECK(purpose='organization_control'), + expires_at INTEGER NOT NULL, + created_at INTEGER NOT NULL + ) STRICT; + CREATE INDEX IF NOT EXISTS platform_target_refs_expiry ON platform_target_refs(expires_at); + CREATE TABLE IF NOT EXISTS platform_mutation_requests ( + id TEXT PRIMARY KEY, + actor_user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + action TEXT NOT NULL CHECK(action IN ( + 'organization.create','organization.status','owner_invitation.create','owner_invitation.revoke' + )), + idempotency_key TEXT NOT NULL, + request_digest TEXT NOT NULL, + organization_id TEXT REFERENCES organizations(id) ON DELETE CASCADE, + result_json TEXT NOT NULL, + created_at INTEGER NOT NULL + ) STRICT; + CREATE UNIQUE INDEX IF NOT EXISTS platform_mutation_idempotency + ON platform_mutation_requests(actor_user_id,action,idempotency_key); + CREATE TRIGGER IF NOT EXISTS installations_backend_immutable + BEFORE UPDATE OF backend ON installations + FOR EACH ROW WHEN NEW.backend IS NOT OLD.backend + BEGIN + SELECT RAISE(ABORT, 'installation_backend_immutable'); + END; + CREATE TABLE IF NOT EXISTS installation_onboarding_requests ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL REFERENCES organizations(id), + actor_user_id TEXT NOT NULL REFERENCES users(id), + idempotency_key TEXT NOT NULL, + intent TEXT NOT NULL CHECK(intent IN ('create','replace')), + manifest_revision INTEGER NOT NULL CHECK(manifest_revision>=1), + status TEXT NOT NULL CHECK(status IN ('enrolling','queued','running','succeeded','failed')), + worker_id TEXT, + fence INTEGER NOT NULL DEFAULT 0 CHECK(fence>=0), + lease_expires_at INTEGER, + attempt INTEGER NOT NULL DEFAULT 0 CHECK(attempt BETWEEN 0 AND 3), + failure_code TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(organization_id,actor_user_id,idempotency_key) + ) STRICT; + CREATE UNIQUE INDEX IF NOT EXISTS one_active_installation_onboarding + ON installation_onboarding_requests(organization_id) WHERE status IN ('enrolling','queued','running'); + CREATE TABLE IF NOT EXISTS organization_profiles ( + organization_id TEXT PRIMARY KEY REFERENCES organizations(id), + owner_email TEXT NOT NULL, + details_json TEXT, + applied_at INTEGER + ) STRICT; + CREATE TABLE IF NOT EXISTS platform_rollouts ( + id TEXT PRIMARY KEY, + release_id TEXT NOT NULL, + actor_user_id TEXT NOT NULL REFERENCES users(id), + idempotency_key TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('running','paused','completed')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + UNIQUE(actor_user_id,idempotency_key) + ) STRICT; + CREATE UNIQUE INDEX IF NOT EXISTS one_active_platform_rollout + ON platform_rollouts((1)) WHERE status IN ('running','paused'); + CREATE TABLE IF NOT EXISTS platform_rollout_members ( + rollout_id TEXT NOT NULL REFERENCES platform_rollouts(id), + organization_id TEXT NOT NULL REFERENCES organizations(id), + position INTEGER NOT NULL, + status TEXT NOT NULL CHECK(status IN ('queued','updating','updated','blocked','removed')), + job_id TEXT, + attempt INTEGER NOT NULL DEFAULT 0, + message TEXT, + PRIMARY KEY(rollout_id,organization_id) + ) STRICT; + CREATE TABLE IF NOT EXISTS platform_rollout_core ( + rollout_id TEXT PRIMARY KEY REFERENCES platform_rollouts(id), + status TEXT NOT NULL CHECK(status IN ('queued','updating','verifying','succeeded','failed')), + release_json TEXT NOT NULL, + attempt INTEGER NOT NULL DEFAULT 0, + failure_code TEXT, + updated_at INTEGER NOT NULL + ) STRICT; + `); + migrateRuntimeBackends(db); + db.exec(`CREATE TABLE IF NOT EXISTS diagnostic_dsps ( + organization_id TEXT PRIMARY KEY REFERENCES organizations(id) ON DELETE CASCADE, + actor_user_id TEXT NOT NULL REFERENCES users(id), + idempotency_key TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('pending','ready','failed')), + created_at INTEGER NOT NULL, + UNIQUE(actor_user_id,idempotency_key) + ) STRICT;`); + require('./backup-schema').initializeBackupSchema(db); + require('./directory-lifecycle').initializeDirectoryLifecycleSchema(db); + db.exec(`CREATE TABLE IF NOT EXISTS dsp_removals ( + organization_id TEXT PRIMARY KEY REFERENCES organizations(id) ON DELETE CASCADE, + installation_state TEXT NOT NULL, + organization_status TEXT NOT NULL, + sync_running INTEGER, + removed_at INTEGER NOT NULL, + actor_user_id TEXT REFERENCES users(id), + legacy_services INTEGER NOT NULL DEFAULT 0 + ) STRICT; + INSERT OR IGNORE INTO dsp_removals + (organization_id,installation_state,organization_status,sync_running,removed_at,legacy_services) + SELECT i.organization_id,CASE WHEN j.starting_state='suspended' THEN 'ready' ELSE j.starting_state END, + CASE WHEN j.starting_state IN ('ready','suspended') THEN 'active' + WHEN EXISTS (SELECT 1 FROM memberships m WHERE m.organization_id=i.organization_id) THEN 'setup_required' ELSE 'pending_owner' END, + json_extract(j.stage_receipts_json, '$.inspect_schedule.syncWasRunning'),j.created_at,1 + FROM installations i JOIN installation_lifecycle_jobs j ON j.id=COALESCE(i.current_job_id, + (SELECT id FROM installation_lifecycle_jobs old WHERE old.organization_id=i.organization_id AND old.operation='decommission' AND old.status='succeeded' ORDER BY old.updated_at DESC,old.rowid DESC LIMIT 1)) + WHERE j.operation='decommission' AND instr(j.stages_json,'remove_services')>0 + AND (i.current_job_id IS NOT NULL OR i.status='decommissioned') + AND NOT EXISTS (SELECT 1 FROM installation_lifecycle_jobs dead WHERE dead.organization_id=i.organization_id AND dead.operation='destroy' AND dead.status='succeeded'); + CREATE TRIGGER IF NOT EXISTS membership_single_dsp_insert + BEFORE INSERT ON memberships WHEN EXISTS ( + SELECT 1 FROM memberships WHERE user_id=NEW.user_id AND organization_id<>NEW.organization_id + ) BEGIN SELECT RAISE(ABORT, 'user_already_belongs_to_dsp'); END; + CREATE TRIGGER IF NOT EXISTS membership_single_dsp_update + BEFORE UPDATE OF user_id,organization_id ON memberships WHEN EXISTS ( + SELECT 1 FROM memberships WHERE user_id=NEW.user_id AND organization_id<>NEW.organization_id AND id<>OLD.id + ) BEGIN SELECT RAISE(ABORT, 'user_already_belongs_to_dsp'); END;`); + db.exec(`CREATE TABLE IF NOT EXISTS dsp_plugins ( + organization_id TEXT NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, + plugin_id TEXT NOT NULL, version TEXT NOT NULL, + desired_state TEXT NOT NULL CHECK(desired_state IN ('enabled','disabled','uninstalled')), + applied_state TEXT NOT NULL CHECK(applied_state IN ('enabled','disabled','uninstalled')), + revision INTEGER NOT NULL CHECK(revision>=1), applied_revision INTEGER NOT NULL CHECK(applied_revision>=0), + failure_code TEXT, actor_user_id TEXT REFERENCES users(id), updated_at INTEGER NOT NULL, + PRIMARY KEY(organization_id,plugin_id) + ) STRICT; + CREATE TABLE IF NOT EXISTS dsp_plugin_requests ( + organization_id TEXT NOT NULL REFERENCES organizations(id) ON DELETE CASCADE, plugin_id TEXT NOT NULL, + idempotency_key TEXT NOT NULL, action TEXT NOT NULL, expected_revision INTEGER NOT NULL, + actor_user_id TEXT NOT NULL REFERENCES users(id), PRIMARY KEY(organization_id,plugin_id,idempotency_key) + ) STRICT; + CREATE TABLE IF NOT EXISTS plugin_migration_checks ( + organization_id TEXT PRIMARY KEY REFERENCES organizations(id) ON DELETE CASCADE + ) STRICT;`); + // One-time adoption records intent from real setup requests, including failed + // enrollment. Empty provider directories and shipped code are not evidence. + if (initialVersion > 0 && initialVersion < 17) { + const paycom = require('../../../shared/plugin-sdk/catalog').plugin('paycom'); + if (paycom) db.prepare(`INSERT OR IGNORE INTO dsp_plugins(organization_id,plugin_id,version, + desired_state,applied_state,revision,applied_revision,failure_code,actor_user_id,updated_at) + SELECT i.organization_id,'paycom',?,'enabled','uninstalled',1,0,NULL,NULL,i.updated_at FROM installations i + WHERE EXISTS(SELECT 1 FROM installation_onboarding_requests q WHERE q.organization_id=i.organization_id) + OR EXISTS(SELECT 1 FROM installation_activation_jobs a WHERE a.organization_id=i.organization_id AND a.status='succeeded') + OR EXISTS(SELECT 1 FROM audit_events a WHERE a.organization_id=i.organization_id AND a.target_id='paycom' + AND a.action='connection.save' AND a.result='succeeded')`).run(paycom.version); + } + // Catalog and version advance together so a failed role migration can be retried. + db.exec('BEGIN IMMEDIATE'); + try { + if (initialVersion < 13) require('./fixed-roles-migration').migrateFixedRoles(db); + db.exec(`PRAGMA user_version=${SCHEMA_VERSION}; COMMIT`); + } catch (error) { db.exec('ROLLBACK'); throw error; } +} + +function migrateRuntimeBackends(db) { + const row = db.prepare("SELECT sql FROM sqlite_schema WHERE type='table' AND name='installations'").get(); + const target = "CHECK(backend IN ('local_reference','systemd_user','oci_container_v1','native_service_v1','directory_service_v1'))"; + if (row.sql.includes(target)) return; + const check = ["CHECK(backend IN ('local_reference','systemd_user','oci_container_v1'))", + "CHECK(backend IN ('local_reference','systemd_user','oci_container_v1','native_service_v1'))"] + .find(value => row.sql.includes(value)); + if (!check) fail('access_schema_incompatible'); + requireColumns(db, 'installations', CRITICAL_SCHEMA_COLUMNS.installations); + const dependents = db.prepare("SELECT sql FROM sqlite_schema WHERE tbl_name='installations' AND type IN ('index','trigger') AND sql IS NOT NULL").all(); + const foreignKeys = db.prepare('PRAGMA foreign_keys').get().foreign_keys; + db.exec('PRAGMA foreign_keys=OFF; BEGIN IMMEDIATE'); + try { + db.exec(row.sql.replace(/^CREATE TABLE (?:IF NOT EXISTS )?"?installations"?\s*\(/i, 'CREATE TABLE installations_updated (') + .replace(check, target)); + const columns = CRITICAL_SCHEMA_COLUMNS.installations.join(','); + db.exec(`INSERT INTO installations_updated(${columns}) SELECT ${columns} FROM installations; + DROP TABLE installations; ALTER TABLE installations_updated RENAME TO installations;`); + for (const item of dependents) db.exec(item.sql); + if (db.prepare('PRAGMA foreign_key_check').all().length) fail('access_schema_incompatible'); + db.exec('COMMIT'); + } catch (error) { db.exec('ROLLBACK'); throw error; } + finally { db.exec(`PRAGMA foreign_keys=${foreignKeys ? 'ON' : 'OFF'}`); } +} + +module.exports = { SCHEMA_VERSION, REVIEWED_UPGRADE_SCHEMAS, LEGACY_INSTALLATION_COLUMNS, CRITICAL_SCHEMA_COLUMNS, + CRITICAL_SCHEMA_INDEXES, CRITICAL_SCHEMA_TRIGGERS, requireColumns, requireIndex, requireTrigger, initializeAccessSchema }; diff --git a/core/core/accounts/src/service.js b/core/core/accounts/src/service.js new file mode 100644 index 0000000..b54d2bf --- /dev/null +++ b/core/core/accounts/src/service.js @@ -0,0 +1,1436 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { + AccessError, exact, text, identifier, email, password, token, controlReference, idempotencyKey, + timezone, station, abbreviation, +} = require('./validation'); +const { hashPassword, verifyPassword, consumeEquivalentPasswordWork } = require('./passwords'); +const { TENANT_PERMISSIONS, SYSTEM_ROLES, PLATFORM_PERMISSIONS } = require('./permissions'); +const { + installationTransition, + installationFailure, + platformInstallationStatus, + platformOrganization, + platformInstallationReceipt, + organizationSetupStatus, +} = require('../../../shared/contracts/src'); +const { createAccessInstallationProvisioningAuthority } = require('./installation-provisioning'); +const { createAccessInstallationLifecycleAuthority } = require('./installation-lifecycle'); +const { runtimeBackend } = require('../../runtime-deployment'); + +const DEFAULT_SESSION_TTL_MS = 12 * 60 * 60 * 1000; +const DEFAULT_INVITATION_TTL_MS = 72 * 60 * 60 * 1000; +const PLATFORM_CONTROL_TTL_MS = 15 * 60 * 1000; + +function opaqueToken() { return crypto.randomBytes(32).toString('base64url'); } +function tokenHash(value) { return crypto.createHash('sha256').update(value).digest('hex'); } +function id(prefix) { return `${prefix}_${crypto.randomUUID().replaceAll('-', '')}`; } +function iso(timestamp) { return new Date(timestamp).toISOString(); } +function valueDigest(value) { return crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex'); } +function platformContinuityRef(sessionHash, organizationId) { + return crypto.createHmac('sha256', sessionHash) + .update(`dispatch_platform_row\0${organizationId}`) + .digest('base64url'); +} +function platformControlMac(session, organizationId, expiry, purpose = 'dispatch_platform_control_v1') { + return crypto.createHmac('sha256', session.tokenHash) + .update(`${purpose}\0${session.user.id}\0${organizationId}\0`) + .update(expiry).digest().subarray(0, 24); +} +function storedResult(row) { + try { + const value = JSON.parse(row.result_json); + if (!value || typeof value !== 'object' || Array.isArray(value)) throw new Error('invalid'); + return value; + } catch { throw new AccessError('access_schema_incompatible', 500); } +} + +function maskedEmail(value) { + const [local, domain] = value.split('@'); + const shown = local.length <= 2 ? local[0] : `${local[0]}${'*'.repeat(Math.min(6, local.length - 2))}${local.at(-1)}`; + return `${shown}@${domain}`; +} + +function safeConflict(error) { + if (error instanceof AccessError) return error; + if (/UNIQUE constraint failed/i.test(String(error?.message || ''))) return new AccessError('conflict', 409); + return error; +} + +class AccessControlService { + constructor(store, { + clock = () => new Date(), + sessionTtlMs = DEFAULT_SESSION_TTL_MS, + invitationTtlMs = DEFAULT_INVITATION_TTL_MS, + installationOperatorEnabled = false, + installationBackend = 'systemd_user', + } = {}) { + if (!store || typeof store.transaction !== 'function' || typeof clock !== 'function' + || !Number.isSafeInteger(sessionTtlMs) || sessionTtlMs < 15 * 60 * 1000 || sessionTtlMs > 30 * 24 * 60 * 60 * 1000 + || !Number.isSafeInteger(invitationTtlMs) || invitationTtlMs < 15 * 60 * 1000 || invitationTtlMs > 30 * 24 * 60 * 60 * 1000 + || typeof installationOperatorEnabled !== 'boolean') { + throw new TypeError('access_control_dependencies_required'); + } + this.store = store; + this.clock = clock; + this.sessionTtlMs = sessionTtlMs; + this.invitationTtlMs = invitationTtlMs; + this.installationOperatorEnabled = installationOperatorEnabled; + try { this.installationBackend = runtimeBackend(installationBackend); } + catch { throw new TypeError('access_control_dependencies_required'); } + if (this.installationBackend === 'local_reference') throw new TypeError('access_control_dependencies_required'); + } + + now() { return this.clock().getTime(); } + + audit({ actorUserId = null, organizationId = null, action, targetType, targetId = null, result = 'succeeded', timestamp = this.now() }) { + this.store.createAudit({ + id: id('aud'), actorUserId, organizationId, action, targetType, targetId, result, timestamp, + }); + } + + ensureSystemRoles(organizationId, createdBy, timestamp) { + const roles = {}; + for (const definition of SYSTEM_ROLES) { + let role = this.store.roleByKey(organizationId, definition.key); + if (!role) { + role = this.store.createRole({ + id: id('role'), organizationId, key: definition.key, name: definition.name, + description: definition.description, system: true, permissions: definition.permissions, + createdBy, timestamp, + }); + } + roles[definition.key] = role; + } + return roles; + } + + ensureLocalOrganization(config) { + const organizationId = identifier(config.organization.id); + const name = text(config.organization.name, 'name', { minimum: 2, maximum: 120 }); + const code = station(config.site.code); + const zone = timezone(config.timezone); + const timestamp = this.now(); + return this.store.transaction(() => { + let organization = this.store.organization(organizationId); + if (!organization) { + this.store.createOrganization({ + id: organizationId, name, abbreviation: null, timezone: zone, status: 'active', + createdBy: null, timestamp, + }); + this.store.insertStation(organizationId, code, true, timestamp); + this.store.createInstallation( + organizationId, 'local', 'ready', timestamp, 'dispatch_current_1', 'local_reference', + ); + } + this.ensureSystemRoles(organizationId, null, timestamp); + organization = this.store.organization(organizationId); + return organization; + }); + } + + bootstrapStatus() { + this.store.expireInvitations(this.now()); + return { + initialized: this.store.platformOwnerCount() > 0, + invitationPending: Boolean(this.store.pendingPlatformInvitation()), + }; + } + + createPlatformBootstrap({ email: emailValue, organizationId = null }) { + const selectedEmail = email(emailValue); + const selectedOrganization = organizationId === null ? null : identifier(organizationId); + const timestamp = this.now(); + const rawToken = opaqueToken(); + let invitation; + try { + this.store.transaction(() => { + this.store.expireInvitations(timestamp); + if (this.store.platformOwnerCount() > 0) throw new AccessError('platform_already_initialized', 409); + if (this.store.pendingPlatformInvitation()) throw new AccessError('platform_invitation_pending', 409); + let roleId = null; + if (selectedOrganization) { + const organization = this.store.organization(selectedOrganization); + if (!organization) throw new AccessError('organization_not_found', 404); + roleId = this.store.roleByKey(selectedOrganization, 'owner')?.id; + if (!roleId) throw new AccessError('role_not_found', 500); + } + invitation = this.store.createInvitation({ + id: id('inv'), kind: 'platform_owner', organizationId: selectedOrganization, roleId, + email: selectedEmail, tokenHash: tokenHash(rawToken), expiresAt: timestamp + this.invitationTtlMs, + createdBy: null, timestamp, + }); + this.audit({ + organizationId: selectedOrganization, action: 'platform.bootstrap.invitation.create', + targetType: 'invitation', targetId: invitation.id, timestamp, + }); + }); + } catch (error) { throw safeConflict(error); } + return { invitation, token: rawToken }; + } + + revokePlatformBootstrap() { + const timestamp = this.now(); + return this.store.transaction(() => { + this.store.expireInvitations(timestamp); + const invitation = this.store.pendingPlatformInvitation(); + if (!invitation) return null; + this.store.revokeInvitation(invitation.id); + this.audit({ + organizationId: invitation.organization_id, action: 'platform.bootstrap.invitation.revoke', + targetType: 'invitation', targetId: invitation.id, timestamp, + }); + return this.store.invitationById(invitation.id); + }); + } + + invitation(rawToken) { + token(rawToken); + const row = this.store.invitationByHash(tokenHash(rawToken)); + const timestamp = this.now(); + if (!row || row.status !== 'pending' || row.expires_at <= timestamp) throw new AccessError('invitation_invalid', 404); + if (row.organization_id && this.removalStarted(row.organization_id)) throw new AccessError('invitation_invalid', 404); + return row; + } + + inspectInvitation(rawToken) { + const row = this.invitation(rawToken); + return { + kind: row.kind, + organization: row.organization_id ? { name: row.organization_name } : null, + role: row.role_id ? { name: row.role_name } : null, + email: maskedEmail(row.email), + accountExists: Boolean(this.store.userByEmail(row.email)), + expiresAt: iso(row.expires_at), + }; + } + + createSession(userId) { + const user = this.store.userById(userId); + if (!user || user.status !== 'active') throw new AccessError('account_disabled', 403); + const memberships = this.availableMemberships(userId); + if (user.platform_role !== 'owner' && this.membershipAccessBlocked(userId, memberships)) throw new AccessError('account_disabled', 403); + const rawToken = opaqueToken(); + const csrfToken = opaqueToken(); + const timestamp = this.now(); + this.store.deleteExpiredSessions(timestamp); + this.store.createSession({ + tokenHash: tokenHash(rawToken), userId, csrfToken, + activeOrganizationId: user.platform_role === 'owner' ? null : memberships[0]?.organizationId || null, + authVersion: user.auth_version, expiresAt: timestamp + this.sessionTtlMs, timestamp, + }); + return { token: rawToken, expiresAt: timestamp + this.sessionTtlMs, session: this.session(rawToken) }; + } + + session(rawToken) { + if (typeof rawToken !== 'string' || !/^[A-Za-z0-9_-]{43}$/.test(rawToken)) return null; + const hash = tokenHash(rawToken); + const session = this.store.session(hash, this.now()); + if (!session) return null; + const memberships = this.availableMemberships(session.userId); + if (session.user.platformRole !== 'owner' && this.membershipAccessBlocked(session.userId, memberships)) { + this.store.deleteSession(hash); + return null; + } + if (session.activeOrganizationId && !memberships.some(item => item.organizationId === session.activeOrganizationId)) { + session.activeOrganizationId = memberships[0]?.organizationId || null; + this.store.selectOrganization(hash, session.activeOrganizationId, this.now()); + } + this.store.touchSession(hash, this.now()); + return { + tokenHash: hash, + csrfToken: session.csrfToken, + expiresAt: iso(session.expiresAt), + user: session.user, + platformPermissions: session.user.platformRole === 'owner' ? PLATFORM_PERMISSIONS : [], + activeOrganizationId: session.activeOrganizationId, + memberships, + }; + } + + requireSession(rawToken) { + const session = this.session(rawToken); + if (!session) throw new AccessError('authentication_required', 401); + return session; + } + + availableMemberships(userId) { + return this.store.membershipsForUser(userId).filter(item => item.organization?.status !== 'suspended' + && !this.removalStarted(item.organizationId)); + } + + membershipAccessBlocked(userId, available) { + return available.length === 0 && Boolean(this.store.db.prepare('SELECT 1 FROM memberships WHERE user_id=? LIMIT 1').get(userId)); + } + + async signIn({ email: emailValue, password: passwordValue }) { + let selectedEmail; + try { selectedEmail = email(emailValue); } catch { + await consumeEquivalentPasswordWork(typeof passwordValue === 'string' ? passwordValue : ''); + throw new AccessError('invalid_credentials', 401); + } + const user = this.store.userByEmail(selectedEmail); + if (!user) { + await consumeEquivalentPasswordWork(typeof passwordValue === 'string' ? passwordValue : ''); + throw new AccessError('invalid_credentials', 401); + } + if (typeof passwordValue !== 'string') { + await consumeEquivalentPasswordWork(''); + throw new AccessError('invalid_credentials', 401); + } + const valid = await verifyPassword(passwordValue, user.password_hash); + if (!valid || user.status !== 'active') throw new AccessError('invalid_credentials', 401); + return this.store.transaction(() => { + const current = this.store.userById(user.id); + if (!current || current.status !== 'active' || current.email !== selectedEmail + || current.auth_version !== user.auth_version) throw new AccessError('invalid_credentials', 401); + this.audit({ actorUserId: user.id, action: 'session.login', targetType: 'user', targetId: user.id }); + return this.createSession(user.id); + }); + } + + signOut(session) { + this.store.deleteSession(session.tokenHash); + this.audit({ actorUserId: session.user.id, organizationId: session.activeOrganizationId, action: 'session.logout', targetType: 'user', targetId: session.user.id }); + } + + requestPasswordReset(input) { return require('./password-recovery').requestPasswordReset.call(this, input); } + + resetPassword(input) { return require('./password-recovery').resetPassword.call(this, input); } + + async changePassword(session, input) { + exact(input, ['currentPassword', 'newPassword', 'confirmPassword']); + const user = this.store.userById(session.user.id); + if (!user || typeof input.currentPassword !== 'string' || !await verifyPassword(input.currentPassword, user.password_hash)) { + throw new AccessError('current_password_invalid', 403); + } + password(input.newPassword); + if (input.newPassword !== input.confirmPassword) throw new AccessError('password_confirmation_mismatch'); + if (input.newPassword === input.currentPassword) throw new AccessError('password_unchanged', 409); + const passwordHash = await hashPassword(input.newPassword); + const timestamp = this.now(); + this.store.transaction(() => { + const current = this.store.userById(user.id); + if (!current || current.auth_version !== user.auth_version || current.status !== 'active') { + throw new AccessError('authentication_required', 401); + } + this.store.updatePassword(user.id, passwordHash, timestamp); + this.store.deleteUserSessions(user.id); + this.audit({ actorUserId: user.id, organizationId: session.activeOrganizationId, action: 'account.password.change', targetType: 'user', targetId: user.id, timestamp }); + }); + return this.createSession(user.id); + } + + selectMembership(session, membershipId) { + const selectedMembershipId = identifier(membershipId); + const membership = session.memberships.find(candidate => candidate.id === selectedMembershipId && candidate.status === 'active'); + if (!membership) throw new AccessError('membership_not_found', 404); + const organizationId = membership.organization.id; + this.store.selectOrganization(session.tokenHash, organizationId, this.now()); + return this.sessionByHash(session.tokenHash); + } + + sessionByHash(hash) { + const record = this.store.session(hash, this.now()); + if (!record) throw new AccessError('authentication_required', 401); + const memberships = this.store.membershipsForUser(record.userId); + return { + tokenHash: hash, csrfToken: record.csrfToken, expiresAt: iso(record.expiresAt), user: record.user, + platformPermissions: record.user.platformRole === 'owner' ? PLATFORM_PERMISSIONS : [], + activeOrganizationId: record.activeOrganizationId, memberships, + }; + } + + async acceptNewUser({ token: rawToken, firstName, lastName, password: passwordValue, confirmPassword }) { + const invitation = this.invitation(rawToken); + const selectedFirstName = text(firstName, 'firstName', { maximum: 80 }); + const selectedLastName = text(lastName, 'lastName', { maximum: 80 }); + password(passwordValue); + if (passwordValue !== confirmPassword) throw new AccessError('password_confirmation_mismatch'); + if (this.store.userByEmail(invitation.email)) throw new AccessError('account_exists', 409); + const passwordHash = await hashPassword(passwordValue); + const timestamp = this.now(); + let user; + try { + user = this.store.transaction(() => { + const current = this.invitation(rawToken); + if (this.store.userByEmail(current.email)) throw new AccessError('account_exists', 409); + const userId = id('usr'); + const created = this.store.insertUser({ + id: userId, email: current.email, firstName: selectedFirstName, lastName: selectedLastName, + passwordHash, platformRole: current.kind === 'platform_owner' ? 'owner' : null, timestamp, + }); + this.applyInvitationMembership(current, userId, timestamp); + this.store.acceptInvitation(current.id, userId, timestamp); + this.audit({ + actorUserId: userId, organizationId: current.organization_id, action: 'invitation.accept', + targetType: 'invitation', targetId: current.id, timestamp, + }); + return created; + }); + } catch (error) { throw safeConflict(error); } + return this.createSession(user.id); + } + + acceptExistingUser(session, rawToken) { + const invitation = this.invitation(rawToken); + if (session.user.email !== invitation.email) throw new AccessError('invitation_email_mismatch', 403); + const timestamp = this.now(); + this.store.transaction(() => { + const current = this.invitation(rawToken); + if (current.kind === 'platform_owner') this.store.setPlatformRole(session.user.id, 'owner', timestamp); + this.applyInvitationMembership(current, session.user.id, timestamp); + this.store.acceptInvitation(current.id, session.user.id, timestamp); + if (current.organization_id) this.store.selectOrganization(session.tokenHash, current.organization_id, timestamp); + this.audit({ + actorUserId: session.user.id, organizationId: current.organization_id, action: 'invitation.accept', + targetType: 'invitation', targetId: current.id, timestamp, + }); + }); + return this.sessionByHash(session.tokenHash); + } + + applyInvitationMembership(invitation, userId, timestamp) { + if (!invitation.organization_id) return; + if (this.store.membership(userId, invitation.organization_id)) throw new AccessError('membership_exists', 409); + this.store.createMembership({ + id: id('mem'), organizationId: invitation.organization_id, userId, roleId: invitation.role_id, + createdBy: invitation.created_by, timestamp, + }); + if (invitation.kind === 'organization_owner' || invitation.kind === 'platform_owner') { + const organization = this.store.organization(invitation.organization_id); + let installation = this.store.installationControl(invitation.organization_id); + if (installation?.status === 'waiting_for_owner' && installation.currentJobId === null) { + installationTransition('waiting_for_owner', 'waiting_for_provider_auth'); + installation = this.store.updateInstallationControl({ + organizationId: invitation.organization_id, + expectedStatus: 'waiting_for_owner', + expectedRevision: installation.revision, + status: 'waiting_for_provider_auth', + revision: installation.revision + 1, + currentJobId: null, + timestamp, + }); + } + if (organization?.status !== 'suspended') { + this.store.updateOrganizationStatus(invitation.organization_id, installation?.status === 'ready' ? 'active' : 'setup_required', timestamp); + require('./workspace-readiness').completeWorkspaceSetup(this.store, () => timestamp); + } + } + } + + requirePlatform(session, permission) { + if (session.dspView) throw new AccessError('dsp_view_scope', 403); + if (!session.platformPermissions.includes(permission)) throw new AccessError('platform_forbidden', 403); + } + + beginDspView(session, input) { + this.requirePlatform(session, 'platform.organizations.read'); + exact(input, ['controlRef']); + const { organizationId } = this.resolvePlatformControl(session, input.controlRef); + const viewRef = this.issuePlatformControlRef(session, organizationId, 'dispatch_dsp_owner_view_v1'); + const viewed = this.dspViewSession(session, viewRef); + this.audit({ actorUserId: session.user.id, organizationId, + action: 'organization.view.start', targetType: 'organization', targetId: organizationId }); + return viewed; + } + + dspViewSession(session, viewRef) { + let target; + try { + this.requirePlatform(session, 'platform.organizations.read'); + target = this.resolvePlatformControl(session, viewRef, 'dispatch_dsp_owner_view_v1'); + } + catch (error) { + if (error instanceof AccessError) throw new AccessError('dsp_view_unavailable', 403); + throw error; + } + const { organizationId, organization } = target; + const role = this.store.roleByKey(organizationId, 'owner'); + if (!role || organization.status === 'suspended' || this.removalStarted(organizationId)) { + throw new AccessError('dsp_view_unavailable', 403); + } + const membership = { + id: `view_${organizationId}`, organizationId, roleId: role.id, + roleKey: 'owner', roleName: 'Owner', status: 'active', + permissions: role.permissions, organization, + }; + return { ...session, activeOrganizationId: organizationId, memberships: [membership], + dspView: { viewRef, access: 'owner', + expiresAt: iso(Number(Buffer.from(viewRef, 'base64url').readBigUInt64BE())) } }; + } + + dspViewContext(session, organizationId, permission = null) { + // Revalidate the signed, session-bound scope and current DSP state on every access. + const current = this.sessionByHash(session.tokenHash); + const viewed = this.dspViewSession(current, session.dspView.viewRef); + if (viewed.activeOrganizationId !== organizationId) throw new AccessError('organization_forbidden', 403); + const membership = viewed.memberships[0]; + if (permission !== null && !membership.permissions.includes(permission)) throw new AccessError('organization_forbidden', 403); + if (permission !== null && !permission.endsWith('.read')) this.requireBackupIdle(organizationId); + return { membership, organization: membership.organization }; + } + + removalStarted(organizationId) { + if (this.store.db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(organizationId)) return true; + const installation = this.store.installationControl(organizationId); + if (!installation) return false; + if (['decommissioning', 'decommissioned'].includes(installation.status)) return true; + const job = installation.currentJobId ? this.store.lifecycleJob(installation.currentJobId) : null; + return Boolean(job && ['decommission', 'destroy'].includes(job.operation)); + } + + requireBackupIdle(organizationId) { + if (this.store.db.prepare("SELECT 1 FROM platform_backup_requests WHERE organization_id=? AND status IN ('queued','running')").get(organizationId) + || ['backup','restore','upgrade','decommission','destroy'].includes(this.store.activeLifecycleJob(organizationId)?.operation)) { + throw new AccessError('backup_operation_in_progress', 409); + } + } + + requirePermission(session, organizationId, permission) { + if (session.dspView) return this.dspViewContext(session, organizationId, permission); + const selected = identifier(organizationId); + const membership = this.store.membership(session.user.id, selected); + const organization = this.store.organization(selected); + if (!membership || membership.status !== 'active' || !organization || organization.status === 'suspended' || this.removalStarted(selected) + || !membership.permissions.includes(permission)) throw new AccessError('organization_forbidden', 403); + if (!permission.endsWith('.read')) this.requireBackupIdle(selected); + return { membership, organization }; + } + + organizationMembership(session) { + if (session.dspView) return this.dspViewContext(session, session.activeOrganizationId); + if (!session.activeOrganizationId) throw new AccessError('organization_required', 409); + const selected = identifier(session.activeOrganizationId); + const membership = this.store.membership(session.user.id, selected); + const organization = this.store.organization(selected); + if (!membership || membership.status !== 'active' || !organization || organization.status === 'suspended' || this.removalStarted(selected)) { + throw new AccessError('organization_forbidden', 403); + } + return { membership, organization }; + } + + organizationFor(session, permission) { + if (session.dspView) return this.dspViewContext(session, session.activeOrganizationId, permission); + const context = this.organizationMembership(session); + if (!permission.endsWith('.read')) this.requireBackupIdle(context.organization.id); + if (!context.membership.permissions.includes(permission)) throw new AccessError('organization_forbidden', 403); + return context; + } + + requireDspOwner(session) { + // Credential operations must use a current login and the authenticated DSP + // scope. Platform owners receive the same authority through their signed view. + const current = this.sessionByHash(session.tokenHash); + const selected = session.dspView ? this.dspViewSession(current, session.dspView.viewRef) : current; + if (current.user.id !== session.user.id || selected.activeOrganizationId !== session.activeOrganizationId) { + throw new AccessError('organization_forbidden', 403); + } + const context = this.organizationFor(selected, 'organization.owner'); + if (context.membership.roleKey !== 'owner') throw new AccessError('permission_denied', 403); + return context; + } + + runtimeFor(session, permission = null) { + const context = permission === null ? this.organizationMembership(session) : this.organizationFor(session, permission); + const installation = this.store.installation(context.organization.id); + if (context.organization.status !== 'active' || !installation || installation.status !== 'ready') { + throw new AccessError('installation_not_ready', 409); + } + return { ...context, installation }; + } + + issuePlatformControlRef(session, organizationId, purpose = 'dispatch_platform_control_v1') { + const timestamp = this.now(); + const sessionExpiry = Date.parse(session.expiresAt); + const expiresAt = Math.min(timestamp + PLATFORM_CONTROL_TTL_MS, sessionExpiry); + if (!Number.isSafeInteger(expiresAt) || expiresAt <= timestamp) throw new AccessError('authentication_required', 401); + // Keep listing read-only while lifecycle workers hold the database writer lock. + // The session-bound MAC hides the organization identity and authenticates expiry. + const expiry = Buffer.alloc(8); + expiry.writeBigUInt64BE(BigInt(expiresAt)); + return Buffer.concat([expiry, platformControlMac(session, organizationId, expiry, purpose)]).toString('base64url'); + } + + resolvePlatformControl(session, rawReference, purpose = 'dispatch_platform_control_v1') { + controlReference(rawReference); + const row = purpose === 'dispatch_platform_control_v1' ? this.store.platformTargetRef(tokenHash(rawReference)) : null; + // Honor references issued before this upgrade until their original expiry. + if (row) { + if (row.session_hash !== session.tokenHash || row.user_id !== session.user.id + || row.purpose !== 'organization_control' || row.expires_at <= this.now()) { + throw new AccessError('platform_control_invalid', 404); + } + const organization = this.store.organization(row.organization_id); + if (!organization) throw new AccessError('platform_control_invalid', 404); + return { organizationId: row.organization_id, organization }; + } + const bytes = Buffer.from(rawReference, 'base64url'); + if (bytes.length === 32 && bytes.toString('base64url') === rawReference) { + const expiresAt = Number(bytes.readBigUInt64BE()); + if (Number.isSafeInteger(expiresAt) && expiresAt > this.now() && expiresAt <= Date.parse(session.expiresAt)) { + for (const { id: organizationId } of this.store.db.prepare('SELECT id FROM organizations').all()) { + if (crypto.timingSafeEqual(bytes.subarray(8), platformControlMac(session, organizationId, bytes.subarray(0, 8), purpose))) { + return { organizationId, organization: this.store.organization(organizationId) }; + } + } + } + } + throw new AccessError('platform_control_invalid', 404); + } + + runPlatformMutation({ session, action, idempotencyKey: keyValue, digestValue, organizationId = null, execute, replay }) { + const key = idempotencyKey(keyValue); + const digest = valueDigest(digestValue); + return this.store.transaction(() => { + const prior = this.store.platformMutationRequest(session.user.id, action, key); + if (prior) { + if (prior.request_digest !== digest) throw new AccessError('idempotency_conflict', 409); + return replay(storedResult(prior), true); + } + const completed = execute(); + this.store.createPlatformMutationRequest({ + id: id('pmr'), + actorUserId: session.user.id, + action, + idempotencyKey: key, + requestDigest: digest, + organizationId: completed.organizationId ?? organizationId, + result: completed.record, + timestamp: this.now(), + }); + return completed.result; + }); + } + + installationConsoleStatus(organizationId) { + const control = this.store.installationControl(organizationId); + if (!control) throw new AccessError('installation_not_found', 404); + const provisioning = this.store.latestProvisioningRequest(organizationId); + const latestActivation = this.store.latestActivationJob(organizationId); + const activation = latestActivation?.authority_scope === 'optional_paycom' ? null : latestActivation; + let provisioningOperation = null; + if (provisioning) { + let operation; + try { operation = JSON.parse(provisioning.request_json)?.operation; } catch {} + if (!['provision', 'retry'].includes(operation)) throw new AccessError('access_schema_incompatible', 500); + provisioningOperation = { + kind: operation, + status: provisioning.status, + failureCode: provisioning.failure_code, + updatedAt: provisioning.updated_at, + }; + } + const activationOperation = activation ? { + kind: 'activation', + status: activation.status, + failureCode: activation.failure_code, + updatedAt: activation.updated_at, + } : null; + let selected = null; + if (control.status === 'provisioning') selected = provisioningOperation; + else if (['waiting_for_owner', 'waiting_for_provider_auth'].includes(control.status)) { + selected = provisioningOperation || activationOperation; + } else if (['verifying', 'ready'].includes(control.status)) { + selected = activationOperation || provisioningOperation; + } else if (control.status === 'failed' + && provisioning?.provisioner_job_id === control.currentJobId) { + selected = provisioningOperation; + } else if (control.status === 'failed' && activation?.id === control.currentJobId) { + selected = activationOperation; + } else if (activationOperation + && (!provisioningOperation || activationOperation.updatedAt >= provisioningOperation.updatedAt)) { + selected = activationOperation; + } else { + selected = provisioningOperation; + } + const lifecycle = control.currentJobId ? this.store.lifecycleJob(control.currentJobId) : null; + if (lifecycle && ['decommission', 'destroy', 'upgrade', 'resume'].includes(lifecycle.operation)) { + selected = { kind: lifecycle.operation === 'resume' && this.removalStarted(organizationId) ? 'restore_dsp' : lifecycle.operation, status: lifecycle.status, failureCode: lifecycle.failure_code }; + } + let failure = selected?.status === 'failed' ? installationFailure(selected.failureCode) : null; + if (control.status === 'failed' && failure === null) { + failure = installationFailure('installation_operation_failed'); + } + const availableActions = []; + if (this.installationOperatorEnabled && control.status === 'pending') availableActions.push('provision'); + const provisionFailure = provisioning?.status === 'failed' ? installationFailure(provisioning.failure_code) : null; + if (this.installationOperatorEnabled && control.status === 'failed' && provisioning?.status === 'failed' + && provisioning.provisioner_job_id === control.currentJobId + && provisionFailure.recoverable && provisionFailure.category === 'infrastructure') { + availableActions.push('retry_provision'); + } + const managed = !['local_reference', 'directory_service_v1'].includes(this.store.installationBackend(organizationId)); + if (this.installationOperatorEnabled && managed && (!this.store.activeLifecycleJob(organizationId) || this.store.activeLifecycleJob(organizationId).operation === 'backup')) { + const removal = this.store.db.prepare('SELECT * FROM dsp_removals WHERE organization_id=?').get(organizationId); + if ((['pending', 'waiting_for_owner', 'waiting_for_provider_auth', 'ready', 'suspended', 'failed'].includes(control.status) || this.store.activeLifecycleJob(organizationId)?.operation === 'backup') + && !removal && lifecycle?.operation !== 'destroy') availableActions.push('decommission'); + if (removal && lifecycle?.operation === 'decommission' && lifecycle.status === 'failed') availableActions.push('decommission'); + if ((control.status === 'decommissioned' || removal && lifecycle?.operation === 'destroy' && lifecycle.status === 'failed') && !(lifecycle?.operation === 'destroy' && lifecycle.status === 'succeeded')) availableActions.push('destroy'); + if (removal && control.status === 'decommissioned' && lifecycle?.operation !== 'destroy') availableActions.push('restore_dsp'); + } + const projection = { + state: control.status, + revision: control.revision, + operation: selected ? { kind: selected.kind, status: selected.status } : null, + failure, + availableActions, + }; + const selectedProjection = this.directoryLifecycleFor(organizationId)?.projection( + organizationId, projection, this.installationOperatorEnabled) || projection; + return platformInstallationStatus(this.installationOperatorEnabled && this.directoryLifecycleFor(organizationId) + && this.directoryDeletion ? this.directoryDeletion.projection(organizationId, selectedProjection) : selectedProjection); + } + + directoryLifecycleFor(organizationId) { + return this.store.installationBackend(organizationId) === 'directory_service_v1' + ? require('./directory-lifecycle').createDirectoryLifecycle({ store: this.store, clock: () => this.now() }) : null; + } + + requestPlatformRuntime(session, input, action) { + this.requirePlatform(session, 'platform.installations.manage'); + if (!this.installationOperatorEnabled) throw new AccessError('installation_operator_disabled', 503); + exact(input, ['controlRef', 'idempotencyKey', 'expectedRevision']); + const target = this.resolvePlatformControl(session, input.controlRef); + const authority = this.directoryLifecycleFor(target.organizationId); + if (!authority) throw new AccessError('installation_operation_not_allowed', 409); + return authority.request({ organizationId: target.organizationId, actorUserId: session.user.id, + action, expectedRevision: input.expectedRevision, requestId: input.idempotencyKey }); + } + + platformOrganizationView(session, organization) { + const invitations = this.store.invitations(organization.id) + .filter(invitation => invitation.kind === 'organization_owner' && invitation.status === 'pending' + && Date.parse(invitation.expiresAt) > this.now()); + const invitation = invitations[0] || null; + const ownerActive = this.store.activeOwnerCount(organization.id) > 0; + const platformPermissions = session.platformPermissions; + const canManageInvitations = platformPermissions.includes('platform.invitations.manage'); + const canManageInstallation = platformPermissions.includes('platform.installations.manage'); + const projectedInstallation = this.installationConsoleStatus(organization.id); + const installation = canManageInstallation + ? platformInstallationStatus({ ...projectedInstallation, availableActions: projectedInstallation.availableActions + .filter(action => organization.status !== 'suspended' || ['decommission', 'destroy', 'restore_dsp', 'resume', 'suspend'].includes(action)) }) + : platformInstallationStatus({ ...projectedInstallation, availableActions: [] }); + const availableActions = []; + const removed = this.removalStarted(organization.id); + if (!removed && !ownerActive && canManageInvitations) { + availableActions.push(invitation ? 'revoke_owner_invitation' : 'issue_owner_invitation'); + } + const profile = this.store.db.prepare('SELECT * FROM organization_profiles WHERE organization_id=?').get(organization.id); + const owner = this.store.db.prepare("SELECT u.email FROM users u JOIN memberships m ON m.user_id=u.id JOIN roles r ON r.id=m.role_id WHERE m.organization_id=? AND m.status='active' AND r.key='owner' ORDER BY m.created_at LIMIT 1").get(organization.id); + return platformOrganization({ + ownerEmail: owner?.email || invitation?.email || profile?.owner_email || null, + detailsStatus: !profile || profile.applied_at !== null ? 'complete' : profile.details_json ? 'submitted' : 'required', + controlRef: this.issuePlatformControlRef(session, organization.id), + continuityRef: platformContinuityRef(session.tokenHash, organization.id), + name: organization.name, + abbreviation: organization.abbreviation, + timezone: organization.timezone, + stations: organization.stations, + memberCount: organization.memberCount || 0, + organizationStatus: organization.status, + ownerStatus: ownerActive ? 'active' : invitation ? 'pending' : 'missing', + ownerInvitation: invitation ? { email: invitation.email, expiresAt: invitation.expiresAt } : null, + installation, + availableActions, + }); + } + + organizationSetup(session) { + if (!session.activeOrganizationId) throw new AccessError('organization_required', 409); + const membership = session.dspView + ? this.dspViewContext(session, session.activeOrganizationId, 'organization.owner').membership + : this.store.membership(session.user.id, session.activeOrganizationId); + const organization = this.store.organization(session.activeOrganizationId); + if (!membership || membership.status !== 'active' || !organization + || this.removalStarted(organization.id) + || !membership.permissions.includes('organization.owner')) throw new AccessError('organization_forbidden', 403); + const installation = this.installationConsoleStatus(organization.id); + const setupState = ['pending', 'provisioning', 'waiting_for_owner'].includes(installation.state) + ? 'waiting_for_platform' + : installation.state === 'waiting_for_provider_auth' + ? (['oci_container_v1', 'native_service_v1', 'directory_service_v1'].includes(this.store.installationBackend(organization.id)) ? 'owner_required' : 'server_owner_required') + : installation.state === 'verifying' ? 'verification_in_progress' + : installation.state === 'ready' ? 'ready' : 'unavailable'; + return organizationSetupStatus({ + organization: { + name: organization.name, + abbreviation: organization.abbreviation, + stationCode: organization.stations[0].code, + timezone: organization.timezone, + }, + organizationStatus: organization.status, + installationState: installation.state, + setupState, + handoff: setupState === 'server_owner_required' + ? { status: 'required', audience: 'server_owner', channel: 'private_terminal' } + : setupState === 'owner_required' ? { status: 'required', audience: 'dsp_owner', channel: 'dashboard' } : null, + operationalAccess: organization.status === 'active' && installation.state === 'ready' ? 'available' : 'unavailable', + failure: installation.state === 'failed' + ? (installation.failure || installationFailure('installation_operation_failed')) : null, + }); + } + + organizationProfile(session, input) { + const { organization } = this.organizationFor(session, 'organization.owner'); + const db = this.store.db; + if (input !== undefined) { + exact(input, ['name', 'abbreviation', 'stationCode', 'timezone']); + const details = { name: text(input.name, 'name', { minimum: 2, maximum: 120 }), + abbreviation: abbreviation(input.abbreviation), stationCode: station(input.stationCode), timezone: timezone(input.timezone) }; + this.store.transaction(() => { + this.organizationFor(session, 'organization.owner'); + const profile = db.prepare('SELECT * FROM organization_profiles WHERE organization_id=?').get(organization.id); + if (!profile || profile.applied_at !== null) throw new AccessError('organization_details_complete', 409); + db.prepare('UPDATE organization_profiles SET details_json=? WHERE organization_id=?').run(JSON.stringify(details), organization.id); + this.audit({ actorUserId: session.user.id, organizationId: organization.id, + action: 'organization.details.submit', targetType: 'organization', targetId: organization.id }); + }); + require('./organization-profile').applyOrganizationProfiles(this.store, () => this.now()); + } + const profile = db.prepare('SELECT * FROM organization_profiles WHERE organization_id=?').get(organization.id); + return { status: !profile || profile.applied_at !== null ? 'complete' : profile.details_json ? 'submitted' : 'required', + details: profile?.details_json ? JSON.parse(profile.details_json) : null }; + } + + createOrganization(session, input) { + this.requirePlatform(session, 'platform.organizations.create'); + exact(input, ['idempotencyKey', 'name', 'abbreviation', 'stationCode', 'timezone', 'ownerEmail']); + const emailOnly = input.name === undefined && input.stationCode === undefined && input.timezone === undefined; + if (emailOnly) { + this.requirePlatform(session, 'platform.installations.manage'); + if (!this.installationOperatorEnabled) throw new AccessError('installation_operator_disabled', 503); + if (!['oci_container_v1', 'native_service_v1', 'directory_service_v1'].includes(this.installationBackend)) throw new AccessError('container_provisioning_required', 409); + input = { ...input, name: 'New DSP', stationCode: 'NEW', timezone: 'UTC' }; + } + const selected = { + emailOnly, + idempotencyKey: idempotencyKey(input.idempotencyKey), + name: text(input.name, 'name', { minimum: 2, maximum: 120 }), + abbreviation: abbreviation(input.abbreviation), + stationCode: station(input.stationCode), + timezone: timezone(input.timezone), + ownerEmail: email(input.ownerEmail), + }; + try { + return this.runPlatformMutation({ + session, + action: 'organization.create', + idempotencyKey: selected.idempotencyKey, + digestValue: selected, + execute: () => { + const timestamp = this.now(); + const organizationId = id('org'); + const rawToken = opaqueToken(); + this.store.expireInvitations(timestamp); + this.store.createOrganization({ + id: organizationId, name: selected.name, abbreviation: selected.abbreviation, + timezone: selected.timezone, status: 'pending_owner', createdBy: session.user.id, timestamp, + }); + this.store.insertStation(organizationId, selected.stationCode, true, timestamp); + this.store.createInstallation( + organizationId, + `${this.installationBackend === 'directory_service_v1' ? 'dsp' : 'runtime'}_${organizationId.slice(4)}`, + 'pending', + timestamp, + this.store.db.prepare("SELECT r.release_id FROM platform_rollouts r JOIN platform_rollout_core c ON c.rollout_id=r.id WHERE c.status='succeeded' ORDER BY r.created_at DESC,r.rowid DESC LIMIT 1").get()?.release_id || 'dispatch_current_1', + this.installationBackend, + ); + if (emailOnly) { + this.store.db.prepare('INSERT INTO organization_profiles(organization_id,owner_email) VALUES(?,?)').run(organizationId, selected.ownerEmail); + const request = this.store.createProvisioningRequest({ + id: id('prq'), organizationId, authorityScope: 'platform_installation', + operation: { operation: 'provision', idempotencyKey: `create:${organizationId}`, expectedRevision: 1 }, timestamp, + }); + this.audit({ actorUserId: session.user.id, organizationId, action: 'installation.provision.request', + targetType: 'installation_request', targetId: request.row.id, timestamp }); + } + const roles = this.ensureSystemRoles(organizationId, session.user.id, timestamp); + const invitation = this.store.createInvitation({ + id: id('inv'), kind: 'organization_owner', organizationId, roleId: roles.owner.id, + email: selected.ownerEmail, tokenHash: tokenHash(rawToken), expiresAt: timestamp + this.invitationTtlMs, + createdBy: session.user.id, timestamp, + }); + this.audit({ + actorUserId: session.user.id, organizationId, action: 'organization.create', + targetType: 'organization', targetId: organizationId, timestamp, + }); + return { + organizationId, + record: { organizationId, invitationId: invitation.id }, + result: { organization: this.store.organization(organizationId), invitation, token: rawToken, replayed: false }, + }; + }, + replay: record => { + if (Object.keys(record).sort().join(',') !== 'invitationId,organizationId') { + throw new AccessError('access_schema_incompatible', 500); + } + const organization = this.store.organization(record.organizationId); + const invitation = this.store.invitationById(record.invitationId); + if (!organization || !invitation || invitation.organizationId !== organization.id) { + throw new AccessError('access_schema_incompatible', 500); + } + return { organization, invitation, token: null, replayed: true }; + }, + }); + } catch (error) { throw safeConflict(error); } + } + + platformOrganizations(session) { + this.requirePlatform(session, 'platform.organizations.read'); + return this.store.organizations().filter(organization => { + const control = this.store.installationControl(organization.id); + const job = control?.currentJobId ? this.store.lifecycleJob(control.currentJobId) : null; + return !(job?.operation === 'destroy' && job.status === 'succeeded' && this.store.installationBackend(organization.id) !== 'native_service_v1'); + }).map(organization => this.platformOrganizationView(session, organization)); + } + + platformDiagnostics(session, input) { + this.requirePlatform(session, 'platform.installations.manage'); + if (session.user.platformRole !== 'owner') throw new AccessError('platform_forbidden', 403); + if (input !== undefined) { + exact(input, ['idempotencyKey']); + const key = idempotencyKey(input.idempotencyKey); + if (!this.installationOperatorEnabled || !['native_service_v1', 'directory_service_v1'].includes(this.installationBackend)) { + throw new AccessError('installation_operator_disabled', 503); + } + this.store.transaction(() => { + if (this.store.db.prepare('SELECT 1 FROM diagnostic_dsps WHERE actor_user_id=? AND idempotency_key=?') + .get(session.user.id, key)) return; + const directoryMode = this.installationBackend === 'directory_service_v1'; + const ownerEmail = directoryMode ? `diagnostic-${crypto.randomBytes(16).toString('hex')}@example.test` : session.user.email; + const created = this.createOrganization(session, { ownerEmail, + idempotencyKey: `diagnostic:${tokenHash(key)}` }); + const organizationId = created.organization.id; + if (created.replayed) throw new AccessError('idempotency_conflict', 409); + const timestamp = this.now(); + const invitation = this.store.invitationByHash(tokenHash(created.token)); + let ownerId = session.user.id; + if (directoryMode) { + // A dedicated synthetic app identity lets the platform owner create + // multiple test DSPs without joining their tenant memberships. There + // is no known password and no invitation delivery or Linux account. + ownerId = id('usr'); + this.store.insertUser({ id: ownerId, email: ownerEmail, firstName: 'Synthetic', lastName: 'Owner', + passwordHash: `scrypt-v1$32768$8$1$${crypto.randomBytes(24).toString('base64url')}$${crypto.randomBytes(64).toString('base64url')}`, + platformRole: null, timestamp }); + } + this.applyInvitationMembership(invitation, ownerId, timestamp); + this.store.acceptInvitation(invitation.id, ownerId, timestamp); + const details = { name: `TEST DSP ${organizationId.slice(-8)}`, abbreviation: 'TEST', stationCode: 'TST1', timezone: 'UTC' }; + this.store.db.prepare('UPDATE organization_profiles SET details_json=? WHERE organization_id=?') + .run(JSON.stringify(details), organizationId); + this.store.db.prepare('UPDATE organizations SET name=? WHERE id=?').run(details.name, organizationId); + this.store.db.prepare("INSERT INTO diagnostic_dsps VALUES(?,?,?,'pending',?)") + .run(organizationId, session.user.id, key, timestamp); + this.audit({ actorUserId: session.user.id, organizationId, action: 'diagnostics.create', + targetType: 'organization', targetId: organizationId, timestamp }); + }); + } + return { + enabled: this.installationOperatorEnabled && ['native_service_v1', 'directory_service_v1'].includes(this.installationBackend), + dsps: this.store.db.prepare(`SELECT d.*,o.name,i.status AS installation_status FROM diagnostic_dsps d + JOIN organizations o ON o.id=d.organization_id JOIN installations i ON i.organization_id=d.organization_id + ORDER BY d.created_at DESC`).all().map(row => ({ + name: row.name, createdAt: iso(row.created_at), + status: row.status === 'pending' && row.installation_status === 'failed' ? 'failed' : row.status, + installation: this.installationConsoleStatus(row.organization_id), + })), + }; + } + + requestInstallationProvisioning(session, organizationId, input) { + this.requirePlatform(session, 'platform.installations.manage'); + if (!this.installationOperatorEnabled) throw new AccessError('installation_operator_disabled', 503); + exact(input, ['idempotencyKey', 'expectedRevision']); + const authority = createAccessInstallationProvisioningAuthority({ + store: this.store, + organizationId: identifier(organizationId), + authorityScope: 'platform_installation', + actorUserId: session.user.id, + clock: () => this.now(), + }); + return authority.request({ + operation: 'provision', + idempotencyKey: input.idempotencyKey, + expectedRevision: input.expectedRevision, + }); + } + + requestInstallationRetry(session, organizationId, input) { + this.requirePlatform(session, 'platform.installations.manage'); + if (!this.installationOperatorEnabled) throw new AccessError('installation_operator_disabled', 503); + exact(input, ['idempotencyKey', 'expectedRevision']); + const selectedOrganizationId = identifier(organizationId); + const selectedIdempotencyKey = idempotencyKey(input.idempotencyKey); + const authorityScope = 'platform_installation'; + const authority = createAccessInstallationProvisioningAuthority({ + store: this.store, + organizationId: selectedOrganizationId, + authorityScope, + actorUserId: session.user.id, + clock: () => this.now(), + }); + const existing = this.store.provisioningRequestByKey( + selectedOrganizationId, authorityScope, selectedIdempotencyKey, + ); + if (existing === null + && !this.installationConsoleStatus(selectedOrganizationId).availableActions.includes('retry_provision')) { + throw new AccessError('installation_operation_not_allowed', 409); + } + return authority.request({ + operation: 'retry', + idempotencyKey: selectedIdempotencyKey, + expectedRevision: input.expectedRevision, + }); + } + + requestPlatformInstallationProvisioning(session, input) { + this.requirePlatform(session, 'platform.installations.manage'); + exact(input, ['controlRef', 'idempotencyKey', 'expectedRevision']); + const target = this.resolvePlatformControl(session, input.controlRef); + const request = this.requestInstallationProvisioning(session, target.organizationId, { + idempotencyKey: input.idempotencyKey, + expectedRevision: input.expectedRevision, + }); + const current = this.store.installationControl(target.organizationId); + return platformInstallationReceipt({ + action: 'provision', + status: request.replayed ? 'replayed' : 'accepted', + installationState: current.status, + installationRevision: current.revision, + replayed: request.replayed, + }); + } + + requestPlatformInstallationRetry(session, input) { + this.requirePlatform(session, 'platform.installations.manage'); + exact(input, ['controlRef', 'idempotencyKey', 'expectedRevision']); + const target = this.resolvePlatformControl(session, input.controlRef); + const request = this.requestInstallationRetry(session, target.organizationId, { + idempotencyKey: input.idempotencyKey, + expectedRevision: input.expectedRevision, + }); + const current = this.store.installationControl(target.organizationId); + return platformInstallationReceipt({ + action: 'retry_provision', + status: request.replayed ? 'replayed' : 'accepted', + installationState: current.status, + installationRevision: current.revision, + replayed: request.replayed, + }); + } + + async requestPlatformRemoval(session, input, operation) { + this.requirePlatform(session, 'platform.installations.manage'); + if (!this.installationOperatorEnabled) throw new AccessError('installation_operator_disabled', 503); + if (!['decommission', 'destroy', 'resume'].includes(operation)) throw new AccessError('invalid_input'); + exact(input, ['controlRef', 'idempotencyKey', 'expectedRevision', ...(operation === 'destroy' ? ['password'] : [])]); + let user; + if (operation === 'destroy') { + user = this.store.userById(session.user.id); + if (!user || typeof input.password !== 'string' || !await verifyPassword(input.password, user.password_hash)) { + throw new AccessError('current_password_invalid', 403); + } + } + return this.store.transaction(() => { + // Password hashing yields: recheck session, credentials and permissions before mutation. + if (user) { + const current = this.store.userById(user.id); + if (!current || current.auth_version !== user.auth_version || current.status !== 'active' + || current.platform_role !== 'owner' || !this.store.session(session.tokenHash, this.now())) { + throw new AccessError('authentication_required', 401); + } + } + const target = this.resolvePlatformControl(session, input.controlRef); + const directory = this.directoryLifecycleFor(target.organizationId); + if (directory && operation === 'destroy' && this.directoryDeletion) return this.directoryDeletion.request({ + organizationId: target.organizationId, actorUserId: session.user.id, expectedRevision: input.expectedRevision, + requestId: input.idempotencyKey }); + if (directory) return directory.request({ organizationId: target.organizationId, actorUserId: session.user.id, + action: operation === 'resume' ? 'restore_dsp' : operation, + expectedRevision: input.expectedRevision, requestId: input.idempotencyKey }); + if (operation === 'resume' && !this.store.db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(target.organizationId) + && !this.store.lifecycleJobByRequest(target.organizationId, 'platform_removal', input.idempotencyKey)) { + throw new AccessError('installation_operation_not_allowed', 409); + } + const authority = createAccessInstallationLifecycleAuthority({ + store: this.store, organizationId: target.organizationId, + authorityScope: 'platform_removal', actorUserId: session.user.id, + destructionEnabled: operation === 'destroy', clock: () => this.now(), + }); + const result = authority.request({ operation, idempotencyKey: input.idempotencyKey, expectedRevision: input.expectedRevision }); + const current = this.store.installationControl(target.organizationId); + return platformInstallationReceipt({ + action: operation === 'resume' ? 'restore_dsp' : operation, status: result.replayed ? 'replayed' : 'accepted', + installationState: current.status, installationRevision: current.revision, replayed: result.replayed, + }); + }); + } + + setOrganizationSuspended(session, organizationId, input) { + this.requirePlatform(session, 'platform.organizations.suspend'); + exact(input, ['suspended']); + if (typeof input.suspended !== 'boolean') throw new AccessError('invalid_request'); + const selected = identifier(organizationId); + const timestamp = this.now(); + return this.store.transaction(() => { + const organization = this.store.organization(selected); + if (!organization) throw new AccessError('organization_not_found', 404); + const installation = this.store.installationControl(selected); + if (this.removalStarted(selected)) throw new AccessError('installation_operation_not_allowed', 409); + this.requireBackupIdle(selected); + if (['provisioning', 'waiting_for_provider_auth', 'verifying'].includes(installation?.status)) { + throw new AccessError('installation_operation_in_progress', 409); + } + const resumedStatus = this.store.activeOwnerCount(selected) === 0 + ? 'pending_owner' + : (installation?.status === 'ready' || installation?.status === 'suspended' && this.store.latestReadyEvidence(selected)) ? 'active' : 'setup_required'; + this.changeOrganizationRuntimeStatus(selected, installation, input.suspended ? 'suspended' : resumedStatus, session.user.id); + this.audit({ + actorUserId: session.user.id, organizationId: selected, + action: input.suspended ? 'organization.suspend' : 'organization.resume', + targetType: 'organization', targetId: selected, timestamp, + }); + return this.store.organization(selected); + }); + } + + queueOrganizationStatusLifecycle(organizationId, installation, status) { + if (!this.installationOperatorEnabled || !installation + || !['oci_container_v1', 'native_service_v1'].includes(this.store.installationBackend(organizationId))) return; + const operation = status === 'suspended' && installation.status === 'ready' ? 'suspend' + : status === 'active' && installation.status === 'suspended' ? 'resume' : null; + if (!operation) return; + // Commit access revocation and its lifecycle job together. Otherwise the + // agent loses its authority before the worker can inspect and stop it. + createAccessInstallationLifecycleAuthority({ store: this.store, organizationId, + authorityScope: 'organization_status_lifecycle', clock: () => this.now() }).request({ + operation, expectedRevision: installation.revision, + idempotencyKey: `organization-status:${operation}:${installation.revision}`, + }); + } + + changeOrganizationRuntimeStatus(organizationId, installation, status, actorUserId) { + const directory = this.directoryLifecycleFor(organizationId); + if (directory) { + if (!this.installationOperatorEnabled) throw new AccessError('installation_operator_disabled', 503); + const operation = status === 'suspended' ? 'suspend' : 'resume'; + directory.request({ organizationId, actorUserId, action: operation, expectedRevision: installation.revision, + requestId: `organization-status:${operation}:${installation.revision}` }); + } else { + this.store.updateOrganizationStatus(organizationId, status, this.now()); + this.queueOrganizationStatusLifecycle(organizationId, installation, status); + } + return this.store.organization(organizationId).status; + } + + revokePlatformInvitation(session, invitationId) { + this.requirePlatform(session, 'platform.invitations.manage'); + const selected = identifier(invitationId); + const invitation = this.store.invitationById(selected); + if (!invitation || invitation.kind !== 'organization_owner') throw new AccessError('invitation_not_found', 404); + this.store.transaction(() => { + this.store.revokeInvitation(selected); + this.audit({ + actorUserId: session.user.id, organizationId: invitation.organizationId, + action: 'invitation.revoke', targetType: 'invitation', targetId: selected, + }); + }); + } + + createOwnerInvitation(session, organizationId, input) { + this.requirePlatform(session, 'platform.invitations.manage'); + const selectedOrganizationId = identifier(organizationId); + exact(input, ['ownerEmail']); + const selectedEmail = email(input.ownerEmail); + const rawToken = opaqueToken(); + const timestamp = this.now(); + let invitation; + try { + this.store.transaction(() => { + this.store.expireInvitations(timestamp); + const organization = this.store.organization(selectedOrganizationId); + if (!organization) throw new AccessError('organization_not_found', 404); + if (this.removalStarted(selectedOrganizationId)) throw new AccessError('installation_operation_not_allowed', 409); + if (this.store.activeOwnerCount(selectedOrganizationId) > 0) throw new AccessError('organization_owner_exists', 409); + if (this.store.invitations(selectedOrganizationId).some(candidate => candidate.kind === 'organization_owner' && candidate.status === 'pending')) { + throw new AccessError('invitation_pending', 409); + } + if (this.store.pendingInvitationForEmail(selectedOrganizationId, selectedEmail)) throw new AccessError('invitation_pending', 409); + const ownerRole = this.store.roleByKey(selectedOrganizationId, 'owner'); + if (!ownerRole) throw new AccessError('role_not_found', 500); + invitation = this.store.createInvitation({ + id: id('inv'), kind: 'organization_owner', organizationId: selectedOrganizationId, + roleId: ownerRole.id, email: selectedEmail, tokenHash: tokenHash(rawToken), + expiresAt: timestamp + this.invitationTtlMs, createdBy: session.user.id, timestamp, + }); + this.audit({ + actorUserId: session.user.id, organizationId: selectedOrganizationId, + action: 'invitation.create', targetType: 'invitation', targetId: invitation.id, timestamp, + }); + }); + } catch (error) { throw safeConflict(error); } + return { invitation, token: rawToken }; + } + + setPlatformOrganizationSuspended(session, input) { + this.requirePlatform(session, 'platform.organizations.suspend'); + exact(input, ['controlRef', 'idempotencyKey', 'suspended']); + if (typeof input.suspended !== 'boolean') throw new AccessError('invalid_input'); + const target = this.resolvePlatformControl(session, input.controlRef); + return this.runPlatformMutation({ + session, + action: 'organization.status', + idempotencyKey: input.idempotencyKey, + digestValue: { organizationId: target.organizationId, suspended: input.suspended }, + organizationId: target.organizationId, + execute: () => { + const timestamp = this.now(); + const organization = this.store.organization(target.organizationId); + const installation = this.store.installationControl(target.organizationId); + if (!organization || !installation) throw new AccessError('platform_control_invalid', 404); + if (this.removalStarted(target.organizationId)) throw new AccessError('installation_operation_not_allowed', 409); + if (['provisioning', 'waiting_for_provider_auth', 'verifying'].includes(installation.status)) { + throw new AccessError('installation_operation_in_progress', 409); + } + const resumedStatus = this.store.activeOwnerCount(target.organizationId) === 0 + ? 'pending_owner' : (installation.status === 'ready' || installation.status === 'suspended' && this.store.latestReadyEvidence(target.organizationId)) ? 'active' : 'setup_required'; + const status = this.changeOrganizationRuntimeStatus(target.organizationId, installation, + input.suspended ? 'suspended' : resumedStatus, session.user.id); + this.audit({ + actorUserId: session.user.id, organizationId: target.organizationId, + action: input.suspended ? 'organization.suspend' : 'organization.resume', + targetType: 'organization', targetId: target.organizationId, timestamp, + }); + return { + record: { organizationId: target.organizationId, status }, + result: { status, replayed: false }, + }; + }, + replay: record => { + if (Object.keys(record).sort().join(',') !== 'organizationId,status' + || record.organizationId !== target.organizationId || typeof record.status !== 'string') { + throw new AccessError('access_schema_incompatible', 500); + } + return { status: record.status, replayed: true }; + }, + }); + } + + createPlatformOwnerInvitation(session, input) { + this.requirePlatform(session, 'platform.invitations.manage'); + exact(input, ['controlRef', 'idempotencyKey', 'ownerEmail']); + const target = this.resolvePlatformControl(session, input.controlRef); + const selectedEmail = email(input.ownerEmail); + try { + return this.runPlatformMutation({ + session, + action: 'owner_invitation.create', + idempotencyKey: input.idempotencyKey, + digestValue: { organizationId: target.organizationId, ownerEmail: selectedEmail }, + organizationId: target.organizationId, + execute: () => { + const timestamp = this.now(); + const rawToken = opaqueToken(); + this.store.expireInvitations(timestamp); + if (this.removalStarted(target.organizationId)) throw new AccessError('installation_operation_not_allowed', 409); + if (this.store.activeOwnerCount(target.organizationId) > 0) throw new AccessError('organization_owner_exists', 409); + if (this.store.invitations(target.organizationId) + .some(candidate => candidate.kind === 'organization_owner' && candidate.status === 'pending')) { + throw new AccessError('invitation_pending', 409); + } + const ownerRole = this.store.roleByKey(target.organizationId, 'owner'); + if (!ownerRole) throw new AccessError('role_not_found', 500); + const invitation = this.store.createInvitation({ + id: id('inv'), kind: 'organization_owner', organizationId: target.organizationId, + roleId: ownerRole.id, email: selectedEmail, tokenHash: tokenHash(rawToken), + expiresAt: timestamp + this.invitationTtlMs, createdBy: session.user.id, timestamp, + }); + this.audit({ + actorUserId: session.user.id, organizationId: target.organizationId, + action: 'invitation.create', targetType: 'invitation', targetId: invitation.id, timestamp, + }); + return { + record: { organizationId: target.organizationId, invitationId: invitation.id }, + result: { invitation, token: rawToken, replayed: false }, + }; + }, + replay: record => { + if (Object.keys(record).sort().join(',') !== 'invitationId,organizationId' + || record.organizationId !== target.organizationId) throw new AccessError('access_schema_incompatible', 500); + const invitation = this.store.invitationById(record.invitationId); + if (!invitation || invitation.organizationId !== target.organizationId) { + throw new AccessError('access_schema_incompatible', 500); + } + return { invitation, token: null, replayed: true }; + }, + }); + } catch (error) { throw safeConflict(error); } + } + + revokePlatformOwnerInvitation(session, input) { + this.requirePlatform(session, 'platform.invitations.manage'); + exact(input, ['controlRef', 'idempotencyKey']); + const target = this.resolvePlatformControl(session, input.controlRef); + return this.runPlatformMutation({ + session, + action: 'owner_invitation.revoke', + idempotencyKey: input.idempotencyKey, + digestValue: { organizationId: target.organizationId }, + organizationId: target.organizationId, + execute: () => { + const timestamp = this.now(); + this.store.expireInvitations(timestamp); + const invitation = this.store.invitations(target.organizationId) + .find(candidate => candidate.kind === 'organization_owner' && candidate.status === 'pending'); + if (!invitation) throw new AccessError('invitation_not_found', 404); + this.store.revokeInvitation(invitation.id); + this.audit({ + actorUserId: session.user.id, organizationId: target.organizationId, + action: 'invitation.revoke', targetType: 'invitation', targetId: invitation.id, timestamp, + }); + return { + record: { organizationId: target.organizationId, invitationId: invitation.id }, + result: { status: 'revoked', replayed: false }, + }; + }, + replay: record => { + if (Object.keys(record).sort().join(',') !== 'invitationId,organizationId' + || record.organizationId !== target.organizationId) throw new AccessError('access_schema_incompatible', 500); + return { status: 'revoked', replayed: true }; + }, + }); + } + + organizationAudit(session, organizationId) { + this.requirePermission(session, organizationId, 'audit.read'); + return { audit: this.store.audits(organizationId, 100, { excludePlatformAccess: true }) }; + } + + organizationAdministration(session, organizationId) { + const { organization, membership } = this.requirePermission(session, organizationId, 'members.read'); + this.store.expireInvitations(this.now()); + this.requirePermission(session, organizationId, 'roles.read'); + return { + organization, + membership, + permissionCatalog: TENANT_PERMISSIONS, + roles: this.store.roles(organizationId), + members: this.store.members(organizationId), + invitations: this.store.invitations(organizationId), + audit: membership.permissions.includes('audit.read') + ? this.organizationAudit(session, organizationId).audit : [], + }; + } + + createRole(session, organizationId, input) { + this.requirePermission(session, organizationId, 'roles.read'); + throw new AccessError('fixed_roles_only', 409); + } + + updateRole(session, organizationId, roleId, input) { + this.requirePermission(session, organizationId, 'roles.read'); + const role = this.store.role(identifier(roleId)); + if (!role || role.organizationId !== organizationId) throw new AccessError('role_not_found', 404); + throw new AccessError('fixed_roles_only', 409); + } + + deleteRole(session, organizationId, roleId) { + this.requirePermission(session, organizationId, 'roles.read'); + const role = this.store.role(identifier(roleId)); + if (!role || role.organizationId !== organizationId) throw new AccessError('role_not_found', 404); + throw new AccessError('fixed_roles_only', 409); + } + + createMemberInvitation(session, organizationId, input) { + const { membership } = this.requirePermission(session, organizationId, 'members.invite'); + exact(input, ['email', 'roleId']); + const selectedEmail = email(input.email); + const roleId = identifier(input.roleId); + const rawToken = opaqueToken(); + const timestamp = this.now(); + let invitation; + try { + this.store.transaction(() => { + this.store.expireInvitations(timestamp); + const role = this.store.role(roleId); + if (!role || role.organizationId !== organizationId || !role.system || !SYSTEM_ROLES.some(definition => definition.key === role.key)) throw new AccessError('role_not_assignable', 409); + if (role.permissions.some(permission => !membership.permissions.includes(permission))) throw new AccessError('permission_escalation_forbidden', 403); + const existingUser = this.store.userByEmail(selectedEmail); + if (existingUser && this.store.membership(existingUser.id, organizationId)) throw new AccessError('membership_exists', 409); + if (this.store.pendingInvitationForEmail(organizationId, selectedEmail)) throw new AccessError('invitation_pending', 409); + invitation = this.store.createInvitation({ + id: id('inv'), kind: 'organization_member', organizationId, roleId, + email: selectedEmail, tokenHash: tokenHash(rawToken), expiresAt: timestamp + this.invitationTtlMs, + createdBy: session.user.id, timestamp, + }); + this.audit({ actorUserId: session.user.id, organizationId, action: 'invitation.create', targetType: 'invitation', targetId: invitation.id, timestamp }); + }); + } catch (error) { throw safeConflict(error); } + return { invitation, token: rawToken }; + } + + revokeMemberInvitation(session, organizationId, invitationId) { + this.requirePermission(session, organizationId, 'members.invite'); + const selected = identifier(invitationId); + const invitation = this.store.invitationById(selected); + if (!invitation || invitation.organizationId !== organizationId || invitation.kind !== 'organization_member') throw new AccessError('invitation_not_found', 404); + this.store.transaction(() => { + this.store.revokeInvitation(selected); + this.audit({ actorUserId: session.user.id, organizationId, action: 'invitation.revoke', targetType: 'invitation', targetId: selected }); + }); + } + + updateMemberRole(session, organizationId, membershipId, roleId) { + this.store.transaction(() => { + const { membership: actor } = this.requirePermission(session, organizationId, 'members.manage'); + const selectedMembershipId = identifier(membershipId); + const selectedRoleId = identifier(roleId); + const target = this.store.membershipById(selectedMembershipId); + const currentRole = target ? this.store.role(target.role_id) : null; + const role = this.store.role(selectedRoleId); + if (!target || target.organization_id !== organizationId || !currentRole) throw new AccessError('member_not_found', 404); + if (!role || role.organizationId !== organizationId || !role.system || !SYSTEM_ROLES.some(definition => definition.key === role.key)) throw new AccessError('role_not_assignable', 409); + if (target.user_id === session.user.id) throw new AccessError('self_role_change_forbidden', 409); + if (currentRole.permissions.some(permission => !actor.permissions.includes(permission)) + || role.permissions.some(permission => !actor.permissions.includes(permission))) { + throw new AccessError('permission_escalation_forbidden', 403); + } + const timestamp = this.now(); + if (currentRole.key === 'owner' && role.key !== 'owner' && this.store.activeOwnerCount(organizationId) <= 1) { + throw new AccessError('last_owner_protected', 409); + } + this.store.updateMembershipRole(selectedMembershipId, selectedRoleId, timestamp); + this.audit({ actorUserId: session.user.id, organizationId, action: 'membership.role.update', targetType: 'membership', targetId: selectedMembershipId, timestamp }); + }); + } + + removeMember(session, organizationId, membershipId) { + this.store.transaction(() => { + const { membership: actor } = this.requirePermission(session, organizationId, 'members.manage'); + const selected = identifier(membershipId); + const target = this.store.membershipById(selected); + const role = target ? this.store.role(target.role_id) : null; + if (!target || target.organization_id !== organizationId || !role) throw new AccessError('member_not_found', 404); + if (target.user_id === session.user.id) throw new AccessError('self_removal_forbidden', 409); + if (role.permissions.some(permission => !actor.permissions.includes(permission))) throw new AccessError('permission_escalation_forbidden', 403); + if (role.key === 'owner' && this.store.activeOwnerCount(organizationId) <= 1) { + throw new AccessError('last_owner_protected', 409); + } + this.store.removeMembership(selected); + this.audit({ actorUserId: session.user.id, organizationId, action: 'membership.remove', targetType: 'membership', targetId: selected }); + }); + } +} + +module.exports = { + DEFAULT_SESSION_TTL_MS, + DEFAULT_INVITATION_TTL_MS, + AccessControlService, + opaqueToken, + tokenHash, + maskedEmail, +}; diff --git a/core/core/accounts/src/store.js b/core/core/accounts/src/store.js new file mode 100644 index 0000000..fd42a22 --- /dev/null +++ b/core/core/accounts/src/store.js @@ -0,0 +1,1302 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const { installationFailure } = require('../../../shared/contracts/src'); +const { runtimeBackend } = require('../../runtime-deployment'); +const { AccessError } = require('./validation'); + +const { SCHEMA_VERSION, LEGACY_INSTALLATION_COLUMNS, CRITICAL_SCHEMA_COLUMNS, + CRITICAL_SCHEMA_INDEXES, CRITICAL_SCHEMA_TRIGGERS, requireColumns, requireIndex, requireTrigger, + initializeAccessSchema } = require('./schema'); +const MAX_DATABASE_BYTES = 64 * 1024 * 1024; + +function mode(info) { return info.mode & 0o777; } +function fail(code = 'unsafe_access_storage') { throw new AccessError(code, 500); } +function ensurePrivateDirectory(directory) { + const selected = path.resolve(directory); + if (selected !== directory) fail(); + const parent = path.dirname(selected); + let parentInfo; + try { parentInfo = fs.lstatSync(parent); } catch { fail(); } + if (!parentInfo.isDirectory() || parentInfo.isSymbolicLink() || parentInfo.uid !== process.geteuid() + || (mode(parentInfo) & 0o022) !== 0 || fs.realpathSync(parent) !== parent) fail(); + try { fs.mkdirSync(selected, { mode: 0o700 }); } catch (error) { if (error?.code !== 'EEXIST') throw error; } + const info = fs.lstatSync(selected); + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== process.geteuid() + || mode(info) !== 0o700 || fs.realpathSync(selected) !== selected) fail(); +} + +function safeDatabase(file) { + const info = fs.lstatSync(file); + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() || info.nlink !== 1 + || mode(info) !== 0o600 || info.size < 1 || info.size > MAX_DATABASE_BYTES + || fs.realpathSync(file) !== file) fail(); +} + +function roleView(row, permissions = []) { + if (!row) return null; + return { + id: row.id, + organizationId: row.organization_id, + key: row.key, + name: row.name, + description: row.description, + system: Boolean(row.is_system), + permissions, + createdAt: new Date(row.created_at).toISOString(), + updatedAt: new Date(row.updated_at).toISOString(), + }; +} + +function organizationView(row, stations = []) { + if (!row) return null; + return { + id: row.id, + name: row.name, + abbreviation: row.abbreviation, + timezone: row.timezone, + status: row.status, + stations, + installation: row.installation_status ? { + status: row.installation_status, + } : null, + createdAt: new Date(row.created_at).toISOString(), + updatedAt: new Date(row.updated_at).toISOString(), + }; +} + +function userView(row) { + if (!row) return null; + return { + id: row.id, + email: row.email, + firstName: row.first_name, + lastName: row.last_name, + name: `${row.first_name} ${row.last_name}`, + status: row.status, + platformRole: row.platform_role, + createdAt: new Date(row.created_at).toISOString(), + }; +} + +function invitationView(row) { + if (!row) return null; + return { + id: row.id, + kind: row.kind, + organizationId: row.organization_id, + organizationName: row.organization_name || null, + roleId: row.role_id, + roleName: row.role_name || null, + email: row.email, + status: row.status, + expiresAt: new Date(row.expires_at).toISOString(), + createdAt: new Date(row.created_at).toISOString(), + acceptedAt: row.accepted_at === null ? null : new Date(row.accepted_at).toISOString(), + }; +} + +class AccessStore { + constructor(paths, { readOnly = false } = {}) { + if (!paths || path.resolve(paths.databaseRoot) !== paths.databaseRoot + || path.resolve(paths.database) !== paths.database + || path.dirname(paths.database) !== paths.databaseRoot) fail(); + this.paths = paths; + this.db = null; + if (readOnly && !fs.existsSync(paths.databaseRoot)) throw new AccessError('access_control_not_initialized', 503); + ensurePrivateDirectory(paths.databaseRoot); + const existing = fs.existsSync(paths.database); + if (readOnly && !existing) throw new AccessError('access_control_not_initialized', 503); + if (existing) safeDatabase(paths.database); + try { + this.db = new DatabaseSync(paths.database, { readOnly }); + if (!existing) fs.chmodSync(paths.database, 0o600); + if (existing) safeDatabase(paths.database); + this.db.exec(readOnly + ? 'PRAGMA query_only=ON; PRAGMA foreign_keys=ON; PRAGMA trusted_schema=OFF; PRAGMA busy_timeout=3000;' + : 'PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON; PRAGMA trusted_schema=OFF; PRAGMA busy_timeout=3000;'); + const initialVersion = this.db.prepare('PRAGMA user_version').get().user_version; + if (!Number.isSafeInteger(initialVersion) || initialVersion > SCHEMA_VERSION) fail('access_schema_incompatible'); + if (initialVersion === 2) requireColumns(this.db, 'installations', LEGACY_INSTALLATION_COLUMNS); + if (!readOnly) this.initialize(initialVersion); + safeDatabase(paths.database); + const version = this.db.prepare('PRAGMA user_version').get().user_version; + if (version !== SCHEMA_VERSION) fail('access_schema_incompatible'); + for (const [table, columns] of Object.entries(CRITICAL_SCHEMA_COLUMNS)) { + requireColumns(this.db, table, columns); + } + for (const [name, expected] of Object.entries(CRITICAL_SCHEMA_INDEXES)) { + requireIndex(this.db, name, expected); + } + for (const [name, expected] of Object.entries(CRITICAL_SCHEMA_TRIGGERS)) { + requireTrigger(this.db, name, expected); + } + const integrity = this.db.prepare('PRAGMA quick_check(1)').all(); + if (integrity.length !== 1 || integrity[0].quick_check !== 'ok' + || this.db.prepare('PRAGMA foreign_key_check').all().length !== 0) fail('access_schema_incompatible'); + } catch (error) { + try { this.db?.close(); } catch {} + this.db = null; + if (error instanceof AccessError) throw error; + fail('access_storage_unavailable'); + } + } + + initialize(initialVersion) { initializeAccessSchema(this.db, initialVersion); } + + close() { if (this.db) this.db.close(); this.db = null; } + + afterCommit(callback) { + if (this.transactionDepth) (this.commitCallbacks ||= []).push(callback); + else { try { callback(); } catch {} } + } + + transaction(callback) { + const depth = this.transactionDepth || 0; + const savepoint = `dispatch_nested_${depth}`; + const callbackOffset = (this.commitCallbacks ||= []).length; + this.db.exec(depth ? `SAVEPOINT ${savepoint}` : 'BEGIN IMMEDIATE'); + this.transactionDepth = depth + 1; + try { + const result = callback(); + this.db.exec(depth ? `RELEASE SAVEPOINT ${savepoint}` : 'COMMIT'); + if (!depth) { + this.transactionDepth = 0; + const callbacks = this.commitCallbacks.splice(0); + for (const callback of callbacks) { try { callback(); } catch {} } + } + return result; + } catch (error) { + this.commitCallbacks.splice(callbackOffset); + try { + this.db.exec(depth ? `ROLLBACK TO SAVEPOINT ${savepoint}; RELEASE SAVEPOINT ${savepoint}` : 'ROLLBACK'); + } catch {} + throw error; + } finally { this.transactionDepth = depth; } + } + + platformOwnerCount() { + return this.db.prepare("SELECT count(*) AS count FROM users WHERE status='active' AND platform_role='owner'").get().count; + } + + userByEmail(email) { return this.db.prepare('SELECT * FROM users WHERE email=?').get(email) || null; } + userById(id) { return this.db.prepare('SELECT * FROM users WHERE id=?').get(id) || null; } + + insertUser(user) { + this.db.prepare(`INSERT INTO users(id,email,first_name,last_name,password_hash,status,platform_role,auth_version,created_at,updated_at) + VALUES(?,?,?,?,?,'active',?,1,?,?)`).run( + user.id, user.email, user.firstName, user.lastName, user.passwordHash, user.platformRole, user.timestamp, user.timestamp, + ); + return this.userById(user.id); + } + + setPlatformRole(id, role, timestamp) { + this.db.prepare('UPDATE users SET platform_role=?,updated_at=? WHERE id=?').run(role, timestamp, id); + } + + updatePassword(id, passwordHash, timestamp) { + this.db.prepare('UPDATE users SET password_hash=?,auth_version=auth_version+1,updated_at=? WHERE id=?') + .run(passwordHash, timestamp, id); + } + + deleteUserSessions(userId) { this.db.prepare('DELETE FROM sessions WHERE user_id=?').run(userId); } + + createOrganization(organization) { + this.db.prepare(`INSERT INTO organizations(id,name,abbreviation,timezone,status,created_by,created_at,updated_at) + VALUES(?,?,?,?,?,?,?,?)`).run( + organization.id, organization.name, organization.abbreviation, organization.timezone, organization.status, + organization.createdBy, organization.timestamp, organization.timestamp, + ); + } + + updateOrganizationStatus(id, status, timestamp) { + this.db.prepare('UPDATE organizations SET status=?,updated_at=? WHERE id=?').run(status, timestamp, id); + if (status === 'suspended') { + this.db.prepare(`DELETE FROM sessions WHERE user_id IN (SELECT user_id FROM memberships WHERE organization_id=?) + AND user_id IN (SELECT id FROM users WHERE platform_role IS NULL) + AND NOT EXISTS (SELECT 1 FROM memberships m JOIN organizations o ON o.id=m.organization_id + WHERE m.user_id=sessions.user_id AND m.status='active' AND o.status<>'suspended')`).run(id); + this.db.prepare('UPDATE sessions SET active_organization_id=NULL WHERE active_organization_id=?').run(id); + } + } + + organization(id) { + const row = this.db.prepare(`SELECT o.*,i.runtime_key,i.status AS installation_status + FROM organizations o LEFT JOIN installations i ON i.organization_id=o.id WHERE o.id=?`).get(id); + if (!row) return null; + const stations = this.db.prepare('SELECT code,is_primary FROM stations WHERE organization_id=? ORDER BY is_primary DESC,code').all(id) + .map(station => ({ code: station.code, primary: Boolean(station.is_primary) })); + return organizationView(row, stations); + } + + organizations() { + return this.db.prepare(`SELECT o.*,i.runtime_key,i.status AS installation_status + FROM organizations o LEFT JOIN installations i ON i.organization_id=o.id ORDER BY lower(o.name),o.id`).all() + .map(row => { + const result = organizationView(row, this.db.prepare('SELECT code,is_primary FROM stations WHERE organization_id=? ORDER BY is_primary DESC,code').all(row.id) + .map(station => ({ code: station.code, primary: Boolean(station.is_primary) }))); + result.memberCount = this.db.prepare("SELECT count(*) AS count FROM memberships WHERE organization_id=? AND status='active'").get(row.id).count; + return result; + }); + } + + insertStation(organizationId, code, primary, timestamp) { + this.db.prepare('INSERT INTO stations(organization_id,code,is_primary,created_at) VALUES(?,?,?,?)') + .run(organizationId, code, primary ? 1 : 0, timestamp); + } + + createInstallation( + organizationId, runtimeKey, status, timestamp, releaseId = 'dispatch_current_1', + backend = 'systemd_user', + ) { + const selectedBackend = runtimeBackend(backend); + if (selectedBackend === 'directory_service_v1') require('../../../shared/paths/platform-paths').validateDspId(runtimeKey); + this.db.prepare(`INSERT INTO installations( + organization_id,runtime_key,status,revision,manifest_revision,release_id,backend,current_job_id,created_at,updated_at + ) VALUES(?,?,?,1,1,?,?,NULL,?,?)`).run( + organizationId, runtimeKey, status, releaseId, selectedBackend, timestamp, timestamp, + ); + } + + installation(organizationId) { + const row = this.db.prepare('SELECT * FROM installations WHERE organization_id=?').get(organizationId); + return row ? { organizationId: row.organization_id, runtimeKey: row.runtime_key, status: row.status } : null; + } + + installationControl(organizationId) { + const row = this.db.prepare('SELECT * FROM installations WHERE organization_id=?').get(organizationId); + return row ? { + organizationId: row.organization_id, + runtimeKey: row.runtime_key, + status: row.status, + revision: row.revision, + manifestRevision: row.manifest_revision, + releaseId: row.release_id, + currentJobId: row.current_job_id, + } : null; + } + + installationBackend(organizationId) { + const row = this.db.prepare('SELECT backend FROM installations WHERE organization_id=?').get(organizationId); + return row ? runtimeBackend(row.backend) : null; + } + + runtimeAgentAuthority(runtimeKey) { + return this.db.prepare(`SELECT a.*,i.status AS installation_status,o.status AS organization_status + FROM runtime_agent_authorities a + JOIN installations i ON i.organization_id=a.organization_id AND i.runtime_key=a.runtime_key + JOIN organizations o ON o.id=a.organization_id + WHERE a.runtime_key=?`).get(runtimeKey) || null; + } + + activeRuntimeAgentAuthority(runtimeKey) { + const row = this.runtimeAgentAuthority(runtimeKey); + if (!row || row.status !== 'active' + || row.organization_status === 'suspended' && !this.activeLifecycleJob(row.organization_id) + || !['provisioning', 'waiting_for_owner', 'waiting_for_provider_auth', 'verifying', 'ready', 'suspended', 'decommissioning'] + .includes(row.installation_status)) return null; + return { + organizationId: row.organization_id, + runtimeKey: row.runtime_key, + tokenHash: row.token_hash, + generation: row.generation, + }; + } + + activeRuntimeAgentAuthorityCount() { + return this.db.prepare(`SELECT count(*) AS count FROM runtime_agent_authorities a + JOIN installations i ON i.organization_id=a.organization_id AND i.runtime_key=a.runtime_key + JOIN organizations o ON o.id=a.organization_id + WHERE a.status='active' AND (o.status<>'suspended' OR EXISTS ( + SELECT 1 FROM installation_lifecycle_jobs j WHERE j.organization_id=i.organization_id AND j.status IN ('queued','running') + )) + AND i.status IN ('provisioning','waiting_for_owner','waiting_for_provider_auth','verifying','ready','suspended','decommissioning')`).get().count; + } + + recordRuntimeAgentAuthority({ organizationId, runtimeKey, tokenHash, timestamp }) { + if (!/^[a-f0-9]{64}$/.test(tokenHash) || !Number.isSafeInteger(timestamp) || timestamp < 0) { + throw new AccessError('runtime_boundary_violation', 500); + } + const control = this.installationControl(organizationId); + if (!control || control.runtimeKey !== runtimeKey || runtimeKey === 'local') { + throw new AccessError('runtime_identity_mismatch', 409); + } + const prior = this.db.prepare('SELECT * FROM runtime_agent_authorities WHERE organization_id=?') + .get(organizationId); + if (prior) { + if (prior.runtime_key !== runtimeKey) throw new AccessError('runtime_identity_mismatch', 409); + if (prior.token_hash !== tokenHash || prior.status !== 'active') { + throw new AccessError('runtime_agent_unauthorized', 409); + } + return { + organizationId: prior.organization_id, + runtimeKey: prior.runtime_key, + tokenHash: prior.token_hash, + generation: prior.generation, + status: prior.status, + changed: false, + }; + } + this.db.prepare(`INSERT INTO runtime_agent_authorities( + organization_id,runtime_key,token_hash,generation,status,created_at,updated_at,revoked_at + ) VALUES(?,?,?,1,'active',?,?,NULL)`).run( + organizationId, runtimeKey, tokenHash, timestamp, timestamp, + ); + return { + organizationId, + runtimeKey, + tokenHash, + generation: 1, + status: 'active', + changed: true, + }; + } + + replaceRuntimeAgentAuthority({ + organizationId, runtimeKey, tokenHash, expectedGeneration, expectedStatus, timestamp, + }) { + if (!/^[a-f0-9]{64}$/.test(tokenHash) + || !Number.isSafeInteger(expectedGeneration) || expectedGeneration < 1 + || !['active', 'revoked'].includes(expectedStatus) + || !Number.isSafeInteger(timestamp) || timestamp < 0) { + throw new AccessError('runtime_boundary_violation', 500); + } + const control = this.installationControl(organizationId); + if (!control || control.runtimeKey !== runtimeKey || runtimeKey === 'local') { + throw new AccessError('runtime_identity_mismatch', 409); + } + const changed = this.db.prepare(`UPDATE runtime_agent_authorities + SET token_hash=?,generation=generation+1,status='active',updated_at=?,revoked_at=NULL + WHERE organization_id=? AND runtime_key=? AND generation=? AND status=?`).run( + tokenHash, timestamp, organizationId, runtimeKey, expectedGeneration, expectedStatus, + ).changes; + if (changed !== 1) throw new AccessError('runtime_agent_authority_conflict', 409); + const current = this.runtimeAgentAuthority(runtimeKey); + if (!current || current.organization_id !== organizationId || current.token_hash !== tokenHash + || current.generation !== expectedGeneration + 1 || current.status !== 'active') { + throw new AccessError('runtime_boundary_violation', 500); + } + return { + organizationId, + runtimeKey, + tokenHash, + generation: current.generation, + status: current.status, + changed: true, + }; + } + + revokeRuntimeAgentAuthority({ organizationId, runtimeKey, expectedGeneration, timestamp }) { + const control = this.installationControl(organizationId); + if (!control || control.runtimeKey !== runtimeKey || !Number.isSafeInteger(expectedGeneration) + || expectedGeneration < 1 || !Number.isSafeInteger(timestamp) || timestamp < 0) { + throw new AccessError('runtime_identity_mismatch', 409); + } + const changed = this.db.prepare(`UPDATE runtime_agent_authorities + SET generation=generation+1,status='revoked',updated_at=?,revoked_at=? + WHERE organization_id=? AND runtime_key=? AND generation=? AND status='active'`).run( + timestamp, timestamp, organizationId, runtimeKey, expectedGeneration, + ).changes; + if (changed !== 1) throw new AccessError('runtime_agent_authority_conflict', 409); + return true; + } + + installationSetup(organizationId) { + const row = this.db.prepare(`SELECT setup_worker_id,setup_fence,setup_lease_expires_at + FROM installations WHERE organization_id=?`).get(organizationId); + return row ? { + workerId: row.setup_worker_id, + fence: row.setup_fence, + leaseExpiresAt: row.setup_lease_expires_at, + } : null; + } + + updateInstallationControl({ + organizationId, expectedStatus, expectedRevision, status, revision, currentJobId, timestamp, + manifestRevision, releaseId, + }) { + const clearSetup = status !== 'waiting_for_provider_auth'; + const changed = this.db.prepare(`UPDATE installations SET status=?,revision=?,current_job_id=?, + manifest_revision=COALESCE(?,manifest_revision),release_id=COALESCE(?,release_id), + setup_worker_id=CASE WHEN ? THEN NULL ELSE setup_worker_id END, + setup_fence=setup_fence, + setup_lease_expires_at=CASE WHEN ? THEN NULL ELSE setup_lease_expires_at END,updated_at=? + WHERE organization_id=? AND status=? AND revision=?`).run( + status, revision, currentJobId, manifestRevision ?? null, releaseId ?? null, clearSetup ? 1 : 0, + clearSetup ? 1 : 0, timestamp, organizationId, expectedStatus, expectedRevision, + ).changes; + if (changed !== 1) throw new AccessError('installation_revision_conflict', 409); + return this.installationControl(organizationId); + } + + claimInstallationSetup(organizationId, workerId, expectedFence, leaseExpiresAt, timestamp) { + const changed = this.db.prepare(`UPDATE installations + SET setup_worker_id=?,setup_fence=setup_fence+1,setup_lease_expires_at=?,updated_at=? + WHERE organization_id=? AND status='waiting_for_provider_auth' AND current_job_id IS NULL + AND setup_fence=? AND (setup_worker_id IS NULL OR setup_lease_expires_at<=?)`).run( + workerId, leaseExpiresAt, timestamp, organizationId, expectedFence, timestamp, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + return this.installationControl(organizationId); + } + + renewInstallationSetup(organizationId, workerId, fence, leaseExpiresAt, timestamp) { + const changed = this.db.prepare(`UPDATE installations SET setup_lease_expires_at=?,updated_at=? + WHERE organization_id=? AND status='waiting_for_provider_auth' AND current_job_id IS NULL + AND setup_worker_id=? AND setup_fence=? AND setup_lease_expires_at>?`).run( + leaseExpiresAt, timestamp, organizationId, workerId, fence, timestamp, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + return this.installationControl(organizationId); + } + + releaseInstallationSetup(organizationId, workerId, fence, timestamp) { + const changed = this.db.prepare(`UPDATE installations + SET setup_worker_id=NULL,setup_lease_expires_at=NULL,updated_at=? + WHERE organization_id=? AND status='waiting_for_provider_auth' AND current_job_id IS NULL + AND setup_worker_id=? AND setup_fence=? AND setup_lease_expires_at>?`).run( + timestamp, organizationId, workerId, fence, timestamp, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + return this.installationControl(organizationId); + } + + provisioningRequest(id) { + return this.db.prepare('SELECT * FROM installation_provisioning_requests WHERE id=?').get(id) || null; + } + + provisioningRequestByKey(organizationId, authorityScope, idempotencyKey) { + return this.db.prepare(`SELECT * FROM installation_provisioning_requests + WHERE organization_id=? AND authority_scope=? AND idempotency_key=?`).get( + organizationId, authorityScope, idempotencyKey, + ) || null; + } + + pendingProvisioningRequests(limit = 20, backends = null) { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 50) throw new AccessError('invalid_input', 400); + if (backends !== null && (!Array.isArray(backends) || backends.length < 1)) throw new AccessError('invalid_input', 400); + backends?.forEach(runtimeBackend); + return this.db.prepare(`SELECT r.* FROM installation_provisioning_requests r + JOIN installations i ON i.organization_id=r.organization_id + WHERE r.status IN ('pending','dispatched') + ${backends === null ? '' : `AND i.backend IN (${backends.map(() => '?').join(',')})`} + ORDER BY r.created_at,r.id LIMIT ?`).all(...(backends || []), limit); + } + + latestProvisioningRequest(organizationId) { + return this.db.prepare(`SELECT * FROM installation_provisioning_requests + WHERE organization_id=? ORDER BY updated_at DESC,rowid DESC LIMIT 1`).get(organizationId) || null; + } + + createProvisioningRequest(request) { + const requestJson = JSON.stringify(request.operation); + const prior = this.provisioningRequestByKey( + request.organizationId, request.authorityScope, request.operation.idempotencyKey, + ); + if (prior) { + if (prior.request_json !== requestJson) throw new AccessError('idempotency_conflict', 409); + return { row: prior, replayed: true }; + } + const control = this.installationControl(request.organizationId); + const startingState = request.operation.operation === 'provision' ? 'pending' + : request.operation.operation === 'retry' ? 'failed' : null; + if (!control || startingState === null || control.status !== startingState + || control.revision !== request.operation.expectedRevision || control.runtimeKey === 'local' + || startingState === 'pending' && control.currentJobId !== null) { + throw new AccessError(control?.revision !== request.operation.expectedRevision + ? 'installation_revision_conflict' : 'installation_operation_not_allowed', 409); + } + if (startingState === 'failed') { + const failed = this.db.prepare(`SELECT id,failure_code FROM installation_provisioning_requests + WHERE organization_id=? AND provisioner_job_id=? AND status='failed'`).get( + request.organizationId, control.currentJobId, + ); + const failure = failed?.failure_code === null || failed === undefined + ? null : installationFailure(failed.failure_code); + if (!failed || !failure?.recoverable || failure.category !== 'infrastructure') { + throw new AccessError('installation_operation_not_allowed', 409); + } + } + const active = this.db.prepare(`SELECT id FROM installation_provisioning_requests + WHERE organization_id=? AND status IN ('pending','dispatched')`).get(request.organizationId); + if (active) throw new AccessError('installation_operation_in_progress', 409); + const nextRevision = control.revision + 1; + this.updateInstallationControl({ + organizationId: request.organizationId, + expectedStatus: startingState, + expectedRevision: control.revision, + status: 'provisioning', + revision: nextRevision, + currentJobId: null, + timestamp: request.timestamp, + }); + this.db.prepare(`INSERT INTO installation_provisioning_requests( + id,organization_id,authority_scope,idempotency_key,request_json,starting_state,installation_revision, + manifest_revision,runtime_key,status,provisioner_job_id,failure_code,created_at,updated_at,finished_at + ) VALUES(?,?,?,?,?,?,?, ?,?,'pending',NULL,NULL,?,?,NULL)`).run( + request.id, + request.organizationId, + request.authorityScope, + request.operation.idempotencyKey, + requestJson, + startingState, + nextRevision, + control.manifestRevision, + control.runtimeKey, + request.timestamp, + request.timestamp, + ); + require('./worker-wakeup').afterCommit(this, ['reconcile']); + return { row: this.provisioningRequest(request.id), replayed: false }; + } + + acknowledgeProvisioningRequest(id, job, timestamp) { + const request = this.provisioningRequest(id); + if (!request) throw new AccessError('installation_operation_not_found', 404); + if (request.status === 'dispatched') { + if (request.provisioner_job_id !== job.id) throw new AccessError('idempotency_conflict', 409); + return request; + } + const control = this.installationControl(request.organization_id); + const operation = JSON.parse(request.request_json); + if (request.status !== 'pending' || !control || control.status !== 'provisioning' + || control.revision !== request.installation_revision || control.currentJobId !== null + || control.runtimeKey !== request.runtime_key || control.manifestRevision !== request.manifest_revision + || job.operation !== operation.operation || job.status !== 'queued' + || job.installationState !== 'provisioning' || job.revision !== request.installation_revision) { + throw new AccessError('installation_operation_in_progress', 409); + } + const changed = this.db.prepare(`UPDATE installation_provisioning_requests + SET status='dispatched',provisioner_job_id=?,updated_at=? WHERE id=? AND status='pending'`).run( + job.id, timestamp, id, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + this.db.prepare(`UPDATE installations SET current_job_id=?,updated_at=? + WHERE organization_id=? AND status='provisioning' AND revision=? AND current_job_id IS NULL`).run( + job.id, timestamp, request.organization_id, request.installation_revision, + ); + if (this.db.prepare('SELECT changes() AS count').get().count !== 1) { + throw new AccessError('installation_revision_conflict', 409); + } + return this.provisioningRequest(id); + } + + finishProvisioningRequest(id, job, timestamp) { + const request = this.provisioningRequest(id); + if (!request) throw new AccessError('installation_operation_not_found', 404); + if (request.status === 'completed' || request.status === 'failed') return request; + const control = this.installationControl(request.organization_id); + const operation = JSON.parse(request.request_json); + if (request.status !== 'dispatched' || request.provisioner_job_id !== job.id || !control + || control.status !== 'provisioning' || control.revision !== request.installation_revision + || control.currentJobId !== job.id || control.runtimeKey !== request.runtime_key + || control.manifestRevision !== request.manifest_revision || job.operation !== operation.operation) { + throw new AccessError('installation_operation_in_progress', 409); + } + let destination; + let revision; + let requestStatus; + let failureCode = null; + let currentJobId = null; + if (job.status === 'succeeded' && job.installationState === 'provisioning' + && job.revision === control.revision) { + destination = this.activeOwnerCount(request.organization_id) > 0 + ? 'waiting_for_provider_auth' : 'waiting_for_owner'; + revision = control.revision + 1; + requestStatus = 'completed'; + } else if (job.status === 'failed' && job.installationState === 'failed' + && job.revision === control.revision + 1 && job.failure?.code) { + destination = 'failed'; + revision = job.revision; + requestStatus = 'failed'; + failureCode = job.failure.code; + currentJobId = job.id; + } else { + throw new AccessError('installation_operation_in_progress', 409); + } + this.updateInstallationControl({ + organizationId: request.organization_id, + expectedStatus: 'provisioning', + expectedRevision: control.revision, + status: destination, + revision, + currentJobId, + timestamp, + }); + const changed = this.db.prepare(`UPDATE installation_provisioning_requests + SET status=?,failure_code=?,updated_at=?,finished_at=? WHERE id=? AND status='dispatched'`).run( + requestStatus, failureCode, timestamp, timestamp, id, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + return this.provisioningRequest(id); + } + + activationJob(id) { + return this.db.prepare('SELECT * FROM installation_activation_jobs WHERE id=?').get(id) || null; + } + + activationJobByRequest(organizationId, authorityScope, idempotencyKey) { + return this.db.prepare(`SELECT * FROM installation_activation_jobs + WHERE organization_id=? AND authority_scope=? AND idempotency_key=?`).get( + organizationId, authorityScope, idempotencyKey, + ) || null; + } + + runningActivationJob(organizationId) { + return this.db.prepare("SELECT * FROM installation_activation_jobs WHERE organization_id=? AND status='running'") + .get(organizationId) || null; + } + + latestActivationJob(organizationId) { + return this.db.prepare(`SELECT * FROM installation_activation_jobs + WHERE organization_id=? ORDER BY updated_at DESC,rowid DESC LIMIT 1`).get(organizationId) || null; + } + + createActivationJob(job) { + this.db.prepare(`INSERT INTO installation_activation_jobs( + id,organization_id,operation,status,installation_state,installation_revision,manifest_revision, + runtime_key,authority_scope,idempotency_key,worker_id,fence,lease_expires_at,provider,profile_id, + provider_tested_at,evidence_json,evidence_digest,failure_code,created_at,started_at,finished_at,updated_at + ) VALUES(?,?,'resume','running','verifying',?,?,?,?,?,?,1,?,'paycom','paycom-main',?,NULL,NULL,NULL,?,?,NULL,?)`).run( + job.id, + job.organizationId, + job.installationRevision, + job.manifestRevision, + job.runtimeKey, + job.authorityScope, + job.idempotencyKey, + job.workerId, + job.leaseExpiresAt, + job.providerTestedAt, + job.timestamp, + job.timestamp, + job.timestamp, + ); + return this.activationJob(job.id); + } + + claimActivationJob(id, workerId, expectedFence, leaseExpiresAt, timestamp) { + const changed = this.db.prepare(`UPDATE installation_activation_jobs + SET worker_id=?,fence=fence+1,lease_expires_at=?,updated_at=? + WHERE id=? AND status='running' AND fence=? AND lease_expires_at<=?`).run( + workerId, leaseExpiresAt, timestamp, id, expectedFence, timestamp, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + return this.activationJob(id); + } + + renewActivationJob(id, workerId, fence, leaseExpiresAt, timestamp) { + const changed = this.db.prepare(`UPDATE installation_activation_jobs SET lease_expires_at=?,updated_at=? + WHERE id=? AND status='running' AND worker_id=? AND fence=? AND lease_expires_at>?`).run( + leaseExpiresAt, timestamp, id, workerId, fence, timestamp, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + return this.activationJob(id); + } + + finishActivationJob(id, workerId, fence, status, installationState, revision, failureCode, evidence, timestamp) { + const evidenceJson = evidence === null ? null : JSON.stringify(evidence); + const evidenceDigest = evidence === null ? null : evidence.evidenceDigest; + if ((status === 'succeeded') !== (evidenceJson !== null) + || (status === 'failed') !== (failureCode !== null)) { + throw new AccessError('installation_operation_failed', 500); + } + const changed = this.db.prepare(`UPDATE installation_activation_jobs + SET status=?,installation_state=?,installation_revision=?,lease_expires_at=NULL, + failure_code=?,evidence_json=?,evidence_digest=?, + finished_at=?,updated_at=? + WHERE id=? AND status='running' AND worker_id=? AND fence=? AND lease_expires_at>?`).run( + status, installationState, revision, failureCode, evidenceJson, evidenceDigest, timestamp, timestamp, + id, workerId, fence, timestamp, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + return this.activationJob(id); + } + + lifecycleJob(id) { + return this.db.prepare('SELECT * FROM installation_lifecycle_jobs WHERE id=?').get(id) || null; + } + + lifecycleJobByRequest(organizationId, authorityScope, idempotencyKey) { + return this.db.prepare(`SELECT * FROM installation_lifecycle_jobs + WHERE organization_id=? AND authority_scope=? AND idempotency_key=?`).get( + organizationId, authorityScope, idempotencyKey, + ) || null; + } + + activeLifecycleJob(organizationId) { + return this.db.prepare(`SELECT * FROM installation_lifecycle_jobs + WHERE organization_id=? AND status IN ('queued','running')`).get(organizationId) || null; + } + + lifecycleExecutionCandidates(timestamp, limit) { + return this.db.prepare(`SELECT id,organization_id,authority_scope,operation + FROM installation_lifecycle_jobs + WHERE attempt=max_attempts AND lease_expires_at<=? + ORDER BY updated_at,id LIMIT ?`).all(timestamp, limit); + } + + lifecycleOutstandingCount() { + return this.db.prepare(`SELECT count(*) AS count FROM installation_lifecycle_jobs + WHERE status IN ('queued','running')`).get().count; + } + + statusLifecycleMismatches(limit) { + return this.db.prepare(`SELECT o.id AS organization_id,o.status AS organization_status, + i.status AS installation_status,i.revision AS installation_revision + FROM organizations o JOIN installations i ON i.organization_id=o.id + WHERE i.runtime_key!='local' + AND NOT EXISTS (SELECT 1 FROM dsp_removals d WHERE d.organization_id=o.id) + AND ((o.status='suspended' AND i.status='ready') + OR (o.status='active' AND i.status='suspended')) + AND NOT EXISTS ( + SELECT 1 FROM installation_lifecycle_jobs j + WHERE j.organization_id=o.id AND j.status IN ('queued','running') + ) + AND NOT EXISTS ( + SELECT 1 FROM installation_lifecycle_jobs f + WHERE f.organization_id=o.id AND f.status='failed' AND f.updated_at>=o.updated_at + AND ((o.status='suspended' AND f.operation='suspend') + OR (o.status='active' AND f.operation='resume')) + ) + ORDER BY o.updated_at,o.id LIMIT ?`).all(limit); + } + + createLifecycleJob(job) { + this.db.prepare(`INSERT INTO installation_lifecycle_jobs( + id,organization_id,operation,status,starting_state,installation_state,installation_revision, + manifest_revision,runtime_key,release_id,target_release_id,backup_id,safety_backup_id, + authority_scope,idempotency_key,stages_json,next_stage,stage_receipts_json,worker_id,fence, + lease_expires_at,failure_code,result_json,created_at,started_at,finished_at,updated_at + ) VALUES(?,?,?,'queued',?,?,?,?,?,?,?,?,?,?,?, ?,0,'{}',NULL,0,NULL,NULL,NULL,?,NULL,NULL,?)`).run( + job.id, job.organizationId, job.operation, job.startingState, job.installationState, + job.installationRevision, job.manifestRevision, job.runtimeKey, job.releaseId, + job.targetReleaseId, job.backupId, job.safetyBackupId, job.authorityScope, + job.idempotencyKey, JSON.stringify(job.stages), job.timestamp, job.timestamp, + ); + require('./worker-wakeup').afterCommit(this, ['reconcile']); + return this.lifecycleJob(job.id); + } + + claimLifecycleJob(id, workerId, leaseExpiresAt, timestamp) { + const row = this.lifecycleJob(id); + if (!row || !['queued', 'running'].includes(row.status)) { + throw new AccessError(row ? 'installation_operation_not_allowed' : 'installation_operation_not_found', 409); + } + if (row.status === 'running' && row.worker_id === workerId && row.lease_expires_at > timestamp) return row; + if (row.attempt >= row.max_attempts) throw new AccessError('installation_operation_failed', 409); + const changed = row.status === 'queued' + ? this.db.prepare(`UPDATE installation_lifecycle_jobs SET status='running',worker_id=?,attempt=attempt+1, + fence=fence+1,lease_expires_at=?,started_at=COALESCE(started_at,?),updated_at=? + WHERE id=? AND status='queued' AND attempt?`).run( + leaseExpiresAt, timestamp, id, workerId, fence, timestamp, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + return this.lifecycleJob(id); + } + + completeLifecycleStage(id, workerId, fence, stage, receipt, timestamp) { + const { row } = this.lifecycleClaim(id, workerId, fence, timestamp); + let stages; + let receipts; + try { + stages = JSON.parse(row.stages_json); + receipts = JSON.parse(row.stage_receipts_json); + } catch { throw new AccessError('installation_operation_failed', 500); } + if (!Array.isArray(stages) || stages[row.next_stage] !== stage + || !receipts || typeof receipts !== 'object' || Array.isArray(receipts)) { + throw new AccessError('installation_operation_in_progress', 409); + } + receipts[stage] = receipt; + const changed = this.db.prepare(`UPDATE installation_lifecycle_jobs + SET next_stage=next_stage+1,stage_receipts_json=?,updated_at=? + WHERE id=? AND status='running' AND worker_id=? AND fence=? AND next_stage=? AND lease_expires_at>?`).run( + JSON.stringify(receipts), timestamp, id, workerId, fence, row.next_stage, timestamp, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + return this.lifecycleJob(id); + } + + finishLifecycleJob(id, workerId, fence, destination, result, timestamp, options = {}) { + const { row, control } = this.lifecycleClaim(id, workerId, fence, timestamp); + let stages; + try { stages = JSON.parse(row.stages_json); } catch { throw new AccessError('installation_operation_failed', 500); } + if (!Array.isArray(stages) || row.next_stage !== stages.length) { + throw new AccessError('installation_operation_in_progress', 409); + } + const nextRevision = control.revision + 1; + const nextManifestRevision = options.manifestRevision ?? control.manifestRevision; + const nextReleaseId = options.releaseId ?? control.releaseId; + const changed = this.db.prepare(`UPDATE installation_lifecycle_jobs SET status='succeeded', + installation_state=?,installation_revision=?,lease_expires_at=NULL,result_json=?,finished_at=?,updated_at=? + WHERE id=? AND status='running' AND worker_id=? AND fence=? AND lease_expires_at>?`).run( + destination, nextRevision, JSON.stringify(result), timestamp, timestamp, + id, workerId, fence, timestamp, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + const installed = this.db.prepare(`UPDATE installations SET status=?,revision=?,manifest_revision=?,release_id=?, + current_job_id=?,updated_at=? WHERE organization_id=? AND status=? AND revision=? AND current_job_id=?`).run( + destination, nextRevision, nextManifestRevision, nextReleaseId, ['decommission', 'destroy'].includes(row.operation) ? row.id : null, timestamp, + row.organization_id, control.status, control.revision, row.id, + ).changes; + if (installed !== 1) throw new AccessError('installation_revision_conflict', 409); + return this.lifecycleJob(id); + } + + failLifecycleJob(id, workerId, fence, destination, failureCode, timestamp) { + const { row, control } = this.lifecycleClaim(id, workerId, fence, timestamp); + const nextRevision = control.revision + 1; + const changed = this.db.prepare(`UPDATE installation_lifecycle_jobs SET status='failed', + installation_state=?,installation_revision=?,lease_expires_at=NULL,failure_code=?,finished_at=?,updated_at=? + WHERE id=? AND status='running' AND worker_id=? AND fence=? AND lease_expires_at>?`).run( + destination, nextRevision, failureCode, timestamp, timestamp, + id, workerId, fence, timestamp, + ).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + const installed = this.db.prepare(`UPDATE installations SET status=?,revision=?,current_job_id=?,updated_at=? + WHERE organization_id=? AND status=? AND revision=? AND current_job_id=?`).run( + destination, nextRevision, destination === 'failed' || row.operation === 'resume' && this.db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(row.organization_id) ? row.id : null, timestamp, + row.organization_id, control.status, control.revision, row.id, + ).changes; + if (installed !== 1) throw new AccessError('installation_revision_conflict', 409); + return this.lifecycleJob(id); + } + + reopenExhaustedLifecycleJob(id, timestamp) { + const row = this.lifecycleJob(id); + const control = row ? this.installationControl(row.organization_id) : null; + if (!row || row.status !== 'running' || row.attempt < row.max_attempts + || row.lease_expires_at > timestamp || !control || control.currentJobId !== row.id + || control.status !== row.installation_state || control.revision !== row.installation_revision + || control.runtimeKey !== row.runtime_key || control.manifestRevision !== row.manifest_revision + || control.releaseId !== row.release_id) throw new AccessError('installation_operation_in_progress', 409); + const changed = this.db.prepare(`UPDATE installation_lifecycle_jobs SET status='queued',worker_id=NULL, + attempt=0,lease_expires_at=NULL,updated_at=? WHERE id=? AND status='running' + AND attempt>=max_attempts AND lease_expires_at<=?`).run(timestamp, id, timestamp).changes; + if (changed !== 1) throw new AccessError('installation_operation_in_progress', 409); + return this.lifecycleJob(id); + } + + reserveInstallationBackup(backup) { + this.db.prepare(`INSERT INTO installation_backups( + id,organization_id,runtime_key,manifest_revision,release_id,purpose,status,tree_digest, + file_count,total_bytes,lifecycle_job_id,created_at,completed_at,destroyed_at + ) VALUES(?,?,?,?,?,?,'reserved',NULL,NULL,NULL,?,?,NULL,NULL)`).run( + backup.id, backup.organizationId, backup.runtimeKey, backup.manifestRevision, backup.releaseId, + backup.purpose, backup.lifecycleJobId, backup.timestamp, + ); + return this.installationBackup(backup.id); + } + + installationBackup(id) { + return this.db.prepare('SELECT * FROM installation_backups WHERE id=?').get(id) || null; + } + + installationBackups(organizationId) { + return this.db.prepare(`SELECT * FROM installation_backups + WHERE organization_id=? AND status='available' ORDER BY created_at DESC,id`).all(organizationId); + } + + completeInstallationBackup(id, lifecycleJobId, receipt, timestamp) { + const changed = this.db.prepare(`UPDATE installation_backups SET status='available',tree_digest=?, + file_count=?,total_bytes=?,completed_at=? WHERE id=? AND lifecycle_job_id=? AND status='reserved'`).run( + receipt.treeDigest, receipt.fileCount, receipt.totalBytes, timestamp, id, lifecycleJobId, + ).changes; + if (changed !== 1) { + const row = this.installationBackup(id); + if (!row || row.status !== 'available' || row.lifecycle_job_id !== lifecycleJobId + || row.tree_digest !== receipt.treeDigest || row.file_count !== receipt.fileCount + || row.total_bytes !== receipt.totalBytes) throw new AccessError('backup_failed', 409); + } + return this.installationBackup(id); + } + + discardReservedInstallationBackup(id, lifecycleJobId) { + const detached = this.db.prepare(`UPDATE installation_lifecycle_jobs SET backup_id=NULL + WHERE id=? AND backup_id=? AND status='running'`).run(lifecycleJobId, id).changes; + const removed = this.db.prepare(`DELETE FROM installation_backups + WHERE id=? AND lifecycle_job_id=? AND status='reserved'`).run(id, lifecycleJobId).changes; + if (detached !== 1 || removed !== 1) throw new AccessError('decommission_failed', 409); + } + + destroyOrganizationAccess(organizationId) { + this.db.prepare('UPDATE sessions SET active_organization_id=NULL WHERE active_organization_id=?').run(organizationId); + for (const table of ['invitations', 'memberships', 'roles', 'organization_profiles']) { + this.db.prepare(`DELETE FROM ${table} WHERE organization_id=?`).run(organizationId); + } + } + + eraseOrganization(organizationId) { + const installation = this.installationControl(organizationId); + if (!installation) return; + if (this.installationBackend(organizationId) !== 'native_service_v1' || installation.status !== 'decommissioned' + || !this.db.prepare("SELECT 1 FROM installation_lifecycle_jobs WHERE organization_id=? AND operation='destroy' AND status='succeeded'").get(organizationId)) { + throw new AccessError('destruction_failed', 409); + } + this.db.exec('PRAGMA secure_delete=ON'); + this.transaction(() => { + this.db.exec('PRAGMA defer_foreign_keys=ON'); + const users = this.db.prepare(`SELECT u.id FROM users u JOIN memberships m ON m.user_id=u.id + WHERE m.organization_id=? AND u.platform_role IS NULL + AND NOT EXISTS(SELECT 1 FROM memberships other WHERE other.user_id=u.id AND other.organization_id!=?)`).all(organizationId, organizationId); + const tables = this.db.prepare("SELECT name FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'sqlite_%'").all(); + const quote = name => '"' + name.replaceAll('"', '""') + '"'; + this.db.prepare('UPDATE sessions SET active_organization_id=NULL WHERE active_organization_id=?').run(organizationId); + this.db.prepare('DELETE FROM audit_events WHERE target_id=?').run(organizationId); + for (const { name } of tables) { + if (this.db.prepare(`PRAGMA table_info(${quote(name)})`).all().some(c => c.name === 'organization_id')) + this.db.prepare(`DELETE FROM ${quote(name)} WHERE organization_id=?`).run(organizationId); + } + this.db.prepare('DELETE FROM backup_scope_settings WHERE scope=?').run(organizationId); + this.db.prepare('DELETE FROM backup_scope_slots WHERE scope=?').run(organizationId); + for(const set of this.db.prepare('SELECT * FROM backup_sets').all()) { + if(!JSON.parse(set.members_json).some(member=>member.organizationId===organizationId))continue; + this.db.prepare('DELETE FROM backup_set_settings WHERE set_id=?').run(set.id); + this.db.prepare('DELETE FROM backup_sets WHERE id=?').run(set.id); + } + this.db.prepare('DELETE FROM organizations WHERE id=?').run(organizationId); + for (const user of users) { + for (const { name } of tables) { + const columns = this.db.prepare(`PRAGMA table_info(${quote(name)})`).all(); + for (const reference of this.db.prepare(`PRAGMA foreign_key_list(${quote(name)})`).all().filter(f => f.table === 'users')) { + const column = columns.find(c => c.name === reference.from); + if (column && !column.notnull) this.db.prepare(`UPDATE ${quote(name)} SET ${quote(column.name)}=NULL WHERE ${quote(column.name)}=?`).run(user.id); + } + } + this.db.prepare('DELETE FROM audit_events WHERE target_id=?').run(user.id); + this.db.prepare('DELETE FROM users WHERE id=?').run(user.id); + } + if (this.db.prepare('PRAGMA foreign_key_check').all().length) throw new AccessError('destruction_failed', 409); + }); + this.db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); + } + + destroyInstallationBackups(organizationId, timestamp) { + this.db.prepare("UPDATE platform_backup_records SET deleted_at=?,metadata_json='{}' WHERE organization_id=? AND deleted_at IS NULL").run(timestamp, organizationId); + this.db.prepare(`UPDATE installation_backups SET status='destroyed',tree_digest=NULL,file_count=NULL, + total_bytes=NULL,completed_at=NULL,destroyed_at=? WHERE organization_id=? AND status!='destroyed'`).run( + timestamp, organizationId, + ); + } + + latestReadyEvidence(organizationId) { + const activation = this.db.prepare(`SELECT evidence_json,updated_at FROM installation_activation_jobs + WHERE organization_id=? AND status='succeeded' AND evidence_json IS NOT NULL + ORDER BY updated_at DESC,rowid DESC LIMIT 1`).get(organizationId); + const resumed = this.db.prepare(`SELECT result_json,updated_at FROM installation_lifecycle_jobs + WHERE organization_id=? AND operation IN ('resume','upgrade') AND status='succeeded' AND result_json IS NOT NULL + ORDER BY updated_at DESC,rowid DESC LIMIT 1`).get(organizationId); + if (!activation && !resumed) return null; + if (resumed && (!activation || resumed.updated_at >= activation.updated_at)) { + try { return JSON.parse(resumed.result_json).activationEvidence || null; } catch { return null; } + } + try { return JSON.parse(activation.evidence_json); } catch { return null; } + } + + latestSuspensionResult(organizationId) { + const row = this.db.prepare(`SELECT result_json FROM installation_lifecycle_jobs + WHERE organization_id=? AND operation='suspend' AND status='succeeded' AND result_json IS NOT NULL + ORDER BY updated_at DESC,id DESC LIMIT 1`).get(organizationId); + if (!row) return null; + try { return JSON.parse(row.result_json); } catch { return null; } + } + + createRole(role) { + this.db.prepare(`INSERT INTO roles(id,organization_id,key,name,description,is_system,created_by,created_at,updated_at) + VALUES(?,?,?,?,?,?,?,?,?)`).run( + role.id, role.organizationId, role.key, role.name, role.description, role.system ? 1 : 0, + role.createdBy, role.timestamp, role.timestamp, + ); + const statement = this.db.prepare('INSERT INTO role_permissions(role_id,permission) VALUES(?,?)'); + for (const permission of role.permissions) statement.run(role.id, permission); + return this.role(role.id); + } + + role(id) { + const row = this.db.prepare('SELECT * FROM roles WHERE id=?').get(id); + if (!row) return null; + const permissions = this.db.prepare('SELECT permission FROM role_permissions WHERE role_id=? ORDER BY permission').all(id).map(item => item.permission); + return roleView(row, permissions); + } + + roleByKey(organizationId, key) { + const row = this.db.prepare('SELECT * FROM roles WHERE organization_id=? AND key=?').get(organizationId, key); + return row ? this.role(row.id) : null; + } + + roles(organizationId) { + return this.db.prepare(`SELECT id FROM roles WHERE organization_id=? ORDER BY CASE key WHEN 'owner' THEN 0 WHEN 'manager' THEN 1 WHEN 'dispatcher' THEN 2 WHEN 'driver' THEN 3 ELSE 4 END,lower(name),id`).all(organizationId) + .map(row => this.role(row.id)); + } + + updateRole(id, name, description, permissions, timestamp) { + const role = this.role(id); + if (!role || role.system) throw new AccessError(role ? 'system_role_protected' : 'role_not_found', role ? 409 : 404); + this.db.prepare('UPDATE roles SET name=?,description=?,updated_at=? WHERE id=?').run(name, description, timestamp, id); + this.db.prepare('DELETE FROM role_permissions WHERE role_id=?').run(id); + const statement = this.db.prepare('INSERT INTO role_permissions(role_id,permission) VALUES(?,?)'); + for (const permission of permissions) statement.run(id, permission); + return this.role(id); + } + + deleteRole(id) { + const role = this.role(id); + if (!role || role.system) throw new AccessError(role ? 'system_role_protected' : 'role_not_found', role ? 409 : 404); + const count = this.db.prepare('SELECT count(*) AS count FROM memberships WHERE role_id=?').get(id).count; + if (count > 0) throw new AccessError('role_in_use', 409); + this.db.prepare('DELETE FROM roles WHERE id=?').run(id); + } + + createMembership(membership) { + if (this.db.prepare('SELECT 1 FROM memberships WHERE user_id=? AND organization_id<>?').get(membership.userId, membership.organizationId)) { + throw new AccessError('user_already_belongs_to_dsp', 409); + } + this.db.prepare(`INSERT INTO memberships(id,organization_id,user_id,role_id,status,created_by,created_at,updated_at) + VALUES(?,?,?,?,'active',?,?,?)`).run( + membership.id, membership.organizationId, membership.userId, membership.roleId, + membership.createdBy, membership.timestamp, membership.timestamp, + ); + } + + membership(userId, organizationId) { + const row = this.db.prepare(`SELECT m.*,r.key AS role_key,r.name AS role_name + FROM memberships m JOIN roles r ON r.id=m.role_id WHERE m.user_id=? AND m.organization_id=?`).get(userId, organizationId); + if (!row) return null; + return { + id: row.id, organizationId: row.organization_id, userId: row.user_id, roleId: row.role_id, + roleKey: row.role_key, roleName: row.role_name, status: row.status, + permissions: this.role(row.role_id).permissions, + }; + } + + membershipsForUser(userId) { + return this.db.prepare(`SELECT organization_id FROM memberships WHERE user_id=? AND status='active' ORDER BY created_at,id`).all(userId) + .map(row => { + const membership = this.membership(userId, row.organization_id); + return { ...membership, organization: this.organization(row.organization_id) }; + }); + } + + members(organizationId) { + return this.db.prepare(`SELECT m.id AS membership_id,m.status AS membership_status,u.*,r.id AS role_id,r.key AS role_key,r.name AS role_name + FROM memberships m JOIN users u ON u.id=m.user_id JOIN roles r ON r.id=m.role_id + WHERE m.organization_id=? ORDER BY lower(u.first_name),lower(u.last_name),u.id`).all(organizationId).map(row => ({ + id: row.membership_id, + status: row.membership_status, + user: userView(row), + role: { id: row.role_id, key: row.role_key, name: row.role_name }, + })); + } + + membershipById(id) { + const row = this.db.prepare('SELECT * FROM memberships WHERE id=?').get(id); + return row || null; + } + + updateMembershipRole(id, roleId, timestamp) { + this.db.prepare('UPDATE memberships SET role_id=?,updated_at=? WHERE id=?').run(roleId, timestamp, id); + } + + removeMembership(id) { this.db.prepare('DELETE FROM memberships WHERE id=?').run(id); } + + activeOwnerCount(organizationId) { + return this.db.prepare(`SELECT count(*) AS count FROM memberships m JOIN roles r ON r.id=m.role_id + WHERE m.organization_id=? AND m.status='active' AND r.key='owner'`).get(organizationId).count; + } + + createInvitation(invitation) { + this.db.prepare(`INSERT INTO invitations(id,kind,organization_id,role_id,email,token_hash,status,expires_at,created_by,accepted_by,created_at,accepted_at) + VALUES(?,?,?,?,?,?,'pending',?,?,NULL,?,NULL)`).run( + invitation.id, invitation.kind, invitation.organizationId, invitation.roleId, invitation.email, + invitation.tokenHash, invitation.expiresAt, invitation.createdBy, invitation.timestamp, + ); + return this.invitationById(invitation.id); + } + + invitationByHash(tokenHash) { + return this.db.prepare(`SELECT i.*,o.name AS organization_name,r.name AS role_name + FROM invitations i LEFT JOIN organizations o ON o.id=i.organization_id LEFT JOIN roles r ON r.id=i.role_id + WHERE i.token_hash=?`).get(tokenHash) || null; + } + + invitationById(id) { + const row = this.db.prepare(`SELECT i.*,o.name AS organization_name,r.name AS role_name + FROM invitations i LEFT JOIN organizations o ON o.id=i.organization_id LEFT JOIN roles r ON r.id=i.role_id + WHERE i.id=?`).get(id); + return invitationView(row); + } + + invitations(organizationId) { + return this.db.prepare(`SELECT i.*,o.name AS organization_name,r.name AS role_name + FROM invitations i LEFT JOIN organizations o ON o.id=i.organization_id LEFT JOIN roles r ON r.id=i.role_id + WHERE i.organization_id=? ORDER BY i.created_at DESC`).all(organizationId).map(invitationView); + } + + pendingInvitationForEmail(organizationId, email) { + return this.db.prepare(`SELECT * FROM invitations WHERE organization_id=? AND email=? AND status='pending' ORDER BY created_at DESC LIMIT 1`) + .get(organizationId, email) || null; + } + + pendingPlatformInvitation() { + return this.db.prepare("SELECT * FROM invitations WHERE kind='platform_owner' AND status='pending' LIMIT 1").get() || null; + } + + acceptInvitation(id, userId, timestamp) { + const result = this.db.prepare(`UPDATE invitations SET status='accepted',accepted_by=?,accepted_at=? WHERE id=? AND status='pending'`) + .run(userId, timestamp, id); + if (result.changes !== 1) throw new AccessError('invitation_invalid', 404); + } + + revokeInvitation(id) { + const result = this.db.prepare("UPDATE invitations SET status='revoked' WHERE id=? AND status='pending'").run(id); + if (result.changes !== 1) throw new AccessError('invitation_not_pending', 409); + } + + expireInvitations(timestamp) { + this.db.prepare("UPDATE invitations SET status='revoked' WHERE status='pending' AND expires_at<=?").run(timestamp); + } + + createSession(session) { + this.db.prepare(`INSERT INTO sessions(token_hash,user_id,csrf_token,active_organization_id,auth_version,expires_at,created_at,last_seen_at) + VALUES(?,?,?,?,?,?,?,?)`).run( + session.tokenHash, session.userId, session.csrfToken, session.activeOrganizationId, + session.authVersion, session.expiresAt, session.timestamp, session.timestamp, + ); + } + + session(tokenHash, timestamp) { + const row = this.db.prepare(`SELECT s.*,u.email,u.first_name,u.last_name,u.status AS user_status,u.platform_role,u.auth_version AS current_auth_version,u.created_at AS user_created_at + FROM sessions s JOIN users u ON u.id=s.user_id WHERE s.token_hash=?`).get(tokenHash); + if (!row || row.expires_at <= timestamp || row.user_status !== 'active' || row.auth_version !== row.current_auth_version) return null; + return { + tokenHash: row.token_hash, + userId: row.user_id, + csrfToken: row.csrf_token, + activeOrganizationId: row.active_organization_id, + expiresAt: row.expires_at, + user: userView({ + id: row.user_id, email: row.email, first_name: row.first_name, last_name: row.last_name, + status: row.user_status, platform_role: row.platform_role, created_at: row.user_created_at, + }), + }; + } + + selectOrganization(tokenHash, organizationId, timestamp) { + this.db.prepare('UPDATE sessions SET active_organization_id=?,last_seen_at=? WHERE token_hash=?') + .run(organizationId, timestamp, tokenHash); + } + + touchSession(tokenHash, timestamp) { + // Activity metadata must not block Core's event loop while a fenced host + // operation holds the writer lock. Authentication and expiry use reads. + const timeout = this.db.prepare('PRAGMA busy_timeout').get().timeout; + this.db.exec('PRAGMA busy_timeout=0'); + try { + this.db.prepare('UPDATE sessions SET last_seen_at=? WHERE token_hash=?').run(timestamp, tokenHash); + } catch (error) { + if (error.code !== 'ERR_SQLITE_ERROR' || (error.errcode & 0xff) !== 5) throw error; + } finally { this.db.exec(`PRAGMA busy_timeout=${timeout}`); } + } + deleteSession(tokenHash) { this.db.prepare('DELETE FROM sessions WHERE token_hash=?').run(tokenHash); } + deleteExpiredSessions(timestamp) { this.db.prepare('DELETE FROM sessions WHERE expires_at<=?').run(timestamp); } + + createPlatformTargetRef(reference) { + this.db.prepare(`INSERT INTO platform_target_refs( + reference_hash,session_hash,user_id,organization_id,purpose,expires_at,created_at + ) VALUES(?,?,?,?,?,?,?)`).run( + reference.referenceHash, reference.sessionHash, reference.userId, reference.organizationId, + reference.purpose, reference.expiresAt, reference.timestamp, + ); + return this.platformTargetRef(reference.referenceHash); + } + + platformTargetRef(referenceHash) { + return this.db.prepare('SELECT * FROM platform_target_refs WHERE reference_hash=?').get(referenceHash) || null; + } + + deleteExpiredPlatformTargetRefs(timestamp) { + this.db.prepare('DELETE FROM platform_target_refs WHERE expires_at<=?').run(timestamp); + } + + platformMutationRequest(actorUserId, action, idempotencyKey) { + return this.db.prepare(`SELECT * FROM platform_mutation_requests + WHERE actor_user_id=? AND action=? AND idempotency_key=?`).get(actorUserId, action, idempotencyKey) || null; + } + + createPlatformMutationRequest(request) { + this.db.prepare(`INSERT INTO platform_mutation_requests( + id,actor_user_id,action,idempotency_key,request_digest,organization_id,result_json,created_at + ) VALUES(?,?,?,?,?,?,?,?)`).run( + request.id, request.actorUserId, request.action, request.idempotencyKey, request.requestDigest, + request.organizationId, JSON.stringify(request.result), request.timestamp, + ); + return this.platformMutationRequest(request.actorUserId, request.action, request.idempotencyKey); + } + + createAudit(event) { + this.db.prepare(`INSERT INTO audit_events(id,actor_user_id,organization_id,action,target_type,target_id,result,created_at) + VALUES(?,?,?,?,?,?,?,?)`).run( + event.id, event.actorUserId, event.organizationId, event.action, event.targetType, event.targetId, event.result, event.timestamp, + ); + } + + audits(organizationId, limit = 100, { excludePlatformAccess = false } = {}) { + return this.db.prepare(`SELECT a.*,u.email AS actor_email FROM audit_events a LEFT JOIN users u ON u.id=a.actor_user_id + WHERE a.organization_id=? + ${excludePlatformAccess ? "AND a.action NOT LIKE 'organization.view.%'" : ''} + ORDER BY a.created_at DESC,a.id DESC LIMIT ?`).all(organizationId, limit).map(row => ({ + actor: row.actor_email || 'System', + action: row.action, + targetType: row.target_type, + result: row.result, + createdAt: new Date(row.created_at).toISOString(), + })); + } +} + +module.exports = { + SCHEMA_VERSION, + AccessStore, + roleView, + organizationView, + userView, + invitationView, + ensurePrivateDirectory, +}; diff --git a/core/core/accounts/src/validation.js b/core/core/accounts/src/validation.js new file mode 100644 index 0000000..5aa8792 --- /dev/null +++ b/core/core/accounts/src/validation.js @@ -0,0 +1,112 @@ +'use strict'; + +const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; +const ID_RE = /^[a-z][a-z0-9_-]{2,95}$/; +const TOKEN_RE = /^[A-Za-z0-9_-]{43}$/; +const IDEMPOTENCY_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{15,127}$/; +const STATION_RE = /^[A-Z0-9]{3,8}$/; + +class AccessError extends Error { + constructor(code, statusCode = 400) { + super(code); + this.code = code; + this.statusCode = statusCode; + } +} + +function plain(value) { + return value && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, keys) { + if (!plain(value) || Object.keys(value).some(key => !keys.includes(key))) throw new AccessError('invalid_input'); + return value; +} + +function text(value, name, { minimum = 1, maximum = 120, optional = false } = {}) { + if (optional && (value === null || value === undefined || value === '')) return null; + if (typeof value !== 'string') throw new AccessError('invalid_input'); + const selected = value.trim(); + if (selected.length < minimum || selected.length > maximum || /[\0\r\n]/.test(selected)) throw new AccessError('invalid_input'); + return selected; +} + +function identifier(value) { + if (typeof value !== 'string' || !ID_RE.test(value)) throw new AccessError('invalid_input'); + return value; +} + +function email(value) { + const selected = text(value, 'email', { maximum: 254 }).toLowerCase(); + if (!EMAIL_RE.test(selected)) throw new AccessError('invalid_input'); + return selected; +} + +function password(value) { + if (typeof value !== 'string' || value.length < 12 || value.length > 128 || Buffer.byteLength(value, 'utf8') > 512) { + throw new AccessError('password_policy_failed'); + } + return value; +} + +function token(value) { + if (typeof value !== 'string' || !TOKEN_RE.test(value)) throw new AccessError('invitation_invalid', 404); + return value; +} + +function controlReference(value) { + if (typeof value !== 'string' || !TOKEN_RE.test(value)) throw new AccessError('platform_control_invalid', 404); + return value; +} + +function idempotencyKey(value) { + if (typeof value !== 'string' || !IDEMPOTENCY_RE.test(value)) throw new AccessError('invalid_input'); + return value; +} + +function timezone(value, fallback = 'America/Los_Angeles') { + const selected = text(value === undefined ? fallback : value, 'timezone', { maximum: 64 }); + try { new Intl.DateTimeFormat('en-US', { timeZone: selected }).format(); } + catch { throw new AccessError('invalid_input'); } + return selected; +} + +function station(value) { + const selected = text(value, 'station', { maximum: 8 }).toUpperCase(); + if (!STATION_RE.test(selected)) throw new AccessError('invalid_input'); + return selected; +} + +function abbreviation(value) { + const selected = text(value, 'abbreviation', { minimum: 2, maximum: 16, optional: true }); + if (selected === null) return null; + const normalized = selected.toUpperCase(); + if (!/^[A-Z0-9][A-Z0-9 -]{0,14}[A-Z0-9]$/.test(normalized)) throw new AccessError('invalid_input'); + return normalized; +} + +function permissions(value, allowed) { + if (!Array.isArray(value) || value.length < 1 || value.length > allowed.length || new Set(value).size !== value.length + || value.some(permission => typeof permission !== 'string' || !allowed.includes(permission))) { + throw new AccessError('invalid_input'); + } + return [...value].sort(); +} + +module.exports = { + AccessError, + plain, + exact, + text, + identifier, + email, + password, + token, + controlReference, + idempotencyKey, + timezone, + station, + abbreviation, + permissions, +}; diff --git a/core/core/accounts/src/worker-wakeup.js b/core/core/accounts/src/worker-wakeup.js new file mode 100644 index 0000000..f6fe6c2 --- /dev/null +++ b/core/core/accounts/src/worker-wakeup.js @@ -0,0 +1,24 @@ +'use strict'; +const path = require('node:path'); +// Wakeups are hints after durable commits. Supervised fallback timers recover +// a missed hint; a notification failure must never undo a successful operation. +function wake(workers, { databaseRoot, localRoot = process.env.DISPATCH_LOCAL_ROOT, + send = require('node:child_process').spawnSync, uid = process.geteuid() } = {}) { + if (!localRoot || !path.isAbsolute(localRoot) || path.resolve(localRoot) !== localRoot + || databaseRoot && path.resolve(localRoot, 'data/access-control') !== databaseRoot) return; + const units = { reconcile: 'dispatch-installation-reconcile.service', core: 'dispatch-platform-update.service' }; + if (!Array.isArray(workers) || workers.some(worker => !units[worker])) return; + try { + send('/usr/bin/systemctl', ['--user', '--no-block', 'start', ...new Set(workers.map(worker => units[worker]))], { + timeout: 3000, stdio: 'ignore', env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', + XDG_RUNTIME_DIR: `/run/user/${uid}`, DBUS_SESSION_BUS_ADDRESS: `unix:path=/run/user/${uid}/bus` }, + }); + } catch {} // A fallback timer will read the committed queue. +} +function afterCommit(store, workers) { + store.afterCommit?.(() => { + if (typeof store.wakeWorkers === 'function') store.wakeWorkers(workers); + else wake(workers, { databaseRoot: store.paths?.databaseRoot }); + }); +} +module.exports = { wake, afterCommit }; diff --git a/core/core/accounts/src/workspace-readiness.js b/core/core/accounts/src/workspace-readiness.js new file mode 100644 index 0000000..957f3d0 --- /dev/null +++ b/core/core/accounts/src/workspace-readiness.js @@ -0,0 +1,48 @@ +'use strict'; + +const crypto = require('node:crypto'); + +// Provisioning already verifies the isolated runtime. Paycom authentication and +// publication evidence belong to the optional integration, not DSP onboarding. +function provisionedWorkspace(store, organizationId) { + return ['oci_container_v1', 'native_service_v1', 'directory_service_v1'].includes(store.installationBackend(organizationId)) + && store.latestProvisioningRequest(organizationId)?.status === 'completed'; +} + +function workspaceWithoutPaycom(store, organizationId) { + return provisionedWorkspace(store, organizationId) + && store.activeOwnerCount(organizationId) > 0 + && !store.db.prepare('SELECT 1 FROM organization_profiles WHERE organization_id=? AND applied_at IS NULL').get(organizationId) + && !store.db.prepare("SELECT 1 FROM installation_activation_jobs WHERE organization_id=? AND status='succeeded'").get(organizationId); +} + +function completeWorkspaceSetup(store, clock = Date.now) { + return store.transaction(() => { + const rows = store.db.prepare(`SELECT i.organization_id FROM installations i + JOIN organizations o ON o.id=i.organization_id + LEFT JOIN organization_profiles p ON p.organization_id=o.id + WHERE i.status IN ('waiting_for_owner','waiting_for_provider_auth') + AND i.current_job_id IS NULL AND i.setup_worker_id IS NULL + AND o.status='setup_required' AND (p.organization_id IS NULL OR p.applied_at IS NOT NULL) + AND NOT EXISTS (SELECT 1 FROM dsp_removals d WHERE d.organization_id=o.id) + AND NOT EXISTS (SELECT 1 FROM installation_onboarding_requests r WHERE r.organization_id=o.id AND r.status IN ('enrolling','queued','running'))`).all(); + let completed = 0; + for (const { organization_id: organizationId } of rows) { + if (!provisionedWorkspace(store, organizationId) || !store.activeOwnerCount(organizationId) + || store.activeLifecycleJob(organizationId) || store.runningActivationJob(organizationId)) continue; + const control = store.installationControl(organizationId); + const timestamp = clock(); + store.updateInstallationControl({ organizationId, expectedStatus: control.status, + expectedRevision: control.revision, status: 'ready', revision: control.revision + 1, + currentJobId: null, timestamp }); + store.updateOrganizationStatus(organizationId, 'active', timestamp); + store.createAudit({ id: `aud_${crypto.randomBytes(16).toString('hex')}`, actorUserId: null, + organizationId, action: 'installation.workspace.ready', targetType: 'organization', + targetId: organizationId, result: 'succeeded', timestamp }); + completed += 1; + } + return completed; + }); +} + +module.exports = { completeWorkspaceSetup, provisionedWorkspace, workspaceWithoutPaycom }; diff --git a/core/core/accounts/tests/access-control.test.js b/core/core/accounts/tests/access-control.test.js new file mode 100644 index 0000000..830249a --- /dev/null +++ b/core/core/accounts/tests/access-control.test.js @@ -0,0 +1,708 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { AccessStore, AccessControlService } = require('../src'); + +function fixture(installationOperatorEnabled = false) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-access-control-')); + fs.chmodSync(root, 0o700); + const paths = { databaseRoot: path.join(root, 'access-control'), database: path.join(root, 'access-control', 'access-control.sqlite3') }; + const time = { value: Date.parse('2026-09-02T12:00:00.000Z') }; + const store = new AccessStore(paths); + const service = new AccessControlService(store, { + clock: () => new Date(time.value), installationOperatorEnabled, + }); + service.ensureLocalOrganization({ + organization: { id: 'local-dsp', name: 'EXMP' }, + site: { id: 'local-site', code: 'TST1' }, + timezone: 'America/Los_Angeles', + }); + return { root, paths, time, store, service, close() { store.close(); fs.rmSync(root, { recursive: true, force: true }); } }; +} + +async function platformOwner(service) { + const bootstrap = service.createPlatformBootstrap({ email: 'fixture-owner@example.test', organizationId: 'local-dsp' }); + return service.acceptNewUser({ + token: bootstrap.token, + firstName: 'Fixture Owner', + lastName: 'Owner', + password: 'correct horse battery staple', + confirmPassword: 'correct horse battery staple', + }); +} + +test('bootstrap creates a hashed-credential platform owner with an isolated local-DSP membership', async () => { + const context = fixture(); + try { + assert.deepEqual(context.service.bootstrapStatus(), { initialized: false, invitationPending: false }); + const revokedBootstrap = context.service.createPlatformBootstrap({ email: 'Fixture-owner@Example.test', organizationId: 'local-dsp' }); + assert.equal(context.service.revokePlatformBootstrap().status, 'revoked'); + assert.equal(context.service.revokePlatformBootstrap(), null); + assert.throws(() => context.service.inspectInvitation(revokedBootstrap.token), /invitation_invalid/); + assert.deepEqual(context.service.bootstrapStatus(), { initialized: false, invitationPending: false }); + const bootstrap = context.service.createPlatformBootstrap({ email: 'Fixture-owner@Example.test', organizationId: 'local-dsp' }); + const inspectedInvitation = context.service.inspectInvitation(bootstrap.token); + assert.equal(inspectedInvitation.organization.name, 'EXMP'); + assert.equal(Object.hasOwn(inspectedInvitation, 'id'), false); + assert.equal(Object.hasOwn(inspectedInvitation.organization, 'id'), false); + assert.equal(Object.hasOwn(inspectedInvitation.role, 'id'), false); + const accepted = await context.service.acceptNewUser({ + token: bootstrap.token, + firstName: 'Fixture Owner', + lastName: 'Owner', + password: 'correct horse battery staple', + confirmPassword: 'correct horse battery staple', + }); + assert.equal(accepted.session.user.email, 'fixture-owner@example.test'); + assert.equal(accepted.session.user.platformRole, 'owner'); + assert.throws(() => context.service.requestInstallationProvisioning(accepted.session, 'local-dsp', { + idempotencyKey: 'disabled:provision:local', expectedRevision: 1, + }), /installation_operator_disabled/); + assert.ok(accepted.session.platformPermissions.includes('platform.installations.read')); + assert.ok(accepted.session.platformPermissions.includes('platform.installations.manage')); + assert.equal(accepted.session.memberships[0].organizationId, 'local-dsp'); + assert.equal(accepted.session.memberships[0].roleKey, 'owner'); + assert.ok(accepted.session.memberships[0].permissions.includes('workforce.read')); + assert.throws(() => context.service.inspectInvitation(bootstrap.token), /invitation_invalid/); + + const signedIn = await context.service.signIn({ email: 'fixture-owner@example.test', password: 'correct horse battery staple' }); + assert.equal(signedIn.session.user.id, accepted.session.user.id); + assert.notEqual(signedIn.token, accepted.token); + await assert.rejects(context.service.signIn({ email: 'fixture-owner@example.test', password: null }), /invalid_credentials/); + await assert.rejects(context.service.signIn({ email: 'missing@example.test', password: null }), /invalid_credentials/); + await assert.rejects(context.service.signIn({ email: null, password: null }), /invalid_credentials/); + const changed = await context.service.changePassword(signedIn.session, { + currentPassword: 'correct horse battery staple', + newPassword: 'replacement secure passphrase', + confirmPassword: 'replacement secure passphrase', + }); + assert.equal(context.service.session(signedIn.token), null); + await assert.rejects(context.service.signIn({ email: 'fixture-owner@example.test', password: 'correct horse battery staple' }), /invalid_credentials/); + assert.equal((await context.service.signIn({ email: 'fixture-owner@example.test', password: 'replacement secure passphrase' })).session.user.id, changed.session.user.id); + context.store.close(); + const bytes = fs.readFileSync(context.paths.database); + assert.equal(bytes.includes(Buffer.from('correct horse battery staple')), false); + assert.equal(bytes.includes(Buffer.from('replacement secure passphrase')), false); + assert.equal(bytes.includes(Buffer.from(bootstrap.token)), false); + assert.equal(fs.statSync(context.paths.database).mode & 0o777, 0o600); + assert.equal(fs.statSync(context.paths.databaseRoot).mode & 0o777, 0o700); + } finally { + try { context.store.close(); } catch {} + fs.rmSync(context.root, { recursive: true, force: true }); + } +}); + +test('organization invitations create tenant-scoped owners and prevent horizontal access', async () => { + const context = fixture(); + try { + const platform = await platformOwner(context.service); + const created = context.service.createOrganization(platform.session, { + idempotencyKey: 'test:organization:create:second', + name: 'Second Delivery LLC', abbreviation: 'SECOND', stationCode: 'DWA1', + timezone: 'America/Los_Angeles', ownerEmail: 'owner@second.test', + }); + assert.equal(created.organization.status, 'pending_owner'); + assert.equal(created.organization.installation.status, 'pending'); + const second = await context.service.acceptNewUser({ + token: created.token, firstName: 'Second', lastName: 'Owner', + password: 'a second secure passphrase', confirmPassword: 'a second secure passphrase', + }); + const secondOrg = second.session.activeOrganizationId; + assert.notEqual(secondOrg, 'local-dsp'); + assert.equal(second.session.platformPermissions.includes('platform.installations.manage'), false); + assert.equal(context.store.organization(secondOrg).status, 'setup_required'); + assert.throws(() => context.service.requestInstallationProvisioning(second.session, secondOrg, { + idempotencyKey: 'tenant:provision:forbidden', expectedRevision: 1, + }), /platform_forbidden/); + assert.throws(() => context.service.requirePermission(second.session, 'local-dsp', 'workforce.read'), /organization_forbidden/); + assert.throws(() => context.service.requirePermission(platform.session, secondOrg, 'workforce.read'), /organization_forbidden/); + assert.equal(context.service.setOrganizationSuspended(platform.session, secondOrg, { suspended: true }).status, 'suspended'); + assert.throws(() => context.service.requirePermission(second.session, secondOrg, 'workforce.read'), /organization_forbidden/); + assert.equal(context.service.session(second.token), null); + await assert.rejects(context.service.signIn({ email: 'owner@second.test', password: 'a second secure passphrase' }), /account_disabled/); + assert.ok(context.service.session(platform.token)); + assert.equal(context.service.setOrganizationSuspended(platform.session, secondOrg, { suspended: false }).status, 'setup_required'); + assert.equal(context.service.session(second.token), null); + assert.ok((await context.service.signIn({ email: 'owner@second.test', password: 'a second secure passphrase' })).session); + assert.equal(context.service.requirePermission(second.session, secondOrg, 'workforce.read').membership.roleKey, 'owner'); + const viewer = context.store.roleByKey(secondOrg, 'driver'); + const platformInvitation = context.service.createMemberInvitation(second.session, secondOrg, { + email: platform.session.user.email, roleId: viewer.id, + }); + assert.throws(() => context.service.acceptExistingUser(platform.session, platformInvitation.token), /user_already_belongs_to_dsp/); + assert.throws(() => context.service.createOrganization(second.session, { + idempotencyKey: 'test:organization:create:unauthorized', + name: 'Unauthorized DSP', abbreviation: null, stationCode: 'DWA2', timezone: 'UTC', ownerEmail: 'bad@example.test', + }), /platform_forbidden/); + } finally { context.close(); } +}); + +test('only platform authority can create a durable managed installation provisioning request', async () => { + const context = fixture(true); + try { + const platform = await platformOwner(context.service); + const created = context.service.createOrganization(platform.session, { + idempotencyKey: 'test:organization:create:provisioned', + name: 'Provisioned Delivery LLC', abbreviation: 'PROV', stationCode: 'DWA3', + timezone: 'America/Denver', ownerEmail: 'owner@provisioned.test', + }); + const request = context.service.requestInstallationProvisioning( + platform.session, + created.organization.id, + { idempotencyKey: 'platform:provision:request', expectedRevision: 1 }, + ); + assert.equal(request.status, 'pending'); + assert.equal(request.jobId, null); + assert.equal(context.store.installationControl(created.organization.id).status, 'provisioning'); + const replay = context.service.requestInstallationProvisioning( + platform.session, + created.organization.id, + { idempotencyKey: 'platform:provision:request', expectedRevision: 1 }, + ); + assert.equal(replay.id, request.id); + assert.equal(replay.replayed, true); + } finally { context.close(); } +}); + +test('opaque platform controls are session-bound, idempotent, and expose only setup-safe state', async () => { + const context = fixture(true); + try { + const platform = await platformOwner(context.service); + const input = { + idempotencyKey: 'test:organization:create:console', + name: 'Console Delivery LLC', abbreviation: 'CONSOLE', stationCode: 'DWA5', + timezone: 'America/Los_Angeles', ownerEmail: 'owner@console.test', + }; + const created = context.service.createOrganization(platform.session, input); + assert.equal(created.replayed, false); + assert.equal(typeof created.token, 'string'); + const replayedCreate = context.service.createOrganization(platform.session, input); + assert.equal(replayedCreate.replayed, true); + assert.equal(replayedCreate.token, null); + assert.equal(replayedCreate.organization.id, created.organization.id); + assert.equal(context.store.organizations().filter(item => item.name === input.name).length, 1); + assert.throws(() => context.service.createOrganization(platform.session, { + ...input, name: 'Changed Delivery LLC', + }), /idempotency_conflict/); + + const rows = context.service.platformOrganizations(platform.session); + const row = rows.find(item => item.name === input.name); + assert.match(row.controlRef, /^[A-Za-z0-9_-]{43}$/); + assert.match(row.continuityRef, /^[A-Za-z0-9_-]{43}$/); + assert.equal(context.service.platformOrganizations(platform.session) + .find(item => item.name === input.name).continuityRef, row.continuityRef); + assert.deepEqual(row.installation.availableActions, ['provision', 'decommission']); + assert.equal(row.availableActions.includes('suspend'), false); + const forbiddenKeys = new Set(['id', 'organizationId', 'runtimeKey', 'jobId', 'manifestRevision', 'path', 'socket']); + const visit = value => { + if (!value || typeof value !== 'object') return; + for (const [key, child] of Object.entries(value)) { + assert.equal(forbiddenKeys.has(key), false, `forbidden platform key: ${key}`); + visit(child); + } + }; + visit(row); + + const anotherSession = await context.service.signIn({ + email: 'fixture-owner@example.test', password: 'correct horse battery staple', + }); + assert.throws(() => context.service.requestPlatformInstallationProvisioning(anotherSession.session, { + controlRef: row.controlRef, + idempotencyKey: 'test:platform:provision:cross-session', + expectedRevision: row.installation.revision, + }), /platform_control_invalid/); + + const revoked = context.service.revokePlatformOwnerInvitation(platform.session, { + controlRef: row.controlRef, idempotencyKey: 'test:owner-invitation:revoke', + }); + assert.deepEqual(revoked, { status: 'revoked', replayed: false }); + assert.deepEqual(context.service.revokePlatformOwnerInvitation(platform.session, { + controlRef: row.controlRef, idempotencyKey: 'test:owner-invitation:revoke', + }), { status: 'revoked', replayed: true }); + const replacement = context.service.createPlatformOwnerInvitation(platform.session, { + controlRef: row.controlRef, + idempotencyKey: 'test:owner-invitation:create', + ownerEmail: 'replacement@console.test', + }); + assert.equal(typeof replacement.token, 'string'); + assert.equal(context.service.createPlatformOwnerInvitation(platform.session, { + controlRef: row.controlRef, + idempotencyKey: 'test:owner-invitation:create', + ownerEmail: 'replacement@console.test', + }).token, null); + + assert.deepEqual(context.service.setPlatformOrganizationSuspended(platform.session, { + controlRef: row.controlRef, idempotencyKey: 'test:organization:status:suspend', suspended: true, + }), { status: 'suspended', replayed: false }); + const suspendedRow = context.service.platformOrganizations(platform.session) + .find(organization => organization.name === input.name); + assert.equal(suspendedRow.availableActions.includes('resume'), false); + assert.deepEqual(suspendedRow.installation.availableActions, ['decommission']); + assert.deepEqual(context.service.setPlatformOrganizationSuspended(platform.session, { + controlRef: row.controlRef, idempotencyKey: 'test:organization:status:suspend', suspended: true, + }), { status: 'suspended', replayed: true }); + assert.throws(() => context.service.setPlatformOrganizationSuspended(platform.session, { + controlRef: row.controlRef, idempotencyKey: 'test:organization:status:suspend', suspended: false, + }), /idempotency_conflict/); + context.service.setPlatformOrganizationSuspended(platform.session, { + controlRef: row.controlRef, idempotencyKey: 'test:organization:status:resume', suspended: false, + }); + assert.deepEqual(context.service.setPlatformOrganizationSuspended(platform.session, { + controlRef: row.controlRef, idempotencyKey: 'test:organization:status:suspend', suspended: true, + }), { status: 'suspended', replayed: true }); + + const provisioned = context.service.requestPlatformInstallationProvisioning(platform.session, { + controlRef: row.controlRef, + idempotencyKey: 'test:platform:provision:console', + expectedRevision: row.installation.revision, + }); + assert.deepEqual(provisioned, { + action: 'provision', status: 'accepted', installationState: 'provisioning', + installationRevision: 2, replayed: false, + }); + assert.equal(context.service.requestPlatformInstallationProvisioning(platform.session, { + controlRef: row.controlRef, + idempotencyKey: 'test:platform:provision:console', + expectedRevision: row.installation.revision, + }).status, 'replayed'); + + const owner = await context.service.acceptNewUser({ + token: replacement.token, + firstName: 'Console', lastName: 'Owner', + password: 'console owner secure passphrase', confirmPassword: 'console owner secure passphrase', + }); + const setup = context.service.organizationSetup(owner.session); + assert.equal(setup.installationState, 'provisioning'); + assert.equal(setup.setupState, 'waiting_for_platform'); + assert.equal(setup.operationalAccess, 'unavailable'); + assert.equal(Object.hasOwn(setup, 'organizationId'), false); + assert.equal(Object.hasOwn(setup, 'runtimeKey'), false); + + const firstRequest = context.store.latestProvisioningRequest(created.organization.id); + const failedProvisionJob = { + id: 'job_console_provision_failure', operation: 'provision', status: 'queued', + installationState: 'provisioning', revision: firstRequest.installation_revision, + }; + context.store.transaction(() => { + context.store.acknowledgeProvisioningRequest(firstRequest.id, failedProvisionJob, context.time.value); + context.store.finishProvisioningRequest(firstRequest.id, { + ...failedProvisionJob, + status: 'failed', installationState: 'failed', revision: firstRequest.installation_revision + 1, + failure: { code: 'runtime_health_failed' }, + }, context.time.value); + }); + const failedControl = context.store.installationControl(created.organization.id); + const retryInput = { + controlRef: row.controlRef, + idempotencyKey: 'test:platform:retry:console', + expectedRevision: failedControl.revision, + }; + assert.equal(context.service.requestPlatformInstallationRetry(platform.session, retryInput).status, 'accepted'); + assert.equal(context.service.requestPlatformInstallationRetry(platform.session, retryInput).status, 'replayed'); + + const retryRequest = context.store.latestProvisioningRequest(created.organization.id); + const failedRetryJob = { + id: 'job_console_retry_failure', operation: 'retry', status: 'queued', + installationState: 'provisioning', revision: retryRequest.installation_revision, + }; + context.store.transaction(() => { + context.store.acknowledgeProvisioningRequest(retryRequest.id, failedRetryJob, context.time.value); + context.store.finishProvisioningRequest(retryRequest.id, { + ...failedRetryJob, + status: 'failed', installationState: 'failed', revision: retryRequest.installation_revision + 1, + failure: { code: 'first_publication_failed' }, + }, context.time.value); + }); + const nonInfrastructureFailure = context.service.platformOrganizations(platform.session) + .find(organization => organization.name === input.name); + assert.equal(nonInfrastructureFailure.installation.failure.category, 'activation'); + assert.equal(nonInfrastructureFailure.installation.availableActions.includes('retry_provision'), false); + assert.throws(() => context.service.requestPlatformInstallationRetry(platform.session, { + controlRef: nonInfrastructureFailure.controlRef, + idempotencyKey: 'test:platform:retry:non-infrastructure', + expectedRevision: nonInfrastructureFailure.installation.revision, + }), /installation_operation_not_allowed/); + assert.throws(() => context.store.transaction(() => context.store.createProvisioningRequest({ + id: 'prq_non_infrastructure_retry', + organizationId: created.organization.id, + authorityScope: 'platform_installation', + operation: { + operation: 'retry', + idempotencyKey: 'test:authority:retry:non-infrastructure', + expectedRevision: nonInfrastructureFailure.installation.revision, + }, + timestamp: context.time.value, + })), /installation_operation_not_allowed/); + + context.time.value += 16 * 60 * 1000; + assert.throws(() => context.service.setPlatformOrganizationSuspended(platform.session, { + controlRef: row.controlRef, + idempotencyKey: 'test:organization:status:expired-control', + suspended: true, + }), /platform_control_invalid/); + context.store.close(); + for (const file of [context.paths.database, `${context.paths.database}-wal`].filter(fs.existsSync)) { + const bytes = fs.readFileSync(file); + assert.equal(bytes.includes(Buffer.from(row.controlRef)), false); + assert.equal(bytes.includes(Buffer.from(row.continuityRef)), false); + } + } finally { context.close(); } +}); + +test('owner acceptance advances a provisioned installation from owner wait to provider setup atomically', async () => { + const context = fixture(); + try { + const platform = await platformOwner(context.service); + const created = context.service.createOrganization(platform.session, { + idempotencyKey: 'test:organization:create:owner-wait', + name: 'Owner Wait Delivery LLC', abbreviation: 'WAIT', stationCode: 'DWA4', + timezone: 'America/Chicago', ownerEmail: 'owner@waiting.test', + }); + const installation = context.store.installationControl(created.organization.id); + context.store.transaction(() => context.store.updateInstallationControl({ + organizationId: created.organization.id, + expectedStatus: 'pending', + expectedRevision: installation.revision, + status: 'waiting_for_owner', + revision: installation.revision + 1, + currentJobId: null, + timestamp: context.time.value, + })); + await context.service.acceptNewUser({ + token: created.token, + firstName: 'Waiting', + lastName: 'Owner', + password: 'waiting owner secure passphrase', + confirmPassword: 'waiting owner secure passphrase', + }); + assert.equal(context.store.organization(created.organization.id).status, 'setup_required'); + assert.deepEqual(context.store.installationControl(created.organization.id), { + organizationId: created.organization.id, + runtimeKey: `runtime_${created.organization.id.slice(4)}`, + status: 'waiting_for_provider_auth', + revision: 3, + manifestRevision: 1, + releaseId: 'dispatch_current_1', + currentJobId: null, + }); + } finally { context.close(); } +}); + +test('all four fixed roles have equal DSP permissions and can invite and assign every role', async t => { + const context = fixture(); + t.after(() => context.close()); + const { service, store } = context; + const owner = await platformOwner(service); + const roles = store.roles('local-dsp'); + assert.deepEqual(roles.map(role => role.name), ['Owner', 'Manager', 'Dispatcher', 'Driver']); + const actors = []; + for (const role of roles) { + assert.equal(role.system, true); + assert.deepEqual(role.permissions, roles[0].permissions); + assert.equal(role.permissions.includes('roles.manage'), false); + const invite = service.createMemberInvitation(owner.session, 'local-dsp', { email: `${role.key}@example.test`, roleId: role.id }); + const member = await service.acceptNewUser({ token: invite.token, firstName: 'Team', lastName: role.name, + password: 'another secure passphrase', confirmPassword: 'another secure passphrase' }); + assert.equal(member.session.memberships[0].roleName, role.name); + assert.deepEqual(member.session.platformPermissions, []); + actors.push(member); + } + const ownerMembership = store.membership(owner.session.user.id, 'local-dsp'); + const target = store.membership(actors[0].session.user.id, 'local-dsp'); + for (const actor of actors) { + assert.equal(service.organizationAdministration(actor.session, 'local-dsp').roles.length, 4); + for (const permission of roles[0].permissions) service.requirePermission(actor.session, 'local-dsp', permission); + assert.throws(() => service.createRole(actor.session, 'local-dsp', { name: 'Custom', permissions: [] }), /fixed_roles_only/); + for (const role of roles) { + assert.throws(() => service.updateRole(actor.session, 'local-dsp', role.id, {}), /fixed_roles_only/); + assert.throws(() => service.deleteRole(actor.session, 'local-dsp', role.id), /fixed_roles_only/); + const invite = service.createMemberInvitation(actor.session, 'local-dsp', { + email: `${actor.session.user.id}-${role.key}@example.test`, roleId: role.id, + }); + service.revokeMemberInvitation(actor.session, 'local-dsp', invite.invitation.id); + if (actor !== actors[0]) service.updateMemberRole(actor.session, 'local-dsp', target.id, role.id); + } + } + // Drivers can assign ownership and manage Owners, but nobody can leave the + // DSP without an Owner or change/remove their own membership. + const driver = actors[3]; + service.updateMemberRole(driver.session, 'local-dsp', target.id, roles[0].id); + service.removeMember(driver.session, 'local-dsp', target.id); + assert.throws(() => service.updateMemberRole(driver.session, 'local-dsp', ownerMembership.id, roles[3].id), /last_owner_protected/); + assert.throws(() => service.removeMember(driver.session, 'local-dsp', ownerMembership.id), /last_owner_protected/); + const driverMembership = store.membership(driver.session.user.id, 'local-dsp'); + assert.throws(() => service.updateMemberRole(driver.session, 'local-dsp', driverMembership.id, roles[0].id), /self_role_change_forbidden/); + assert.throws(() => service.removeMember(driver.session, 'local-dsp', driverMembership.id), /self_removal_forbidden/); + const custom = store.createRole({ id: 'legacy_custom', organizationId: 'local-dsp', key: null, name: 'Legacy', description: '', + system: false, permissions: [], createdBy: owner.session.user.id, timestamp: service.now() }); + assert.throws(() => service.createMemberInvitation(driver.session, 'local-dsp', { email: 'legacy@example.test', roleId: custom.id }), /role_not_assignable/); + assert.throws(() => service.updateMemberRole(driver.session, 'local-dsp', ownerMembership.id, custom.id), /role_not_assignable/); +}); + +test('platform removal revokes invitation and tenant administration immediately, preserves peers, and replays once', async t => { + const context = fixture(true); + t.after(() => context.close()); + const { service, store } = context; + const platform = await platformOwner(service); + const create = (suffix, ownerEmail) => service.createOrganization(platform.session, { + idempotencyKey: `removal:create:${suffix}`, name: `Removal ${suffix}`, abbreviation: suffix.toUpperCase(), + stationCode: 'DWA1', timezone: 'America/Chicago', ownerEmail, + }); + const alpha = create('alpha', 'alpha@example.test'); + const beta = create('beta', 'beta@example.test'); + const owner = await service.acceptNewUser({ token: alpha.token, firstName: 'Alpha', lastName: 'Owner', + password: 'a secure fixture password', confirmPassword: 'a secure fixture password' }); + const memberInvite = service.createMemberInvitation(owner.session, alpha.organization.id, { + email: 'member@example.test', roleId: store.roleByKey(alpha.organization.id, 'driver').id, + }); + const row = service.platformOrganizations(platform.session).find(row => row.name === alpha.organization.name); + const input = { controlRef: row.controlRef, idempotencyKey: 'removal:alpha:request', + expectedRevision: row.installation.revision }; + await assert.rejects(service.requestPlatformRemoval(owner.session, input, 'decommission'), /platform_forbidden/); + await assert.rejects(service.requestPlatformRemoval(platform.session, { ...input, confirmation: 'wrong' }, 'decommission'), /invalid_input/); + const removal = await service.requestPlatformRemoval(platform.session, input, 'decommission'); + assert.equal(removal.installationState, 'decommissioning'); + assert.equal((await service.requestPlatformRemoval(platform.session, input, 'decommission')).replayed, true); + assert.equal(service.session(owner.token), null); + await assert.rejects(service.signIn({ email: 'alpha@example.test', password: 'a secure fixture password' }), /account_disabled/); + assert.equal(store.invitationById(memberInvite.invitation.id).status, 'pending'); + assert.equal(store.lifecycleExecutionCandidates(context.time.value, 20).length, 1); + assert.throws(() => service.organizationMembership(owner.session), /organization_forbidden/); + assert.throws(() => service.organizationSetup(owner.session), /organization_forbidden/); + assert.throws(() => service.inspectInvitation(memberInvite.token), /invitation_invalid/); + assert.equal(service.inspectInvitation(beta.token).organization.name, beta.organization.name); + assert.equal(store.organization(beta.organization.id).status, 'pending_owner'); + assert.throws(() => service.setOrganizationSuspended(platform.session, alpha.organization.id, { suspended: false }), /installation_operation_not_allowed/); + const removed = service.platformOrganizations(platform.session).find(row => row.name === alpha.organization.name); + assert.deepEqual(removed.availableActions, []); + assert.deepEqual(removed.installation.operation, { kind: 'decommission', status: 'queued' }); + assert.deepEqual(removed.installation.availableActions, []); +}); + +// An access revocation must not disconnect the agent before its stop job exists. +for (const route of ['direct', 'platform']) test(`${route} suspension preserves agent authority until its stop job completes`, async () => { + const c = fixture(true); + try { + const platform = await platformOwner(c.service); + c.service = new AccessControlService(c.store, { clock: () => new Date(c.time.value), installationOperatorEnabled: true, installationBackend: 'native_service_v1' }); + const created = c.service.createOrganization(platform.session, { ownerEmail: 'suspension@example.test', idempotencyKey: 'regression:create:native' }); + const organizationId = created.organization.id; + c.store.db.prepare("UPDATE installations SET status='ready' WHERE organization_id=?").run(organizationId); + c.store.updateOrganizationStatus(organizationId, 'active', c.time.value); + const installation = c.store.installationControl(organizationId); + c.store.recordRuntimeAgentAuthority({ organizationId: organizationId, runtimeKey: installation.runtimeKey, + tokenHash: 'a'.repeat(64), timestamp: c.time.value }); + assert.ok(c.store.activeRuntimeAgentAuthority(installation.runtimeKey)); + if (route === 'direct') c.service.setOrganizationSuspended(platform.session, organizationId, { suspended: true }); + else { + const row = c.service.platformOrganizations(platform.session).find(r => r.ownerEmail === 'suspension@example.test'); + c.service.setPlatformOrganizationSuspended(platform.session, { + controlRef: row.controlRef, idempotencyKey: 'regression:suspend:authority', suspended: true, + }); + } + assert.equal(c.store.organization(organizationId).status, 'suspended'); + assert.equal(c.store.activeLifecycleJob(organizationId).operation, 'suspend'); + assert.ok(c.store.activeRuntimeAgentAuthority(installation.runtimeKey)); + assert.equal(c.store.activeRuntimeAgentAuthorityCount(), 1); + } finally { c.close(); } +}); + +// Model a successfully activated DSP after its suspension worker has stopped it. +test('both dashboard resume routes reactivate an installed suspended DSP', async () => { + const c = fixture(true); + try { + const platform = await platformOwner(c.service); + c.store.db.prepare("UPDATE installations SET status='suspended' WHERE organization_id='local-dsp'").run(); + c.store.latestReadyEvidence = () => ({ previouslyVerified: true }); + c.store.updateOrganizationStatus('local-dsp', 'suspended', c.time.value); + assert.equal(c.service.setOrganizationSuspended(platform.session, 'local-dsp', { suspended: false }).status, 'active'); + c.store.updateOrganizationStatus('local-dsp', 'suspended', c.time.value); + const row = c.service.platformOrganizations(platform.session).find(r => r.name === 'EXMP'); + assert.equal(c.service.setPlatformOrganizationSuspended(platform.session, { + controlRef: row.controlRef, idempotencyKey: 'regression:resume:activated', suspended: false, + }).status, 'active'); + c.store.latestReadyEvidence = () => null; + c.store.updateOrganizationStatus('local-dsp', 'suspended', c.time.value); + assert.equal(c.service.setOrganizationSuspended(platform.session, 'local-dsp', { suspended: false }).status, 'setup_required'); + } finally { c.close(); } +}); + + +test('DSP listing and opaque controls remain read-only under a lifecycle writer lock', async t => { + const context = fixture(); t.after(() => context.close()); + const owner = await platformOwner(context.service); + const created = context.service.createOrganization(owner.session, { + idempotencyKey: 'listing:expired:invitation', name: 'Expired Invitation', abbreviation: 'EXP', + stationCode: 'DWA5', timezone: 'America/Los_Angeles', ownerEmail: 'expired@example.test', + }); + context.store.db.prepare('UPDATE invitations SET expires_at=? WHERE organization_id=?') + .run(context.time.value, created.organization.id); + const { DatabaseSync } = require('node:sqlite'); + const writer = new DatabaseSync(context.paths.database); t.after(() => writer.close()); + writer.exec('BEGIN IMMEDIATE'); + try { + const changes = context.store.db.prepare('SELECT total_changes() AS count').get().count; + const start = performance.now(); + const rows = context.service.platformOrganizations(owner.session); + for (const row of rows) { + assert.equal(context.service.resolvePlatformControl(owner.session, row.controlRef).organization.name, row.name); + } + assert.ok(performance.now() - start < 250, 'DSP polling must not stall Runtime Agent requests'); + assert.equal(context.store.db.prepare('SELECT total_changes() AS count').get().count, changes); + const expired = rows.find(row => row.name === 'Expired Invitation'); + assert.equal(expired.ownerStatus, 'missing'); + assert.equal(expired.ownerInvitation, null); + assert.ok(expired.availableActions.includes('issue_owner_invitation')); + } finally { writer.exec('ROLLBACK'); } +}); + +test('stateless platform controls reject tampering and support existing stored references', async t => { + const context = fixture(); t.after(() => context.close()); + const owner = await platformOwner(context.service); + const reference = context.service.issuePlatformControlRef(owner.session, 'local-dsp'); + const invalid = value => assert.throws(() => context.service.resolvePlatformControl(owner.session, value), /platform_control_invalid/); + for (const offset of [0, 7, 8, 31]) { + const bytes = Buffer.from(reference, 'base64url'); bytes[offset] ^= 1; + invalid(bytes.toString('base64url')); + } + const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'; + invalid(reference.slice(0, -1) + alphabet[alphabet.indexOf(reference.at(-1)) + 1]); + invalid(context.service.issuePlatformControlRef(owner.session, 'missing-organization')); + const crypto = require('node:crypto'); + const legacy = crypto.randomBytes(32).toString('base64url'); + context.store.createPlatformTargetRef({ referenceHash: crypto.createHash('sha256').update(legacy).digest('hex'), + sessionHash: owner.session.tokenHash, userId: owner.session.user.id, organizationId: 'local-dsp', + purpose: 'organization_control', expiresAt: context.time.value + 1000, timestamp: context.time.value }); + assert.equal(context.service.resolvePlatformControl(owner.session, legacy).organizationId, 'local-dsp'); + const another = context.service.createSession(owner.session.user.id); + for (const value of [reference, legacy]) { + assert.throws(() => context.service.resolvePlatformControl(another.session, value), /platform_control_invalid/); + } + context.time.value += 1000; + invalid(legacy); + context.time.value += 15 * 60 * 1000; + invalid(reference); + context.time.value = Date.parse(owner.session.expiresAt) - 1000; + const lastReference = context.service.issuePlatformControlRef(owner.session, 'local-dsp'); + assert.equal(Number(Buffer.from(lastReference, 'base64url').readBigUInt64BE()), Date.parse(owner.session.expiresAt)); + context.time.value += 1000; + invalid(lastReference); +}); + +test('session activity never waits for a DSP writer lock and still enforces revocation and expiry', async t => { + const context = fixture(); t.after(() => context.close()); + const owner = await platformOwner(context.service); + const { DatabaseSync } = require('node:sqlite'); + const writer = new DatabaseSync(context.paths.database); t.after(() => writer.close()); + const hash = owner.session.tokenHash; + const before = context.store.db.prepare('SELECT last_seen_at FROM sessions WHERE token_hash=?').get(hash).last_seen_at; + context.time.value += 1000; + writer.exec('BEGIN IMMEDIATE'); + try { + const start = performance.now(); + assert.equal(context.service.requireSession(owner.token).user.id, owner.session.user.id); + assert.ok(performance.now() - start < 250, 'session reads must not stall the Runtime Agent connection'); + assert.equal(context.store.db.prepare('PRAGMA busy_timeout').get().timeout, 3000); + assert.equal(context.store.db.prepare('SELECT last_seen_at FROM sessions WHERE token_hash=?').get(hash).last_seen_at, before); + } finally { writer.exec('ROLLBACK'); } + context.service.requireSession(owner.token); + assert.equal(context.store.db.prepare('SELECT last_seen_at FROM sessions WHERE token_hash=?').get(hash).last_seen_at, context.time.value); + writer.prepare('UPDATE sessions SET expires_at=? WHERE token_hash=?').run(context.time.value, hash); + assert.equal(context.service.session(owner.token), null); + writer.prepare('UPDATE sessions SET expires_at=? WHERE token_hash=?').run(context.time.value + 10000, hash); + writer.prepare('UPDATE users SET auth_version=auth_version+1 WHERE id=?').run(owner.session.user.id); + assert.equal(context.service.session(owner.token), null); +}); + +test('schema 13 migrates every DSP and preserves memberships, sessions, and invitation links on reopen', async t => { + const context = fixture(); + t.after(() => context.close()); + const { store, service } = context; + const owner = await platformOwner(service); + store.createOrganization({ id: 'second-dsp', name: 'Second DSP', abbreviation: null, timezone: 'UTC', status: 'suspended', createdBy: owner.session.user.id, timestamp: service.now() }); + service.ensureSystemRoles('second-dsp', owner.session.user.id, service.now()); + const canonical = ['owner', 'manager'].map(key => store.roleByKey('local-dsp', key)); + const dispatcher = store.roleByKey('local-dsp', 'dispatcher'); + const memberInvite = service.createMemberInvitation(owner.session, 'local-dsp', { email: 'legacy-member@example.test', roleId: dispatcher.id }); + const member = await service.acceptNewUser({ token: memberInvite.token, firstName: 'Legacy', lastName: 'Member', + password: 'migration secure password', confirmPassword: 'migration secure password' }); + const membershipId = store.membership(member.session.user.id, 'local-dsp').id; + const invitations = []; + for (const org of store.organizations()) { + // Recreate the previous catalog, including a custom role named Dispatcher. + store.db.prepare("UPDATE roles SET key='administrator',name='Administrator' WHERE organization_id=? AND key='dispatcher'").run(org.id); + store.db.prepare("UPDATE roles SET key='viewer',name='Viewer' WHERE organization_id=? AND key='driver'").run(org.id); + for (const [suffix, name] of [['matching', 'Dispatcher'], ['custom', 'Route lead']]) { + store.createRole({ id: `${org.id}_${suffix}`, organizationId: org.id, key: null, name, description: 'Legacy custom role', + system: false, permissions: ['dashboard.view'], createdBy: owner.session.user.id, timestamp: service.now() }); + } + for (const role of store.roles(org.id)) { + const rawToken = require('../src/service').opaqueToken(); + const invitation = store.createInvitation({ id: `legacy_${role.id}`, kind: 'organization_member', organizationId: org.id, + roleId: role.id, email: `${role.id}@example.test`, tokenHash: require('../src/service').tokenHash(rawToken), + expiresAt: service.now() + 3600000, createdBy: owner.session.user.id, timestamp: service.now() }); + if (role.name === 'Route lead') store.revokeInvitation(invitation.id); + invitations.push({ ...store.invitationById(invitation.id), rawToken, expectedRole: role.key === 'administrator' ? 'Manager' + : role.key === 'viewer' ? 'Driver' : role.system ? role.name : 'Dispatcher' }); + } + } + store.db.exec("DELETE FROM role_permissions WHERE permission <> 'dashboard.view'; PRAGMA user_version=12;"); + store.close(); + const migrated = new AccessStore(context.paths); + t.after(() => migrated.close()); + const access = new AccessControlService(migrated, { clock: () => new Date(context.time.value) }); + assert.equal(migrated.db.prepare('PRAGMA user_version').get().user_version, require('../src/schema').SCHEMA_VERSION); + assert.deepEqual(migrated.db.prepare('PRAGMA foreign_key_check').all(), []); + for (const org of migrated.organizations()) { + const roles = migrated.roles(org.id); + assert.deepEqual(roles.map(role => role.name), ['Owner', 'Manager', 'Dispatcher', 'Driver']); + for (const role of roles) assert.deepEqual(role.permissions, roles[0].permissions); + } + for (const role of canonical) assert.equal(migrated.roleByKey('local-dsp', role.key).id, role.id); + assert.equal(migrated.membership(member.session.user.id, 'local-dsp').id, membershipId); + assert.equal(access.session(member.token).memberships[0].roleName, 'Manager'); + assert.equal(migrated.invitationById(memberInvite.invitation.id).status, 'accepted'); + for (const original of invitations) { + const current = migrated.invitationById(original.id); + assert.equal(current.roleName, original.expectedRole); + for (const field of ['status', 'expiresAt', 'email', 'createdAt', 'acceptedAt']) assert.equal(current[field], original[field]); + assert.equal(migrated.invitationByHash(require('../src/service').tokenHash(original.rawToken)).id, original.id); + if (original.organizationId === 'local-dsp' && original.status === 'pending') { + assert.equal(access.inspectInvitation(original.rawToken).role.name, original.expectedRole); + } + } + const selected = invitations.find(invite => invite.organizationId === 'local-dsp' && invite.expectedRole === 'Driver'); + const accepted = await access.acceptNewUser({ token: selected.rawToken, firstName: 'Migrated', lastName: 'Driver', + password: 'migration secure password', confirmPassword: 'migration secure password' }); + assert.equal(accepted.session.memberships[0].roleName, 'Driver'); + const snapshot = migrated.db.prepare('SELECT * FROM roles ORDER BY id').all(); + migrated.close(); + const reopened = new AccessStore(context.paths); + assert.deepEqual(reopened.db.prepare('SELECT * FROM roles ORDER BY id').all(), snapshot); + reopened.close(); +}); + +test('failed fixed-role migration rolls back role references and schema version', t => { + const context = fixture(); + t.after(() => context.close()); + const { store } = context; + store.db.exec("UPDATE roles SET key='viewer',name='Viewer' WHERE key='driver'; PRAGMA user_version=12;"); + const original = store.db.prepare('SELECT * FROM roles ORDER BY id').all(); + store.db.exec("CREATE TRIGGER fail_fixed_roles BEFORE DELETE ON roles BEGIN SELECT RAISE(ABORT, 'migration_fixture_failure'); END;"); + store.close(); + assert.throws(() => new AccessStore(context.paths), /access_storage_unavailable/); + const { DatabaseSync } = require('node:sqlite'); + const raw = new DatabaseSync(context.paths.database); + assert.equal(raw.prepare('PRAGMA user_version').get().user_version, 12); + assert.deepEqual(raw.prepare('SELECT * FROM roles ORDER BY id').all(), original); + raw.exec('DROP TRIGGER fail_fixed_roles'); + raw.close(); + const retried = new AccessStore(context.paths); + assert.equal(retried.db.prepare('PRAGMA user_version').get().user_version, require('../src/schema').SCHEMA_VERSION); + assert.deepEqual(retried.roles('local-dsp').map(role => role.name), ['Owner', 'Manager', 'Dispatcher', 'Driver']); + retried.close(); +}); diff --git a/core/core/accounts/tests/installation-activation.test.js b/core/core/accounts/tests/installation-activation.test.js new file mode 100644 index 0000000..9cffc01 --- /dev/null +++ b/core/core/accounts/tests/installation-activation.test.js @@ -0,0 +1,370 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { DatabaseSync } = require('node:sqlite'); +const { + AccessStore, + AccessControlService, + createAccessInstallationActivationAuthority, +} = require('../src'); +const { runManagedPaycomActivation } = require('../../../compatibility/provisioner/src/activation.js'); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-access-activation-')); + fs.chmodSync(root, 0o700); + const paths = { + databaseRoot: path.join(root, 'access-control'), + database: path.join(root, 'access-control', 'access-control.sqlite3'), + }; + const time = { value: Date.parse('2026-09-02T21:30:00.000Z') }; + const store = new AccessStore(paths); + const service = new AccessControlService(store, { clock: () => new Date(time.value) }); + service.ensureLocalOrganization({ + organization: { id: 'local-dsp', name: 'EXMP' }, + site: { id: 'local-site', code: 'TST1' }, + timezone: 'America/Los_Angeles', + }); + return { root, paths, time, store, service }; +} + +async function managedOwner(context) { + const bootstrap = context.service.createPlatformBootstrap({ email: 'platform@example.test', organizationId: 'local-dsp' }); + const platform = await context.service.acceptNewUser({ + token: bootstrap.token, + firstName: 'Platform', + lastName: 'Owner', + password: 'platform owner passphrase', + confirmPassword: 'platform owner passphrase', + }); + const created = context.service.createOrganization(platform.session, { + idempotencyKey: 'test:organization:create:activation', + name: 'Activation DSP', + abbreviation: 'ACT', + stationCode: 'TST1', + timezone: 'America/Chicago', + ownerEmail: 'owner@activation.test', + }); + const owner = await context.service.acceptNewUser({ + token: created.token, + firstName: 'Activation', + lastName: 'Owner', + password: 'activation owner passphrase', + confirmPassword: 'activation owner passphrase', + }); + const control = context.store.installationControl(created.organization.id); + context.store.updateInstallationControl({ + organizationId: created.organization.id, + expectedStatus: 'pending', + expectedRevision: control.revision, + status: 'waiting_for_provider_auth', + revision: control.revision + 1, + currentJobId: null, + timestamp: context.time.value, + }); + return { organizationId: created.organization.id, owner }; +} + +function runtime(time) { + let definitionDigest = null; + let requestDigest = null; + return { + verifyInfrastructure: async manifest => ({ + runtimeKey: manifest.runtime.key, + runtime_layout: true, + service_supervision: true, + auth_broker: true, + collection_manager: true, + runtime_gateway: true, + }), + configure: async definition => { + definitionDigest = definition.digest; + return { + digest: definition.digest, + collectors: 1, + sources: 1, + plans: 15, + syncs: 1, + }; + }, + testProvider: async profileId => ({ + profileId, + provider: 'paycom', + status: 'authenticated', + testedAt: new Date(time.value).toISOString(), + }), + publishFirst: async request => { + requestDigest = crypto.createHash('sha256').update(JSON.stringify(request)).digest('hex'); + return { + batchId: 'batch_activation_001', + preparationRunId: 'run_periods', + status: 'succeeded', + runCount: 5, + succeededRuns: 5, + failedRuns: 0, + cancelledRuns: 0, + }; + }, + verifyPublication: async batchId => ({ + definitionDigest, + requestDigest, + previewDigest: 'a'.repeat(64), + batchId, + preparationRunId: 'run_periods', + target: '2026-09-05', + runs: [ + { id: 'run_roster', taskId: 'roster', plan: 'paycom-period-roster', method: 'roster.period' }, + { id: 'run_timecards', taskId: 'timecards', plan: 'paycom-period-timecards-from-roster', method: 'timecards.from-published-roster' }, + { id: 'run_timecards_audit', taskId: 'timecards-audit', plan: 'paycom-period-timecards-audit', method: 'timecards.audit' }, + { id: 'run_links', taskId: 'links', plan: 'paycom-period-resource-links', method: 'resource-links.period' }, + { id: 'run_links_audit', taskId: 'links-audit', plan: 'paycom-period-resource-links-audit', method: 'resource-links.audit' }, + ], + publications: { + payPeriods: { id: 'pub_periods', runId: 'run_periods', originRunId: 'run_periods', contentSha256: '1'.repeat(64), batchBound: false }, + roster: { id: 'pub_roster', runId: 'run_roster', originRunId: 'run_roster', contentSha256: '2'.repeat(64), batchBound: true }, + timecards: { id: 'pub_timecards', runId: 'run_timecards', originRunId: 'run_timecards', contentSha256: '3'.repeat(64), batchBound: true }, + resourceLinks: { id: 'pub_links', runId: 'run_links', originRunId: 'run_links', contentSha256: '4'.repeat(64), batchBound: true }, + }, + capturedAt: new Date(time.value).toISOString(), + }), + }; +} + +test('Access Control atomically binds provider activation to owner, runtime, job, and readiness evidence', async () => { + const context = fixture(); + try { + const { organizationId, owner } = await managedOwner(context); + const authority = createAccessInstallationActivationAuthority({ + store: context.store, + organizationId, + authorityScope: 'platform_activation', + idempotencyKey: 'activation:fixture:0001', + workerId: 'worker_activation_a', + clock: () => context.time.value, + leaseMs: 10_000, + jobFactory: () => 'job_activation_001', + releaseId: 'dispatch_fixture_1', + }); + const result = await runManagedPaycomActivation({ authority, runtime: runtime(context.time) }); + assert.deepEqual(result, { + ok: true, + status: 'ready', + state: 'ready', + revision: 4, + manifestRevision: 1, + gates: 9, + }); + assert.equal(context.store.installationControl(organizationId).status, 'ready'); + assert.equal(context.store.organization(organizationId).status, 'active'); + assert.equal(context.service.runtimeFor(owner.session, 'workforce.read').installation.runtimeKey.startsWith('runtime_'), true); + const job = context.store.activationJob('job_activation_001'); + assert.equal(job.status, 'succeeded'); + assert.equal(job.installation_state, 'ready'); + assert.equal(job.failure_code, null); + assert.match(job.evidence_digest, /^[a-f0-9]{64}$/); + const evidence = JSON.parse(job.evidence_json); + assert.equal(evidence.jobId, job.id); + assert.equal(evidence.batchId, 'batch_activation_001'); + assert.equal(evidence.evidenceDigest, job.evidence_digest); + assert.equal((await authority.inspect()).installation.state, 'ready'); + } finally { + context.store.close(); + fs.rmSync(context.root, { recursive: true, force: true }); + } +}); + +test('a durable setup lease fences credential mutation and activation across workers', async () => { + const context = fixture(); + try { + const { organizationId } = await managedOwner(context); + const common = { + store: context.store, + organizationId, + authorityScope: 'platform_activation', + clock: () => context.time.value, + leaseMs: 10_000, + setupLeaseMs: 10_000, + releaseId: 'dispatch_fixture_1', + }; + const first = createAccessInstallationActivationAuthority({ + ...common, + idempotencyKey: 'activation:fixture:setup:a', + workerId: 'worker_setup_a', + jobFactory: () => 'job_setup_blocked_a', + }); + const second = createAccessInstallationActivationAuthority({ + ...common, + idempotencyKey: 'activation:fixture:setup:b', + workerId: 'worker_setup_b', + jobFactory: () => 'job_setup_blocked_b', + }); + const activation = createAccessInstallationActivationAuthority({ + ...common, + idempotencyKey: 'activation:fixture:setup:activate', + workerId: 'worker_activation_c', + jobFactory: () => 'job_setup_activation_c', + }); + assert.throws(() => first.guard(() => true), /installation_operation_in_progress/); + first.beginSetup(); + assert.deepEqual(context.store.installationSetup(organizationId), { + workerId: 'worker_setup_a', fence: 1, leaseExpiresAt: context.time.value + 10_000, + }); + assert.throws(() => second.beginSetup(), /installation_operation_in_progress/); + assert.throws(() => activation.begin({ + profileId: 'paycom-main', provider: 'paycom', status: 'authenticated', + testedAt: new Date(context.time.value).toISOString(), + }), /installation_operation_in_progress/); + assert.equal(first.guard(() => 'mutated'), 'mutated'); + + context.time.value += 10_001; + second.beginSetup(); + assert.equal(context.store.installationSetup(organizationId).fence, 2); + assert.throws(() => first.guard(() => true), /installation_operation_in_progress/); + second.endSetup(); + const running = activation.begin({ + profileId: 'paycom-main', provider: 'paycom', status: 'authenticated', + testedAt: new Date(context.time.value).toISOString(), + }); + assert.equal(running.installation.state, 'verifying'); + assert.equal(context.store.installationSetup(organizationId).workerId, null); + } finally { + context.store.close(); + fs.rmSync(context.root, { recursive: true, force: true }); + } +}); + +test('an expired activation claim is fenced before stale completion', async () => { + const context = fixture(); + try { + const { organizationId } = await managedOwner(context); + const common = { + store: context.store, + organizationId, + authorityScope: 'platform_activation', + idempotencyKey: 'activation:fixture:0002', + clock: () => context.time.value, + leaseMs: 10_000, + jobFactory: () => 'job_activation_002', + releaseId: 'dispatch_fixture_1', + }; + const first = createAccessInstallationActivationAuthority({ ...common, workerId: 'worker_activation_a' }); + const initial = await first.inspect(); + const running = await first.begin({ + profileId: 'paycom-main', provider: 'paycom', status: 'authenticated', + testedAt: new Date(context.time.value).toISOString(), + }); + assert.equal(initial.installation.state, 'waiting_for_provider_auth'); + assert.equal(running.installation.state, 'verifying'); + + const replacement = createAccessInstallationActivationAuthority({ ...common, workerId: 'worker_activation_b' }); + const observed = await replacement.peek(); + assert.equal(observed.installation.state, 'verifying'); + assert.equal(context.store.activationJob('job_activation_002').worker_id, 'worker_activation_a'); + assert.equal(context.store.activationJob('job_activation_002').fence, 1); + + context.time.value += 10_001; + const reclaimed = await replacement.inspect(); + assert.equal(reclaimed.job.id, 'job_activation_002'); + assert.throws(() => first.fail('job_activation_002', { code: 'first_publication_failed' }), + /installation_operation_in_progress/); + const failed = await replacement.fail('job_activation_002', { code: 'first_publication_failed' }); + assert.equal(failed.status, 'failed'); + assert.equal(context.store.installationControl(organizationId).status, 'failed'); + const retry = await replacement.retry(); + assert.equal(retry.installation.state, 'waiting_for_provider_auth'); + assert.equal(retry.installation.currentJobId, null); + } finally { + context.store.close(); + fs.rmSync(context.root, { recursive: true, force: true }); + } +}); + +test('schema version 2 installation bindings migrate conservatively to the full lifecycle', () => { + const context = fixture(); + const local = context.store.installationControl('local-dsp'); + assert.equal(local.status, 'ready'); + context.store.close(); + const db = new DatabaseSync(context.paths.database); + try { + db.exec(`INSERT INTO organizations(id,name,abbreviation,timezone,status,created_at,updated_at) + VALUES('org_unverified_ready','Unverified Ready','UR','America/Los_Angeles','setup_required',1000,1000); + INSERT INTO stations(organization_id,code,is_primary,created_at) + VALUES('org_unverified_ready','BAD1',1,1000); + INSERT INTO installations( + organization_id,runtime_key,status,revision,manifest_revision,current_job_id,created_at,updated_at + ) VALUES('org_unverified_ready','runtime_unverified_ready','ready',1,1,NULL,1000,1000); + PRAGMA foreign_keys=OFF; + DROP INDEX one_running_installation_activation; + DROP TABLE installation_activation_jobs; + DROP INDEX one_active_installation_provisioning; + DROP TABLE installation_provisioning_requests; + ALTER TABLE installations RENAME TO installations_v3; + CREATE TABLE installations ( + organization_id TEXT PRIMARY KEY REFERENCES organizations(id) ON DELETE CASCADE, + runtime_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL CHECK(status IN ('pending','ready','disabled')), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) STRICT; + INSERT INTO installations(organization_id,runtime_key,status,created_at,updated_at) + SELECT organization_id,runtime_key,status,created_at,updated_at FROM installations_v3; + DROP TABLE installations_v3; + PRAGMA user_version=2; + PRAGMA foreign_keys=ON;`); + } finally { db.close(); } + const migrated = new AccessStore(context.paths); + try { + assert.equal(migrated.db.prepare('PRAGMA user_version').get().user_version, require('../src/schema').SCHEMA_VERSION); + assert.deepEqual(migrated.installationControl('local-dsp'), { + organizationId: 'local-dsp', + runtimeKey: 'local', + status: 'ready', + revision: 1, + manifestRevision: 1, + releaseId: 'dispatch_current_1', + currentJobId: null, + }); + assert.equal(migrated.installationBackend('local-dsp'), 'local_reference'); + assert.equal(migrated.installationBackend('org_unverified_ready'), 'systemd_user'); + assert.throws(() => migrated.db.prepare("UPDATE installations SET backend='oci_container_v1' WHERE organization_id='org_unverified_ready'").run(), + /installation_backend_immutable/); + assert.equal(migrated.installationControl('org_unverified_ready').status, 'failed'); + } finally { + migrated.close(); + fs.rmSync(context.root, { recursive: true, force: true }); + } +}); + +test('schema version 4 upgrades add lifecycle, backup, release, and Runtime Agent authority tables', () => { + const context = fixture(); + context.store.close(); + const legacy = new DatabaseSync(context.paths.database); + try { + legacy.exec(` + DROP INDEX installation_backups_by_organization; + DROP TABLE installation_backups; + DROP INDEX one_active_installation_lifecycle; + DROP TABLE installation_lifecycle_jobs; + DROP TRIGGER installations_backend_immutable; + ALTER TABLE installations DROP COLUMN release_id; + ALTER TABLE installations DROP COLUMN backend; + PRAGMA user_version=4; + `); + } finally { legacy.close(); } + + const migrated = new AccessStore(context.paths); + try { + assert.equal(migrated.db.prepare('PRAGMA user_version').get().user_version, require('../src/schema').SCHEMA_VERSION); + assert.equal(migrated.db.prepare("SELECT count(*) AS count FROM sqlite_schema WHERE type='table' AND name IN ('installation_lifecycle_jobs','installation_backups')").get().count, 2); + assert.equal(migrated.installationControl('local-dsp').releaseId, 'dispatch_current_1'); + assert.equal(migrated.installationBackend('local-dsp'), 'local_reference'); + assert.equal(migrated.installationControl('local-dsp').status, 'ready'); + } finally { + migrated.close(); + fs.rmSync(context.root, { recursive: true, force: true }); + } +}); diff --git a/core/core/accounts/tests/owner-admin-terminal.py b/core/core/accounts/tests/owner-admin-terminal.py new file mode 100644 index 0000000..42112f6 --- /dev/null +++ b/core/core/accounts/tests/owner-admin-terminal.py @@ -0,0 +1,63 @@ +"""Exercise the real CLI with a controlling terminal; fixture credentials only.""" +import errno +import fcntl +import os +import pty +import select +import subprocess +import sys +import termios +import time + +binary, root = sys.argv[1:] + +def invoke(action, entries, expected): + master, slave = pty.openpty() + original = termios.tcgetattr(slave) + def controlling_terminal(): + os.setsid() + fcntl.ioctl(slave, termios.TIOCSCTTY, 0) + env = dict(os.environ, DISPATCH_LOCAL_ROOT=root) + child = subprocess.Popen([binary, action], stdin=slave, stdout=slave, stderr=slave, + env=env, preexec_fn=controlling_terminal) + transcript = b'' + position = 0 + deadline = time.monotonic() + 8 + try: + while time.monotonic() < deadline: + ready, _, _ = select.select([master], [], [], 0.1) + if ready: + try: + chunk = os.read(master, 8192) + except OSError as error: + if error.errno == errno.EIO: + break + raise + transcript += chunk + if position < len(entries): + prompt, value = entries[position] + if prompt.encode() in transcript: + assert not termios.tcgetattr(slave)[3] & termios.ECHO, 'input echo enabled' + os.write(master, (value + '\n').encode()) + position += 1 + if child.poll() is not None: + break + assert child.wait(timeout=1) == 0, transcript.decode() + assert expected.encode() in transcript, transcript.decode() + for _, value in entries: + if value: + assert value.encode() not in transcript, 'credential echoed' + assert termios.tcgetattr(slave) == original, 'terminal mode not restored' + finally: + if child.poll() is None: + child.kill() + child.wait() + os.close(master) + os.close(slave) + +invoke('owner-create', [('Owner email:', 'terminal@example.test'), ('First name:', 'Fixture'), + ('Last name:', 'Administrator'), ('New password (12', 'terminal fixture initial password'), + ('Confirm new password:', 'terminal fixture initial password')], 'platform_owner_created') +invoke('owner-recover', [('Current owner email:', 'terminal@example.test'), ('New owner email (', 'terminal-new@example.test'), + ('New password (12', 'terminal fixture replacement password'), + ('Confirm new password:', 'terminal fixture replacement password')], 'platform_owner_recovered') diff --git a/core/core/accounts/tests/owner-admin.test.js b/core/core/accounts/tests/owner-admin.test.js new file mode 100644 index 0000000..2e55d3d --- /dev/null +++ b/core/core/accounts/tests/owner-admin.test.js @@ -0,0 +1,121 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const test = require('node:test'); +const { AccessStore, AccessControlService } = require('../src'); +const { administerOwner } = require('../src/owner-admin'); +const { main } = require('../src/owner-admin-cli'); +const { resolveLocalRuntimePaths } = require('../../../shared/paths/runtime-paths'); +const { hashPassword } = require('../src/passwords'); +const PASSWORD = 'fixture initial owner password'; +const NEXT = 'fixture replacement owner password'; +const BIN = path.resolve(__dirname, "../../../bin/dispatch-access-admin"); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-owner-admin-')); + const paths = resolveLocalRuntimePaths({ localRoot: root }); + fs.mkdirSync(paths.dataRoot, { recursive: true, mode: 0o700 }); + const store = new AccessStore(paths.accessControl); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + return { root, paths, store, service: new AccessControlService(store) }; +} +const creation = () => ({ email: 'platform@example.test', firstName: 'Platform', lastName: 'Owner', password: PASSWORD, confirmPassword: PASSWORD }); + +test('CLI creates a platform-only owner without any DSP and invalidates old bootstrap invitations', async t => { + const { store, service } = fixture(t); + const pending = service.createPlatformBootstrap({ email: 'bootstrap@example.test' }); + await administerOwner(store, 'owner-create', creation()); + const login = await service.signIn({ email: 'platform@example.test', password: PASSWORD }); + assert.equal(login.session.activeOrganizationId, null); + assert.deepEqual(login.session.memberships, []); + assert.ok(login.session.platformPermissions.includes('platform.organizations.read')); + assert.equal(store.organizations().length, 0); + assert.throws(() => service.inspectInvitation(pending.token), /invitation_invalid/); + assert.notEqual(store.userByEmail('platform@example.test').password_hash, PASSWORD); + await assert.rejects(administerOwner(store, 'owner-create', { ...creation(), email: 'second@example.test' }), /platform_owner_exists/); +}); + +test('recovery changes email/password, restores disabled owner and invalidates sessions without altering membership', async t => { + const { store, service } = fixture(t); + service.ensureLocalOrganization({ organization: { id: 'local-dsp', name: 'Legacy DSP' }, site: { code: 'TST1' }, timezone: 'UTC' }); + const bootstrap = service.createPlatformBootstrap({ email: 'platform@example.test', organizationId: 'local-dsp' }); + const accepted = await service.acceptNewUser({ token: bootstrap.token, firstName: 'Platform', lastName: 'Owner', password: PASSWORD, confirmPassword: PASSWORD }); + assert.equal(accepted.session.activeOrganizationId, null, 'platform login must not select a legacy DSP'); + const second = await service.signIn({ email: 'platform@example.test', password: PASSWORD }); + const memberships = store.membershipsForUser(accepted.session.user.id); + store.db.prepare("UPDATE users SET status='disabled' WHERE id=?").run(accepted.session.user.id); + await administerOwner(store, 'owner-recover', { email: 'platform@example.test', newEmail: 'recovered@example.test', password: NEXT, confirmPassword: NEXT }); + assert.equal(service.session(accepted.token), null); + assert.equal(service.session(second.token), null); + await assert.rejects(service.signIn({ email: 'platform@example.test', password: PASSWORD }), /invalid_credentials/); + await assert.rejects(service.signIn({ email: 'recovered@example.test', password: PASSWORD }), /invalid_credentials/); + const recovered = await service.signIn({ email: 'recovered@example.test', password: NEXT }); + assert.equal(recovered.session.user.id, accepted.session.user.id); + assert.equal(recovered.session.activeOrganizationId, null); + assert.deepEqual(store.membershipsForUser(recovered.session.user.id), memberships); + const audit = JSON.stringify(store.db.prepare("SELECT * FROM audit_events WHERE action LIKE 'platform.owner.%'").all()); + assert.match(audit, /recover_cli/); + assert.equal(audit.includes(NEXT), false); +}); + +test('recovery cannot promote a DSP user or take another account email, and failed recovery is atomic', async t => { + const { store, service } = fixture(t); + await administerOwner(store, 'owner-create', creation()); + store.insertUser({ id: 'usr_tenant', email: 'tenant@example.test', firstName: 'DSP', lastName: 'Owner', passwordHash: await hashPassword(PASSWORD), platformRole: null, timestamp: Date.now() }); + const login = await service.signIn({ email: 'platform@example.test', password: PASSWORD }); + const recovery = { email: 'platform@example.test', newEmail: 'tenant@example.test', password: NEXT, confirmPassword: NEXT }; + await assert.rejects(administerOwner(store, 'owner-recover', recovery), /email_in_use/); + await assert.rejects(administerOwner(store, 'owner-recover', { ...recovery, email: 'tenant@example.test', newEmail: '' }), /platform_owner_not_found/); + await assert.rejects(administerOwner(store, 'owner-recover', { ...recovery, newEmail: '', confirmPassword: 'wrong' }), /password_confirmation_mismatch/); + assert.ok(service.session(login.token)); + assert.equal(store.userByEmail('tenant@example.test').platform_role, null); +}); + +test('owner CLI rejects credential arguments, refuses missing recovery databases and does not leak failures', async t => { + const { paths } = fixture(t); + let output = ''; + const write = value => { output += value; }; + const collect = () => { throw new Error(NEXT); }; + assert.equal(await main(['owner-recover', '--password', NEXT], { paths, collect, write }), 2); + assert.equal(output.includes(NEXT), false); + output = ''; + assert.equal(await main(['owner-recover'], { paths, collect, write }), 1); + assert.equal(output.includes(NEXT), false); + assert.match(output, /owner_admin_failed/); + const result = spawnSync(BIN, ['owner-create'], { detached: true, encoding: 'utf8', env: { ...process.env, DISPATCH_LOCAL_ROOT: paths.localRoot || path.dirname(paths.dataRoot) }, input: NEXT }); + assert.equal(result.status, 1); + assert.match(result.stdout, /tty_required/); + assert.equal((result.stdout + result.stderr).includes(NEXT), false); +}); + +test('actual private-terminal create and recovery hide inputs and restore terminal echo', async t => { + const { root } = fixture(t); + const freshRoot = path.join(root, 'fresh'); + const result = spawnSync('python3', [path.join(__dirname, "./owner-admin-terminal.py"), BIN, freshRoot], { encoding: 'utf8', timeout: 20000 }); + assert.equal(result.status, 0, result.stderr + result.stdout); + const store = new AccessStore(resolveLocalRuntimePaths({ localRoot: freshRoot }).accessControl); + try { + const login = await new AccessControlService(store).signIn({ email: 'terminal-new@example.test', password: 'terminal fixture replacement password' }); + assert.equal(login.session.activeOrganizationId, null); + assert.deepEqual(login.session.memberships, []); + } finally { store.close(); } +}); + +test('credential replacement fences in-flight login and password-change requests', async t => { + const { store, service } = fixture(t); + await administerOwner(store, 'owner-create', creation()); + const session = (await service.signIn({ email: 'platform@example.test', password: PASSWORD })).session; + const replacementHash = await hashPassword(NEXT); + const login = service.signIn({ email: 'platform@example.test', password: PASSWORD }); + const change = service.changePassword(session, { currentPassword: PASSWORD, newPassword: 'stale competing new password', confirmPassword: 'stale competing new password' }); + // Recovery commits while both requests are awaiting their password hash work. + store.transaction(() => { + store.updatePassword(session.user.id, replacementHash, Date.now()); + store.deleteUserSessions(session.user.id); + }); + await assert.rejects(login, /invalid_credentials/); + await assert.rejects(change, /authentication_required/); + assert.ok(await service.signIn({ email: 'platform@example.test', password: NEXT })); +}); diff --git a/core/core/accounts/tests/owner-connections.test.js b/core/core/accounts/tests/owner-connections.test.js new file mode 100644 index 0000000..cda4696 --- /dev/null +++ b/core/core/accounts/tests/owner-connections.test.js @@ -0,0 +1,161 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { AccessStore, AccessControlService } = require('../src'); +const { createOwnerPaycomSetup } = require('../src/owner-paycom-setup'); +const { createOwnerConnections } = require('../src/owner-connections'); +const { createOnboardingStore } = require('../src/onboarding-store'); +const { success } = require('../../../shared/contracts/src'); +const CREDENTIALS = { clientCode: 'fixture-code', username: 'fixture-user', password: 'fixture-secret-never-persist', + pin1: 'one', pin2: 'two', pin3: 'three', pin4: 'four', pin5: 'five' }; +async function fixture(t, backend = 'oci_container_v1') { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-owner-setup-'));fs.chmodSync(root, 0o700); + const paths = { databaseRoot: path.join(root, 'access'), database: path.join(root, 'access', 'access-control.sqlite3') }; + const store = new AccessStore(paths); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true }); }); + const service = new AccessControlService(store, { installationOperatorEnabled: true, installationBackend: backend }); + const bootstrap = service.createPlatformBootstrap({ email: 'platform@example.test' }); + const platform = await service.acceptNewUser({ token: bootstrap.token, firstName: 'Platform', lastName: 'Owner', + password: 'fixture platform password', confirmPassword: 'fixture platform password' }); + const created = service.createOrganization(platform.session, { idempotencyKey: 'fixture:owner:setup', name: 'Fixture Setup DSP', + abbreviation: 'FIX', stationCode: 'DWA1', timezone: 'America/Chicago', ownerEmail: 'dsp@example.test' }); + const owner = await service.acceptNewUser({ token: created.token, firstName: 'DSP', lastName: 'Owner', + password: 'fixture dsp password', confirmPassword: 'fixture dsp password' }); + const organizationId = created.organization.id; + store.updateOrganizationStatus(organizationId, 'active', Date.now()); + store.updateInstallationControl({ organizationId, expectedStatus: 'pending', expectedRevision: 1, + status: 'waiting_for_provider_auth', revision: 2, currentJobId: null, timestamp: Date.now() }); + require('./plugin-fixture').enableFixturePlugin(store, organizationId); + return { root, paths, store, service, platform, owner, organizationId }; +} + +const blank = service => ({ service, configured: false, state: 'not_connected', checkedAt: null, reason: null, retryAt: null }); +const connected = service => ({ ...blank(service), configured: true, state: 'checking' }); +const list = () => success('found', { items: ['cortex', 'paycom'].map(blank) }); + +test('connection ownership is scoped to the selected DSP and credentials never enter Core storage', async t => { + const f = await fixture(t, 'native_service_v1'); const calls = []; + const connections = createOwnerConnections({ store: f.store, access: f.service, invoke: async (...args) => { + calls.push(args); return args[2].command === 'list' ? list() : success('accepted', connected(args[2].service)); + } }); + assert.equal((await connections.list(f.owner.session)).items.length, 2); + await connections.change(f.owner.session, 'cortex', 'save', { credentials: { username: 'owner', password: 'do-not-persist-cortex' } }); + assert.equal(calls[1][0], f.store.installationControl(f.organizationId).runtimeKey); + assert.equal(calls[1][1], 'connections.manage'); + assert.equal(calls[1][2].credentials.password, 'do-not-persist-cortex'); + assert.ok(calls[1][2].expiresAt <= Date.now() + 30_000); + assert.equal(fs.readFileSync(f.paths.database).includes(Buffer.from('do-not-persist-cortex')), false); + const before = calls.length; + await assert.rejects(connections.list(f.platform.session)); + await assert.rejects(connections.list({ ...f.owner.session, activeOrganizationId: 'org_nonexistent' })); + await assert.rejects(connections.list({ ...f.owner.session, dspView: { viewRef: 'support' } }), /dsp_view_unavailable/); + await assert.rejects(connections.change(f.owner.session, 'cortex', 'save', { credentials: { username: 'a', password: 'b' }, runtimeKey: 'another-dsp' }), /invalid_input/); + assert.equal(calls.length, before); + const membership = f.store.membership(f.owner.session.user.id, f.organizationId); + f.store.updateMembershipRole(membership.id, f.store.roleByKey(f.organizationId, 'manager').id, Date.now()); + await assert.rejects(connections.list(f.owner.session), /permission_denied/); + await assert.rejects(connections.change(f.owner.session, 'cortex', 'test', {}), /permission_denied/); + const paycomSetup = createOwnerPaycomSetup({ store: f.store, access: f.service, invoke: async () => list() }); + await assert.rejects(paycomSetup.submit(f.owner.session, { credentials: CREDENTIALS }), /permission_denied/); + await assert.rejects(paycomSetup.retry(f.owner.session, {}), /permission_denied/); + assert.equal(calls.length, before); +}); + +test('connection responses reject secret fields and authorization is checked again after transport', async t => { + const f = await fixture(t); let scenario = 'secret'; + const connections = createOwnerConnections({ store: f.store, access: f.service, invoke: async () => { + if (scenario === 'secret') return success('accepted', { ...connected('cortex'), password: 'must-not-return' }); + f.store.updateOrganizationStatus(f.organizationId, 'suspended', Date.now()); + return list(); + } }); + await assert.rejects(connections.change(f.owner.session, 'cortex', 'test', {}), /auth_unavailable/); + scenario = 'suspend'; + await assert.rejects(connections.list(f.owner.session)); +}); + +test('Paycom first enrollment still queues its existing collection onboarding', async t => { + const f = await fixture(t); const calls = []; let configured = false; + const invoke = async (key, action, input) => { + calls.push({ key, action, input }); + if (action === 'paycom.setup') { configured = true; return success('succeeded', { configured: true }); } + return success('found', { items: [blank('cortex'), configured ? { ...connected('paycom'), state: 'not_verified' } : blank('paycom')] }); + }; + const paycomSetup = createOwnerPaycomSetup({ store: f.store, access: f.service, invoke }); + const connections = createOwnerConnections({ store: f.store, access: f.service, invoke, paycomSetup }); + const result = await connections.change(f.owner.session, 'paycom', 'save', { credentials: CREDENTIALS }); + assert.equal(result.configured, true); + assert.equal(createOnboardingStore(f.store).latest(f.organizationId).status, 'queued'); + assert.equal(calls.filter(call => call.action === 'paycom.setup').length, 1); + assert.equal(fs.readFileSync(f.paths.database).includes(Buffer.from(CREDENTIALS.password)), false); +}); + +function platformView(f, organizationId = f.organizationId) { + const session = f.service.session(f.platform.token); + return f.service.beginDspView(session, { controlRef: f.service.issuePlatformControlRef(session, organizationId) }); +} + +test('platform owners manage each viewed DSP connection with scoped routing and attributed audit records', async t => { + const f = await fixture(t, 'directory_service_v1'), calls = []; + const other = f.service.createOrganization(f.platform.session, { idempotencyKey: 'fixture:connections:other', + name: 'Other Fixture DSP', abbreviation: 'OTH', stationCode: 'TST2', timezone: 'UTC', ownerEmail: 'other@example.test' }); + f.store.updateOrganizationStatus(other.organization.id, 'active', Date.now()); + f.store.updateInstallationControl({ organizationId: other.organization.id, expectedStatus: 'pending', expectedRevision: 1, + status: 'ready', revision: 2, currentJobId: null, timestamp: Date.now() }); + const connections = createOwnerConnections({ store: f.store, access: f.service, invoke: async (key, _action, input) => { + calls.push({ key, input }); return input.command === 'list' ? list() : success('accepted', connected(input.service)); + } }); + const first = platformView(f), second = platformView(f, other.organization.id); + for (const selected of [first, second]) { + await connections.list(selected); + await connections.change(selected, 'cortex', 'save', { credentials: { username: 'fixture', password: 'synthetic support secret' } }); + await connections.change(selected, 'cortex', 'test', {}); + await connections.change(selected, 'cortex', 'disconnect', {}); + assert.equal(calls.at(-1).key, f.store.installationControl(selected.activeOrganizationId).runtimeKey); + } + assert.equal(calls.length, 8); + const audited = f.store.db.prepare("SELECT actor_user_id,organization_id FROM audit_events WHERE action LIKE 'connection.%'").all(); + assert.equal(audited.length, 6); + assert.ok(audited.every(row => row.actor_user_id === f.platform.session.user.id)); + assert.equal(f.store.membership(f.platform.session.user.id, f.organizationId), null); + await assert.rejects(connections.list({ ...first, activeOrganizationId: other.organization.id }), /organization_forbidden/); + await assert.rejects(connections.list({ ...f.owner.session, dspView: first.dspView }), /dsp_view_unavailable/); + await assert.rejects(connections.change(first, 'cortex', 'save', { + credentials: { username: 'fixture', password: 'synthetic' }, organizationId: other.organization.id, + }), /invalid_input/); + f.store.db.prepare('UPDATE users SET platform_role=NULL WHERE id=?').run(f.platform.session.user.id); + await assert.rejects(connections.list(first), /dsp_view_unavailable/); + assert.equal(calls.length, 8); +}); + +test('platform view is reauthorized after awaited connection work and expired sessions receive no result', async t => { + const f = await fixture(t, 'directory_service_v1'), viewed = platformView(f); + const connections = createOwnerConnections({ store: f.store, access: f.service, invoke: async () => { + f.service.signOut(f.service.session(f.platform.token)); return list(); + } }); + await assert.rejects(connections.list(viewed), /authentication_required/); +}); + +test('platform DSP view can enroll Paycom and retry its existing onboarding request', async t => { + const f = await fixture(t, 'directory_service_v1'), calls = []; let configured = false; + const viewed = platformView(f); + const invoke = async (key, action, input) => { + calls.push({ key, action, input }); + if (input.command === 'enroll') { configured = true; return success('succeeded', { configured: true }); } + if (action === 'paycom.setup') return success('succeeded', { state: 'ready', retryAllowed: true, retryAt: null }); + return success('found', { items: [blank('cortex'), configured ? { ...connected('paycom'), state: 'not_verified' } : blank('paycom')] }); + }; + const paycomSetup = createOwnerPaycomSetup({ store: f.store, access: f.service, invoke }); + const connections = createOwnerConnections({ store: f.store, access: f.service, invoke, paycomSetup }); + assert.equal((await connections.change(viewed, 'paycom', 'save', { credentials: CREDENTIALS })).configured, true); + const requests = createOnboardingStore(f.store), job = requests.latest(f.organizationId); + assert.equal(job.status, 'queued'); assert.equal(job.actor_user_id, viewed.user.id); + f.store.db.prepare("UPDATE installation_onboarding_requests SET status='failed',failure_code='provider_auth_required' WHERE id=?").run(job.id); + assert.equal((await paycomSetup.status(viewed)).canRetry, true); + await connections.change(viewed, 'paycom', 'test', {}); + assert.equal(requests.latest(f.organizationId).status, 'queued'); + assert.equal(calls.filter(call => call.input.command === 'enroll').length, 1); + assert.ok(calls.every(call => call.key === f.store.installationControl(f.organizationId).runtimeKey)); +}); diff --git a/core/core/accounts/tests/owner-paycom-setup.test.js b/core/core/accounts/tests/owner-paycom-setup.test.js new file mode 100644 index 0000000..eb6e69d --- /dev/null +++ b/core/core/accounts/tests/owner-paycom-setup.test.js @@ -0,0 +1,233 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { AccessStore, AccessControlService } = require('../src'); +const { createOwnerPaycomSetup } = require('../src/owner-paycom-setup'); +const { createOnboardingStore } = require('../src/onboarding-store'); +const { success } = require('../../../shared/contracts/src'); +const CREDENTIALS = { clientCode: 'fixture-code', username: 'fixture-user', password: 'fixture-secret-never-persist', + pin1: 'one', pin2: 'two', pin3: 'three', pin4: 'four', pin5: 'five' }; +async function fixture(t, backend = 'oci_container_v1') { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-owner-setup-'));fs.chmodSync(root, 0o700); + const paths = { databaseRoot: path.join(root, 'access'), database: path.join(root, 'access', 'access-control.sqlite3') }; + const store = new AccessStore(paths); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true }); }); + const service = new AccessControlService(store, { installationOperatorEnabled: true, installationBackend: backend }); + const bootstrap = service.createPlatformBootstrap({ email: 'platform@example.test' }); + const platform = await service.acceptNewUser({ token: bootstrap.token, firstName: 'Platform', lastName: 'Owner', + password: 'fixture platform password', confirmPassword: 'fixture platform password' }); + const created = service.createOrganization(platform.session, { idempotencyKey: 'fixture:owner:setup', name: 'Fixture Setup DSP', + abbreviation: 'FIX', stationCode: 'DWA1', timezone: 'America/Chicago', ownerEmail: 'dsp@example.test' }); + const owner = await service.acceptNewUser({ token: created.token, firstName: 'DSP', lastName: 'Owner', + password: 'fixture dsp password', confirmPassword: 'fixture dsp password' }); + const organizationId = created.organization.id; + store.updateInstallationControl({ organizationId, expectedStatus: 'pending', expectedRevision: 1, + status: 'waiting_for_provider_auth', revision: 2, currentJobId: null, timestamp: Date.now() }); + require('./plugin-fixture').enableFixturePlugin(store, organizationId); + return { root, paths, store, service, platform, owner, organizationId }; +} +test('only the selected DSP owner can enroll; credentials bypass durable control state and request replay does not resend', async t => { + const f = await fixture(t); const calls = []; + const setup = createOwnerPaycomSetup({ store: f.store, access: f.service, invoke: async (...args) => { + calls.push(args); return success('succeeded', { configured: true }); + } }); + const input = { idempotencyKey: 'owner:setup:credentials', intent: 'create', credentials: CREDENTIALS }; + await assert.rejects(setup.submit(f.platform.session, input), /organization_required|organization_forbidden/); + const result = await setup.submit(f.owner.session, input); + assert.equal(result.status, 'queued'); assert.equal(result.canSubmit, false); + assert.equal((await setup.submit(f.owner.session, input)).replayed, true); + assert.equal(calls.length, 1); + assert.equal(calls[0][0], f.store.installationControl(f.organizationId).runtimeKey); + assert.equal(calls[0][2].credentials.password, CREDENTIALS.password); + const row = createOnboardingStore(f.store).latest(f.organizationId); + assert.equal(JSON.stringify(row).includes(CREDENTIALS.password), false); + for (const file of fs.readdirSync(f.paths.databaseRoot)) { + assert.equal(fs.readFileSync(path.join(f.paths.databaseRoot, file)).includes(Buffer.from(CREDENTIALS.password)), false); + } + await assert.rejects(() => setup.status({ ...f.owner.session, activeOrganizationId: 'org_nonexistent' }), /organization_forbidden/); +}); +test('failed credential delivery remains retryable and cannot mark a DSP ready', async t => { + const f = await fixture(t); + const setup = createOwnerPaycomSetup({ store: f.store, access: f.service, invoke: async () => { throw new Error('transport'); } }); + await assert.rejects(setup.submit(f.owner.session, { idempotencyKey: 'owner:setup:failure', intent: 'create', credentials: CREDENTIALS }), /provider_setup_failed/); + assert.equal((await setup.status(f.owner.session)).status, 'failed'); + assert.equal((await setup.status(f.owner.session)).canSubmit, true); + assert.equal(f.store.installationControl(f.organizationId).status, 'waiting_for_provider_auth'); + assert.equal(f.store.installationSetup(f.organizationId).workerId, null); +}); + +test('onboarding validates login before starting hourly collection and marking success', async t => { + const f = await fixture(t); + const { createOwnerOnboardingWorker } = require('../../installations/src/owner-onboarding'); + const calls = []; + let valid = false; + const invoke = async (key, action, input) => { + assert.equal(action, 'paycom.setup'); + if (input.command === 'enroll') return success('succeeded', { configured: true }); + if (input.step === 'readiness') return success('succeeded', { state: 'ready', retryAllowed: true, retryAt: null }); + calls.push(input); + if (input.step === 'sync') return success('succeeded', { syncId: 'paycom-main-workforce', intervalSeconds: 3600, desiredState: 'running' }); + assert.equal(input.step, 'test'); + return success('succeeded', { profileId: valid ? 'paycom-main' : 'wrong-profile', + provider: 'paycom', status: 'authenticated', testedAt: new Date().toISOString() }); + }; + const setup = createOwnerPaycomSetup({ store: f.store, access: f.service, invoke }); + await setup.submit(f.owner.session, { idempotencyKey: 'owner:complete:setup', intent: 'create', credentials: CREDENTIALS }); + const worker = createOwnerOnboardingWorker({ store: f.store, invoke }); + const before = f.store.installationControl(f.organizationId); + assert.deepEqual(await worker.runPending('worker_onboarding'), { processed: 1, completed: 0, failed: 1 }); + assert.equal((await setup.status(f.owner.session)).failureCode, 'provider_auth_required'); + assert.equal((await setup.status(f.owner.session)).canRetry, true); + await setup.retry(f.owner.session, {}); valid = true; + assert.deepEqual(await worker.runPending('worker_retry'), { processed: 1, completed: 1, failed: 0 }); + assert.notEqual(calls[0].requestId, calls[1].requestId); + assert.deepEqual(calls.map(call => call.step), ['test', 'test', 'sync']); + assert.equal((await setup.status(f.owner.session)).status, 'succeeded'); + assert.equal((await setup.status(f.owner.session)).canSubmit, false); + assert.equal((await setup.status(f.owner.session)).workforceAvailable, false); + assert.deepEqual(f.store.installationControl(f.organizationId), before); + assert.equal(f.store.latestReadyEvidence(f.organizationId), null); + assert.deepEqual(await worker.runPending('worker_replayed'), { processed: 0, completed: 0, failed: 0 }); +}); + +test('schema 7 migration adds onboarding without changing DSPs or memberships', async t => { + const f = await fixture(t); + const before = f.store.organization(f.organizationId); + f.store.db.exec('DROP TABLE installation_onboarding_requests; PRAGMA user_version=7;'); + f.store.close(); + const reopened = new AccessStore(f.paths); + t.after(() => reopened.close()); + assert.deepEqual(reopened.organization(f.organizationId), before); + assert.equal(reopened.db.prepare('PRAGMA user_version').get().user_version, require('../src/schema').SCHEMA_VERSION); + assert.equal(reopened.membershipsForUser(f.owner.session.user.id).length, 1); + assert.equal(createOnboardingStore(reopened).latest(f.organizationId), null); +}); + +for (const backend of ['oci_container_v1', 'native_service_v1']) for (const initialState of ['pending', 'waiting_for_provider_auth', 'failed']) { + test(`remove and permanently delete an unallocated ${backend} ${initialState} DSP without creating a host account`, async t => { + const f = await fixture(t, backend); + let peer; + if (backend === 'native_service_v1') { + peer = f.service.createOrganization(f.platform.session, { idempotencyKey: 'fixture:retained:peer', name: 'Retained DSP', + abbreviation: 'PEER', stationCode: 'DWA2', timezone: 'America/Chicago', ownerEmail: 'peer@example.test' }); + } + f.store.db.prepare('UPDATE installations SET status=? WHERE organization_id=?').run(initialState, f.organizationId); + const { createAccessInstallationLifecycleAuthority } = require('../src/installation-lifecycle'); + const { createOciInstallationLifecycle } = require('../../installations/src/oci-lifecycle'); + const { createOciRuntimeAgentCredentialPort } = require('../../installations/src/oci-runtime-agent-credential'); + const { retireOciCredentials } = require('../../installations/src/retire-oci-credentials'); + const credentialRoot = path.join(f.root, 'credentials'); fs.mkdirSync(credentialRoot, { mode: 0o700 }); + const credentialPort = createOciRuntimeAgentCredentialPort({ credentialRoot }); + const runtimeKey = f.store.installationControl(f.organizationId).runtimeKey; + const credential = credentialPort.issue(runtimeKey); + f.store.recordRuntimeAgentAuthority({ organizationId: f.organizationId, runtimeKey, tokenHash: credential.tokenHash, timestamp: Date.now() }); + const requests = createOnboardingStore(f.store); + const request = requests.begin(f.organizationId, f.owner.session.user.id, 'removal:pending:onboarding', 'create', 1); + requests.enrolled(request.id); + const authority = createAccessInstallationLifecycleAuthority({ store: f.store, organizationId: f.organizationId, + authorityScope: 'fixture_removal', destructionEnabled: true }); + const unexpected = () => { throw new Error('must_not_allocate_or_touch_another_runtime'); }; + const hostExecutor = Object.fromEntries(['start','stop','disable','health','inspectInactive','render','validate','install', + 'commit','rollback','rollbackStopped','settleRollback','removeServices','inspectRemoved','settleRemoved', + 'destroyAccount','verifyDestroyed','verifyPublication'].map(key => [key, unexpected])); + let inspections = 0; + const lifecycle = createOciInstallationLifecycle({ authority, + offsitePolicy: { offsiteRequired: () => false, waitForDspBackupDeletion: async () => {} }, + adapter: { plan: unexpected, inspectUnallocated: manifest => { assert.equal(manifest.runtime.key, runtimeKey); inspections += 1; return true; } }, + hostExecutor, backupManagerFactory: unexpected, runtimeFactory: unexpected }); + const job = authority.request({ operation: 'decommission', idempotencyKey: 'fixture:remove:unallocated', expectedRevision: 2 }); + assert.equal((await lifecycle.run(job.id, 'worker_remove')).status, 'succeeded'); + assert.equal(f.store.installationControl(f.organizationId).status, 'decommissioned'); + assert.equal(f.store.runtimeAgentAuthority(runtimeKey).status, 'active'); + assert.equal(requests.get(request.id).status, 'queued'); + assert.equal(retireOciCredentials({ store: f.store, credentialPort }), 0); + assert.equal(retireOciCredentials({ store: f.store, credentialPort }), 0); + assert.equal(f.store.installationBackups(f.organizationId).length, 0); + const destroy = authority.request({ operation: 'destroy', idempotencyKey: 'fixture:destroy:unallocated', + expectedRevision: f.store.installationControl(f.organizationId).revision }); + assert.equal((await lifecycle.run(destroy.id, 'worker_destroy')).status, 'succeeded'); + assert.equal(authority.request({ operation: 'destroy', idempotencyKey: 'fixture:destroy:unallocated', + expectedRevision: destroy.installationRevision - 1 }).id, destroy.id); + assert.ok(inspections >= 10); + if (backend === 'native_service_v1') { + retireOciCredentials({ store: f.store, credentialPort }); + assert.equal(f.store.installationControl(f.organizationId), null); + assert.equal(f.store.organization(f.organizationId), null); + assert.equal(Boolean(f.store.userByEmail('dsp@example.test')), false); + assert.ok(f.store.userByEmail('platform@example.test')); + assert.ok(f.store.organization(peer.organization.id)); + assert.equal(f.store.db.prepare('PRAGMA foreign_key_check').all().length, 0); + for (const table of f.store.db.prepare("SELECT name FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'sqlite_%'").all()) { + if (f.store.db.prepare(`PRAGMA table_info(${table.name})`).all().some(c => c.name === 'organization_id')) + assert.equal(f.store.db.prepare(`SELECT count(*) AS count FROM ${table.name} WHERE organization_id=?`).get(f.organizationId).count, 0); + } + } + }); +} + +test('owner retry checks current broker recovery state and cannot enqueue a blocked attempt', async t => { + const f = await fixture(t); + let readiness = { state: 'manual', retryAllowed: false, retryAt: null }; + const requests = createOnboardingStore(f.store); + let checks = 0; + const setup = createOwnerPaycomSetup({ store: f.store, access: f.service, invoke: async (_key, _action, input) => { + if (input.command === 'enroll') return success('succeeded', { configured: true }); + assert.equal(input.step, 'readiness'); checks++; + if (!readiness) throw new Error('offline'); + return success('succeeded', readiness); + } }); + await setup.submit(f.owner.session, { idempotencyKey: 'fixture:retry:guard', intent: 'create', credentials: CREDENTIALS }); + const row = requests.claim(requests.latest(f.organizationId).id, 'fixture_worker'); + requests.finish(row, 'security_answers_rejected'); + for (const state of ['manual', 'cooldown', 'busy', 'not_configured', 'unavailable']) { + readiness = state === 'unavailable' ? null : { state, retryAllowed: false, + retryAt: state === 'cooldown' ? new Date(Date.now() + 300_000).toISOString() : null }; + const status = await setup.status(f.owner.session); + assert.equal(status.canRetry, false); + assert.equal(status.retryState, state); + assert.equal(status.retryAt, readiness?.retryAt || null); + await assert.rejects(setup.retry(f.owner.session, {}), /installation_operation_not_allowed/); + assert.equal(requests.latest(f.organizationId).status, 'failed'); + } + readiness = { state: 'ready', retryAllowed: true, retryAt: null }; + assert.equal((await setup.status(f.owner.session)).canRetry, true); + await setup.retry(f.owner.session, {}); + assert.equal(requests.latest(f.organizationId).status, 'queued'); + assert.equal(checks, 12); +}); + +test('an onboarding state change during readiness cannot authorize a stale retry', async t => { + const f = await fixture(t); + const requests = createOnboardingStore(f.store); + const setup = createOwnerPaycomSetup({ store: f.store, access: f.service, invoke: async (_key, _action, input) => { + if (input.command === 'enroll') return success('succeeded', { configured: true }); + requests.requeue(requests.latest(f.organizationId).id); + return success('succeeded', { state: 'ready', retryAllowed: true, retryAt: null }); + } }); + await setup.submit(f.owner.session, { idempotencyKey: 'fixture:retry:stale', intent: 'create', credentials: CREDENTIALS }); + const row = requests.claim(requests.latest(f.organizationId).id, 'fixture_worker'); + requests.finish(row, 'provider_setup_failed'); + await assert.rejects(setup.retry(f.owner.session, {}), /installation_operation_not_allowed/); + assert.equal(requests.latest(f.organizationId).status, 'queued'); +}); + +test('a completed lifecycle revision change invalidates an in-flight retry readiness check', async t => { + const f = await fixture(t); + const requests = createOnboardingStore(f.store); + const setup = createOwnerPaycomSetup({ store: f.store, access: f.service, invoke: async (_key, _action, input) => { + if (input.command === 'enroll') return success('succeeded', { configured: true }); + const before = f.store.installationControl(f.organizationId); + f.store.updateInstallationControl({ organizationId: f.organizationId, expectedStatus: before.status, + expectedRevision: before.revision, status: before.status, revision: before.revision + 1, + currentJobId: null, timestamp: Date.now() }); + return success('succeeded', { state: 'ready', retryAllowed: true, retryAt: null }); + } }); + await setup.submit(f.owner.session, { idempotencyKey: 'fixture:retry:revision', intent: 'create', credentials: CREDENTIALS }); + const row = requests.claim(requests.latest(f.organizationId).id, 'fixture_worker'); + requests.finish(row, 'provider_setup_failed'); + await assert.rejects(setup.retry(f.owner.session, {}), /installation_operation_not_allowed/); + assert.equal(requests.latest(f.organizationId).status, 'failed'); +}); diff --git a/core/core/accounts/tests/password-recovery.test.js b/core/core/accounts/tests/password-recovery.test.js new file mode 100644 index 0000000..cd21db7 --- /dev/null +++ b/core/core/accounts/tests/password-recovery.test.js @@ -0,0 +1,205 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const test = require('node:test'); +const { AccessStore, AccessControlService } = require('../src'); +const { consumeRecoveryLimits, RESET_TTL_MS } = require('../src/password-recovery'); +const PASSWORD = 'original account passphrase'; +const NEXT = 'replacement account passphrase'; +const input = token => ({ token, newPassword: NEXT, confirmPassword: NEXT }); + +async function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-recovery-')); + fs.chmodSync(root, 0o700); + const paths = { databaseRoot: path.join(root, 'access'), database: path.join(root, 'access/db.sqlite') }; + const store = new AccessStore(paths); + const time = { value: Date.now() }; + const clock = () => new Date(time.value); + const service = new AccessControlService(store, { clock, installationOperatorEnabled: true }); + const bootstrap = service.createPlatformBootstrap({ email: 'owner@example.test' }); + const owner = await service.acceptNewUser({ token: bootstrap.token, firstName: 'Test', lastName: 'Owner', password: PASSWORD, confirmPassword: PASSWORD }); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const issue = () => service.requestPasswordReset({ email: ' OWNER@example.test ' }); + return { root, paths, store, service, time, clock, owner, issue }; +} + +test('recovery stores only hashes, preserves sessions until completion, revokes every link/session and permits normal login', async t => { + const c = await fixture(t); + const first = c.issue(); + c.time.value += 60000; + const second = c.issue(); + assert.equal(Buffer.from(first.token, 'base64url').length, 32); + const row = c.store.db.prepare('SELECT * FROM password_reset_tokens WHERE token_hash=?') + .get(crypto.createHash('sha256').update(first.token).digest('hex')); + assert.equal(row.expires_at - row.created_at, RESET_TTL_MS); + assert.equal(JSON.stringify(row).includes(first.token), false); + assert.ok(c.service.session(c.owner.token)); + const otherSession = await c.service.signIn({ email: first.email, password: PASSWORD }); + assert.deepEqual(await c.service.resetPassword(input(first.token)), { email: first.email, userId: c.owner.session.user.id }); + assert.equal(c.service.session(c.owner.token), null); + assert.equal(c.service.session(otherSession.token), null); + assert.equal(c.store.db.prepare('SELECT count(*) AS n FROM sessions').get().n, 0); + assert.equal(c.store.db.prepare('SELECT count(*) AS n FROM password_reset_tokens').get().n, 0); + for (const token of [first.token, second.token]) await assert.rejects(c.service.resetPassword(input(token)), /password_reset_invalid/); + await assert.rejects(c.service.signIn({ email: first.email, password: PASSWORD }), /invalid_credentials/); + assert.equal((await c.service.signIn({ email: first.email, password: NEXT })).session.user.id, c.owner.session.user.id); + const audits = JSON.stringify(c.store.db.prepare('SELECT * FROM audit_events').all()); + assert.ok(audits.includes('account.password.reset.complete')); + for (const secret of [first.token, second.token, PASSWORD, NEXT]) assert.equal(audits.includes(secret), false); + c.store.close(); + const bytes = fs.readFileSync(c.paths.database); + for (const secret of [first.token, second.token, PASSWORD, NEXT]) assert.equal(bytes.includes(Buffer.from(secret)), false); +}); + +test('malformed, expired, invitation and unknown tokens fail; password-policy failures do not consume a valid link', async t => { + const c = await fixture(t), reset = c.issue(); + for (const token of [null, '', 'x'.repeat(42), crypto.randomBytes(32).toString('base64url')]) { + await assert.rejects(c.service.resetPassword(input(token)), /password_reset_invalid/); + } + await assert.rejects(c.service.resetPassword({ ...input(reset.token), newPassword: 'short' }), /password_policy_failed/); + await assert.rejects(c.service.resetPassword({ ...input(reset.token), confirmPassword: 'does not match' }), /password_confirmation_mismatch/); + await assert.rejects(c.service.resetPassword({ ...input(reset.token), email: 'attacker@example.test' }), /invalid_input/); + assert.ok(c.service.session(c.owner.token)); + c.time.value += RESET_TTL_MS; + await assert.rejects(c.service.resetPassword(input(reset.token)), /password_reset_invalid/); + const invite = c.service.createOrganization(c.owner.session, { idempotencyKey: 'recovery:test:invitation', ownerEmail: 'other@example.test', name: 'Recovery DSP', abbreviation: 'RST', stationCode: 'TST1', timezone: 'America/Chicago' }); + await assert.rejects(c.service.resetPassword(input(invite.token)), /password_reset_invalid/); +}); + +test('parallel resets from different connections have exactly one winner, even with different valid links', async t => { + const c = await fixture(t), first = c.issue(); + c.time.value += 60000; + const second = c.issue(); + const store2 = new AccessStore(c.paths); + t.after(() => store2.close()); + const service2 = new AccessControlService(store2, { clock: c.clock }); + const outcomes = await Promise.allSettled([c.service.resetPassword(input(first.token)), service2.resetPassword(input(second.token))]); + assert.equal(outcomes.filter(result => result.status === 'fulfilled').length, 1); + assert.match(outcomes.find(result => result.status === 'rejected').reason.message, /password_reset_invalid/); + assert.equal(c.store.userById(first.userId).auth_version, 2); +}); + +test('simultaneous replay of one token succeeds once, and recovering a tenant never changes another account or membership', async t => { + const c = await fixture(t); + const invitation = c.service.createOrganization(c.owner.session, { + idempotencyKey: 'recovery:tenant:scope', ownerEmail: 'tenant@example.test', name: 'Recovery DSP', + abbreviation: 'RST', stationCode: 'TST1', timezone: 'America/Chicago', + }); + const tenant = await c.service.acceptNewUser({ token: invitation.token, firstName: 'Tenant', lastName: 'Owner', + password: PASSWORD, confirmPassword: PASSWORD }); + const memberships = c.store.db.prepare('SELECT * FROM memberships').all(); + const platform = c.store.userById(c.owner.session.user.id); + const reset = c.service.requestPasswordReset({ email: 'tenant@example.test' }); + const outcomes = await Promise.allSettled([c.service.resetPassword(input(reset.token)), c.service.resetPassword(input(reset.token))]); + assert.equal(outcomes.filter(result => result.status === 'fulfilled').length, 1); + assert.match(outcomes.find(result => result.status === 'rejected').reason.message, /password_reset_invalid/); + assert.deepEqual(c.store.userById(platform.id), platform); + assert.deepEqual(c.store.db.prepare('SELECT * FROM memberships').all(), memberships); + assert.ok(c.service.session(c.owner.token)); + assert.equal(c.service.session(tenant.token), null); + assert.equal(c.store.userById(tenant.session.user.id).platform_role, null); +}); + +test('expiry, disabling an account, and administrator credential changes during hashing abort the reset', async t => { + for (const mutate of [c => { c.time.value += RESET_TTL_MS; }, + c => c.store.db.prepare("UPDATE users SET status='disabled' WHERE id=?").run(c.owner.session.user.id), + c => c.store.db.prepare('UPDATE users SET auth_version=auth_version+1 WHERE id=?').run(c.owner.session.user.id)]) { + const c = await fixture(t), reset = c.issue(); + const previousHash = c.store.userById(reset.userId).password_hash; + const pending = c.service.resetPassword(input(reset.token)); + mutate(c); + await assert.rejects(pending, /password_reset_invalid/); + assert.equal(c.store.userById(reset.userId).password_hash, previousHash); + } +}); + +test('normal password changes and email changes invalidate recovery links', async t => { + const c = await fixture(t), reset = c.issue(); + await c.service.changePassword(c.owner.session, { currentPassword: PASSWORD, newPassword: NEXT, confirmPassword: NEXT }); + await assert.rejects(c.service.resetPassword(input(reset.token)), /password_reset_invalid/); + c.time.value += 60000; + const next = c.issue(); + c.store.db.prepare('UPDATE users SET email=? WHERE id=?').run('renamed@example.test', reset.userId); + await assert.rejects(c.service.resetPassword(input(next.token)), /password_reset_invalid/); +}); + +test('a database failure rolls back the password, token invalidation and session revocation together', async t => { + const c = await fixture(t), reset = c.issue(); + const previous = c.store.userById(reset.userId); + const originalDelete = c.store.deleteUserSessions; + c.store.deleteUserSessions = () => { throw Error('synthetic database failure'); }; + await assert.rejects(c.service.resetPassword(input(reset.token)), /synthetic database failure/); + assert.deepEqual(c.store.userById(reset.userId), previous); + assert.ok(c.service.session(c.owner.token)); + c.store.deleteUserSessions = originalDelete; + await c.service.resetPassword(input(reset.token)); + assert.equal(c.service.session(c.owner.token), null); +}); + +test('email limits cover unknown/disabled accounts and persist across restart without storing addresses', async t => { + const c = await fixture(t); + assert.ok(c.issue()); + assert.equal(c.issue(), null); + const missing = 'missing@example.test'; + assert.equal(c.service.requestPasswordReset({ email: missing }), null); + c.store.close(); + const reopened = new AccessStore(c.paths); + t.after(() => reopened.close()); + const service = new AccessControlService(reopened, { clock: c.clock }); + assert.equal(service.requestPasswordReset({ email: 'owner@example.test' }), null); + for (let i = 0; i < 4; i++) { + c.time.value += 60000; + assert.ok(service.requestPasswordReset({ email: 'owner@example.test' })); + } + c.time.value += 60000; + assert.equal(service.requestPasswordReset({ email: 'owner@example.test' }), null); + const limits = JSON.stringify(reopened.db.prepare('SELECT * FROM password_recovery_limits').all()); + assert.equal(limits.includes(missing), false); + assert.equal(limits.includes('owner@example.test'), false); + c.time.value += 3600000; + reopened.db.prepare("UPDATE users SET status='disabled'").run(); + assert.equal(service.requestPasswordReset({ email: 'owner@example.test' }), null); + assert.equal(reopened.db.prepare('SELECT count(*) AS n FROM password_reset_tokens').get().n, 0); +}); + +test('rate limits survive another connection, expire, and fail closed when storage is full', async t => { + const c = await fixture(t); + const limits = [{ key: 'reset:ip:127.0.0.1', count: 2, window: 1000 }]; + assert.ok(consumeRecoveryLimits(c.store, limits, c.time.value)); + const store2 = new AccessStore(c.paths); + t.after(() => store2.close()); + assert.ok(consumeRecoveryLimits(store2, limits, c.time.value)); + assert.equal(consumeRecoveryLimits(c.store, limits, c.time.value), false); + c.time.value += 1000; + assert.ok(consumeRecoveryLimits(c.store, limits, c.time.value)); + c.store.transaction(() => { + const insert = c.store.db.prepare('INSERT OR IGNORE INTO password_recovery_limits VALUES(?,1,?,?)'); + for (let i = 0; i < 9999; i++) insert.run(i.toString(16).padStart(64, '0'), c.time.value + 10000, c.time.value); + }); + assert.equal(consumeRecoveryLimits(c.store, [{ key: 'new-ip', count: 2, window: 1000 }], c.time.value), false); +}); + +test('schema 13 upgrades with user credentials intact; recovery tokens are absent from sanitized core backups', async t => { + const c = await fixture(t); + c.store.db.exec('DROP TRIGGER password_reset_invalidate; DROP TABLE password_reset_tokens; DROP TABLE password_recovery_limits; PRAGMA user_version=13'); + const hash = c.store.userById(c.owner.session.user.id).password_hash; + c.store.close(); + const upgraded = new AccessStore(c.paths); + t.after(() => upgraded.close()); + assert.equal(upgraded.db.prepare('PRAGMA user_version').get().user_version, require('../src/schema').SCHEMA_VERSION); + assert.equal(upgraded.userById(c.owner.session.user.id).password_hash, hash); + new AccessControlService(upgraded, { clock: c.clock }).requestPasswordReset({ email: 'owner@example.test' }); + upgraded.db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); + const backup = path.join(c.root, 'backup.sqlite'); + fs.copyFileSync(c.paths.database, backup); + require('../src/core-backup').sanitizeCoreDatabase(backup); + const { DatabaseSync } = require('node:sqlite'); + const saved = new DatabaseSync(backup); + try { + assert.equal(saved.prepare('SELECT count(*) AS n FROM password_reset_tokens').get().n, 0); + assert.equal(saved.prepare('SELECT count(*) AS n FROM password_recovery_limits').get().n, 0); + } finally { saved.close(); } +}); diff --git a/core/core/accounts/tests/platform-administration.test.js b/core/core/accounts/tests/platform-administration.test.js new file mode 100644 index 0000000..5ce5055 --- /dev/null +++ b/core/core/accounts/tests/platform-administration.test.js @@ -0,0 +1,528 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const test = require('node:test'); +const { AccessStore, AccessControlService } = require('../src'); +const { completeRolloutBackups } = require('../../installations/tests/helpers/rollout-backups'); +const { createPlatformUpdates } = require('../src/platform-updates'); +const { applyOrganizationProfiles } = require('../src/organization-profile'); +const { managedInstallationContext } = require('../src/installation-authority'); +const { createOwnerPaycomSetup } = require('../src/owner-paycom-setup'); +async function fixture(t, backend = 'oci_container_v1') { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-platform-admin-')); fs.chmodSync(root, 0o700); + const paths = { databaseRoot: path.join(root, 'access'), database: path.join(root, 'access/access-control.sqlite3') }; + const store = new AccessStore(paths); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const access = new AccessControlService(store, { installationOperatorEnabled: true, installationBackend: backend }); + const invitation = access.createPlatformBootstrap({ email: 'platform@example.test' }); + const platform = await access.acceptNewUser({ token: invitation.token, firstName: 'Platform', lastName: 'Owner', + password: 'test platform password', confirmPassword: 'test platform password' }); + const create = (n, emailOnly = true) => access.createOrganization(platform.session, { ownerEmail: `owner${n}@example.test`, + idempotencyKey: `admin:creation:${n}`, ...(emailOnly ? {} : { name: `DSP ${n}`, stationCode: 'TST1', timezone: 'UTC' }) }); + const platformReleases = { dispatch_update_2: { version: '0.0.2', publishedAt: '2026-09-05T00:00:00.000Z', + changelog: [{ kind: 'fixed', title: 'Example fix', description: '' }], core: {} } }; + const updates = (finishCore = true) => { + const coordinator = createPlatformUpdates({ store, releases: { dispatch_update_2: {} }, platformReleases, enabled: true }); + // Fleet tests simulate the external updater's terminal receipt; dedicated tests below exercise that worker. + return { ...coordinator, command(session, input) { + const result = coordinator.command(session, input); + if (input.action === 'start') completeRolloutBackups(store); + return result; + }, tick() { + if (finishCore) store.db.prepare("UPDATE platform_rollout_core SET status='succeeded' WHERE status='queued'").run(); + return coordinator.tick(); + } }; + }; + return { store, access, platform, create, updates, paths, platformReleases }; +} +test('email creation atomically queues exactly one runtime and invitation; replay cannot duplicate either', async t => { + const f = await fixture(t); + const first = f.create(100), replay = f.create(100); + assert.equal(replay.organization.id, first.organization.id); + assert.equal(replay.token, null); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM installation_provisioning_requests').get().n, 1); + assert.equal(f.store.installationControl(first.organization.id).status, 'provisioning'); + const view = f.access.platformOrganizations(f.platform.session)[0]; + assert.equal(view.detailsStatus, 'required'); assert.equal(view.ownerEmail, 'owner100@example.test'); + assert.equal(view.name, 'New DSP'); + const original = f.store.createProvisioningRequest; + f.store.createProvisioningRequest = () => { throw new Error('queue unavailable'); }; + assert.throws(() => f.create(101), /queue unavailable/); + f.store.createProvisioningRequest = original; + assert.equal(f.store.organizations().length, 1); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM invitations WHERE kind='organization_owner'").get().n, 1); +}); +test('owner details wait for provisioning, preserve runtime identity and gate provider setup', async t => { + const f = await fixture(t); const created = f.create(200); + const owner = await f.access.acceptNewUser({ token: created.token, firstName: 'DSP', lastName: 'Owner', + password: 'test owner password', confirmPassword: 'test owner password' }); + const before = managedInstallationContext(f.store, created.organization.id); + const details = { name: 'Northstar Delivery', abbreviation: 'NS', stationCode: 'DWA1', timezone: 'America/Chicago' }; + assert.throws(() => f.access.organizationProfile(f.platform.session, details), /organization_required|organization_forbidden/); + assert.throws(() => f.access.organizationProfile(owner.session, { ...details, organizationId: 'org_other' }), /invalid_input/); + assert.throws(() => f.access.organizationProfile(owner.session, { ...details, timezone: 'Invalid/Zone' }), /invalid_input/); + assert.equal(f.access.organizationProfile(owner.session, details).status, 'submitted'); + assert.equal(f.store.organization(created.organization.id).name, 'New DSP'); + require('./plugin-fixture').enableFixturePlugin(f.store, created.organization.id); + const setup = createOwnerPaycomSetup({ store: f.store, access: f.access, invoke: async () => { throw new Error('must not call'); } }); + await assert.rejects(() => setup.status(owner.session), /organization_details_required/); + // The infrastructure worker has finished; applying details requires no container recreation. + f.store.db.prepare("UPDATE installations SET status='waiting_for_provider_auth' WHERE organization_id=?").run(created.organization.id); + assert.equal(applyOrganizationProfiles(f.store), 1); + const after = managedInstallationContext(f.store, created.organization.id); + assert.deepEqual(after.manifest.runtime, before.manifest.runtime); + assert.equal(after.organization.id, before.organization.id); + assert.equal(after.manifest.organization.stationCode, 'DWA1'); + assert.equal(after.manifest.organization.timezone, 'America/Chicago'); + assert.equal(f.access.organizationProfile(owner.session).status, 'complete'); + assert.equal((await setup.status(owner.session)).canSubmit, true); + assert.throws(() => f.access.organizationProfile(owner.session, details), /organization_details_complete/); + assert.equal(applyOrganizationProfiles(f.store), 0); +}); +function ready(f, created) { + f.store.db.prepare("UPDATE installations SET status='ready' WHERE organization_id=?").run(created.organization.id); + f.store.updateOrganizationStatus(created.organization.id, 'active', Date.now()); +} +function finishWorkerJob(f, id, status) { + // Simulate the private lifecycle worker's terminal receipt; its real stage/evidence + // verification is covered by the provisioner lifecycle suite. + f.store.db.prepare('UPDATE installation_lifecycle_jobs SET status=?,failure_code=?,finished_at=?,result_json=? WHERE id=?').run(status, status === 'failed' ? 'upgrade_failed' : null, Date.now(), status === 'succeeded' ? '{}' : null, id); + const job = f.store.lifecycleJob(id); + f.store.db.prepare("UPDATE installations SET status='ready',release_id=? WHERE organization_id=?") + .run(status === 'succeeded' ? 'dispatch_update_2' : 'dispatch_current_1', job.organization_id); +} +test('rollout persists across restart, updates one DSP at a time, pauses and retries without skips', async t => { + const f = await fixture(t); const one = f.create(301, false), two = f.create(302, false); + ready(f, one); ready(f, two); + let coordinator = f.updates(); + const command = { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:test:all-dsps' }; + coordinator.command(f.platform.session, command); coordinator.command(f.platform.session, command); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_rollouts').get().n, 1); + assert.throws(() => coordinator.command(f.platform.session, { ...command, idempotencyKey: 'rollout:test:duplicate' }), /rollout_in_progress/); + coordinator.tick(); coordinator.tick(); + let jobs = f.store.db.prepare("SELECT * FROM installation_lifecycle_jobs WHERE operation='upgrade'").all(); + assert.equal(jobs.length, 1); assert.equal(jobs[0].organization_id, one.organization.id); + assert.equal(coordinator.view().rollout.members[1].status, 'queued'); + // Crash after job persistence but before the link is saved: reconnect by request key. + f.store.db.prepare('UPDATE platform_rollout_members SET job_id=NULL').run(); + coordinator = f.updates(); coordinator.tick(); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM installation_lifecycle_jobs WHERE operation='upgrade'").get().n, 1); + finishWorkerJob(f, jobs[0].id, 'failed'); coordinator.tick(); + assert.equal(coordinator.view().rollout.status, 'paused'); + coordinator.tick(); assert.equal(coordinator.view().rollout.members[1].status, 'queued'); + coordinator.command(f.platform.session, { action: 'resume' }); coordinator.tick(); coordinator.tick(); + jobs = f.store.db.prepare("SELECT * FROM installation_lifecycle_jobs WHERE operation='upgrade' ORDER BY rowid").all(); + assert.equal(jobs.length, 2); assert.equal(jobs[1].organization_id, one.organization.id); + finishWorkerJob(f, jobs[1].id, 'succeeded'); coordinator.tick(); + coordinator.command(f.platform.session, { action: 'pause' }); coordinator.tick(); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM installation_lifecycle_jobs WHERE operation='upgrade'").get().n, 2); + coordinator.command(f.platform.session, { action: 'resume' }); coordinator.tick(); coordinator.tick(); + jobs = f.store.db.prepare("SELECT * FROM installation_lifecycle_jobs WHERE operation='upgrade' ORDER BY rowid").all(); + assert.equal(jobs[2].organization_id, two.organization.id); + finishWorkerJob(f, jobs[2].id, 'succeeded'); coordinator.tick(); coordinator.tick(); + assert.equal(coordinator.view().rollout.status, 'completed'); + assert.equal(coordinator.view().rollout.updated, 2); +}); +test('unready DSPs block completion and new DSPs inherit the fleet target and join the rollout', async t => { + const f = await fixture(t); f.create(400, false); + const c = f.updates(); c.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:test:unready' }); + c.tick(); assert.equal(c.view().rollout.status, 'paused'); assert.equal(c.view().rollout.updated, 0); + const fresh = f.create(401); + assert.equal(f.store.installationControl(fresh.organization.id).releaseId, 'dispatch_update_2'); + c.tick(); assert.equal(c.view().rollout.total, 2); + assert.equal(c.view().rollout.status, 'paused'); +}); +test('nested transactions retain atomic rollback for rollout job creation', async t => { + const f = await fixture(t); + assert.throws(() => f.store.transaction(() => { f.create(501); throw new Error('outer failure'); }), /outer failure/); + assert.equal(f.store.organizations().length, 0); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM installation_provisioning_requests').get().n, 0); + assert.equal(f.create(501).organization.name, 'New DSP'); +}); +test('schema 8 upgrades preserve existing organizations and enable email profiles and durable rollouts', async t => { + const f = await fixture(t); const created = f.create(600, false); + f.store.db.exec('DROP TABLE platform_rollout_core; DROP TABLE platform_rollout_members; DROP TABLE platform_rollouts; DROP TABLE organization_profiles; PRAGMA user_version=8;'); + const migrated = new AccessStore(f.paths); + try { + assert.equal(migrated.organization(created.organization.id).name, 'DSP 600'); + assert.equal(migrated.db.prepare('PRAGMA user_version').get().user_version, require('../src/schema').SCHEMA_VERSION); + assert.equal(migrated.db.prepare('SELECT count(*) n FROM organization_profiles').get().n, 0); + } finally { migrated.close(); } +}); + +test('exhausted worker leases pause and resume the existing job instead of duplicating an upgrade', async t => { + const f = await fixture(t); const created = f.create(701, false); ready(f, created); + const c = f.updates(); c.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:test:exhausted' }); + c.tick(); c.tick(); + const job = f.store.db.prepare("SELECT * FROM installation_lifecycle_jobs WHERE operation='upgrade'").get(); + f.store.db.prepare("UPDATE installation_lifecycle_jobs SET status='running',attempt=max_attempts,lease_expires_at=?,worker_id='worker_old' WHERE id=?").run(Date.now() - 1, job.id); + c.tick(); assert.equal(c.view().rollout.status, 'paused'); + c.command(f.platform.session, { action: 'resume' }); + assert.equal(f.store.lifecycleJob(job.id).status, 'queued'); + assert.equal(f.store.lifecycleJob(job.id).attempt, 0); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM installation_lifecycle_jobs WHERE operation='upgrade'").get().n, 1); +}); +test('a removed DSP remains visible in rollout history and no longer blocks the current fleet', async t => { + const f = await fixture(t); const created = f.create(801, false); + const c = f.updates(); c.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:test:removed' }); + c.tick(); assert.equal(c.view().rollout.status, 'paused'); + f.store.db.prepare("UPDATE installations SET status='decommissioned' WHERE organization_id=?").run(created.organization.id); + c.tick(); c.command(f.platform.session, { action: 'resume' }); c.tick(); + assert.equal(c.view().rollout.status, 'completed'); assert.equal(c.view().rollout.total, 0); + assert.equal(c.view().rollout.members[0].status, 'removed'); +}); + +test('an existing owner cannot accept a second DSP membership', async t => { + const f = await fixture(t); const first = f.create(901); + const owner = await f.access.acceptNewUser({ token: first.token, firstName: 'Multi', lastName: 'Owner', + password: 'multiple dsp password', confirmPassword: 'multiple dsp password' }); + const second = f.access.createOrganization(f.platform.session, { ownerEmail: 'owner901@example.test', idempotencyKey: 'admin:second:dsp:901' }); + assert.throws(() => f.access.acceptExistingUser(owner.session, second.token), /user_already_belongs_to_dsp/); + assert.equal(f.store.membershipsForUser(owner.session.user.id).length, 1); +}); + +test('Core must update and pass verification before the first DSP job can start', async t => { + const f = await fixture(t); const dsp = f.create(901, false); ready(f, dsp); + const c = f.updates(false); + c.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:core:first' }); + c.tick(); c.tick(); + assert.equal(c.view().rollout.phase, 'core'); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM installation_lifecycle_jobs WHERE operation='upgrade'").get().n, 0); + const stages = []; + const worker = require('../../installations/src/platform-core-update').createPlatformCoreUpdater({ store: f.store, + platformReleases: f.platformReleases, execute: async action => { + stages.push(action); c.tick(); + assert.equal(c.view().rollout.phase, action === 'apply' ? 'core' : 'verify_core'); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM installation_lifecycle_jobs WHERE operation='upgrade'").get().n, 0); + } }); + assert.equal((await worker.run()).status, 'core_verified'); + assert.deepEqual(stages, ['apply', 'verify']); + c.tick(); c.tick(); + assert.equal(c.view().rollout.phase, 'dsps'); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM installation_lifecycle_jobs WHERE operation='upgrade'").get().n, 1); +}); + +test('Core failure pauses an empty-fleet rollout; resume retries and completes after verification', async t => { + const f = await fixture(t); const c = f.updates(false); + c.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:core:empty' }); + let broken = true; + const worker = require('../../installations/src/platform-core-update').createPlatformCoreUpdater({ store: f.store, + platformReleases: f.platformReleases, execute: async action => { if (action === 'verify' && broken) throw new Error('private diagnostic'); } }); + assert.equal((await worker.run()).status, 'core_update_failed'); + c.tick(); assert.equal(c.view().rollout.status, 'paused'); + assert.doesNotMatch(JSON.stringify(c.view()), /private diagnostic|artifactPath/); + assert.equal((await worker.run()).status, 'idle'); + broken = false; c.command(f.platform.session, { action: 'resume' }); + assert.equal((await worker.run()).status, 'core_verified'); + c.tick(); assert.equal(c.view().rollout.status, 'completed'); + assert.equal(c.view().releases.length, 0); +}); + +test('Core progress survives interruptions and a pause finishes only an already started stage', async t => { + const f = await fixture(t); const c = f.updates(false); + c.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:core:restart' }); + c.command(f.platform.session, { action: 'pause' }); + const calls = []; + const worker = () => require('../../installations/src/platform-core-update').createPlatformCoreUpdater({ store: f.store, + platformReleases: f.platformReleases, execute: async action => calls.push(action) }); + assert.equal((await worker().run()).status, 'idle'); + f.store.db.prepare("UPDATE platform_rollout_core SET status='verifying',attempt=1").run(); + assert.equal((await worker().run()).status, 'core_verified'); + assert.deepEqual(calls, ['verify']); + c.tick(); assert.equal(c.view().rollout.status, 'paused'); + c.command(f.platform.session, { action: 'resume' }); c.tick(); + assert.equal(c.view().rollout.status, 'completed'); +}); + +test('runtime-only releases cannot start a rollout and queued bundles cannot change underneath the worker', async t => { + const f = await fixture(t); + const incomplete = createPlatformUpdates({ store: f.store, releases: { dispatch_update_2: {} }, enabled: true }); + assert.equal(incomplete.view().releases.length, 0); + assert.throws(() => incomplete.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:core:incomplete' }), /update_unavailable/); + const c = f.updates(false); c.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:core:pinned' }); + const worker = require('../../installations/src/platform-core-update').createPlatformCoreUpdater({ store: f.store, + platformReleases: { dispatch_update_2: { ...f.platformReleases.dispatch_update_2, version: '0.0.3' } }, + execute: async () => assert.fail('Changed bundle must not execute') }); + assert.equal((await worker.run()).status, 'core_update_failed'); + assert.equal(c.view().rollout.status, 'paused'); +}); + +test('schema 9 rollouts require Core verification after migration without changing accounts', async t => { + const f = await fixture(t); const c = f.updates(false); + c.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:core:migrate' }); + const before = f.store.db.prepare('SELECT * FROM users').all(); + f.store.db.exec('DROP TABLE platform_rollout_core; PRAGMA user_version=9;'); + const migrated = new AccessStore(f.paths); + try { + const updates = createPlatformUpdates({ store: migrated, releases: { dispatch_update_2: {} }, platformReleases: f.platformReleases, enabled: true }); + updates.tick(); assert.equal(updates.view().rollout.phase, 'core'); + assert.equal(updates.view().rollout.core.status, 'queued'); + assert.deepEqual(migrated.db.prepare('SELECT * FROM users').all(), before); + assert.equal(migrated.db.prepare("SELECT count(*) n FROM installation_lifecycle_jobs WHERE operation='upgrade'").get().n, 0); + } finally { migrated.close(); } +}); + + +test('newly prepared release catalogs become available without restarting Core', async t => { + const f = await fixture(t); + let catalogs = { releases: {}, platformReleases: {} }; + const updates = createPlatformUpdates({ store: f.store, enabled: true, loadCatalogs: () => catalogs }); + assert.deepEqual(updates.view().releases, []); + catalogs = { releases: { dispatch_update_2: {} }, platformReleases: f.platformReleases }; + assert.equal(updates.view().releases[0].version, '0.0.2'); + updates.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'reload:catalog:1' }); + assert.equal(updates.view().rollout.core.status, 'queued'); +}); + + +test('changing a release after Core verification cannot advance DSPs onto a different bundle', async t => { + const f = await fixture(t); ready(f, f.create(920)); + const updates = f.updates(false); + updates.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'pin:after:core:001' }); + completeRolloutBackups(f.store); + f.store.db.prepare("UPDATE platform_rollout_core SET status='succeeded'").run(); + f.platformReleases.dispatch_update_2.version = '0.0.3'; + updates.tick(); + assert.equal(updates.view().rollout.status, 'paused'); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM installation_lifecycle_jobs WHERE operation='upgrade'").get().n, 0); + assert.equal(updates.view().rollout.core.status, 'failed'); +}); + +test('permanent deletion requires removal and the acting administrator password', async t => { + const f = await fixture(t), target = f.create(1201, false), peer = f.create(1202, false); + ready(f, target); ready(f, peer); + const row = f.access.platformOrganizations(f.platform.session).find(r => r.name === target.organization.name); + assert.ok(!row.installation.availableActions.includes('destroy')); + const command = { controlRef: row.controlRef, password: 'test platform password', + expectedRevision: row.installation.revision, idempotencyKey: 'deletion:removed:1201' }; + await assert.rejects(f.access.requestPlatformRemoval(f.platform.session, command, 'destroy'), /installation_operation_not_allowed/); + f.store.db.prepare("UPDATE installations SET status='decommissioned' WHERE organization_id=?").run(target.organization.id); + await assert.rejects(f.access.requestPlatformRemoval(f.platform.session, { ...command, password: 'wrong' }, 'destroy'), /current_password_invalid/); + await assert.rejects(f.access.requestPlatformRemoval(f.platform.session, { ...command, password: undefined }, 'destroy'), /current_password_invalid/); + await f.access.requestPlatformRemoval(f.platform.session, command, 'destroy'); + assert.equal((await f.access.requestPlatformRemoval(f.platform.session, command, 'destroy')).replayed, true); + assert.ok(!f.store.db.prepare('SELECT stage_receipts_json FROM installation_lifecycle_jobs WHERE organization_id=?').get(target.organization.id).stage_receipts_json.includes(command.password)); + assert.equal(f.store.organization(target.organization.id).status, 'suspended'); + assert.throws(() => f.access.inspectInvitation(target.token), /invitation_invalid/); + assert.equal(f.store.organization(peer.organization.id).status, 'active'); + const job = f.store.db.prepare("SELECT * FROM installation_lifecycle_jobs WHERE organization_id=?").get(target.organization.id); + assert.equal(job.operation, 'destroy');assert.equal(job.backup_id, null); + assert.equal(f.access.platformOrganizations(f.platform.session).length, 2); + f.store.db.prepare("UPDATE installation_lifecycle_jobs SET status='failed',failure_code='destruction_failed',finished_at=1 WHERE id=?").run(job.id); + const failed = f.access.platformOrganizations(f.platform.session).find(r => r.name === row.name); + assert.equal(failed.installation.operation.status, 'failed'); + f.store.db.prepare("UPDATE installation_lifecycle_jobs SET status='succeeded',failure_code=NULL,result_json='{}' WHERE id=?").run(job.id); + assert.deepEqual(f.access.platformOrganizations(f.platform.session).map(r => r.name), [peer.organization.name]); +}); + +test('completed destruction clears only its DSP access and backup metadata while retaining shared users', async t => { + const f = await fixture(t), target=f.create(1301,false), peer=f.create(1302,false); + const owner=await f.access.acceptNewUser({token:target.token,firstName:'DSP',lastName:'Owner',password:'synthetic owner password',confirmPassword:'synthetic owner password'}); + const backup='backup_'+'c'.repeat(32), other='backup_'+'d'.repeat(32); + const insert=f.store.db.prepare("INSERT INTO platform_backup_records(id,organization_id,kind,metadata_json,created_at) VALUES (?,?,'dsp',?,1)"); + insert.run(backup,target.organization.id,JSON.stringify({private:'target DSP backup details'})); + insert.run(other,peer.organization.id,JSON.stringify({private:'peer details'})); + f.store.transaction(()=>{ f.store.destroyInstallationBackups(target.organization.id,2000); f.store.destroyOrganizationAccess(target.organization.id); }); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM memberships WHERE organization_id=?').get(target.organization.id).n,0); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM roles WHERE organization_id=?').get(target.organization.id).n,0); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM organization_profiles WHERE organization_id=?').get(target.organization.id).n,0); + assert.ok(f.store.db.prepare('SELECT id FROM users WHERE id=?').get(owner.session.user.id)); + assert.equal(f.store.db.prepare('SELECT metadata_json FROM platform_backup_records WHERE id=?').get(backup).metadata_json,'{}'); + assert.equal(f.store.db.prepare('SELECT deleted_at FROM platform_backup_records WHERE id=?').get(other).deleted_at,null); + assert.ok(f.store.db.prepare('SELECT count(*) n FROM roles WHERE organization_id=?').get(peer.organization.id).n>0); +}); + + +test('schema 10 migration preserves legacy DSPs and dependent records while enabling native creation', async t => { + const f = await fixture(t); + const created = f.create(898, false); + const tables = ['organizations', 'users', 'sessions', 'memberships', 'invitations', + 'installations', 'installation_provisioning_requests', 'runtime_agent_authorities']; + const before = Object.fromEntries(tables.map(table => [table, f.store.db.prepare(`SELECT * FROM ${table} ORDER BY rowid`).all()])); + // Restore the previous backend CHECK in this disposable database's catalog. + // Closing the connection makes SQLite parse the actual version-10 schema on + // reopen, instead of merely lowering user_version on an already-native table. + f.store.db.exec('PRAGMA writable_schema=ON'); + f.store.db.prepare("UPDATE sqlite_schema SET sql=replace(sql, ?, '') WHERE type='table' AND name='installations'") + .run(",'native_service_v1'"); + f.store.db.prepare("UPDATE sqlite_schema SET sql=replace(sql, ?, '') WHERE type='table' AND name='installations'") + .run(",'directory_service_v1'"); + f.store.db.exec('PRAGMA writable_schema=OFF; PRAGMA user_version=10'); + f.store.close(); + const migrated = new AccessStore(f.paths); + try { + assert.equal(migrated.db.prepare('PRAGMA user_version').get().user_version, require('../src/schema').SCHEMA_VERSION); + for (const table of tables) assert.deepEqual(migrated.db.prepare(`SELECT * FROM ${table} ORDER BY rowid`).all(), before[table], table); + assert.deepEqual(migrated.db.prepare('PRAGMA foreign_key_check').all(), []); + assert.equal(migrated.db.prepare('PRAGMA foreign_keys').get().foreign_keys, 1); + assert.throws(() => migrated.db.prepare("UPDATE installations SET backend='native_service_v1' WHERE organization_id=?") + .run(created.organization.id), /installation_backend_immutable/); + const access = new AccessControlService(migrated, { installationOperatorEnabled: true, installationBackend: 'native_service_v1' }); + const native = access.createOrganization(f.platform.session, { ownerEmail: 'native-after-migration@example.test', + idempotencyKey: 'migration:native:create' }); + assert.equal(migrated.installationBackend(native.organization.id), 'native_service_v1'); + assert.equal(migrated.installationBackend(created.organization.id), 'oci_container_v1'); + } finally { migrated.close(); } +}); + +test('native rollout refuses legacy DSPs before queuing Core and allows decommissioned history', async t => { + const f = await fixture(t); + const created = f.create(899, false); + const coordinator = createPlatformUpdates({ store: f.store, + releases: { dispatch_update_2: { backend: 'native_service_v1' } }, + platformReleases: f.platformReleases, enabled: true }); + const start = { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:native:legacy' }; + assert.throws(() => coordinator.command(f.platform.session, start), /native_migration_required/); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_rollouts').get().n, 0); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_rollout_core').get().n, 0); + f.store.db.prepare("UPDATE installations SET status='decommissioning' WHERE organization_id=?").run(created.organization.id); + assert.throws(() => coordinator.command(f.platform.session, start), /native_migration_required/); + f.store.db.prepare("UPDATE installations SET status='decommissioned' WHERE organization_id=?").run(created.organization.id); + coordinator.command(f.platform.session, start); + coordinator.command(f.platform.session, start); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_rollouts').get().n, 1); + assert.equal(f.store.db.prepare('SELECT status FROM platform_rollout_core').get().status, 'queued'); +}); + +test('native rollout updates suspended and setup DSPs, assigns pending DSPs, and waits for storage cleanup', async t => { + const f = await fixture(t, 'native_service_v1'); + const states = ['pending', 'suspended', 'waiting_for_owner', 'waiting_for_provider_auth']; + const created = states.map((_, i) => f.create(900 + i, false)); + f.store.db.prepare("UPDATE installation_provisioning_requests SET status='completed',finished_at=?").run(Date.now()); + for (let i = 0; i < states.length; i++) { + f.store.db.prepare('UPDATE installations SET status=? WHERE organization_id=?').run(states[i], created[i].organization.id); + if (states[i] === 'suspended') f.store.updateOrganizationStatus(created[i].organization.id, 'suspended', Date.now()); + } + let cleaned = false; + const options = { store: f.store, releases: { dispatch_update_2: { backend: 'native_service_v1' } }, + platformReleases: f.platformReleases, enabled: true, cleanupReady: () => cleaned }; + let coordinator = createPlatformUpdates(options); + coordinator.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:native:offline' }); + completeRolloutBackups(f.store); + f.store.db.prepare("UPDATE platform_rollout_core SET status='succeeded'").run(); + const visited = new Set(); + for (let step = 0; step < 30; step++) { + coordinator = createPlatformUpdates(options); coordinator.tick(); + const job = f.store.db.prepare("SELECT * FROM installation_lifecycle_jobs WHERE status='queued'").get(); + if (!job) continue; + visited.add(job.starting_state); + const authority = require('../src/installation-lifecycle').createAccessInstallationLifecycleAuthority({ + store: f.store, organizationId: job.organization_id, authorityScope: 'platform_rollout', + actorUserId: f.platform.session.user.id, releaseCatalog: ['dispatch_update_2'], + }); + if (job.starting_state !== 'suspended') assert.ok(authority.claim(job.id, 'worker_native_fixture')); + if (job.starting_state === 'suspended') { + // Readiness is needed for a real suspended DSP; this fixture only tests fleet scheduling. + assert.equal(JSON.parse(job.stages_json).includes('start_release'), false); + } else assert.equal(JSON.parse(job.stages_json).includes('verify_release_publication'), false); + f.store.db.prepare("UPDATE installation_lifecycle_jobs SET status='succeeded',lease_expires_at=NULL,finished_at=?,result_json='{}' WHERE id=?").run(Date.now(), job.id); + f.store.db.prepare('UPDATE installations SET status=?,release_id=? WHERE organization_id=?').run(job.starting_state, 'dispatch_update_2', job.organization_id); + } + assert.deepEqual([...visited].sort(), states.filter(s => s !== 'pending').sort()); + assert.equal(coordinator.view().rollout.updated, 4); + assert.equal(coordinator.view().rollout.status, 'running'); + cleaned = true; coordinator.tick(); + assert.equal(coordinator.view().rollout.status, 'completed'); + for (let i = 0; i < states.length; i++) assert.equal(f.store.installationControl(created[i].organization.id).status, states[i]); +}); + +test('removed owner accounts stay signed out until restoration completes and never regain old sessions', async t => { + const f = await fixture(t, 'native_service_v1'), target = f.create(1401, false); + const owner = await f.access.acceptNewUser({ token: target.token, firstName: 'Retained', lastName: 'Owner', + password: 'retained owner password', confirmPassword: 'retained owner password' }); + f.store.db.prepare("UPDATE installations SET status='waiting_for_provider_auth' WHERE organization_id=?").run(target.organization.id); + const row = f.access.platformOrganizations(f.platform.session)[0]; + await f.access.requestPlatformRemoval(f.platform.session, { controlRef: row.controlRef, + expectedRevision: row.installation.revision, idempotencyKey: 'accounts:remove:1401' }, 'decommission'); + assert.equal(f.access.session(owner.token), null); + const { createAccessInstallationLifecycleAuthority } = require('../src/installation-lifecycle'); + const authority = createAccessInstallationLifecycleAuthority({ store: f.store, organizationId: target.organization.id, authorityScope: 'platform_removal' }); + const job = f.store.activeLifecycleJob(target.organization.id), claim = authority.claim(job.id, 'worker_accounts_remove'); + for (const [stage, status] of [['inspect_schedule', 'verified'], ['quiesce_schedule', 'stopped'], ['stop_runtime', 'stopped'], ['disable_runtime', 'disabled'], ['verify_retained', 'retained']]) + authority.checkpoint(claim.claim, stage, { status, ...(stage === 'inspect_schedule' ? { syncWasRunning: false } : {}) }); + authority.succeed(claim.claim); + let removed = f.access.platformOrganizations(f.platform.session)[0]; + assert.deepEqual(removed.installation.availableActions, ['destroy', 'restore_dsp']); + await f.access.requestPlatformRemoval(f.platform.session, { controlRef: removed.controlRef, + expectedRevision: removed.installation.revision, idempotencyKey: 'accounts:restore:1401' }, 'resume'); + await assert.rejects(f.access.signIn({ email: 'owner1401@example.test', password: 'retained owner password' }), /account_disabled/); + const restore = f.store.activeLifecycleJob(target.organization.id), restoreClaim = authority.claim(restore.id, 'worker_accounts_restore'); + authority.checkpoint(restoreClaim.claim, 'start_runtime', { status: 'started' }); + authority.checkpoint(restoreClaim.claim, 'verify_infrastructure', { status: 'verified' }); + authority.succeed(restoreClaim.claim); + assert.ok((await f.access.signIn({ email: 'owner1401@example.test', password: 'retained owner password' })).session); + assert.equal(f.access.session(owner.token), null); + assert.equal(f.store.organization(target.organization.id).status, 'setup_required'); + assert.ok(!f.access.platformOrganizations(f.platform.session)[0].installation.availableActions.includes('destroy')); +}); + +test('release selection preserves installed notes and archived history without permitting historical rollouts', async t => { + const f = await fixture(t); + const old = { ...f.platformReleases.dispatch_update_2, version: '0.0.1', publishedAt: '2026-09-01T00:00:00.000Z' }; + const rich = { groups: [{ id: 'updates', title: 'Updates', icon: 'info' }], changelog: [], afterUpdating: [] }; + const coordinator = createPlatformUpdates({store:f.store,enabled:true,releases:{dispatch_update_2:{}},platformReleases:f.platformReleases, + delivery:{view:()=>null,history:()=>({dispatch_old:old}),notes:id=>id==='dispatch_old'?rich:null}}); + assert.equal(coordinator.view().displayedRelease.id,'dispatch_update_2'); + const history = coordinator.view('dispatch_old'); + assert.equal(history.displayedRelease.state,'historical'); + assert.deepEqual(history.displayedRelease.notes,rich); + assert.equal(history.releaseHistory.length,2); + assert.throws(()=>coordinator.view('dispatch_unknown'),/release_not_found/); + assert.throws(()=>coordinator.command(f.platform.session,{action:'start',releaseId:'dispatch_old',idempotencyKey:'history:forbidden:1'}),/update_unavailable/); + coordinator.command(f.platform.session,{action:'start',releaseId:'dispatch_update_2',idempotencyKey:'history:rollout:start:1'}); + assert.equal(coordinator.view().displayedRelease.state,'rolling_out'); + f.store.db.prepare("UPDATE platform_rollouts SET status='completed'").run(); + f.store.db.prepare("UPDATE platform_rollout_core SET status='succeeded'").run(); + assert.equal(coordinator.view().displayedRelease.state,'installed'); + assert.deepEqual(coordinator.view().displayedRelease.changelog,f.platformReleases.dispatch_update_2.changelog); + assert.throws(()=>coordinator.command(f.platform.session,{action:'start',releaseId:'dispatch_update_2',idempotencyKey:'history:downgrade:1'}),/update_unavailable/); +}); + +test('designated test DSP collects on the candidate before fleet updates, with durable proof', async t => { + const f = await fixture(t, 'native_service_v1'); + const ordinary = f.create(1300, false), canary = f.create(1301, false); + ready(f, ordinary); ready(f, canary); + let resolveCheck; + let checks = 0; + const options = { store: f.store, releases: { dispatch_update_2: { backend: 'native_service_v1' } }, + platformReleases: f.platformReleases, enabled: true, cleanupReady: () => true, + canaryVerifier: () => { checks += 1; return new Promise(resolve => { resolveCheck = resolve; }); } }; + let updates = createPlatformUpdates(options); + updates.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:canary:fixture', canaryOrganizationId: canary.organization.id }); + assert.throws(() => updates.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:canary:fixture', canaryOrganizationId: ordinary.organization.id }), /idempotency_conflict/); + completeRolloutBackups(f.store); + f.store.db.prepare("UPDATE platform_rollout_core SET status='succeeded'").run(); + updates.tick(); + assert.equal(f.store.db.prepare("SELECT organization_id FROM platform_rollout_members WHERE status='updating'").get().organization_id, canary.organization.id); + // Model the lifecycle worker's candidate verification; the collection gate + // must still run even though installation health and release checks passed. + f.store.db.prepare("UPDATE installations SET release_id='dispatch_update_2' WHERE organization_id=?").run(canary.organization.id); + f.store.db.prepare("UPDATE platform_rollout_members SET status='updated' WHERE organization_id=?").run(canary.organization.id); + updates.tick(); await new Promise(resolve => setImmediate(resolve)); + updates.tick(); + assert.equal(checks, 1); + assert.equal(f.store.db.prepare("SELECT status FROM platform_rollout_members WHERE organization_id=?").get(ordinary.organization.id).status, 'queued'); + resolveCheck(true); await new Promise(resolve => setImmediate(resolve)); + updates = createPlatformUpdates({ ...options, canaryVerifier: () => { throw Error('must reuse proof'); } }); + updates.tick(); + assert.equal(f.store.db.prepare("SELECT status FROM platform_rollout_members WHERE organization_id=?").get(ordinary.organization.id).status, 'updating'); +}); + +test('test DSP collection failure pauses the fleet before any other DSP updates', async t => { + const f = await fixture(t, 'native_service_v1'); + const canary = f.create(1302, false), ordinary = f.create(1303, false); + ready(f, canary); ready(f, ordinary); + const updates = createPlatformUpdates({ store: f.store, releases: { dispatch_update_2: { backend: 'native_service_v1' } }, + platformReleases: f.platformReleases, enabled: true, canaryVerifier: async () => { throw Error('synthetic_failure'); } }); + updates.command(f.platform.session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:canary:failure', canaryOrganizationId: canary.organization.id }); + completeRolloutBackups(f.store); + f.store.db.prepare("UPDATE platform_rollout_core SET status='succeeded'").run(); + f.store.db.prepare("UPDATE installations SET release_id='dispatch_update_2' WHERE organization_id=?").run(canary.organization.id); + f.store.db.prepare("UPDATE platform_rollout_members SET status='updated' WHERE organization_id=?").run(canary.organization.id); + updates.tick(); await new Promise(resolve => setImmediate(resolve)); + assert.equal(updates.view().rollout.status, 'paused'); + assert.equal(f.store.db.prepare("SELECT status FROM platform_rollout_members WHERE organization_id=?").get(ordinary.organization.id).status, 'queued'); + assert.doesNotMatch(JSON.stringify(updates.view()), /synthetic_failure/); +}); diff --git a/core/core/accounts/tests/platform-backups.test.js b/core/core/accounts/tests/platform-backups.test.js new file mode 100644 index 0000000..80574ac --- /dev/null +++ b/core/core/accounts/tests/platform-backups.test.js @@ -0,0 +1,508 @@ +'use strict'; +const test = require('node:test'), + assert = require('node:assert/strict'), + fs = require('node:fs'), + os = require('node:os'), + path = require('node:path'), + crypto = require('node:crypto'); +const { AccessStore } = require('../src/store'); +const { createPlatformBackups } = require('../src/platform-backups'); +const { captureDspMetadata, restoreDspMetadata } = require('../src/backup-metadata'); +const { + DEFAULT_BACKUP_SETTINGS, + backupSettings, + scheduledSlot, + nextScheduledAt, +} = require('../src/backup-schedule'); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-backup-control-')); + fs.chmodSync(root, 0o700); + const store = new AccessStore({ + databaseRoot: path.join(root, 'access'), + database: path.join(root, 'access/access-control.sqlite3'), + }); + t.after(() => { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + }); + store.insertUser({ + id: 'user_platform', + email: 'platform@example.test', + firstName: 'Platform', + lastName: 'Owner', + passwordHash: 'synthetic', + platformRole: 'owner', + timestamp: 1000, + }); + for (const suffix of ['one', 'two']) { + const org = `org_${suffix}`, + user = `user_${suffix}`, + role = `role_${suffix}`; + store.createOrganization({ + id: org, + name: `DSP ${suffix}`, + abbreviation: null, + timezone: 'UTC', + status: 'active', + createdBy: null, + timestamp: 1000, + }); + store.insertStation(org, 'TST1', true, 1000); + store.createInstallation( + org, + `runtime_${suffix}`, + 'ready', + 1000, + 'dispatch_current_1', + 'oci_container_v1', + ); + store.insertUser({ + id: user, + email: `${suffix}@example.test`, + firstName: suffix, + lastName: 'Owner', + passwordHash: 'synthetic-old', + platformRole: null, + timestamp: 1000, + }); + store.createRole({ + id: role, + organizationId: org, + key: 'owner', + name: 'Owner', + description: 'Owner', + system: true, + createdBy: null, + timestamp: 1000, + permissions: ['team.read'], + }); + store.createMembership({ + id: `member_${suffix}`, + organizationId: org, + userId: user, + roleId: role, + createdBy: null, + timestamp: 1000, + }); + } + const remote = { status: 'connected', backups: {} }; + let now = Date.parse('2026-09-05T12:00:00Z'); + const control = createPlatformBackups({ + store, + enabled: true, + archive: () => remote, + clock: () => now, + }); + const session = { user: { id: 'user_platform', platformRole: 'owner' } }; + function record(org = 'org_one') { + const id = `backup_${crypto.randomBytes(16).toString('hex')}`, + metadata = JSON.stringify(captureDspMetadata(store, org)); + store.db + .prepare("INSERT INTO platform_backup_records VALUES(?,?,'dsp',?,NULL,?,NULL,NULL)") + .run(id, org, metadata, now); + remote.backups[id] = { + status: 'verified', + format: 2, + metadataDigest: crypto.createHash('sha256').update(metadata).digest('hex'), + localReady: true, + }; + return id; + } + return { + store, + control, + session, + remote, + record, + setNow: (n) => { + now = n; + }, + }; +} +test('usage exposes Core, every DSP and removed retention independently from truncated history, with stale and unavailable states',t=>{ + const f=fixture(t);assert.equal(f.control.view().storageUsage.status,'unavailable'); + f.store.db.prepare("INSERT INTO dsp_removals(organization_id,installation_state,organization_status,sync_running,removed_at) VALUES('org_two','ready','active',1,1000)").run(); + f.remote.usage={status:'ready',checkedAt:Date.parse('2026-09-05T12:00:00Z'),bytes:606,backupCount:3,core:{bytes:100,backupCount:1},dsps:[{organizationId:'org_one',bytes:200,backupCount:1},{organizationId:'org_two',bytes:300,backupCount:1}],other:{bytes:0,backupCount:0},manifestBytes:6,legacyBytes:0,sets:[]}; + f.setNow(f.remote.usage.checkedAt);const usage=f.control.view().storageUsage; + assert.equal(usage.status,'ready');assert.equal(usage.bytes,606);assert.equal(usage.retainedBytes,300);assert.equal(usage.scopes.find(s=>s.scope==='org_two').removed,true);assert.equal(usage.scopes.find(s=>s.scope==='org_two').name,'DSP two');assert.equal(f.control.view().organizations.some(o=>o.id==='org_two'),false); + f.setNow(f.remote.usage.checkedAt+300000);assert.equal(f.control.view().storageUsage.status,'stale'); +}); +test('completed legacy DSP deletions stay out of protection status after the current job pointer is cleared', t => { + const { store, control } = fixture(t); + store.createLifecycleJob({ id: 'life_deleted', organizationId: 'org_one', operation: 'destroy', + startingState: 'decommissioned', installationState: 'decommissioning', installationRevision: 1, + manifestRevision: 1, runtimeKey: 'runtime_one', releaseId: 'dispatch_current_1', + targetReleaseId: null, backupId: null, safetyBackupId: null, authorityScope: 'platform', + idempotencyKey: 'test:destroy', stages: [], timestamp: 1000 }); + store.db.prepare("UPDATE installation_lifecycle_jobs SET status='succeeded',finished_at=2000,result_json='{}' WHERE id='life_deleted'").run(); + store.db.prepare("UPDATE installations SET status='decommissioned',current_job_id=NULL WHERE organization_id='org_one'").run(); + assert.deepEqual(control.view().organizations.map(o => o.id), ['org_two']); +}); + +test('settings reject malformed schedules; DST repeats run once and skipped daily time runs after the gap', () => { + const daily = backupSettings({ + ...DEFAULT_BACKUP_SETTINGS, + enabled: true, + time: '02:30', + timezone: 'America/Los_Angeles', + }); + assert.equal(scheduledSlot(daily, Date.parse('2026-03-08T09:59:00Z')), null); + assert.equal(scheduledSlot(daily, Date.parse('2026-03-08T10:00:00Z')), '2026-03-08'); + const hourly = { ...daily, frequency: 'hourly' }; + assert.equal( + scheduledSlot(hourly, Date.parse('2026-11-01T08:15:00Z')), + scheduledSlot(hourly, Date.parse('2026-11-01T09:15:00Z')), + ); + assert.throws(() => backupSettings({ ...daily, time: '25:00' })); + assert.throws(() => backupSettings({ ...daily, retentionDays: 1 })); + assert.equal( + nextScheduledAt(daily, Date.parse('2026-03-08T09:59:00Z')), + '2026-03-08T10:00:00.000Z', + ); +}); +test('settings are owner-only, persistent, idempotent, revision checked and do not rewrite existing retention', (t) => { + const f = fixture(t), + id = f.record(); + const input = { + action: 'settings', + idempotencyKey: 'settings:test:123456', + revision: 1, + settings: { ...DEFAULT_BACKUP_SETTINGS, enabled: true, retentionDays: 30 }, + }; + assert.throws(() => f.control.command({ user: { id: 'user_one', platformRole: null } }, input), { + code: 'permission_denied', + }); + f.control.command(f.session, input); + f.control.command(f.session, input); + assert.equal(f.control.view().revision, 2); + assert.throws( + () => f.control.command(f.session, { ...input, idempotencyKey: 'settings:stale:123456' }), + { code: 'backup_settings_conflict' }, + ); + assert.equal( + f.store.db.prepare('SELECT retention_days FROM platform_backup_records WHERE id=?').get(id) + .retention_days, + null, + ); + assert.equal(createPlatformBackups({ store: f.store }).settings().retentionDays, 30); +}); +test('manual fleet backup queues Core and every eligible DSP once, and does not disclose credentials', (t) => { + const f = fixture(t); + f.record(); + const command = { action: 'backup', idempotencyKey: 'manual:all:123456789' }; + f.control.command(f.session, command); + f.control.command(f.session, command); + assert.equal(f.control.view().operations.length, 3); + assert.equal(f.control.view().operations.filter((o) => o.kind === 'core').length, 1); + assert.equal(JSON.stringify(f.control.view()).includes('synthetic-old'), false); + assert.equal(JSON.stringify(f.control.view()).includes('password_hash'), false); +}); +test('scheduler persists its slot and queues one fleet batch across repeated ticks', (t) => { + const f = fixture(t); + f.control.command(f.session, { + action: 'settings', + idempotencyKey: 'schedule:test:123456', + revision: 1, + settings: { ...DEFAULT_BACKUP_SETTINGS, enabled: true, frequency: 'hourly', timezone: 'UTC' }, + }); + f.control.schedule(); + f.control.schedule(); + assert.equal(f.control.view().operations.length, 3); + const reopened = createPlatformBackups({ + store: f.store, + enabled: true, + clock: () => Date.parse('2026-09-05T12:30:00Z'), + }); + reopened.schedule(); + assert.equal(f.control.view().operations.length, 3); +}); +test('restore binds the backup to its DSP and rejects missing proof, legacy format, tampering and wrong confirmation', (t) => { + const f = fixture(t), + id = f.record(), + command = { + action: 'restore', + backupId: id, + organizationId: 'org_one', + confirmation: 'DSP one', + idempotencyKey: 'restore:test:1234567', + }; + assert.throws( + () => + f.control.command(f.session, { + ...command, + organizationId: 'org_two', + confirmation: 'DSP two', + }), + { code: 'backup_not_found' }, + ); + assert.throws(() => f.control.command(f.session, { ...command, confirmation: 'wrong' }), { + code: 'backup_confirmation_required', + }); + f.remote.backups[id].format = 1; + assert.throws(() => f.control.command(f.session, command), { + code: 'backup_restore_unavailable', + }); + f.remote.backups[id].format = 2; + const original = f.remote.backups[id].metadataDigest; + f.remote.backups[id].metadataDigest = 'a'.repeat(64); + assert.throws(() => f.control.command(f.session, command), { + code: 'backup_restore_unavailable', + }); + f.remote.backups[id].metadataDigest = original; + f.control.command(f.session, command); + f.control.command(f.session, command); + assert.equal(f.control.view().operations.length, 1); +}); +test('metadata recovery restores DSP permissions while preserving current passwords and other DSPs', (t) => { + const f = fixture(t), + metadata = captureDspMetadata(f.store, 'org_one'); + f.store.updatePassword('user_one', 'synthetic-new', 2000); + f.store.db.prepare("UPDATE roles SET name='Changed' WHERE id='role_one'").run(); + restoreDspMetadata(f.store, 'org_one', metadata, 3000); + assert.equal(f.store.role('role_one').name, 'Owner'); + assert.equal(f.store.userById('user_one').password_hash, 'synthetic-new'); + assert.equal(f.store.role('role_two').name, 'Owner'); + assert.throws(() => restoreDspMetadata(f.store, 'org_two', metadata), { + code: 'backup_identity_conflict', + }); + const bad = structuredClone(metadata); + bad.users[0].platform_role = 'owner'; + assert.throws(() => restoreDspMetadata(f.store, 'org_one', bad), { + code: 'backup_identity_conflict', + }); +}); + +test('a newer runtime release does not hide compatible historical backups', (t) => { + const f = fixture(t), + id = f.record(); + f.store.db + .prepare( + "UPDATE installations SET release_id='dispatch_next_2',manifest_revision=manifest_revision+1 WHERE organization_id='org_one'", + ) + .run(); + assert.equal(f.control.view().backups.find((b) => b.id === id).restoreBlocked, null); +}); + +test('scheduled backups wait for the previous batch instead of accumulating Core jobs', (t) => { + const f = fixture(t); + f.control.command(f.session, { + action: 'settings', + idempotencyKey: 'schedule:busy:123456', + revision: 1, + settings: { ...DEFAULT_BACKUP_SETTINGS, enabled: true, frequency: 'hourly', timezone: 'UTC' }, + }); + f.control.schedule(); + f.setNow(Date.parse('2026-09-05T13:30:00Z')); + f.control.schedule(); + assert.equal(f.control.view().operations.length, 3); + f.store.db.prepare("UPDATE platform_backup_requests SET status='completed'").run(); + f.control.schedule(); + assert.equal(f.control.view().operations.length, 6); +}); + +test('explicit DSP selection is atomic, deduplicated and does not enqueue Platform Core', (t) => { + const f = fixture(t); + const input = { + action: 'backup', + scope: 'dsps', + organizationIds: ['org_one', 'org_two', 'org_one'], + idempotencyKey: 'selected:batch:123456', + }; + f.control.command(f.session, input); + f.control.command(f.session, input); + assert.deepEqual( + f.control + .view() + .operations.map((o) => o.organizationId) + .sort(), + ['org_one', 'org_two'], + ); + assert.equal(f.control.view().canBackupCore, true); + assert.equal( + f.control.view().organizations.every((o) => !o.canBackup), + true, + ); +}); + +test('an unavailable DSP rejects the whole selection without a partially queued batch', (t) => { + const f = fixture(t); + f.store.db + .prepare("UPDATE installations SET status='provisioning' WHERE organization_id='org_two'") + .run(); + assert.throws( + () => + f.control.command(f.session, { + action: 'backup', + scope: 'dsps', + organizationIds: ['org_one', 'org_two'], + idempotencyKey: 'selected:invalid:123456', + }), + { code: 'backup_dsp_unavailable' }, + ); + assert.equal(f.control.view().operations.length, 0); + assert.equal(f.control.view().organizations.find((o) => o.id === 'org_two').canBackup, false); +}); + +test('Platform Core-only backup leaves every DSP available and rejects ambiguous scope', (t) => { + const f = fixture(t); + for (const extra of [ + { scope: 'unknown' }, + { scope: 'core', organizationId: 'org_one' }, + { scope: 'dsps' }, + { scope: 'dsps', organizationIds: [] }, + { organizationIds: ['org_one'] }, + { scope: 'dsps', organizationIds: ['org_one'], organizationId: 'org_two' }, + { scope: 'dsps', organizationIds: ['org_one', '../../org_two'] }, + ]) + assert.throws( + () => + f.control.command(f.session, { + action: 'backup', + idempotencyKey: 'invalid:scope:123456', + ...extra, + }), + { code: 'invalid_input' }, + ); + f.control.command(f.session, { + action: 'backup', + scope: 'core', + idempotencyKey: 'core:only:123456789', + }); + const view = f.control.view(); + assert.equal(view.operations.length, 1); + assert.equal(view.operations[0].kind, 'core'); + assert.equal(view.canBackupCore, false); + assert.equal( + view.organizations.every((o) => o.canBackup), + true, + ); + assert.equal(view.operations[0].backupId, view.operations[0].id); + assert.equal(view.operations[0].updatedAt, view.operations[0].createdAt); +}); + +test('restore operation exposes only the selected backup and safe progress metadata', (t) => { + const f = fixture(t), + backupId = f.record(); + f.remote.backups[backupId].verifiedAt = Date.parse('2026-09-05T12:00:00Z'); + f.control.command(f.session, { + action: 'restore', + organizationId: 'org_one', + backupId, + confirmation: 'DSP one', + idempotencyKey: 'restore:details:123456', + }); + const view = f.control.view(); + assert.equal(view.operations[0].backupId, backupId); + assert.equal(view.operations[0].safetyBackupId, null); + assert.equal(view.backups[0].verifiedAt, '2026-09-05T12:00:00.000Z'); + assert.equal(JSON.stringify(view).includes('input_json'), false); + assert.equal(JSON.stringify(view).includes('synthetic'), false); +}); + +test('expired archive proof stops reporting protection before the archive cleanup runs', (t) => { + const f = fixture(t), + id = f.record(); + f.remote.backups[id].expiresAt = Date.parse('2026-09-05T11:00:00Z'); + const backup = f.control.view().backups.find((b) => b.id === id); + assert.equal(backup.status, 'expired'); + assert.match(backup.restoreBlocked, /retention date/); +}); + +test('busy Core and unresolved DSP issues remain visible beyond the recent operation limit', (t) => { + const f = fixture(t); + f.control.command(f.session, { + action: 'backup', + scope: 'core', + idempotencyKey: 'older:core:123456789', + }); + f.control.command(f.session, { + action: 'backup', + organizationId: 'org_two', + idempotencyKey: 'older:failed:123456789', + }); + f.store.db + .prepare( + "UPDATE platform_backup_requests SET status='failed',phase='failed' WHERE organization_id='org_two'", + ) + .run(); + for (let i = 0; i < 55; i++) { + f.setNow(Date.parse('2026-09-05T13:00:00Z') + i * 1000); + f.control.command(f.session, { + action: 'backup', + organizationId: 'org_one', + idempotencyKey: `newer:backup:123456789:${i}`, + }); + f.store.db + .prepare( + "UPDATE platform_backup_requests SET status='completed',phase='completed' WHERE organization_id='org_one'", + ) + .run(); + } + const view = f.control.view(); + assert.equal(view.canBackupCore, false); + assert.equal(view.operations.find((o) => o.kind === 'core').status, 'queued'); + assert.equal(view.operations.find((o) => o.organizationId === 'org_two').status, 'failed'); +}); + +test('each Core, system and DSP schedule starts off and changes independently with owner authorization',t=>{ + const f=fixture(t);const schedules=f.control.view().schedules; + assert.deepEqual(schedules.map(s=>s.scope),['system','core','org_one','org_two']); + assert.ok(schedules.every(s=>s.settings.enabled===false)); + const input={action:'settings',organizationId:'org_one',revision:1,settings:{...DEFAULT_BACKUP_SETTINGS,enabled:true,frequency:'hourly'},idempotencyKey:'scope:settings:one'}; + assert.throws(()=>f.control.command({user:{id:'user_one',platformRole:null}},input),e=>e.code==='permission_denied'); + f.control.command(f.session,input); + assert.equal(f.control.settings('org_one').enabled,true); + assert.ok(['core','system','org_two'].every(scope=>f.control.settings(scope).enabled===false)); + f.control.schedule(); + assert.deepEqual(f.store.db.prepare('SELECT organization_id FROM platform_backup_requests').all().map(r=>r.organization_id),['org_one']); +}); +test('full system backup is one set containing separately scheduled Core and every DSP',t=>{ + const f=fixture(t);f.control.command(f.session,{action:'backup',scope:'system',idempotencyKey:'scope:system:backup'}); + const set=f.control.view().sets[0];assert.equal(set.members.length,3); + assert.deepEqual(new Set(set.members.map(m=>m.organizationId)),new Set([null,'org_one','org_two'])); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_backup_requests').get().n,3); + assert.ok(f.control.view().schedules.every(s=>!s.settings.enabled)); + +}); +test('DSP archive deletion is bound to the selected DSP and leaves Core and its neighbor untouched',t=>{ + const f=fixture(t),one=f.record('org_one'),two=f.record('org_two'); + assert.throws(()=>f.control.command(f.session,{action:'delete',organizationId:'org_one',backupId:two,confirmation:'DSP one',idempotencyKey:'scope:delete:wrong'}),e=>e.code==='backup_not_found'); + f.control.command(f.session,{action:'delete',organizationId:'org_one',backupId:one,confirmation:'DSP one',idempotencyKey:'scope:delete:right'}); + const rows=f.store.db.prepare('SELECT * FROM backup_deletions').all();assert.equal(rows.length,1);assert.equal(rows[0].backup_id,one); + assert.equal(f.store.db.prepare('SELECT deleted_at FROM platform_backup_records WHERE id=?').get(two).deleted_at,null); +}); +test('DSP restore restores only its own schedule',t=>{ + const f=fixture(t); + f.control.command(f.session,{action:'settings',organizationId:'org_one',revision:1,settings:{...DEFAULT_BACKUP_SETTINGS,enabled:true},idempotencyKey:'schedule:restore:one'}); + const captured=captureDspMetadata(f.store,'org_one'); + f.control.command(f.session,{action:'settings',organizationId:'org_one',revision:2,settings:{...DEFAULT_BACKUP_SETTINGS,enabled:false},idempotencyKey:'schedule:restore:two'}); + restoreDspMetadata(f.store,'org_one',captured); + assert.equal(f.control.settings('org_one').enabled,true);assert.equal(f.control.settings('org_two').enabled,false);assert.equal(f.control.settings('core').enabled,false); +}); +test('a DSP snapshot with no configured schedule restores to off',t=>{ + const f=fixture(t),captured=captureDspMetadata(f.store,'org_one');assert.equal(captured.schedule,null); + f.control.command(f.session,{action:'settings',organizationId:'org_one',revision:1,settings:{...DEFAULT_BACKUP_SETTINGS,enabled:true},idempotencyKey:'schedule:restore:absent'}); + restoreDspMetadata(f.store,'org_one',captured);assert.equal(f.control.settings('org_one').enabled,false); +}); + +test('full-system restore validates every component and enqueues Core first without changing unrelated schedules',t=>{ + const f=fixture(t),one=f.record('org_one'),two=f.record('org_two'),core=`breq_${'1'.repeat(32)}`,set=`breq_${'2'.repeat(32)}`; + const metadata=JSON.stringify({schemaVersion:2,scope:'core',name:'Platform Core'}); + f.store.db.prepare("INSERT INTO platform_backup_records VALUES(?,NULL,'core',?,NULL,1,NULL,NULL)").run(core,metadata); + f.remote.backups[core]={status:'verified',metadataDigest:crypto.createHash('sha256').update(metadata).digest('hex')}; + f.store.db.prepare("INSERT INTO backup_sets VALUES(?,1,?,'verified')").run(set,JSON.stringify([{organizationId:'org_one',backupId:one},{organizationId:null,backupId:core},{organizationId:'org_two',backupId:two}])); + const row=f.store.db.prepare('SELECT * FROM backup_sets WHERE id=?').get(set);f.remote.sets={[set]:{status:'verified',setDigest:crypto.createHash('sha256').update(JSON.stringify(row)).digest('hex')}}; + const command={action:'restore',scope:'system',setId:set,confirmation:'Full system',idempotencyKey:'restore:full:system'}; + delete f.remote.backups[two];assert.throws(()=>f.control.command(f.session,command),e=>e.code==='backup_restore_unavailable');assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_backup_requests').get().n,0); + const meta=f.store.db.prepare('SELECT metadata_json FROM platform_backup_records WHERE id=?').get(two).metadata_json; + f.remote.backups[two]={status:'verified',format:2,metadataDigest:crypto.createHash('sha256').update(meta).digest('hex')};f.control.command(f.session,command); + const requests=f.store.db.prepare("SELECT * FROM platform_backup_requests ORDER BY json_extract(input_json,'$.position')").all();assert.equal(requests.length,3);assert.equal(requests[0].kind,'core');assert.equal(requests[0].organization_id,null);assert.deepEqual(new Set(requests.slice(1).map(r=>r.organization_id)),new Set(['org_one','org_two'])); + assert.ok(f.control.view().schedules.every(s=>!s.settings.enabled)); + assert.equal(f.control.view().sets[0].restore.status,'running'); + f.store.db.prepare("UPDATE platform_backup_requests SET status='failed',phase='failed'").run(); + assert.equal(f.control.view().sets[0].restore.status,'failed'); + f.control.command(f.session,{...command,idempotencyKey:'restore:full:retry'}); + assert.equal(f.control.view().sets[0].restore.status,'running'); + f.store.db.prepare("UPDATE platform_backup_requests SET status='completed',phase='completed' WHERE status='queued'").run(); + assert.equal(f.control.view().sets[0].restore.status,'completed'); +}); diff --git a/core/core/accounts/tests/plugin-fixture.js b/core/core/accounts/tests/plugin-fixture.js new file mode 100644 index 0000000..941031f --- /dev/null +++ b/core/core/accounts/tests/plugin-fixture.js @@ -0,0 +1,50 @@ +'use strict'; +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { AccessStore, AccessControlService } = require('../src'); +const { createPluginService } = require('../src/plugins'); +const { success } = require('../../../shared/contracts/src/result'); +async function fixture(t, { installationCoordinator = null, settingsPort = null, dspCount = 2 } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-plugins-')); + fs.chmodSync(root, 0o700); + const paths = { databaseRoot: path.join(root, 'access'), database: path.join(root, 'access/access.sqlite3') }; + const store = new AccessStore(paths); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const access = new AccessControlService(store, { installationBackend: 'directory_service_v1', installationOperatorEnabled: true }); + const password = 'synthetic plugin password'; + const bootstrap = access.createPlatformBootstrap({ email: 'platform@example.test' }); + const platform = await access.acceptNewUser({ token: bootstrap.token, firstName: 'Platform', lastName: 'Owner', password, confirmPassword: password }); + const dsps = []; + const runtimes = new Map(); + for (let index = 0; index < dspCount; index++) { + const created = access.createOrganization(platform.session, { idempotencyKey: `fixture:plugin:${index}`, name: `Plugin DSP ${index}`, + abbreviation: `FX${index}`, stationCode: 'TST1', timezone: 'America/Chicago', ownerEmail: `owner${index}@example.test` }); + const owner = await access.acceptNewUser({ token: created.token, firstName: 'DSP', lastName: 'Owner', password, confirmPassword: password }); + const id = created.organization.id; + store.db.prepare("UPDATE installations SET status='ready' WHERE organization_id=?").run(id); + store.updateOrganizationStatus(id, 'active', Date.now()); + const runtimeKey = store.installation(id).runtimeKey; + dsps.push({ id, runtimeKey, owner: access.session(owner.token), token: owner.token }); + runtimes.set(runtimeKey, { id: 'paycom', version: '0.18.7', state: 'uninstalled', revision: 0 }); + } + const calls = []; + let unavailable = false; + const plugins = createPluginService({ store, access, installationCoordinator: installationCoordinator ? { latest: () => ({version:'0.18.7'}), ...installationCoordinator } : null, settingsPort, invoke: async (runtimeKey, action, input) => { + calls.push({ runtimeKey, action, input }); + if (unavailable) throw new Error('synthetic runtime unavailable'); + if (input.command === 'status') return success('found', { items: [runtimes.get(runtimeKey)] }); + const current = runtimes.get(runtimeKey); + if (input.revision < current.revision) throw new Error('stale revision'); + const next = { id: input.pluginId, version: input.version, state: input.state, revision: input.revision }; + runtimes.set(runtimeKey, next); + return success('applied', next); + } }); + return { root, paths, store, access, platform, dsps, runtimes, calls, plugins, setUnavailable(value) { unavailable = value; } }; +} +function enableFixturePlugin(store, organizationId) { + store.db.prepare(`INSERT OR REPLACE INTO dsp_plugins(organization_id,plugin_id,version,desired_state,applied_state, + revision,applied_revision,failure_code,actor_user_id,updated_at) VALUES(?,'paycom','0.18.7','enabled','enabled',1,1,NULL,NULL,?)`) + .run(organizationId, Date.now()); +} +module.exports = { fixture, enableFixturePlugin }; diff --git a/core/core/accounts/tests/plugins.test.js b/core/core/accounts/tests/plugins.test.js new file mode 100644 index 0000000..d98adce --- /dev/null +++ b/core/core/accounts/tests/plugins.test.js @@ -0,0 +1,158 @@ +'use strict'; +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { fixture } = require('./plugin-fixture'); +const { requirePlugin } = require('../src/plugins'); +const { AccessStore } = require('../src'); +const command = (action, expectedRevision) => ({ action, expectedRevision, idempotencyKey: `plugin:test:${action}:${expectedRevision}` }); + +test('approved packages update enabled and disabled DSP copies automatically, retry failures, and catch up dormant DSPs', async t => { + let version='0.18.7'; + const f=await fixture(t,{installationCoordinator:{latest:()=>({version}),apply:({runtimeKey,request,invoke})=>invoke(runtimeKey,'plugins.manage',request)}}); + const [a,b]=f.dsps; + for(const dsp of f.dsps)f.plugins.change(dsp.owner,'paycom',command('install',0)); + await f.plugins.runPending(); + f.plugins.change(b.owner,'paycom',command('disable',1));await f.plugins.runPending(); + version='0.19.0';f.setUnavailable(true);await f.plugins.runPending(); + for(const dsp of f.dsps){const plugin=f.plugins.list(dsp.owner).items[0];assert.equal(plugin.pending,true);assert.equal(plugin.available,false);assert.equal(plugin.failureCode,'plugin_unavailable');} + f.setUnavailable(false);await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].available,true); + assert.equal(f.plugins.list(b.owner).items[0].state,'disabled'); + for(const dsp of f.dsps)assert.equal(f.runtimes.get(dsp.runtimeKey).version,version); + f.store.db.prepare("UPDATE installations SET status='suspended' WHERE organization_id=?").run(b.id); + version='0.20.0';await f.plugins.runPending(); + assert.equal(f.runtimes.get(a.runtimeKey).version,version);assert.equal(f.runtimes.get(b.runtimeKey).version,'0.19.0'); + f.store.db.prepare("UPDATE installations SET status='ready' WHERE organization_id=?").run(b.id); + await f.plugins.runPending();assert.equal(f.runtimes.get(b.runtimeKey).version,version);assert.equal(f.runtimes.get(b.runtimeKey).state,'disabled'); + const before=f.plugins.list(b.owner).items[0];f.plugins.change(b.owner,'paycom',command('uninstall',before.revision));await f.plugins.runPending(); + version='0.21.0';await f.plugins.runPending();assert.equal(f.plugins.list(b.owner).items[0].state,'uninstalled'); + assert.equal(f.runtimes.get(b.runtimeKey).version,'0.20.0'); +}); + +test('package reconciliation must finish before Core acknowledges an installation', async t => { + let prepared = false, authority; + const f = await fixture(t, { installationCoordinator: { async apply({ runtimeKey, request, invoke, authorize }) { + authority = authorize; assert.equal(authorize(), true); + if (!prepared) throw new Error('package_not_ready'); + return invoke(runtimeKey, 'plugins.manage', request); + } } }); + const [a] = f.dsps; + f.plugins.change(a.owner, 'paycom', command('install', 0)); + await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].available, false); + assert.equal(f.calls.filter(call => call.input.command === 'apply').length, 0); + prepared = true; await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].available, true); + f.plugins.change(a.owner, 'paycom', command('disable', 1)); + assert.equal(authority(), false); +}); + +test('older installed versions update automatically without an owner version choice', async t => { + const f = await fixture(t), [a] = f.dsps; + f.plugins.change(a.owner, 'paycom', command('install', 0)); await f.plugins.runPending(); + f.store.db.prepare("UPDATE dsp_plugins SET version='0.17.1' WHERE organization_id=?").run(a.id); + f.runtimes.set(a.runtimeKey, { id: 'paycom', version: '0.17.1', state: 'enabled', revision: 1 }); + await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].version, '0.18.7'); + assert.equal(f.plugins.list(a.owner).items[0].latestVersion, '0.18.7'); + f.plugins.change(a.owner, 'paycom', command('disable', 2)); await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].version, '0.18.7'); + f.plugins.change(a.owner, 'paycom', command('enable', 3)); await f.plugins.runPending(); + assert.throws(()=>f.plugins.change(a.owner,'paycom',command('upgrade',4)),/invalid_input/); + assert.equal(f.plugins.list(a.owner).items[0].version, '0.18.7'); + assert.equal(f.plugins.list(a.owner).items[0].available, true); +}); + +test('existing enrollment without installed code is migrated once before availability returns', async t => { + let missing = false, migrations = 0; + const f = await fixture(t, { installationCoordinator: { + needsMigration: () => missing, + async apply({ runtimeKey, request, invoke, authorize }) { + assert.equal(authorize(), true); if (missing) { migrations++; missing = false; } + return invoke(runtimeKey, 'plugins.manage', request); + }, + } }); + const [a] = f.dsps; + f.plugins.change(a.owner, 'paycom', command('install', 0)); await f.plugins.runPending(); + missing = true; await f.plugins.runPending(); await f.plugins.runPending(); + assert.equal(migrations, 1); + assert.equal(f.plugins.list(a.owner).items[0].revision, 2); + assert.equal(f.plugins.list(a.owner).items[0].available, true); +}); + +test('DSPs start without Paycom; installation is durable, owner-scoped and confirmed by its runtime', async t => { + const f = await fixture(t); const [a, b] = f.dsps; + assert.equal(f.plugins.list(a.owner).items[0].state, 'uninstalled'); + assert.throws(() => requirePlugin(f.access, a.owner, 'paycom'), /plugin_disabled/); + const pending = f.plugins.change(a.owner, 'paycom', command('install', 0)); + assert.equal(pending.available, false); assert.equal(pending.pending, true); + assert.equal(f.plugins.change(a.owner, 'paycom', command('install', 0)).revision, 1); + f.setUnavailable(true); await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].failureCode, 'plugin_unavailable'); + assert.equal(f.store.installation(a.id).status, 'ready'); + f.setUnavailable(false); await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].available, true); + assert.equal(f.plugins.list(b.owner).items[0].state, 'uninstalled'); + assert.ok(f.calls.filter(call => call.input.command === 'apply').every(call => call.runtimeKey === a.runtimeKey)); + assert.equal(requirePlugin(f.access, a.owner, 'paycom').id, a.id); + f.plugins.change(a.owner, 'paycom', command('disable', 1)); + assert.throws(() => requirePlugin(f.access, a.owner, 'paycom'), /plugin_disabled/); + await f.plugins.runPending(); + f.plugins.change(a.owner, 'paycom', command('enable', 2)); await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].available, true); + f.plugins.change(a.owner, 'paycom', command('uninstall', 3)); await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].state, 'uninstalled'); + f.plugins.change(a.owner, 'paycom', command('install', 4)); await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].revision, 5); + assert.equal(f.store.installation(b.id).status, 'ready'); +}); + +test('plugin changes reject nonowners, stale revisions, foreign DSP scope and arbitrary fields', async t => { + const f = await fixture(t); const [a, b] = f.dsps; + assert.throws(() => f.plugins.change(f.platform.session, 'paycom', command('install', 0))); + assert.throws(() => f.plugins.change(a.owner, 'paycom', { ...command('install', 0), organizationId: b.id }), /invalid_input/); + assert.throws(() => f.plugins.change(a.owner, '../paycom', command('install', 0)), /invalid_input/); + f.plugins.change(a.owner, 'paycom', command('install', 0)); await f.plugins.runPending(); + assert.throws(() => f.plugins.change(a.owner, 'paycom', command('disable', 0)), /plugin_revision_conflict/); + const manager = f.store.roleByKey(a.id, 'manager'); + f.store.db.prepare('UPDATE memberships SET role_id=? WHERE user_id=? AND organization_id=?').run(manager.id, a.owner.user.id, a.id); + assert.throws(() => f.plugins.change(a.owner, 'paycom', command('disable', 1)), /permission_denied/); +}); + +test('legacy enrollment is adopted once without installing Paycom for untouched DSPs', async t => { + const f = await fixture(t); const [a, b] = f.dsps; + f.access.audit({ actorUserId: a.owner.user.id, organizationId: a.id, action: 'connection.save', targetType: 'connection', targetId: 'paycom' }); + f.store.db.exec('PRAGMA user_version=16'); f.store.close(); + const migrated = new AccessStore(f.paths); + assert.equal(migrated.db.prepare('SELECT desired_state FROM dsp_plugins WHERE organization_id=?').get(a.id).desired_state, 'enabled'); + assert.equal(migrated.db.prepare('SELECT 1 FROM dsp_plugins WHERE organization_id=?').get(b.id), undefined); + migrated.db.prepare("UPDATE dsp_plugins SET desired_state='uninstalled',applied_state='uninstalled',applied_revision=revision WHERE organization_id=?").run(a.id); + migrated.close(); + const reopened = new AccessStore(f.paths); + assert.equal(reopened.db.prepare('SELECT desired_state FROM dsp_plugins WHERE organization_id=?').get(a.id).desired_state, 'uninstalled'); + reopened.close(); +}); + +test('runtime adoption preserves explicit uninstall and does not use the shipped catalog as enrollment', async t => { + const f = await fixture(t); const [a, b] = f.dsps; + f.runtimes.set(a.runtimeKey, { id: 'paycom', version: '0.18.7', state: 'enabled', revision: 0 }); + await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].available, true); + assert.equal(f.plugins.list(b.owner).items[0].available, false); + f.plugins.change(a.owner, 'paycom', command('uninstall', 1)); await f.plugins.runPending(); + f.runtimes.set(a.runtimeKey, { id: 'paycom', version: '0.18.7', state: 'enabled', revision: 0 }); + f.store.db.prepare('DELETE FROM plugin_migration_checks WHERE organization_id=?').run(a.id); + await f.plugins.runPending(); + assert.equal(f.plugins.list(a.owner).items[0].state, 'uninstalled'); +}); + +test('a Dev-only approval updates that DSP while production keeps its approved version', async t => { + const versions=new Map(); + const f=await fixture(t,{installationCoordinator:{latest:(_id,key)=>({version:versions.get(key)||'0.18.7'}),apply:({runtimeKey,request,invoke})=>invoke(runtimeKey,'plugins.manage',request)}}); + for(const dsp of f.dsps)f.plugins.change(dsp.owner,'paycom',command('install',0)); + await f.plugins.runPending(); + versions.set(f.dsps[0].runtimeKey,'0.19.0');await f.plugins.runPending(); + assert.equal(f.runtimes.get(f.dsps[0].runtimeKey).version,'0.19.0'); + assert.equal(f.runtimes.get(f.dsps[1].runtimeKey).version,'0.18.7'); + assert.equal(f.plugins.list(f.dsps[1].owner).items[0].latestVersion,'0.18.7'); +}); diff --git a/core/core/accounts/tests/provisioning-reconciliation.test.js b/core/core/accounts/tests/provisioning-reconciliation.test.js new file mode 100644 index 0000000..c3ac0ac --- /dev/null +++ b/core/core/accounts/tests/provisioning-reconciliation.test.js @@ -0,0 +1,279 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { AccessStore } = require('../src/store'); +const { + createAccessInstallationProvisioningAuthority, + createAccessControlLiveAuthorityResolver, + createInstallationProvisioningReconciler, +} = require('../src/installation-provisioning'); +const { + createDurableInstallationProvisioner, + createRuntimeAgentCredentialManager, +} = require('../../installations/src'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-live-reconcile-')); + fs.chmodSync(root, 0o700); + const accessRoot = path.join(root, 'access'); + const stateRoot = path.join(root, 'provisioner'); + const installationsRoot = path.join(root, 'installations'); + for (const directory of [stateRoot, installationsRoot]) fs.mkdirSync(directory, { mode: 0o700 }); + const store = new AccessStore({ + databaseRoot: accessRoot, + database: path.join(accessRoot, 'access-control.sqlite3'), + }); + store.transaction(() => { + store.insertUser({ + id: 'usr_platform_fixture', + email: 'platform@example.invalid', + firstName: 'Platform', + lastName: 'Fixture', + passwordHash: 'fixture-hash-not-a-secret', + platformRole: 'owner', + timestamp: 1_000, + }); + for (const suffix of ['alpha', 'bravo']) { + const organizationId = `org_live_${suffix}`; + store.createOrganization({ + id: organizationId, + name: `Live ${suffix}`, + abbreviation: suffix.toUpperCase(), + timezone: suffix === 'alpha' ? 'America/Los_Angeles' : 'America/New_York', + status: 'pending_owner', + createdBy: 'usr_platform_fixture', + timestamp: 1_000, + }); + store.insertStation(organizationId, suffix === 'alpha' ? 'TST1' : 'TST2', true, 1_000); + store.createInstallation(organizationId, `runtime_live_${suffix}`, 'pending', 1_000); + } + }); + let now = 2_000; + let nextJob = 0; + const clock = () => { now += 1; return now; }; + const provisioner = createDurableInstallationProvisioner({ + stateRoot, + installationsRoot, + clock, + idFactory: () => `job_live_${++nextJob}`, + liveAuthorityResolver: createAccessControlLiveAuthorityResolver({ store }), + }); + t.after(() => { + provisioner.close(); + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + }); + return { + root, store, provisioner, installationsRoot, clock, + runtimeAgentCredentials: createRuntimeAgentCredentialManager({ installationsRoot }), + }; +} + +test('Access Control outbox acknowledgement makes one live Provisioner job runnable and reconciles success', t => { + const context = fixture(t); + const authority = createAccessInstallationProvisioningAuthority({ + store: context.store, + organizationId: 'org_live_alpha', + authorityScope: 'platform_installation', + actorUserId: 'usr_platform_fixture', + clock: context.clock, + requestFactory: () => 'prq_live_alpha', + }); + const requested = authority.request({ + operation: 'provision', + idempotencyKey: 'live:provision:alpha', + expectedRevision: 1, + }); + assert.equal(requested.status, 'pending'); + assert.deepEqual(context.store.installationControl('org_live_alpha'), { + organizationId: 'org_live_alpha', + runtimeKey: 'runtime_live_alpha', + status: 'provisioning', + revision: 2, + manifestRevision: 1, + releaseId: 'dispatch_current_1', + currentJobId: null, + }); + + const reconciler = createInstallationProvisioningReconciler({ + store: context.store, + provisioner: context.provisioner, + clock: context.clock, + runtimeAgentCredentials: context.runtimeAgentCredentials, + }); + const dispatched = reconciler.dispatch(requested.id); + assert.equal(dispatched.status, 'dispatched'); + assert.equal(dispatched.jobId, 'job_live_1'); + assert.equal(context.store.installationControl('org_live_alpha').currentJobId, 'job_live_1'); + let authorityMutationRan = false; + const resolvedAuthority = createAccessControlLiveAuthorityResolver({ store: context.store })({ + organizationId: 'org_live_alpha', + runtimeKey: 'runtime_live_alpha', + manifestRevision: 1, + installationRevision: 2, + jobId: 'job_live_1', + }, () => { authorityMutationRan = true; }); + assert.equal(authorityMutationRan, true); + assert.equal(resolvedAuthority.installationRevision, 2); + + const completed = reconciler.runNext(requested.id, 'worker_live_alpha'); + assert.equal(completed.status, 'completed'); + assert.deepEqual(context.store.installationControl('org_live_alpha'), { + organizationId: 'org_live_alpha', + runtimeKey: 'runtime_live_alpha', + status: 'waiting_for_owner', + revision: 3, + manifestRevision: 1, + releaseId: 'dispatch_current_1', + currentJobId: null, + }); + assert.equal(reconciler.reconcile(requested.id).status, 'completed'); + assert.equal(fs.lstatSync(path.join(context.installationsRoot, 'runtime_live_alpha')).isDirectory(), true); + assert.match(context.store.activeRuntimeAgentAuthority('runtime_live_alpha').tokenHash, /^[a-f0-9]{64}$/); + assert.equal(fs.lstatSync(path.join( + context.installationsRoot, 'runtime_live_alpha', 'secrets', 'runtime-agent', 'registration-token', + )).mode & 0o7777, 0o600); + assert.equal(fs.existsSync(path.join(context.installationsRoot, 'runtime_live_bravo')), false); + assert.equal(context.store.installationControl('org_live_bravo').status, 'pending'); + + const bravoAuthority = createAccessInstallationProvisioningAuthority({ + store: context.store, + organizationId: 'org_live_bravo', + authorityScope: 'platform_installation', + actorUserId: 'usr_platform_fixture', + clock: context.clock, + requestFactory: () => 'prq_live_bravo', + }); + bravoAuthority.request({ + operation: 'provision', + idempotencyKey: 'live:provision:bravo', + expectedRevision: 1, + }); + assert.deepEqual(reconciler.runPending('worker_live_pending'), { + processed: 1, completed: 1, failed: 0, pending: 0, + }); + assert.equal(context.store.installationControl('org_live_bravo').status, 'waiting_for_owner'); + assert.equal(fs.lstatSync(path.join(context.installationsRoot, 'runtime_live_bravo')).isDirectory(), true); +}); + +test('a delayed provisioning replay cannot reactivate an explicitly revoked Runtime Agent', t => { + const context = fixture(t); + const authority = createAccessInstallationProvisioningAuthority({ + store: context.store, + organizationId: 'org_live_alpha', + authorityScope: 'platform_installation', + actorUserId: 'usr_platform_fixture', + clock: context.clock, + requestFactory: () => 'prq_live_revocation', + }); + const requested = authority.request({ + operation: 'provision', + idempotencyKey: 'live:provision:revocation', + expectedRevision: 1, + }); + const reconciler = createInstallationProvisioningReconciler({ + store: context.store, + provisioner: context.provisioner, + clock: context.clock, + runtimeAgentCredentials: context.runtimeAgentCredentials, + }); + reconciler.dispatch(requested.id); + const current = context.store.runtimeAgentAuthority('runtime_live_alpha'); + context.store.transaction(() => context.store.revokeRuntimeAgentAuthority({ + organizationId: 'org_live_alpha', runtimeKey: 'runtime_live_alpha', + expectedGeneration: current.generation, timestamp: context.clock(), + })); + context.runtimeAgentCredentials.revoke('runtime_live_alpha', current.token_hash); + + assert.throws(() => reconciler.dispatch(requested.id), + error => error?.code === 'runtime_agent_unauthorized'); + const revoked = context.store.runtimeAgentAuthority('runtime_live_alpha'); + assert.equal(revoked.status, 'revoked'); + assert.equal(revoked.generation, current.generation + 1); + assert.equal(fs.existsSync(context.runtimeAgentCredentials.paths('runtime_live_alpha').tokenFile), false); +}); + +test('live authority drift stops before another runtime or filesystem is mutated', t => { + const context = fixture(t); + const authority = createAccessInstallationProvisioningAuthority({ + store: context.store, + organizationId: 'org_live_alpha', + authorityScope: 'platform_installation', + actorUserId: 'usr_platform_fixture', + clock: context.clock, + requestFactory: () => 'prq_live_drift', + }); + const requested = authority.request({ + operation: 'provision', + idempotencyKey: 'live:provision:drift', + expectedRevision: 1, + }); + const reconciler = createInstallationProvisioningReconciler({ + store: context.store, + provisioner: context.provisioner, + clock: context.clock, + }); + reconciler.dispatch(requested.id); + context.store.updateOrganizationStatus('org_live_alpha', 'suspended', context.clock()); + assert.throws(() => context.provisioner.runNext('worker_live_drift'), + error => error?.code === 'installation_operation_in_progress'); + assert.equal(fs.existsSync(path.join(context.installationsRoot, 'runtime_live_alpha')), false); + assert.equal(fs.existsSync(path.join(context.installationsRoot, 'runtime_live_bravo')), false); + assert.equal(context.store.installationControl('org_live_alpha').status, 'provisioning'); + assert.equal(context.store.installationControl('org_live_bravo').status, 'pending'); +}); + +test('an infrastructure failure stays non-ready and an authorized retry preserves the original destination', t => { + const context = fixture(t); + const authority = createAccessInstallationProvisioningAuthority({ + store: context.store, + organizationId: 'org_live_alpha', + authorityScope: 'platform_installation', + actorUserId: 'usr_platform_fixture', + clock: context.clock, + requestFactory: () => 'prq_live_failure', + }); + const requested = authority.request({ + operation: 'provision', + idempotencyKey: 'live:provision:failure', + expectedRevision: 1, + }); + const reconciler = createInstallationProvisioningReconciler({ + store: context.store, + provisioner: context.provisioner, + clock: context.clock, + }); + reconciler.dispatch(requested.id); + const runtimeRoot = path.join(context.installationsRoot, 'runtime_live_alpha'); + fs.mkdirSync(runtimeRoot, { mode: 0o755 }); + const failed = reconciler.runNext(requested.id, 'worker_live_failure'); + assert.equal(failed.status, 'failed'); + const failureState = context.store.installationControl('org_live_alpha'); + assert.equal(failureState.status, 'failed'); + assert.equal(failureState.runtimeKey, 'runtime_live_alpha'); + fs.rmSync(runtimeRoot, { recursive: true, force: true }); + + const retryAuthority = createAccessInstallationProvisioningAuthority({ + store: context.store, + organizationId: 'org_live_alpha', + authorityScope: 'platform_installation', + actorUserId: 'usr_platform_fixture', + clock: context.clock, + requestFactory: () => 'prq_live_retry', + }); + const retry = retryAuthority.request({ + operation: 'retry', + idempotencyKey: 'live:provision:retry', + expectedRevision: failureState.revision, + }); + const completed = reconciler.runNext(retry.id, 'worker_live_retry'); + assert.equal(completed.status, 'completed'); + const finalState = context.store.installationControl('org_live_alpha'); + assert.equal(finalState.status, 'waiting_for_owner'); + assert.equal(finalState.runtimeKey, failureState.runtimeKey); + assert.equal(fs.lstatSync(runtimeRoot).mode & 0o7777, 0o700); +}); diff --git a/core/core/accounts/tests/release-popup.test.js b/core/core/accounts/tests/release-popup.test.js new file mode 100644 index 0000000..31f9277 --- /dev/null +++ b/core/core/accounts/tests/release-popup.test.js @@ -0,0 +1,104 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { AccessStore } = require('../src/store'); +const { createReleasePopup, loadPopup, validatePopup } = require('../src/release-popup'); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'release-popup-')); + const options = { databaseRoot: path.join(root, 'access'), database: path.join(root, 'access/db.sqlite3') }; + let store = new AccessStore(options); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + for (const user of ['platform', 'owner', 'other']) store.insertUser({ id: user, email: `${user}@example.test`, + firstName: user, lastName: 'User', passwordHash: 'synthetic', platformRole: user === 'platform' ? 'owner' : null, timestamp: 1 }); + store.createOrganization({ id: 'org_one', name: 'One', abbreviation: null, timezone: 'UTC', status: 'active', createdBy: null, timestamp: 1 }); + store.createInstallation('org_one', 'runtime_one', 'ready', 1, 'dispatch_1.2.3', 'native_service_v1'); + const release = { schemaVersion: 1, releaseId: 'dispatch_1.2.3', version: '1.2.3', sourceCommit: 'a'.repeat(40), + changelog: [ + { kind: 'added', title: 'Private platform change', description: 'Platform-only details', audience: 'platform' }, + { kind: 'fixed', title: 'DSP search', description: 'Relevant details', audience: 'dsp' }, + ], afterUpdating: [ + { title: 'Private action', description: 'Platform-only action', audience: 'platform' }, + { title: 'DSP action', description: 'Relevant action', audience: 'dsp' }, + ] }; + const platform = { user: { id: 'platform' }, platformPermissions: ['platform.organizations.read'], memberships: [] }; + const owner = { user: { id: 'owner' }, platformPermissions: [], activeOrganizationId: 'org_one', + memberships: [{ organizationId: 'org_one', organization: { status: 'active' }, permissions: ['organization.owner'] }] }; + function rollout(value = release, status = 'completed') { + store.db.prepare('INSERT INTO platform_rollouts VALUES(?,?,?,?,?,?,?)').run(value.releaseId, value.releaseId, 'platform', value.releaseId, status, 1, 1); + store.db.prepare('INSERT INTO platform_rollout_core VALUES(?,?,?,?,?,?)').run(value.releaseId, 'succeeded', + JSON.stringify({ ...value, publishedAt: '2026-09-07T00:00:00.000Z' }), 1, null, 1); + } + return { root, get store() { return store; }, release, platform, owner, rollout, + service: (value = release) => createReleasePopup({ store, release: value }), + reopen() { store.close(); store = new AccessStore(options); } }; +} +test('only completed rollout of running release is announced; DSP must match verified installation', t => { + const f = fixture(t), service = f.service(); + assert.equal(service.pending(f.platform).release, null); + f.rollout(f.release, 'running'); + assert.equal(service.pending(f.platform).release, null); + f.store.db.prepare("UPDATE platform_rollouts SET status='completed'").run(); + f.store.db.prepare("UPDATE platform_rollout_core SET status='verifying'").run(); + assert.equal(service.pending(f.platform).release, null); + f.store.db.prepare("UPDATE platform_rollout_core SET status='succeeded'").run(); + assert.equal(service.pending(f.platform).release.version, '1.2.3'); + f.store.db.prepare("UPDATE installations SET release_id='dispatch_1.2.2'").run(); + assert.equal(service.pending(f.owner).release, null); + f.store.db.prepare("UPDATE installations SET release_id='dispatch_1.2.3',status='verifying'").run(); + assert.equal(service.pending(f.owner).release, null); + f.store.db.prepare("UPDATE installations SET status='ready'").run(); + assert.equal(service.pending(f.owner).release.version, '1.2.3'); +}); +test('DSP response contains no platform text, actions, audience tags or source metadata', t => { + const f = fixture(t); f.rollout(); const service = f.service(); + assert.equal(service.pending(f.platform).release.changelog.length, 2); + const result = service.pending(f.owner); + assert.equal(result.release.changelog.length, 1); + assert.equal(result.release.afterUpdating.length, 1); + assert.doesNotMatch(JSON.stringify(result), /Private|Platform-only|audience|sourceCommit/); + assert.equal(service.pending({ ...f.owner, memberships: [] }).release, null); + assert.equal(service.pending({ ...f.owner, memberships: [{ ...f.owner.memberships[0], permissions: ['dashboard.view'] }] }).release, null); + assert.equal(service.pending({ ...f.platform, dspView: {} }).release, null); + const platformOnly = { ...f.release, changelog: [f.release.changelog[0]] }; + assert.equal(f.service(platformOnly).pending(f.owner).release, null); +}); +test('dismissal is idempotent, per user/release, survives database reopen; new version appears once', t => { + const f = fixture(t); f.rollout(); + const service = f.service(); + service.dismiss(f.owner, { releaseId: f.release.releaseId }); + service.dismiss(f.owner, { releaseId: f.release.releaseId }); + assert.equal(service.pending(f.owner).release, null); + assert.ok(service.pending(f.platform).release); + assert.ok(service.pending({ ...f.owner, user: { id: 'other' } }).release); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM release_popup_dismissals').get().n, 1); + f.reopen(); + assert.equal(f.service().pending(f.owner).release, null); + const next = { ...f.release, releaseId: 'dispatch_1.2.4', version: '1.2.4' }; + f.rollout(next); + f.store.db.prepare('UPDATE installations SET release_id=?').run(next.releaseId); + assert.ok(f.service(next).pending(f.owner).release); + f.service(next).dismiss(f.owner, { releaseId: next.releaseId }); + assert.equal(f.service(next).pending(f.owner).release, null); + assert.throws(() => f.service(next).dismiss(f.owner, { releaseId: f.release.releaseId }), { code: 'release_popup_unavailable' }); + assert.throws(() => f.service().dismiss(f.platform, { releaseId: f.release.releaseId, userId: 'other' })); +}); +test('legacy database receives additive dismissal table without changing existing users', t => { + const f = fixture(t); + f.store.db.exec('DROP TABLE release_popup_dismissals'); + f.reopen(); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM users').get().n, 3); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM release_popup_dismissals').get().n, 0); +}); +test('bundled copy must match deployed identity; missing, invalid and oversized copy stays quiet', t => { + const f = fixture(t), file = path.join(f.root, 'popup.json'); + assert.equal(loadPopup(file, f.release), null); + fs.writeFileSync(file, JSON.stringify(f.release)); + assert.deepEqual(loadPopup(file, f.release), f.release); + assert.equal(loadPopup(file, { ...f.release, sourceCommit: 'b'.repeat(40) }), null); + assert.throws(() => validatePopup({ ...f.release, changelog: [{ ...f.release.changelog[0], audience: 'all' }] }, f.release)); + fs.writeFileSync(file, ' '.repeat(128 * 1024 + 1)); + assert.equal(loadPopup(file, f.release), null); +}); diff --git a/core/core/accounts/tests/rollout-admin.test.js b/core/core/accounts/tests/rollout-admin.test.js new file mode 100644 index 0000000..c36745b --- /dev/null +++ b/core/core/accounts/tests/rollout-admin.test.js @@ -0,0 +1,136 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const { AccessStore } = require('../src/store'); +const { createPlatformUpdates } = require('../src/platform-updates'); +const { parse, operate, main } = require('../src/rollout-admin-cli'); +function fixture(t, backend = 'native_service_v1') { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-rollout-admin-')); + fs.mkdirSync(path.join(root, 'data'), { mode: 0o700 }); + const store = new AccessStore({ databaseRoot: path.join(root, 'data/access-control'), database: path.join(root, 'data/access-control/access-control.sqlite3') }); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + store.insertUser({ id: 'usr_owner', email: 'owner@example.test', firstName: 'Owner', lastName: 'Test', passwordHash: 'synthetic', platformRole: 'owner', timestamp: 1000 }); + store.createOrganization({ id: 'org_test', name: 'Test DSP', abbreviation: null, timezone: 'UTC', status: 'active', createdBy: null, timestamp: 1000 }); + store.insertStation('org_test', 'TST1', true, 1000); + store.createInstallation('org_test', 'runtime_test', 'ready', 1000, 'dispatch_current', backend); + const catalogs = { releases: { dispatch_next: { backend: 'native_service_v1' } }, platformReleases: { dispatch_next: { + version: '0.0.2', sourceCommit: 'a'.repeat(40), publishedAt: '2026-09-08T00:00:00.000Z', changelog: [], core: {}, + } } }; + const updates = createPlatformUpdates({ store, ...catalogs, enabled: true, cleanupReady: () => true }); + const input = { action: 'rollout-start', version: '0.0.2', commit: 'a'.repeat(40) }; + return { root, store, catalogs, updates, input, run: (changes = {}) => operate(store, updates, catalogs, { ...input, ...changes }) }; +} +test('operator starts existing backup-gated coordinator once without creating sessions', t => { + const f = fixture(t); + assert.equal(f.run({ action: 'rollout-status' }).status, 'ready'); + const started = f.run(); + assert.equal(started.status, 'running'); assert.equal(started.rollout.phase, 'backups'); + assert.equal(started.rollout.backups.total, 2); assert.equal(started.rollout.core.status, 'queued'); + assert.equal(started.rollout.members[0].status, 'queued'); + f.run(); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_rollouts').get().n, 1); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_backup_requests').get().n, 2); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM sessions').get().n, 0); + assert.equal(f.store.db.prepare("SELECT actor_user_id FROM audit_events WHERE action='platform.rollout.start'").get().actor_user_id, 'usr_owner'); +}); +test('repeated starts preserve pause and completed state; resume targets the same rollout', t => { + const f = fixture(t); f.run(); + assert.equal(f.run({ action: 'rollout-pause' }).status, 'paused'); + assert.equal(f.run().status, 'paused'); + assert.throws(() => f.run({ action: 'rollout-resume', version: '0.0.3' }), { code: 'rollout_target_mismatch' }); + assert.equal(f.run({ action: 'rollout-status' }).status, 'paused'); + assert.equal(f.run({ action: 'rollout-resume' }).status, 'running'); + f.store.db.prepare("UPDATE platform_rollouts SET status='completed'").run(); + assert.equal(f.run().status, 'completed'); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_rollouts').get().n, 1); +}); +test('unprepared releases, mismatched commits and other active rollouts never enqueue work', t => { + const f = fixture(t); + assert.equal(f.run({ action: 'rollout-status', version: '0.0.3' }).status, 'not_prepared'); + assert.throws(() => f.run({ version: '0.0.3' }), { code: 'release_not_prepared' }); + assert.throws(() => f.run({ commit: 'b'.repeat(40) }), { code: 'release_identity_mismatch' }); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_rollouts').get().n, 0); + f.run(); + assert.throws(() => f.run({ version: '0.0.3' }), { code: 'rollout_in_progress' }); + assert.throws(() => f.run({ action: 'rollout-pause', commit: 'b'.repeat(40) }), { code: 'release_identity_mismatch' }); +}); +test('operator preserves the native fleet preflight gate', t => { + const f = fixture(t, 'oci_container_v1'); + assert.throws(() => f.run(), { code: 'native_migration_required' }); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_rollouts').get().n, 0); +}); +test('inactive owners cannot initiate rollout', t => { + const f = fixture(t); f.store.db.prepare("UPDATE users SET status='disabled'").run(); + assert.throws(() => f.run(), { code: 'platform_owner_required' }); +}); +test('CLI requires an explicit target and never initializes a missing live database', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-rollout-missing-')); t.after(() => fs.rmSync(root, { recursive: true, force: true })); + assert.throws(() => parse(['rollout-start']), { code: 'invalid_input' }); + const args = ['rollout-start', '--local-root', root, '--version', '0.0.2', '--commit', 'a'.repeat(40)]; + assert.throws(() => parse([...args, '--version', '0.0.3']), { code: 'invalid_input' }); + assert.throws(() => parse([...args, '--force', 'true']), { code: 'invalid_input' }); + let output = ''; assert.equal(await main(args, { write: value => { output += value; } }), 1); + assert.equal(JSON.parse(output).status, 'access_not_initialized'); assert.deepEqual(fs.readdirSync(root), []); +}); + +test('CLI reads private matching catalogs, queues a rollout, and rejects unsafe catalog modes', async t => { + const f = fixture(t); + const config = path.join(f.root, 'config'); fs.mkdirSync(config, { mode: 0o700 }); + const runtime = { version: 1, backend: 'native_service_v1', releaseId: 'dispatch_next', channel: 'production', + sourceCommit: f.input.commit, platform: 'linux/amd64', runtimeAgentProtocol: 1, runtimeGatewayProtocol: 1, + artifactSha256: 'b'.repeat(64), embeddedManifestSha256: 'c'.repeat(64), bridgeManifestSha256: 'd'.repeat(64) }; + const platform = { ...f.catalogs.platformReleases.dispatch_next, runtimeImageDigest: `sha256:${runtime.artifactSha256}`, + changelog: [{ kind: 'fixed', title: 'Test release', description: '' }], + core: { artifactPath: '/opt/dispatch-platform/releases/dispatch_next/core-artifact', manifestSha256: 'e'.repeat(64) } }; + const runtimeFile = path.join(config, 'oci-releases.json'); + fs.writeFileSync(runtimeFile, JSON.stringify({ schemaVersion: 1, releases: { dispatch_next: runtime } }), { mode: 0o600 }); + fs.writeFileSync(path.join(config, 'platform-releases.json'), JSON.stringify({ schemaVersion: 1, releases: { dispatch_next: platform } }), { mode: 0o600 }); + const args = ['--local-root', f.root, '--version', f.input.version, '--commit', f.input.commit]; + const invoke = async action => { + let output = ''; const code = await main([action, ...args], { write: value => { output += value; }, wake: () => {} }); + assert.doesNotMatch(output, /password|token|metadata_json|synthetic/); + return { code, value: JSON.parse(output) }; + }; + assert.equal((await invoke('rollout-status')).value.status, 'ready'); + const start = await invoke('rollout-start'); assert.equal(start.code, 0); assert.equal(start.value.rollout.phase, 'backups'); + assert.equal((await invoke('rollout-status')).value.status, 'running'); + fs.chmodSync(runtimeFile, 0o644); + assert.equal((await invoke('rollout-pause')).code, 1); + assert.equal(f.updates.view().rollout.status, 'running'); +}); + +test('hotfix versions produce valid stable idempotency keys', t => { + const f = fixture(t); + f.catalogs.platformReleases.dispatch_next.version = '0.0.2+hotfix.1'; + assert.equal(f.run({ version: '0.0.2+hotfix.1' }).status, 'running'); + assert.equal(f.run({ version: '0.0.2+hotfix.1' }).status, 'running'); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_rollouts').get().n, 1); +}); + +test('operator status includes stage timings only for the requested rollout', t => { + const f = fixture(t); f.run(); + const rollout = f.store.db.prepare('SELECT id FROM platform_rollouts').get(); + const timing = require('../../installations/src/operation-timing'); + let now = 1000; + const finish = timing.start(f.store.db, { jobId: rollout.id, attempt: 1, stage: 'core_verify' }, () => now); + now = 2000; finish(); + timing.start(f.store.db, { jobId: 'rollout_unrelated', attempt: 1, stage: 'core_apply' }); + const result = f.run({ action: 'rollout-status' }); + assert.equal(result.timings.length, 1); assert.equal(result.timings[0].stage, 'core_verify'); assert.equal(result.timings[0].duration_ms, 1000); + assert.deepEqual(f.run({ action: 'rollout-status', version: '0.0.3' }).timings, []); +}); + +test('status remains readable while another connection owns the WAL writer lock', t => { + const f = fixture(t); + const reader = new AccessStore(f.store.paths, {readOnly:true}); + t.after(() => reader.close()); + const updates = createPlatformUpdates({store:reader,...f.catalogs,enabled:true,cleanupReady:()=>true}); + f.store.db.exec('BEGIN IMMEDIATE'); + try { + f.store.db.prepare("UPDATE users SET first_name='Uncommitted'").run(); + const result = operate(reader,updates,f.catalogs,{...f.input,action:'rollout-status'}); + assert.equal(result.status,'ready'); + assert.equal(reader.db.prepare('PRAGMA query_only').get().query_only,1); + } finally { f.store.db.exec('ROLLBACK'); } +}); diff --git a/core/core/accounts/tests/rollout-canary.test.js b/core/core/accounts/tests/rollout-canary.test.js new file mode 100644 index 0000000..b471671 --- /dev/null +++ b/core/core/accounts/tests/rollout-canary.test.js @@ -0,0 +1,22 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { createCanaryVerifier } = require('../src/rollout-canary'); +test('canary requires a fresh successful collection and collection health', async () => { + const calls = []; + let reads = 0; + const verify = createCanaryVerifier({ invoke: async (key, action, input) => { + calls.push([key, action, input]); + return { ok: true, data: action === 'sync.status' ? { desiredState: 'running', lastError: null, + lastSucceededAt: ++reads < 3 ? 900 : 1100 } : {} }; + } }, { clock: () => 1000, pollMs: 1 }); + assert.equal(await verify({ runtimeKey: 'test-dsp', verificationId: 'canary-fixture', startedAt: 1000 }), true); + assert.equal(calls.filter(call => call[1] === 'sync.run_now').length, 1); + assert.equal(calls.at(-1)[1], 'collections.health'); +}); +test('stopped sync and timeout cannot pass a canary gate', async () => { + const stopped = createCanaryVerifier({ invoke: async () => ({ ok: true, data: { desiredState: 'stopped' } }) }); + await assert.rejects(stopped({ runtimeKey: 'test-dsp', verificationId: 'canary-fixture', startedAt: Date.now() }), /canary_sync_stopped/); + const timeout = createCanaryVerifier({ invoke: async () => ({ ok: true, data: { desiredState: 'running' } }) }, { clock: () => 100, timeoutMs: 10 }); + await assert.rejects(timeout({ runtimeKey: 'test-dsp', verificationId: 'canary-fixture', startedAt: 0 }), /canary_collection_timeout/); +}); diff --git a/core/core/accounts/tests/runtime-agent-authority.test.js b/core/core/accounts/tests/runtime-agent-authority.test.js new file mode 100644 index 0000000..1ffa4f2 --- /dev/null +++ b/core/core/accounts/tests/runtime-agent-authority.test.js @@ -0,0 +1,110 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { AccessStore } = require('../src/store'); +const { createAccessRuntimeAgentAuthorityCatalog } = require('../src/runtime-agent-authority'); + +function digest(value) { return crypto.createHash('sha256').update(value).digest('hex'); } + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-agent-authority-')); + fs.chmodSync(root, 0o700); + const databaseRoot = path.join(root, 'access'); + const store = new AccessStore({ databaseRoot, database: path.join(databaseRoot, 'access-control.sqlite3') }); + store.transaction(() => { + store.createOrganization({ + id: 'org_agent_alpha', name: 'Agent Alpha', abbreviation: 'AA', + timezone: 'America/Los_Angeles', status: 'active', createdBy: null, timestamp: 1_000, + }); + store.insertStation('org_agent_alpha', 'TST1', true, 1_000); + store.createInstallation('org_agent_alpha', 'runtime_agent_alpha', 'provisioning', 1_000); + store.createOrganization({ + id: 'org_agent_bravo', name: 'Agent Bravo', abbreviation: 'AB', + timezone: 'America/New_York', status: 'active', createdBy: null, timestamp: 1_000, + }); + store.insertStation('org_agent_bravo', 'TST2', true, 1_000); + store.createInstallation('org_agent_bravo', 'runtime_agent_bravo', 'provisioning', 1_000); + }); + t.after(() => { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + }); + return { store, database: path.join(databaseRoot, 'access-control.sqlite3') }; +} + +test('Access Control owns runtime-agent authority, rotation, and revocation', t => { + const { store, database } = fixture(t); + const alphaToken = crypto.randomBytes(32).toString('base64url'); + const alphaDigest = digest(alphaToken); + const first = store.recordRuntimeAgentAuthority({ + organizationId: 'org_agent_alpha', runtimeKey: 'runtime_agent_alpha', + tokenHash: alphaDigest, timestamp: 2_000, + }); + assert.deepEqual(first, { + organizationId: 'org_agent_alpha', runtimeKey: 'runtime_agent_alpha', + tokenHash: alphaDigest, generation: 1, status: 'active', changed: true, + }); + assert.equal(store.recordRuntimeAgentAuthority({ + organizationId: 'org_agent_alpha', runtimeKey: 'runtime_agent_alpha', + tokenHash: alphaDigest, timestamp: 2_001, + }).changed, false); + + const catalog = createAccessRuntimeAgentAuthorityCatalog({ store }); + assert.deepEqual(catalog.resolve('runtime_agent_alpha'), { digest: alphaDigest, generation: 1 }); + assert.equal(catalog.count(), 1); + assert.equal(catalog.resolve('runtime_agent_bravo'), null); + + const rotatedToken = crypto.randomBytes(32).toString('base64url'); + const rotatedDigest = digest(rotatedToken); + const rotated = store.replaceRuntimeAgentAuthority({ + organizationId: 'org_agent_alpha', runtimeKey: 'runtime_agent_alpha', + tokenHash: rotatedDigest, expectedGeneration: 1, expectedStatus: 'active', timestamp: 3_000, + }); + assert.equal(rotated.generation, 2); + assert.equal(catalog.resolve('runtime_agent_alpha').digest, rotatedDigest); + + store.updateOrganizationStatus('org_agent_alpha', 'suspended', 3_100); + assert.equal(catalog.resolve('runtime_agent_alpha'), null); + assert.equal(catalog.count(), 0); + store.updateOrganizationStatus('org_agent_alpha', 'active', 3_200); + store.updateInstallationControl({ + organizationId: 'org_agent_alpha', expectedStatus: 'provisioning', expectedRevision: 1, + status: 'suspended', revision: 2, currentJobId: null, timestamp: 3_300, + }); + assert.equal(store.installationControl('org_agent_alpha').status, 'suspended'); + assert.equal(catalog.resolve('runtime_agent_alpha').generation, 2); + + assert.equal(store.revokeRuntimeAgentAuthority({ + organizationId: 'org_agent_alpha', runtimeKey: 'runtime_agent_alpha', + expectedGeneration: 2, timestamp: 4_000, + }), true); + assert.throws(() => store.revokeRuntimeAgentAuthority({ + organizationId: 'org_agent_alpha', runtimeKey: 'runtime_agent_alpha', + expectedGeneration: 2, timestamp: 4_001, + }), error => error?.code === 'runtime_agent_authority_conflict'); + assert.throws(() => store.recordRuntimeAgentAuthority({ + organizationId: 'org_agent_alpha', runtimeKey: 'runtime_agent_alpha', + tokenHash: rotatedDigest, timestamp: 4_002, + }), error => error?.code === 'runtime_agent_unauthorized'); + assert.throws(() => store.replaceRuntimeAgentAuthority({ + organizationId: 'org_agent_alpha', runtimeKey: 'runtime_agent_alpha', + tokenHash: digest(crypto.randomBytes(32).toString('base64url')), + expectedGeneration: 2, expectedStatus: 'active', timestamp: 4_003, + }), error => error?.code === 'runtime_agent_authority_conflict'); + assert.equal(catalog.resolve('runtime_agent_alpha'), null); + assert.equal(fs.readFileSync(database).includes(Buffer.from(alphaToken)), false); + assert.equal(fs.readFileSync(database).includes(Buffer.from(rotatedToken)), false); +}); + +test('runtime-agent authority cannot be rebound to another installation', t => { + const { store } = fixture(t); + assert.throws(() => store.recordRuntimeAgentAuthority({ + organizationId: 'org_agent_alpha', runtimeKey: 'runtime_agent_bravo', + tokenHash: digest(crypto.randomBytes(32).toString('base64url')), timestamp: 2_000, + }), error => error?.code === 'runtime_identity_mismatch'); +}); diff --git a/core/core/accounts/tests/workspace-readiness.test.js b/core/core/accounts/tests/workspace-readiness.test.js new file mode 100644 index 0000000..7035ef4 --- /dev/null +++ b/core/core/accounts/tests/workspace-readiness.test.js @@ -0,0 +1,161 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { AccessStore, AccessControlService } = require('../src'); +const { completeWorkspaceSetup, workspaceWithoutPaycom } = require('../src/workspace-readiness'); +const { createOwnerPaycomSetup } = require('../src/owner-paycom-setup'); +const { createOwnerOnboardingWorker } = require('../../installations/src/owner-onboarding'); +const { createAccessInstallationLifecycleAuthority } = require('../src/installation-lifecycle'); +const { success, failure } = require('../../../shared/contracts/src'); +const credentials = { clientCode: 'fixture', username: 'fixture', password: 'fixture secret', pin1: '1', pin2: '2', pin3: '3', pin4: '4', pin5: '5' }; +async function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-workspace-ready-')); + const store = new AccessStore({ databaseRoot: path.join(root, 'access'), database: path.join(root, 'access/db.sqlite3') }); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const access = new AccessControlService(store, { installationOperatorEnabled: true, installationBackend: 'native_service_v1' }); + const boot = access.createPlatformBootstrap({ email: 'platform@example.test' }); + const accept = token => access.acceptNewUser({ token, firstName: 'Fixture', lastName: 'Owner', password: 'fixture password', confirmPassword: 'fixture password' }); + const platform = await accept(boot.token); + const created = access.createOrganization(platform.session, { ownerEmail: 'owner@example.test', idempotencyKey: 'workspace:fixture:create' }); + const org = created.organization.id; + function provision() { + const request = store.latestProvisioningRequest(org); + const job = { id: 'job_workspace_fixture', operation: 'provision', status: 'queued', installationState: 'provisioning', revision: request.installation_revision }; + store.transaction(() => { + store.acknowledgeProvisioningRequest(request.id, job, Date.now()); + store.finishProvisioningRequest(request.id, { ...job, status: 'succeeded' }, Date.now()); + }); + } + const details = { name: 'Example Logistics', abbreviation: 'EXMP', stationCode: 'TST1', timezone: 'UTC' }; + return { store, access, org, provision, details, platform, acceptOwner: () => accept(created.token) }; +} +for (const order of ['details-first', 'provision-first']) test(`DSP becomes usable without Paycom: ${order}`, async t => { + const f = await fixture(t); + if (order === 'provision-first') { f.provision(); assert.equal(completeWorkspaceSetup(f.store), 0); } + const owner = await f.acceptOwner(); + if (order === 'details-first') { + assert.equal(f.access.organizationProfile(owner.session, f.details).status, 'submitted'); + assert.equal(completeWorkspaceSetup(f.store), 0); + f.provision(); + require('../src/organization-profile').applyOrganizationProfiles(f.store); + } else f.access.organizationProfile(owner.session, f.details); + assert.equal(f.store.installationControl(f.org).status, 'ready'); + assert.equal(f.store.organization(f.org).status, 'active'); + assert.equal(f.access.organizationSetup(owner.session).operationalAccess, 'available'); + assert.equal(f.access.platformOrganizations(f.platform.session)[0].installation.state, 'ready'); + assert.equal(f.store.latestReadyEvidence(f.org), null); + assert.equal(workspaceWithoutPaycom(f.store, f.org), true); + assert.equal(completeWorkspaceSetup(f.store), 0); +}); + +test('already completed EXMP-style profiles reconcile without credentials; suspended DSPs stay suspended', async t => { + const f = await fixture(t); const owner = await f.acceptOwner(); f.provision(); + f.store.db.prepare('UPDATE organization_profiles SET details_json=?,applied_at=? WHERE organization_id=?').run(JSON.stringify(f.details), Date.now(), f.org); + f.store.updateOrganizationStatus(f.org, 'suspended', Date.now()); + assert.equal(completeWorkspaceSetup(f.store), 0); + f.store.updateOrganizationStatus(f.org, 'setup_required', Date.now()); + assert.equal(completeWorkspaceSetup(f.store), 1); + assert.equal(f.access.organizationSetup(owner.session).operationalAccess, 'available'); +}); + +test('optional login failure and polling preserve DSP readiness; successful retry starts hourly sync', async t => { + const f = await fixture(t); const owner = await f.acceptOwner(); f.provision(); f.access.organizationProfile(owner.session, f.details); + let rejectProvider = true; + const calls = []; + const lifecycle = createAccessInstallationLifecycleAuthority({ store: f.store, organizationId: f.org, authorityScope: 'optional_test', releaseCatalog: ['dispatch_update_2'] }); + const invoke = async (key, action, input) => { + assert.equal(action, 'paycom.setup'); + assert.equal(f.store.installationControl(f.org).status, 'ready'); + assert.equal(f.store.organization(f.org).status, 'active'); + if (input.step === 'readiness') return success('succeeded', { state: rejectProvider ? 'manual' : 'ready', retryAllowed: !rejectProvider, retryAt: null }); + assert.throws(() => lifecycle.request({ operation: 'backup', idempotencyKey: 'optional:concurrent:backup', expectedRevision: f.store.installationControl(f.org).revision }), /installation_operation_in_progress/); + if (input.command === 'enroll') return success('succeeded', { configured: true }); + assert.ok(['test', 'sync'].includes(input.step)); + require('../../../shared/contracts/src/paycom-setup').setupRequest(input, key); + calls.push(input); + if (input.command === 'start') return success('running', null); + if (input.step === 'sync') { + assert.equal(rejectProvider, false); + return success('succeeded', { syncId: 'paycom-main-workforce', intervalSeconds: 3600, desiredState: 'running' }); + } + return rejectProvider ? failure('manual_verification_required') : success('succeeded', { + provider: 'paycom', profileId: 'paycom-main', status: 'authenticated', testedAt: new Date().toISOString(), + }); + }; + require('./plugin-fixture').enableFixturePlugin(f.store, f.org); + const setup = createOwnerPaycomSetup({ store: f.store, access: f.access, invoke }); + await setup.submit(owner.session, { credentials, intent: 'create', idempotencyKey: 'optional:paycom:enroll' }); + const worker = createOwnerOnboardingWorker({ store: f.store, invoke, delay: async () => {} }); + const before = f.store.installationControl(f.org); + assert.deepEqual(await worker.runPending('worker_optional'), { processed: 1, completed: 0, failed: 1 }); + assert.equal((await setup.status(owner.session)).failureCode, 'manual_verification_required'); + assert.equal((await setup.status(owner.session)).canRetry, false); + await assert.rejects(setup.retry(owner.session, {}), /installation_operation_not_allowed/); + rejectProvider = false; + assert.equal((await setup.status(owner.session)).canRetry, true); + await setup.retry(owner.session, {}); + assert.deepEqual(await worker.runPending('worker_retry'), { processed: 1, completed: 1, failed: 0 }); + assert.equal(calls.length, 6); + assert.deepEqual(calls.map(call => call.step), ['test', 'test', 'test', 'test', 'sync', 'sync']); + assert.equal(calls[0].requestId, calls[1].requestId); + assert.equal(calls[2].requestId, calls[3].requestId); + assert.notEqual(calls[0].requestId, calls[2].requestId); + assert.deepEqual(f.store.installationControl(f.org), before); + assert.equal((await setup.status(owner.session)).status, 'succeeded'); + assert.equal((await setup.status(owner.session)).canSubmit, false); + assert.equal((await setup.status(owner.session)).workforceAvailable, false); + await assert.rejects(setup.submit(owner.session, { credentials, intent: 'replace', idempotencyKey: 'optional:connected:replace' }), /installation_operation_not_allowed/); + assert.equal(f.store.latestReadyEvidence(f.org), null); + assert.equal(workspaceWithoutPaycom(f.store, f.org), true); + const upgrade = lifecycle.request({ operation: 'upgrade', releaseId: 'dispatch_update_2', expectedRevision: before.revision, idempotencyKey: 'optional:connected:upgrade' }); + assert.ok(!JSON.parse(f.store.lifecycleJob(upgrade.id).stages_json).includes('capture_publication')); +}); + +test('unconnected DSP maintenance verifies infrastructure without requiring publication or a Paycom schedule', async t => { + const f = await fixture(t); const owner = await f.acceptOwner(); f.provision(); f.access.organizationProfile(owner.session, f.details); + const authority = createAccessInstallationLifecycleAuthority({ store: f.store, organizationId: f.org, authorityScope: 'optional_test', releaseCatalog: ['dispatch_update_2'] }); + const job = authority.request({ operation: 'upgrade', releaseId: 'dispatch_update_2', expectedRevision: f.store.installationControl(f.org).revision, idempotencyKey: 'optional:unconnected:upgrade' }); + const claimed = authority.claim(job.id, 'worker_upgrade'); + assert.equal(claimed.withoutPaycom, true); + assert.ok(claimed.stages.includes('verify_release')); + assert.ok(!claimed.stages.includes('capture_publication')); + assert.ok(!claimed.stages.includes('verify_release_publication')); +}); + +test('unconnected DSP completes upgrade, backup, suspend, resume, removal and restoration with fenced receipts', async t => { + const f = await fixture(t); const owner = await f.acceptOwner(); f.provision(); f.access.organizationProfile(owner.session, f.details); + const authority = createAccessInstallationLifecycleAuthority({ store: f.store, organizationId: f.org, authorityScope: 'workspace_lifecycle', releaseCatalog: ['dispatch_update_2'] }); + const receipts = { + inspect_schedule: { status: 'verified', syncWasRunning: false }, + quiesce_schedule: { status: 'stopped', syncWasRunning: false }, + stop_runtime: { status: 'stopped' }, stop_if_running: { status: 'stopped' }, + upgrade_backup: { status: 'snapshot', treeDigest: 'a'.repeat(64), fileCount: 1, totalBytes: 10 }, + snapshot: { status: 'snapshot', treeDigest: 'a'.repeat(64), fileCount: 1, totalBytes: 10 }, + install_release: { status: 'installed' }, start_release: { status: 'started' }, + verify_release: { status: 'verified', releaseId: 'dispatch_update_2' }, + restore_schedule: { status: 'started', syncWasRunning: false }, + commit_release: { status: 'committed', releaseId: 'dispatch_update_2' }, + restart_if_needed: { status: 'started' }, verify_runtime: { status: 'healthy' }, + verify_stopped: { status: 'inactive' }, start_runtime: { status: 'started' }, + verify_infrastructure: { status: 'verified' }, disable_runtime: { status: 'disabled' }, verify_retained: { status: 'retained' }, + }; + for (const [i, operation] of ['upgrade', 'backup', 'suspend', 'resume', 'decommission', 'resume'].entries()) { + const job = authority.request({ operation, expectedRevision: f.store.installationControl(f.org).revision, + idempotencyKey: `workspace:lifecycle:${i}`, ...(operation === 'upgrade' ? { releaseId: 'dispatch_update_2' } : {}) }); + const claimed = authority.claim(job.id, `worker_lifecycle_${i}`); + assert.equal(claimed.withoutPaycom, true); + assert.throws(() => authority.succeed(claimed.claim), /installation_operation_failed/); + for (const stage of claimed.stages) { + assert.ok(receipts[stage], `Unexpected provider requirement: ${stage}`); + authority.checkpoint(claimed.claim, stage, receipts[stage]); + } + assert.equal(authority.succeed(claimed.claim).status, 'succeeded'); + } + assert.equal(f.store.installationControl(f.org).status, 'ready'); + assert.equal(f.store.installationControl(f.org).releaseId, 'dispatch_update_2'); + assert.equal(f.store.organization(f.org).status, 'active'); + assert.equal(f.store.latestReadyEvidence(f.org), null); +}); diff --git a/core/core/agent-bridge/OVERVIEW.md b/core/core/agent-bridge/OVERVIEW.md new file mode 100644 index 0000000..fe42af1 --- /dev/null +++ b/core/core/agent-bridge/OVERVIEW.md @@ -0,0 +1,17 @@ +--- +title: Runtime Agent bridge +status: current +last_verified: 2026-09-04 +--- + +# Runtime Agent bridge + +`core/runtime-agent-bridge` is the narrow same-host cross-account transport for the rootless DSP model. One root-owned bridge process serves one server-derived runtime identity. + +The downstream Unix socket is owner `0600` for the DSP account inside a root-owned non-writable directory. The bridge validates strict bounded Runtime Agent frames, rejects a registration for any other runtime, and forwards only to the configured central owner-private Hub socket. It persists and logs no registration token and exposes no TCP listener, path selector, command, shell, URL, generic proxy, or engine operation. + +The central Hub remains the authentication and current-authority boundary. The bridge is transport containment, not an alternate authorization database. + +The executable requires root and receives only systemd/server-owned environment. The current two-account fixture proves the socket boundary; durable production bridge lifecycle belongs to the future privileged host helper. + +See Rootless DSP runtime containers. diff --git a/core/core/agent-bridge/bin/dispatch-runtime-agent-bridge b/core/core/agent-bridge/bin/dispatch-runtime-agent-bridge new file mode 100755 index 0000000..42eddb0 --- /dev/null +++ b/core/core/agent-bridge/bin/dispatch-runtime-agent-bridge @@ -0,0 +1,4 @@ +#!/usr/bin/env node +'use strict'; +process.umask(0o077); +require('../src/service-cli').main().then(code => { if (Number.isInteger(code)) process.exitCode = code; }); diff --git a/core/core/agent-bridge/package.json b/core/core/agent-bridge/package.json new file mode 100644 index 0000000..7bad790 --- /dev/null +++ b/core/core/agent-bridge/package.json @@ -0,0 +1,13 @@ +{ + "name": "dispatch-runtime-agent-bridge", + "version": "0.1.0", + "private": true, + "description": "Per-DSP root-owned Runtime Agent Unix relay", + "type": "commonjs", + "scripts": { + "build": "./scripts/build", + "test": "./scripts/test", + "verify": "./scripts/verify" + }, + "engines": { "node": ">=22" } +} diff --git a/core/core/agent-bridge/scripts/build b/core/core/agent-bridge/scripts/build new file mode 100755 index 0000000..6116eed --- /dev/null +++ b/core/core/agent-bridge/scripts/build @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +for file in "$ROOT"/src/*.js "$ROOT"/tests/*.js; do node --no-warnings --check "$file"; done +printf '%s\n' '{"ok":true,"status":"built"}' diff --git a/core/core/agent-bridge/scripts/test b/core/core/agent-bridge/scripts/test new file mode 100755 index 0000000..d546f4a --- /dev/null +++ b/core/core/agent-bridge/scripts/test @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +exec node --no-warnings --test "$ROOT"/tests/*.test.js diff --git a/core/core/agent-bridge/scripts/verify b/core/core/agent-bridge/scripts/verify new file mode 100755 index 0000000..2a3e236 --- /dev/null +++ b/core/core/agent-bridge/scripts/verify @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +"$ROOT/tooling/build" +"$ROOT/tooling/test" diff --git a/core/core/agent-bridge/src/bridge.js b/core/core/agent-bridge/src/bridge.js new file mode 100644 index 0000000..0bbbd9e --- /dev/null +++ b/core/core/agent-bridge/src/bridge.js @@ -0,0 +1,211 @@ +'use strict'; + +const fs = require('node:fs'); +const net = require('node:net'); +const path = require('node:path'); +const { HOST_BRIDGE_ROOT, opaqueRuntimeSuffix, runtimeKey: checkedRuntimeKey } = require('../../runtime-host-identity'); +const { ForwardingBridge } = require('./forwarding'); +const MAX_UNIX_SOCKET_PATH_BYTES = 107; + +function fail(code = 'runtime_agent_bridge_unavailable') { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exactOptions(value) { + const names = ['runtimeKey', 'downstreamSocket', 'upstreamSocket', 'tenantUid', 'tenantGid', 'controllerUid', 'controllerGid', 'centralUid']; + if (!plain(value) || Object.keys(value).sort().join(',') !== names.sort().join(',')) fail(); +} + +function safeInteger(value) { + if (!Number.isSafeInteger(value) || value < 0 || value > 2 ** 31 - 1) fail(); + return value; +} + +function identity(target, { type, uid, gid = null, mode }) { + let info; + try { info = fs.lstatSync(target); } catch { fail(); } + const correctType = type === 'directory' ? info.isDirectory() : info.isSocket(); + if (!correctType || info.isSymbolicLink() || info.uid !== uid || gid !== null && info.gid !== gid + || (info.mode & 0o7777) !== mode || fs.realpathSync(target) !== target) fail(); + return Object.freeze({ dev: info.dev, ino: info.ino }); +} + +function socketInode(target) { + let info; + try { info = fs.lstatSync(target); } catch { fail(); } + if (!info.isSocket() || info.isSymbolicLink() || info.nlink !== 1 || fs.realpathSync(target) !== target) fail(); + return Object.freeze({ dev: info.dev, ino: info.ino }); +} + +function sameIdentity(left, right) { + return Boolean(left && right && left.dev === right.dev && left.ino === right.ino); +} + +function lexists(target) { + try { fs.lstatSync(target); return true; } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } +} + +function probeUnixSocket(socketPath) { + return new Promise(resolve => { + const socket = net.createConnection(socketPath); + let settled = false; + const finish = value => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + resolve(value); + }; + const timer = setTimeout(() => finish(false), 300); + socket.on('connect', () => finish(true)); + socket.on('error', () => finish(false)); + }); +} + +function socketPath(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || path.basename(value) !== 'runtime-agent-hub.sock' + || Buffer.byteLength(value, 'utf8') > MAX_UNIX_SOCKET_PATH_BYTES) fail(); + return value; +} + +function configuration(options) { + exactOptions(options); + let runtimeKey; + try { runtimeKey = checkedRuntimeKey(options.runtimeKey); } catch { fail(); } + const controllerUid = safeInteger(options.controllerUid); + const controllerGid = safeInteger(options.controllerGid); + if (controllerUid !== process.geteuid() || controllerGid !== process.getegid()) fail(); + const tenantUid = safeInteger(options.tenantUid); + const tenantGid = safeInteger(options.tenantGid); + const centralUid = safeInteger(options.centralUid); + if (tenantUid < 100 || tenantGid < 100 || centralUid < 100 + || tenantUid === centralUid || tenantUid === controllerUid || centralUid === controllerUid) fail(); + const downstreamSocket = socketPath(options.downstreamSocket); + const expectedParent = path.join(HOST_BRIDGE_ROOT, opaqueRuntimeSuffix(runtimeKey)); + if (path.dirname(downstreamSocket) !== expectedParent) fail(); + const upstreamSocket = socketPath(options.upstreamSocket); + if (upstreamSocket === downstreamSocket || path.dirname(upstreamSocket) === expectedParent) fail(); + return Object.freeze({ + runtimeKey, downstreamSocket, upstreamSocket, tenantUid, tenantGid, controllerUid, controllerGid, centralUid, + }); +} + +class RuntimeAgentBridge extends ForwardingBridge { + constructor(options) { + super(configuration(options)); + this.server = null; + this.bridgeRootIdentity = null; + this.parentIdentity = null; + this.socketFileIdentity = null; + } + + validateUpstream() { + const parent = path.dirname(this.config.upstreamSocket); + identity(parent, { type: 'directory', uid: this.config.centralUid, mode: 0o700 }); + identity(this.config.upstreamSocket, { type: 'socket', uid: this.config.centralUid, mode: 0o600 }); + } + + validateParents() { + const parent = path.dirname(this.config.downstreamSocket); + if (!sameIdentity(this.bridgeRootIdentity, identity(HOST_BRIDGE_ROOT, { + type: 'directory', uid: this.config.controllerUid, gid: this.config.controllerGid, mode: 0o711, + })) || !sameIdentity(this.parentIdentity, identity(parent, { + type: 'directory', uid: this.config.controllerUid, gid: this.config.controllerGid, mode: 0o711, + }))) fail(); + } + + async closeServer() { + if (!this.server) return; + const server = this.server; + this.server = null; + await new Promise(resolve => { + try { server.close(() => resolve()); } catch { resolve(); } + }); + } + + async rollbackStart() { + await this.closeServer(); + if (this.socketFileIdentity) { + this.validateParents(); + if (!sameIdentity(this.socketFileIdentity, socketInode(this.config.downstreamSocket))) fail(); + fs.unlinkSync(this.config.downstreamSocket); + } + this.bridgeRootIdentity = null; + this.parentIdentity = null; + this.socketFileIdentity = null; + } + + async start() { + if (this.server) fail(); + const parent = path.dirname(this.config.downstreamSocket); + this.bridgeRootIdentity = identity(HOST_BRIDGE_ROOT, { + type: 'directory', uid: this.config.controllerUid, gid: this.config.controllerGid, mode: 0o711, + }); + this.parentIdentity = identity(parent, { + type: 'directory', uid: this.config.controllerUid, gid: this.config.controllerGid, mode: 0o711, + }); + if (lexists(this.config.downstreamSocket)) { + const before = identity(this.config.downstreamSocket, { + type: 'socket', uid: this.config.tenantUid, gid: this.config.tenantGid, mode: 0o600, + }); + if (await probeUnixSocket(this.config.downstreamSocket)) fail(); + const after = identity(this.config.downstreamSocket, { + type: 'socket', uid: this.config.tenantUid, gid: this.config.tenantGid, mode: 0o600, + }); + if (!sameIdentity(before, after) || !sameIdentity(this.bridgeRootIdentity, identity(HOST_BRIDGE_ROOT, { + type: 'directory', uid: this.config.controllerUid, gid: this.config.controllerGid, mode: 0o711, + })) || !sameIdentity(this.parentIdentity, identity(parent, { + type: 'directory', uid: this.config.controllerUid, gid: this.config.controllerGid, mode: 0o711, + }))) fail(); + fs.unlinkSync(this.config.downstreamSocket); + } + this.server = net.createServer(socket => this.accept(socket)); + this.server.maxConnections = 1; + try { + await new Promise((resolve, reject) => { + const failed = error => { this.server.off('listening', ready); reject(error); }; + const ready = () => { this.server.off('error', failed); resolve(); }; + this.server.once('error', failed); + this.server.once('listening', ready); + this.server.listen(this.config.downstreamSocket); + }); + this.socketFileIdentity = socketInode(this.config.downstreamSocket); + this.validateParents(); + fs.chownSync(this.config.downstreamSocket, this.config.tenantUid, this.config.tenantGid); + fs.chmodSync(this.config.downstreamSocket, 0o600); + const hardened = identity(this.config.downstreamSocket, { + type: 'socket', uid: this.config.tenantUid, gid: this.config.tenantGid, mode: 0o600, + }); + if (!sameIdentity(this.socketFileIdentity, hardened)) fail(); + this.socketFileIdentity = hardened; + } catch (error) { + try { await this.rollbackStart(); } catch (cleanupError) { throw cleanupError; } + throw error; + } + } + + async close() { + if (!this.server) return; + if (this.active) this.closeConnection(this.active); + await this.closeServer(); + this.validateParents(); + if (!sameIdentity(this.socketFileIdentity, identity(this.config.downstreamSocket, { + type: 'socket', uid: this.config.tenantUid, gid: this.config.tenantGid, mode: 0o600, + }))) fail(); + fs.unlinkSync(this.config.downstreamSocket); + this.bridgeRootIdentity = null; + this.parentIdentity = null; + this.socketFileIdentity = null; + } +} + +module.exports = { configuration, RuntimeAgentBridge }; diff --git a/core/core/agent-bridge/src/directory-bridge.js b/core/core/agent-bridge/src/directory-bridge.js new file mode 100644 index 0000000..5962e89 --- /dev/null +++ b/core/core/agent-bridge/src/directory-bridge.js @@ -0,0 +1,73 @@ +'use strict'; + +const net = require('node:net'); +const path = require('node:path'); +const { ForwardingBridge } = require('./forwarding'); +const { DSP_ID } = require('../../../shared/paths/platform-paths'); +const { privateDirectory, socketIdentity, sameIdentity, MAX_UNIX_SOCKET_PATH_BYTES } = require('../../../shared/transport/unix-socket'); + +const { PrivateListener } = require('../../../shared/transport/private-listener'); + +function fail() { throw new Error('directory_bridge_invalid'); } +function configuration(options) { + if (!options || Object.getPrototypeOf(options) !== Object.prototype + || Object.keys(options).sort().join(',') !== 'dspRoot,runtimeKey,upstreamSocket') fail(); + const { runtimeKey, dspRoot, upstreamSocket } = options; + if (typeof runtimeKey !== 'string' || !DSP_ID.test(runtimeKey) || typeof dspRoot !== 'string' + || !path.isAbsolute(dspRoot) || path.resolve(dspRoot) !== dspRoot || path.basename(dspRoot) !== runtimeKey) fail(); + privateDirectory(dspRoot); + const downstreamSocket = path.join(dspRoot, '.control/runtime-agent-hub.sock'); + for (const selected of [downstreamSocket, upstreamSocket]) { + if (typeof selected !== 'string' || !path.isAbsolute(selected) || path.resolve(selected) !== selected + || path.basename(selected) !== 'runtime-agent-hub.sock' || Buffer.byteLength(selected) > MAX_UNIX_SOCKET_PATH_BYTES) fail(); + } + if (upstreamSocket === downstreamSocket || upstreamSocket.startsWith(dspRoot + '/')) fail(); + return Object.freeze({ runtimeKey, dspRoot, downstreamSocket, upstreamSocket }); +} + +// Each read-only mounted socket is bound to exactly one runtime identity. The +// shared host UID grants no ability to register as a sibling through this bridge. +// The Core hub independently validates the registration token and generation. +class DirectoryRuntimeAgentBridge extends ForwardingBridge { + constructor(options) { + super(configuration(options)); + this.server = null; + this.rootIdentity = null; + this.upstreamRootIdentity = null; + } + + validateUpstream() { + const current = privateDirectory(path.dirname(this.config.upstreamSocket)); + if (this.upstreamRootIdentity && !sameIdentity(this.upstreamRootIdentity, current)) fail(); + socketIdentity(this.config.upstreamSocket); + } + + validateParent() { + const current = privateDirectory(path.dirname(this.config.downstreamSocket)); + if (this.rootIdentity && !sameIdentity(this.rootIdentity, current)) fail(); + } + + async start() { + if (this.server) fail(); + const file = this.config.downstreamSocket; + this.rootIdentity = privateDirectory(path.dirname(file)); + this.upstreamRootIdentity = privateDirectory(path.dirname(this.config.upstreamSocket)); + this.validateUpstream(); + this.server = net.createServer(socket => this.accept(socket)); + this.server.maxConnections = 1; + this.listener = new PrivateListener(this.server, file); + try { + await this.listener.start(); + this.server.on('error', () => { if (this.active) this.closeConnection(this.active); }); + } catch (error) { await this.close(); throw error; } + } + + async close() { + if (!this.server) return; + if (this.active) this.closeConnection(this.active); + this.server = null; + await this.listener.close(); + } +} + +module.exports = { DirectoryRuntimeAgentBridge }; diff --git a/core/core/agent-bridge/src/forwarding.js b/core/core/agent-bridge/src/forwarding.js new file mode 100644 index 0000000..1307e01 --- /dev/null +++ b/core/core/agent-bridge/src/forwarding.js @@ -0,0 +1,126 @@ +'use strict'; + +const net = require('node:net'); +const { validateCapacityRequest, validateCapacityResponse } = require('../../../shared/agent/capacity'); +const { MAX_AGENT_FRAME_BYTES, encodeFrame, attachFrameReader } = require('../../../shared/agent/framing'); +const { + validateRegistrationFrame, + validateRegisteredFrame, + validateRejectedFrame, + validateHeartbeatFrame, + validateHeartbeatAckFrame, + validateRequestFrame, + validateResponseFrame, +} = require('../../../shared/agent/protocol'); + +const REGISTRATION_TIMEOUT_MS = 5_000; +const UPSTREAM_CONNECT_TIMEOUT_MS = 5_000; + +function fail(code = 'runtime_agent_bridge_unavailable') { + throw Object.assign(new Error(code), { code }); +} + +function validateDownstreamFrame(value, runtimeKey, registered) { + if (!registered) { + const frame = validateRegistrationFrame(value); + if (frame.runtimeKey !== runtimeKey) fail('runtime_identity_mismatch'); + return frame; + } + if (value?.type === 'capacity_request') return validateCapacityRequest(value); + if (value?.type === 'heartbeat_ack') return validateHeartbeatAckFrame(value); + if (value?.type === 'response') return validateResponseFrame(value); + fail('invalid_runtime_agent_frame'); +} + +function validateUpstreamFrame(value, runtimeKey, registered) { + if (!registered) { + if (value?.type === 'registered') return Object.freeze({ frame: validateRegisteredFrame(value), registered: true, terminal: false }); + if (value?.type === 'rejected') return Object.freeze({ frame: validateRejectedFrame(value), registered: false, terminal: true }); + fail('invalid_runtime_agent_frame'); + } + if (value?.type === 'capacity_response') return Object.freeze({ frame: validateCapacityResponse(value), registered: true, terminal: false }); + if (value?.type === 'heartbeat') return Object.freeze({ frame: validateHeartbeatFrame(value), registered: true, terminal: false }); + if (value?.type === 'request') return Object.freeze({ frame: validateRequestFrame(value, runtimeKey), registered: true, terminal: false }); + fail('invalid_runtime_agent_frame'); +} + +const MAX_BRIDGE_QUEUED_BYTES = MAX_AGENT_FRAME_BYTES * 2; +function writeFrame(socket, frame, source = null) { + if (socket.destroyed || !socket.writable) fail(); + const encoded = encodeFrame(frame); + if (socket.writableLength + Buffer.byteLength(encoded) > MAX_BRIDGE_QUEUED_BYTES) fail(); + if (!socket.write(encoded) && source && !source.isPaused()) { + source.pause(); + socket.once('drain', () => { if (!source.destroyed) source.resume(); }); + } +} + +class ForwardingBridge { + constructor(config) { this.config = config; this.active = null; } + + closeConnection(state) { + clearTimeout(state.registrationTimer); + clearTimeout(state.connectTimer); + if (!state.downstream.destroyed) state.downstream.destroy(); + if (state.upstream && !state.upstream.destroyed) state.upstream.destroy(); + if (this.active === state) this.active = null; + } + + accept(downstream) { + if (this.active) return downstream.destroy(); + const state = { + downstream, + upstream: null, + registrationSeen: false, + registered: false, + registrationTimer: null, + connectTimer: null, + }; + this.active = state; + const close = () => this.closeConnection(state); + downstream.on('error', close); + downstream.on('close', close); + state.registrationTimer = setTimeout(close, REGISTRATION_TIMEOUT_MS); + attachFrameReader(downstream, { + maxFrameBytes: MAX_AGENT_FRAME_BYTES, + onError: close, + onFrame: value => { + try { + if (!state.registrationSeen) { + const registration = validateDownstreamFrame(value, this.config.runtimeKey, false); + state.registrationSeen = true; + this.validateUpstream(); + const upstream = net.createConnection(this.config.upstreamSocket); + state.upstream = upstream; + state.connectTimer = setTimeout(close, UPSTREAM_CONNECT_TIMEOUT_MS); + upstream.on('error', close); + upstream.on('close', close); + attachFrameReader(upstream, { + maxFrameBytes: MAX_AGENT_FRAME_BYTES, + onError: close, + onFrame: upstreamValue => { + try { + const selected = validateUpstreamFrame(upstreamValue, this.config.runtimeKey, state.registered); + state.registered = selected.registered; + if (state.registered || selected.terminal) clearTimeout(state.registrationTimer); + writeFrame(downstream, selected.frame, upstream); + if (selected.terminal) downstream.end(); + } catch { close(); } + }, + }); + upstream.once('connect', () => { + clearTimeout(state.connectTimer); + try { writeFrame(upstream, registration, downstream); } catch { close(); } + }); + return; + } + if (!state.registered || !state.upstream) return close(); + writeFrame(state.upstream, validateDownstreamFrame(value, this.config.runtimeKey, true), downstream); + } catch { close(); } + }, + }); + } + +} + +module.exports = { ForwardingBridge, validateDownstreamFrame, validateUpstreamFrame, writeFrame, MAX_BRIDGE_QUEUED_BYTES }; diff --git a/core/core/agent-bridge/src/service-cli.js b/core/core/agent-bridge/src/service-cli.js new file mode 100644 index 0000000..b01ddb4 --- /dev/null +++ b/core/core/agent-bridge/src/service-cli.js @@ -0,0 +1,51 @@ +'use strict'; + +const path = require('node:path'); +const { HOST_BRIDGE_ROOT, opaqueRuntimeSuffix, runtimeKey } = require('../../runtime-host-identity'); +const { RuntimeAgentBridge } = require('./bridge'); + +function integer(value) { + if (typeof value !== 'string' || !/^(?:0|[1-9][0-9]{0,9})$/.test(value)) throw new Error('runtime_agent_bridge_unavailable'); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed) || parsed > 2 ** 31 - 1) throw new Error('runtime_agent_bridge_unavailable'); + return parsed; +} + +async function main(environment = process.env, write = chunk => process.stdout.write(chunk)) { + process.umask(0o077); + if (process.geteuid() !== 0 || process.getegid() !== 0) return 1; + let bridge; + try { + const selectedRuntimeKey = runtimeKey(environment.DISPATCH_RUNTIME_BRIDGE_KEY); + const upstreamSocket = environment.DISPATCH_RUNTIME_BRIDGE_UPSTREAM_SOCKET; + bridge = new RuntimeAgentBridge({ + runtimeKey: selectedRuntimeKey, + downstreamSocket: path.join(HOST_BRIDGE_ROOT, opaqueRuntimeSuffix(selectedRuntimeKey), 'runtime-agent-hub.sock'), + upstreamSocket, + tenantUid: integer(environment.DISPATCH_RUNTIME_BRIDGE_TENANT_UID), + tenantGid: integer(environment.DISPATCH_RUNTIME_BRIDGE_TENANT_GID), + controllerUid: 0, + controllerGid: 0, + centralUid: integer(environment.DISPATCH_RUNTIME_BRIDGE_CENTRAL_UID), + }); + await bridge.start(); + } catch { + try { await bridge?.close(); } catch {} + return 1; + } + write(`${JSON.stringify({ ok: true, status: 'ready' })}\n`); + let closing = false; + const close = async code => { + if (closing) return; + closing = true; + try { await bridge.close(); } finally { process.exit(code); } + }; + process.once('SIGINT', () => close(0)); + process.once('SIGTERM', () => close(0)); + process.once('uncaughtException', () => close(1)); + process.once('unhandledRejection', () => close(1)); + return new Promise(() => {}); +} + +if (require.main === module) main().then(code => { if (Number.isInteger(code)) process.exitCode = code; }); +module.exports = { main, integer }; diff --git a/core/core/agent-bridge/tests/bridge.test.js b/core/core/agent-bridge/tests/bridge.test.js new file mode 100644 index 0000000..27cefc5 --- /dev/null +++ b/core/core/agent-bridge/tests/bridge.test.js @@ -0,0 +1,123 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const path = require('node:path'); +const test = require('node:test'); +const { HOST_BRIDGE_ROOT, opaqueRuntimeSuffix } = require('../../runtime-host-identity'); +const { RUNTIME_GATEWAY_ACTIONS, gatewayFailure } = require('../../../shared/gateway/protocol'); +const { + RUNTIME_AGENT_PROTOCOL_VERSION, + registeredFrame, + rejectedFrame, + heartbeatFrame, + heartbeatAckFrame, + requestFrame, + responseFrame, +} = require('../../../shared/agent/protocol'); +const { configuration } = require('../src/bridge'); +const { validateDownstreamFrame, validateUpstreamFrame } = require('../src/forwarding'); + +const RUNTIME_KEY = 'runtime_bridge_alpha'; + +function registration(runtimeKey = RUNTIME_KEY) { + return { + protocolVersion: RUNTIME_AGENT_PROTOCOL_VERSION, + type: 'register', + runtimeKey, + registrationToken: 'a'.repeat(43), + actions: [...RUNTIME_GATEWAY_ACTIONS], + }; +} + +test('bridge configuration derives and pins one DSP endpoint', () => { + const downstream = path.join(HOST_BRIDGE_ROOT, opaqueRuntimeSuffix(RUNTIME_KEY), 'runtime-agent-hub.sock'); + const value = configuration({ + runtimeKey: RUNTIME_KEY, + downstreamSocket: downstream, + upstreamSocket: '/tmp/dispatch-central/runtime-agent-hub.sock', + tenantUid: process.geteuid() + 1, + tenantGid: process.getegid() + 1, + controllerUid: process.geteuid(), + controllerGid: process.getegid(), + centralUid: process.geteuid() + 2, + }); + assert.equal(value.runtimeKey, RUNTIME_KEY); + assert.equal(value.downstreamSocket, downstream); + assert.throws(() => configuration({ ...value, downstreamSocket: '/tmp/caller/runtime-agent-hub.sock' }), + error => error.code === 'runtime_agent_bridge_unavailable'); + assert.throws(() => configuration({ ...value, command: '/bin/sh' }), + error => error.code === 'runtime_agent_bridge_unavailable'); + assert.throws(() => configuration({ ...value, tenantUid: value.centralUid }), + error => error.code === 'runtime_agent_bridge_unavailable'); +}); + +test('bridge forwards only the closed Runtime Agent protocol for its pinned identity', () => { + assert.deepEqual(validateDownstreamFrame(registration(), RUNTIME_KEY, false), registration()); + assert.throws(() => validateDownstreamFrame(registration('runtime_bridge_beta'), RUNTIME_KEY, false), + error => error.code === 'runtime_identity_mismatch'); + assert.deepEqual(validateDownstreamFrame(heartbeatAckFrame('b'.repeat(32)), RUNTIME_KEY, true), + heartbeatAckFrame('b'.repeat(32))); + assert.deepEqual(validateDownstreamFrame(responseFrame('c'.repeat(32), gatewayFailure('runtime_gateway_unavailable')), RUNTIME_KEY, true), + responseFrame('c'.repeat(32), gatewayFailure('runtime_gateway_unavailable'))); + assert.throws(() => validateDownstreamFrame({ ...heartbeatAckFrame('b'.repeat(32)), extra: true }, RUNTIME_KEY, true), + error => error.code === 'invalid_runtime_agent_frame'); + + assert.deepEqual(validateUpstreamFrame(registeredFrame(), RUNTIME_KEY, false), { + frame: registeredFrame(), registered: true, terminal: false, + }); + assert.deepEqual(validateUpstreamFrame(rejectedFrame('runtime_agent_unauthorized'), RUNTIME_KEY, false), { + frame: rejectedFrame('runtime_agent_unauthorized'), registered: false, terminal: true, + }); + assert.deepEqual(validateUpstreamFrame(heartbeatFrame('d'.repeat(32)), RUNTIME_KEY, true), { + frame: heartbeatFrame('d'.repeat(32)), registered: true, terminal: false, + }); + const request = requestFrame('e'.repeat(32), { + protocolVersion: 1, + runtimeKey: RUNTIME_KEY, + action: 'health', + input: {}, + }); + assert.deepEqual(validateUpstreamFrame(request, RUNTIME_KEY, true), { + frame: request, registered: true, terminal: false, + }); + assert.throws(() => validateUpstreamFrame(requestFrame('f'.repeat(32), { + protocolVersion: 1, + runtimeKey: 'runtime_bridge_beta', + action: 'health', + input: {}, + }), RUNTIME_KEY, true), error => error.code === 'runtime_identity_mismatch'); +}); + +test('bridge applies backpressure and refuses output beyond its byte bound', () => { + const { EventEmitter } = require('node:events'); + const { writeFrame, MAX_BRIDGE_QUEUED_BYTES } = require('../src/forwarding'); + const socket = new EventEmitter(); + Object.assign(socket, { destroyed: false, writable: true, writableLength: 0, + write(value) { this.writableLength += Buffer.byteLength(value); return false; } }); + let paused = false; + const source = { destroyed: false, isPaused: () => paused, pause() { paused = true; }, resume() { paused = false; } }; + const frame = heartbeatAckFrame('b'.repeat(32)); + writeFrame(socket, frame, source); + assert.equal(paused, true); + assert.equal(socket.listenerCount('drain'), 1); + while (socket.writableLength < MAX_BRIDGE_QUEUED_BYTES) { + const before = socket.writableLength; + try { writeFrame(socket, frame, source); } + catch { assert.equal(socket.writableLength, before); break; } + } + assert.ok(socket.writableLength <= MAX_BRIDGE_QUEUED_BYTES); + assert.equal(socket.listenerCount('drain'), 1); + socket.writableLength = 0; socket.emit('drain'); + assert.equal(paused, false); +}); + +test('capacity frames require prior registration and cannot carry another DSP identity', () => { + const request = { type: 'capacity_request', requestId: 'a'.repeat(32), operation: 'acquire', jobId: 'b'.repeat(32), workers: 2 }; + const response = { type: 'capacity_response', requestId: request.requestId, status: 'granted', workers: 2, leaseMs: 120000 }; + assert.deepEqual(validateDownstreamFrame(request, RUNTIME_KEY, true), request); + assert.deepEqual(validateUpstreamFrame(response, RUNTIME_KEY, true).frame, response); + assert.throws(() => validateDownstreamFrame(request, RUNTIME_KEY, false)); + assert.throws(() => validateUpstreamFrame(response, RUNTIME_KEY, false)); + assert.throws(() => validateDownstreamFrame({ ...request, runtimeKey: 'another-dsp' }, RUNTIME_KEY, true)); + assert.throws(() => validateDownstreamFrame({ ...request, workers: 7 }, RUNTIME_KEY, true)); +}); diff --git a/core/core/agent-bridge/tests/directory-bridge.test.js b/core/core/agent-bridge/tests/directory-bridge.test.js new file mode 100644 index 0000000..c801d4a --- /dev/null +++ b/core/core/agent-bridge/tests/directory-bridge.test.js @@ -0,0 +1,126 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const net = require('node:net'); +const crypto = require('node:crypto'); +const { DirectoryRuntimeAgentBridge } = require('../src/directory-bridge'); +const { CoreRuntimeAgentHub } = require('../../agents/src/hub'); +const { DspRuntimeAgent } = require('dispatch-dsp/runtime/agent/src/agent.js'); +const { success } = require('../../../shared/contracts/src'); +const { RUNTIME_GATEWAY_ACTIONS } = require('../../../shared/gateway/protocol'); +const { encodeFrame } = require('../../../shared/agent/framing'); + +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); +async function until(check) { + for (let i = 0; i < 200; i++) { if (check()) return; await delay(10); } + throw Error('bridge_test_timeout'); +} + +async function fixture(t) { + const roots = [], close = [], ids = [], tokens = {}; + const hubRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'bridge-hub-')); + fs.chmodSync(hubRoot, 0o700); + t.after(async () => { for (const cleanup of close.reverse()) await cleanup(); + for (const root of [...roots, hubRoot]) fs.rmSync(root, { recursive: true, force: true }); }); + for (let i = 0; i < 2; i++) { + const id = `dsp_${crypto.randomBytes(16).toString('hex')}`; + const root = path.join(process.env.DISPATCH_DIRECTORY_TEST_ROOT || os.tmpdir(), id); + fs.mkdirSync(root, { mode: 0o700 }); roots.push(root); ids.push(id); + fs.mkdirSync(path.join(root, '.control'), { mode: 0o700 }); + tokens[id] = crypto.randomBytes(32).toString('base64url'); + } + const hub = new CoreRuntimeAgentHub({ socketPath: path.join(hubRoot, 'runtime-agent-hub.sock'), + authorities: Object.fromEntries(ids.map(id => [id, crypto.createHash('sha256').update(tokens[id]).digest('hex')])) }); + await hub.start(); close.push(() => hub.close()); + const bridges = roots.map((root, i) => new DirectoryRuntimeAgentBridge({ runtimeKey: ids[i], dspRoot: root, upstreamSocket: hub.socketPath })); + for (const bridge of bridges) { await bridge.start(); close.push(() => bridge.close()); } + return { hub, ids, roots, tokens, bridges, close }; +} + +function registration(id, token) { + return { protocolVersion: 1, type: 'register', runtimeKey: id, registrationToken: token, actions: RUNTIME_GATEWAY_ACTIONS }; +} +function rejected(socketPath, frame) { + return new Promise((resolve, reject) => { + const socket = net.createConnection(socketPath); + const timeout = setTimeout(() => { socket.destroy(); reject(Error('rejection_timeout')); }, 2000); + let data = ''; + socket.on('connect', () => socket.write(encodeFrame(frame))); + socket.on('data', chunk => { data += chunk; }); + socket.on('error', () => {}); + socket.on('close', () => { clearTimeout(timeout); resolve(data); }); + }); +} + +test('bridge binds a shared-UID connection to its DSP even with a valid sibling token', async t => { + const c = await fixture(t); + const output = await rejected(c.bridges[0].config.downstreamSocket, registration(c.ids[1], c.tokens[c.ids[1]])); + assert.equal(output.includes('"registered"'), false); + assert.equal(c.hub.connected(c.ids[1]), false); + await until(() => !c.bridges[0].active); + const agent = new DspRuntimeAgent({ socketPath: c.bridges[0].config.downstreamSocket, + runtimeKey: c.ids[0], registrationToken: c.tokens[c.ids[0]], + client: { workforce: { day: async () => success('ready', {}) }, + sync: { status: async () => success('ready', {}), runNow: async () => success('ready', {}) }, + system: { status: async () => success('ready', { fixture: 'alpha', components: { + auth: { healthy: true, ready: true }, + collections: { healthy: true, ready: true, status: 'ready', data: { manager: { running: true } } }, + } }) } } }); + c.close.push(() => agent.close()); + agent.start().catch(() => {}); + await until(() => c.hub.connected(c.ids[0])); + assert.equal(c.hub.connected(c.ids[1]), false); + assert.equal((await c.hub.invoke(c.ids[0], 'health', {})).ok, true); + assert.equal((await c.hub.invoke(c.ids[0], 'system.status', {})).data.fixture, 'alpha'); + const peer = new DspRuntimeAgent({ socketPath: c.bridges[1].config.downstreamSocket, + runtimeKey: c.ids[1], registrationToken: c.tokens[c.ids[1]], client: { ...agent.client, + system: { status: async () => success('ready', { fixture: 'beta' }) } } }); + c.close.push(() => peer.close()); + await peer.start(); + assert.equal((await c.hub.invoke(c.ids[1], 'system.status', {})).data.fixture, 'beta'); + assert.equal((await c.hub.invoke(c.ids[0], 'system.status', {})).data.fixture, 'alpha'); +}); + +test('bridge refuses unsafe socket replacement and duplicate live listeners', async t => { + const c = await fixture(t), bridge = c.bridges[0]; + const duplicate = new DirectoryRuntimeAgentBridge({ runtimeKey: c.ids[0], dspRoot: c.roots[0], upstreamSocket: c.hub.socketPath }); + await assert.rejects(() => duplicate.start()); + await bridge.close(); + const file = bridge.config.downstreamSocket; + fs.symlinkSync(c.hub.socketPath, file); + await assert.rejects(() => duplicate.start()); + fs.unlinkSync(file); + fs.writeFileSync(file, 'preserve', { mode: 0o600 }); + await assert.rejects(() => duplicate.start()); + assert.equal(fs.readFileSync(file, 'utf8'), 'preserve'); + fs.unlinkSync(file); +}); + +test('a bridge can restart without replacing or closing its sibling', async t => { + const c = await fixture(t); + const original = fs.statSync(c.bridges[1].config.downstreamSocket).ino; + await c.bridges[0].close(); + await c.bridges[0].start(); + assert.equal(fs.statSync(c.bridges[1].config.downstreamSocket).ino, original); + assert.equal(fs.statSync(c.bridges[0].config.downstreamSocket).mode & 0o777, 0o600); +}); + +test('closing a replaced socket stops its listener and preserves the replacement', async t => { + const c = await fixture(t), bridge = c.bridges[0], file = bridge.config.downstreamSocket; + fs.unlinkSync(file); + fs.writeFileSync(file, 'preserve', { mode: 0o600 }); + await assert.rejects(() => bridge.close()); + assert.equal(bridge.server, null); + assert.equal(fs.readFileSync(file, 'utf8'), 'preserve'); +}); + +test('the hub rejects an incorrect token even through the correct directory bridge', async t => { + const c = await fixture(t); + const output = await rejected(c.bridges[0].config.downstreamSocket, registration(c.ids[0], c.tokens[c.ids[1]])); + assert.ok(output.includes('"rejected"')); + assert.equal(c.hub.connected(c.ids[0]), false); +}); diff --git a/core/core/agents/OVERVIEW.md b/core/core/agents/OVERVIEW.md new file mode 100644 index 0000000..3af9e20 --- /dev/null +++ b/core/core/agents/OVERVIEW.md @@ -0,0 +1,30 @@ +--- +title: Runtime Agent source +status: current +last_verified: 2026-09-03 +--- + +# Dispatch Runtime Agent + +`core/agents/` contains the central hub and clients used to contact isolated DSP runtimes. The outbound DSP agent lives in `runtime/agent/`; shared wire definitions live in `shared/agent/`. + +## Source map + +- `src/hub.js` — central registration, runtime registry, heartbeat/liveness, request correlation, and failure cleanup. +- `src/client.js` — narrow `DispatchClient` adapter for a server-owned runtime identity. +- `src/control.js` — central private control socket. +- `../../runtime/agent/src/` — outbound agent, health socket, and supervised CLI composition. +- `../../shared/agent/` — registration frames, bounded JSON framing, and private credential loading. +- `examples/` and `tests/` — routing, reconnection, identity, and failure-isolation verification. + +## Boundary + +Core owns human authorization and runtime selection. Access Control stores only runtime registration digests and generations. Provisioner stores each raw token only in that runtime's private `secrets/runtime-agent/registration-token` file. The Agent owns one runtime's private operational connection and proxies only the closed Runtime Gateway action catalog. + +Each production DSP runs in an isolated OCI container. Its outbound connection reaches the Core hub through the host bridge. The opt-in same-user runtime tools remain under `compatibility/`. + +## Verification + +```bash +./core/agents/scripts/verify +``` diff --git a/core/core/agents/examples/durable-service.js b/core/core/agents/examples/durable-service.js new file mode 100644 index 0000000..bc6aab7 --- /dev/null +++ b/core/core/agents/examples/durable-service.js @@ -0,0 +1,151 @@ +'use strict'; + +const { spawn } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { success } = require('../../../shared/contracts/src'); +const { managedInstallationRuntimeEnvironment } = require('../../../shared/paths/runtime-paths'); +const { RuntimeGatewayServer } = require('dispatch-dsp/runtime/gateway/src/index.js'); +const { + createInstallationLayoutManager, + createRuntimeAgentCredentialManager, + INSTALLATION_LAYOUT_TEMPLATE, +} = require('../../installations/src'); +const { CoreRuntimeAgentHub } = require('../src'); +const { queryRuntimeAgentStatus } = require('dispatch-dsp/runtime/agent/src/index.js'); + +function delay(milliseconds) { return new Promise(resolve => setTimeout(resolve, milliseconds)); } +async function waitFor(callback, timeoutMs = 5_000) { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + const value = await callback(); + if (value) return value; + } catch {} + if (Date.now() >= deadline) throw new Error('runtime_agent_fixture_timeout'); + await delay(25); + } +} + +function client() { + return { + workforce: { day: async query => success('found', { businessDate: query.date, items: [] }) }, + sync: { + status: async id => success('found', { id, desiredState: 'stopped' }), + runNow: async id => success('queued', { id, replayed: false }), + start: async id => success('started', { id }), + stop: async id => success('stopped', { id }), + }, + collections: { health: async () => success('ready', { counts: { queued: 0, running: 0 } }) }, + system: { status: async () => success('ready', { fixture: 'durable_service' }) }, + }; +} + +async function run() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-agent-service-')); + fs.chmodSync(root, 0o700); + const installationsRoot = path.join(root, 'installations'); + const centralRuntimeRoot = path.join(root, 'central-run'); + fs.mkdirSync(installationsRoot, { mode: 0o700 }); + fs.mkdirSync(centralRuntimeRoot, { mode: 0o700 }); + const runtimeKey = 'fixture_durable_service'; + const manifest = { + manifestVersion: 1, + revision: 1, + organization: { id: 'org_durable_service', stationCode: 'TST1', timezone: 'America/Los_Angeles' }, + runtime: { key: runtimeKey, templateId: INSTALLATION_LAYOUT_TEMPLATE, releaseId: 'dispatch_fixture_1' }, + }; + const authority = { + revision: manifest.revision, + organization: { ...manifest.organization }, + runtime: { ...manifest.runtime }, + }; + const credentials = createRuntimeAgentCredentialManager({ installationsRoot }); + const credential = credentials.issue(runtimeKey); + const layoutManager = createInstallationLayoutManager({ installationsRoot }); + layoutManager.materialize(manifest, authority); + const layout = layoutManager.derive(manifest, authority); + const paths = layoutManager.runtimePaths(manifest, authority); + const hubSocket = path.join(centralRuntimeRoot, 'runtime-agent-hub.sock'); + let hub = new CoreRuntimeAgentHub({ + socketPath: hubSocket, + authorities: { [runtimeKey]: credential.tokenHash }, + heartbeatIntervalMs: 50, + heartbeatTimeoutMs: 150, + }); + let gateway; + let child; + try { + await hub.start(); + gateway = new RuntimeGatewayServer({ + socketPath: path.join(paths.runtimeRoot, 'runtime-gateway.sock'), + runtimeKey, + client: client(), + }); + await gateway.start(); + const environment = { + ...managedInstallationRuntimeEnvironment(layout), + DISPATCH_RUNTIME_KEY: runtimeKey, + DISPATCH_RUNTIME_GATEWAY_SOCKET: gateway.socketPath, + DISPATCH_RUNTIME_AGENT_HUB_SOCKET: hubSocket, + DISPATCH_RUNTIME_AGENT_TOKEN_FILE: paths.runtimeAgent.registrationToken, + DISPATCH_RUNTIME_AGENT_STATUS_SOCKET: paths.runtimeAgent.statusSocket, + PATH: process.env.PATH, + LANG: 'C.UTF-8', + LC_ALL: 'C.UTF-8', + TZ: 'UTC', + NODE_NO_WARNINGS: '1', + }; + child = spawn(path.join(__dirname, "../../../runtime/agent/bin/dispatch-runtime-agent"), [], { + cwd: path.join(__dirname, ".."), + env: environment, + stdio: ['ignore', 'pipe', 'pipe'], + }); + const exit = new Promise((resolve, reject) => { + child.once('error', reject); + child.once('exit', (code, signal) => resolve({ code, signal })); + }); + await waitFor(async () => (await queryRuntimeAgentStatus(paths.runtimeAgent.statusSocket)).ok); + if (!hub.connected(runtimeKey)) throw new Error('runtime_agent_fixture_failed'); + + await hub.close(); + await waitFor(async () => !(await queryRuntimeAgentStatus(paths.runtimeAgent.statusSocket)).ok); + hub = new CoreRuntimeAgentHub({ + socketPath: hubSocket, + authorities: { [runtimeKey]: credential.tokenHash }, + heartbeatIntervalMs: 50, + heartbeatTimeoutMs: 150, + }); + await hub.start(); + await waitFor(async () => (await queryRuntimeAgentStatus(paths.runtimeAgent.statusSocket)).ok); + if (!hub.connected(runtimeKey)) throw new Error('runtime_agent_fixture_failed'); + + child.kill('SIGTERM'); + const stopped = await exit; + if (stopped.code !== 0) throw new Error('runtime_agent_fixture_failed'); + child = null; + process.stdout.write(`${JSON.stringify({ + ok: true, + status: 'durable_runtime_agent_service_verified', + reconnectAfterHubRestart: true, + privateTokenFile: true, + supervisedUnitReady: true, + realCredentialsUsed: false, + })}\n`); + } finally { + if (child) { + child.kill('SIGKILL'); + await new Promise(resolve => child.once('exit', resolve)); + } + await gateway?.close().catch(() => {}); + await hub.close().catch(() => {}); + fs.rmSync(root, { recursive: true, force: true }); + } +} + +run().catch(() => { + process.stdout.write(`${JSON.stringify({ ok: false, status: 'runtime_agent_fixture_failed' })}\n`); + process.exitCode = 1; +}); diff --git a/core/core/agents/examples/synthetic-core-hub.js b/core/core/agents/examples/synthetic-core-hub.js new file mode 100644 index 0000000..548c022 --- /dev/null +++ b/core/core/agents/examples/synthetic-core-hub.js @@ -0,0 +1,52 @@ +'use strict'; + +const readline = require('node:readline'); +const { CoreRuntimeAgentHub, createRuntimeAgentDispatchClient } = require('../src'); + +function exact(value, required) { + if (!value || typeof value !== 'object' || Array.isArray(value) + || Object.keys(value).length !== required.length + || required.some(key => !Object.prototype.hasOwnProperty.call(value, key))) throw new Error('invalid_runtime_agent_frame'); +} + +async function main(environment = process.env) { + const socketPath = environment.DISPATCH_RUNTIME_AGENT_HUB_SOCKET; + let authorities; + try { authorities = JSON.parse(environment.DISPATCH_RUNTIME_AGENT_AUTHORITIES || ''); } + catch { throw new Error('runtime_agent_unavailable'); } + const hub = new CoreRuntimeAgentHub({ socketPath, authorities }); + await hub.start(); + process.stdout.write(`${JSON.stringify({ ok: true, status: 'ready' })}\n`); + + const input = readline.createInterface({ input: process.stdin, crlfDelay: Infinity }); + input.on('line', line => { + Promise.resolve().then(async () => { + const command = JSON.parse(line); + if (command?.action === 'status') { + exact(command, ['action']); + return { ok: true, status: 'ready', data: hub.status() }; + } + exact(command, ['action', 'runtimeKey']); + if (command.action !== 'system.status') throw new Error('runtime_agent_unavailable'); + return createRuntimeAgentDispatchClient({ hub, runtimeKey: command.runtimeKey }).system.status(); + }).then( + result => process.stdout.write(`${JSON.stringify(result)}\n`), + () => process.stdout.write(`${JSON.stringify({ ok: false, status: 'runtime_agent_unavailable', data: null })}\n`), + ); + }); + let closing = false; + const close = async code => { + if (closing) return; + closing = true; + input.close(); + try { await hub.close(); } finally { process.exit(code); } + }; + process.once('SIGINT', () => close(0)); + process.once('SIGTERM', () => close(0)); + process.once('uncaughtException', () => close(1)); + process.once('unhandledRejection', () => close(1)); + return new Promise(() => {}); +} + +if (require.main === module) main().catch(() => { process.exitCode = 1; }); +module.exports = { main }; diff --git a/core/core/agents/examples/synthetic-runtime-agent.js b/core/core/agents/examples/synthetic-runtime-agent.js new file mode 100644 index 0000000..8199f31 --- /dev/null +++ b/core/core/agents/examples/synthetic-runtime-agent.js @@ -0,0 +1,93 @@ +'use strict'; + +const path = require('node:path'); +const { success } = require('../../../shared/contracts/src'); +const { parseStrictJson } = require('../../../shared/gateway/strict-json'); +const { registrationToken } = require('../src'); +const { DspRuntimeAgent } = require('dispatch-dsp/runtime/agent/src/index.js'); + +const MAX_CONFIG_BYTES = 16 * 1024; + +function fail(code = 'invalid_runtime_agent_frame') { + throw Object.assign(new Error(code), { code }); +} + +function exact(value, keys) { + if (!value || typeof value !== 'object' || Array.isArray(value) + || Object.getPrototypeOf(value) !== Object.prototype + || Object.keys(value).sort().join(',') !== [...keys].sort().join(',')) fail(); + return value; +} + +function runtime(label) { + return { + system: { status: async () => success('ready', { + label, + components: { + auth: { healthy: true, ready: true }, + collections: { + healthy: true, + ready: true, + status: 'ready', + data: { manager: { running: true } }, + }, + }, + }) }, + workforce: { day: async query => success('found', { + label, businessDate: query.date, items: [], + }) }, + sync: { + status: async id => success('found', { label, id, desiredState: 'stopped' }), + runNow: async id => success('queued', { label, id, replayed: false }), + }, + }; +} + +async function main() { + const chunks = []; + let size = 0; + for await (const chunk of process.stdin) { + size += chunk.length; + if (size > MAX_CONFIG_BYTES) fail(); + chunks.push(chunk); + } + const raw = Buffer.concat(chunks).toString('utf8'); + if (!raw.endsWith('\n') || raw.includes('\r') || raw.slice(0, -1).includes('\n')) fail(); + const config = exact(parseStrictJson(raw.slice(0, -1)), [ + 'socketPath', 'runtimeKey', 'registrationToken', 'label', + ]); + if (typeof config.socketPath !== 'string' || !path.isAbsolute(config.socketPath) + || typeof config.runtimeKey !== 'string' || typeof config.label !== 'string' + || !/^[a-z]{3,24}$/.test(config.label)) fail(); + registrationToken(config.registrationToken); + const selectedRegistrationToken = config.registrationToken; + config.registrationToken = null; + const agent = new DspRuntimeAgent({ + socketPath: config.socketPath, + runtimeKey: config.runtimeKey, + registrationToken: selectedRegistrationToken, + client: runtime(config.label), + }); + let stopping = false; + const stop = async () => { + if (stopping) return; + stopping = true; + await agent.close(); + process.exit(0); + }; + process.once('SIGTERM', stop); + process.once('SIGINT', stop); + await agent.start(); + process.stdout.write(`${JSON.stringify({ ok: true, status: 'registered', label: config.label })}\n`); +} + +main().catch(error => { + process.stdout.write(`${JSON.stringify({ + ok: false, + status: [ + 'runtime_agent_unauthorized', 'runtime_agent_conflict', + 'runtime_agent_protocol_mismatch', 'invalid_runtime_agent_frame', + ].includes(error?.code) ? error.code : 'runtime_agent_unavailable', + })}\n`); + process.exitCode = 1; +}); diff --git a/core/core/agents/examples/two-runtime-agents.js b/core/core/agents/examples/two-runtime-agents.js new file mode 100644 index 0000000..05473ce --- /dev/null +++ b/core/core/agents/examples/two-runtime-agents.js @@ -0,0 +1,150 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { spawn } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { parseStrictJson } = require('../../../shared/gateway/strict-json'); +const { + CoreRuntimeAgentHub, + createRuntimeAgentDispatchClient, +} = require('../src'); + +const CHILD = path.join(__dirname, "./synthetic-runtime-agent.js"); +const CHILD_OUTPUT_LIMIT = 4 * 1024; + +function token() { return crypto.randomBytes(32).toString('base64url'); } +function digest(value) { return crypto.createHash('sha256').update(value).digest('hex'); } + +function startAgent(config) { + const child = spawn(process.execPath, ['--no-warnings', CHILD], { + cwd: path.dirname(__dirname), + env: { LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' }, + stdio: ['pipe', 'pipe', 'pipe'], + }); + const ready = new Promise((resolve, reject) => { + let output = Buffer.alloc(0); + let settled = false; + const finish = (error, value) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (error) reject(error); else resolve(value); + }; + const timer = setTimeout(() => finish(new Error('runtime_agent_fixture_failed')), 5_000); + child.once('error', () => finish(new Error('runtime_agent_fixture_failed'))); + child.stdout.on('data', chunk => { + output = Buffer.concat([output, chunk]); + if (output.length > CHILD_OUTPUT_LIMIT) return finish(new Error('runtime_agent_fixture_failed')); + const newline = output.indexOf(0x0a); + if (newline < 0) return; + try { finish(null, parseStrictJson(output.subarray(0, newline).toString('utf8'))); } + catch { finish(new Error('runtime_agent_fixture_failed')); } + }); + child.once('exit', code => { + if (!settled && code !== 0) finish(new Error('runtime_agent_fixture_failed')); + }); + }); + child.stdin.end(`${JSON.stringify(config)}\n`); + return { child, ready }; +} + +async function stopAgent(child) { + if (!child || child.exitCode !== null || child.signalCode !== null) return; + await new Promise(resolve => { + const timer = setTimeout(() => { + child.kill('SIGKILL'); + resolve(); + }, 2_000); + child.once('exit', () => { + clearTimeout(timer); + resolve(); + }); + child.kill('SIGTERM'); + }); +} + +async function waitFor(check) { + const deadline = Date.now() + 1_000; + while (!check()) { + if (Date.now() >= deadline) throw new Error('runtime_agent_fixture_failed'); + await new Promise(resolve => setTimeout(resolve, 10)); + } +} + +async function main() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-two-agents-')); + fs.chmodSync(root, 0o700); + const socketPath = path.join(root, 'runtime-agent-hub.sock'); + const alphaToken = token(); + const bravoToken = token(); + const hub = new CoreRuntimeAgentHub({ + socketPath, + authorities: { + fixture_alpha: digest(alphaToken), + fixture_bravo: digest(bravoToken), + }, + }); + const children = []; + try { + await hub.start(); + const alphaAgent = startAgent({ + socketPath, runtimeKey: 'fixture_alpha', registrationToken: alphaToken, label: 'alpha', + }); + const bravoAgent = startAgent({ + socketPath, runtimeKey: 'fixture_bravo', registrationToken: bravoToken, label: 'bravo', + }); + children.push(alphaAgent.child, bravoAgent.child); + const registrations = await Promise.all([alphaAgent.ready, bravoAgent.ready]); + assert.deepEqual(registrations.map(value => value.status), ['registered', 'registered']); + assert.deepEqual(hub.status(), { protocolVersion: 1, configured: 2, connected: 2 }); + + const alpha = createRuntimeAgentDispatchClient({ hub, runtimeKey: 'fixture_alpha' }); + const bravo = createRuntimeAgentDispatchClient({ hub, runtimeKey: 'fixture_bravo' }); + const [alphaDay, bravoDay] = await Promise.all([ + alpha.workforce.day({ date: '2026-09-03', limit: 10, offset: 0 }), + bravo.workforce.day({ date: '2026-09-03', limit: 10, offset: 0 }), + ]); + assert.equal(alphaDay.data.label, 'alpha'); + assert.equal(bravoDay.data.label, 'bravo'); + assert.equal((await alpha.health()).data.runtimeIdentity, 'matched'); + assert.equal((await bravo.health()).data.runtimeIdentity, 'matched'); + + const crossed = startAgent({ + socketPath, runtimeKey: 'fixture_alpha', registrationToken: bravoToken, label: 'crossed', + }); + children.push(crossed.child); + assert.equal((await crossed.ready).status, 'runtime_agent_unauthorized'); + await stopAgent(crossed.child); + assert.equal(hub.status().connected, 2); + + await stopAgent(alphaAgent.child); + await waitFor(() => !hub.connected('fixture_alpha')); + assert.equal((await alpha.system.status()).status, 'runtime_agent_unavailable'); + assert.equal((await bravo.system.status()).data.label, 'bravo'); + + process.stdout.write(`${JSON.stringify({ + ok: true, + status: 'two_runtime_agents_verified', + centralCoreHubs: 1, + runtimeAgents: 2, + independentProcesses: true, + outboundConnections: true, + crossedIdentityRejected: true, + failureIsolation: true, + realCredentialsUsed: false, + productionStateChanged: false, + })}\n`); + } finally { + await Promise.allSettled(children.map(stopAgent)); + await hub.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(error => { + process.stdout.write(`${JSON.stringify({ ok: false, status: error?.code || 'runtime_agent_fixture_failed' })}\n`); + process.exitCode = 1; +}); diff --git a/core/core/agents/package.json b/core/core/agents/package.json new file mode 100644 index 0000000..4edb1d1 --- /dev/null +++ b/core/core/agents/package.json @@ -0,0 +1,20 @@ +{ + "name": "dispatch-runtime-agent", + "version": "0.1.0", + "private": true, + "description": "Core connection hub, authenticated runtime routing and local control endpoint", + "type": "commonjs", + "main": "src/index.js", + "bin": { + "dispatch-runtime-agent": "bin/dispatch-runtime-agent", + "dispatch-runtime-agentctl": "bin/dispatch-runtime-agentctl" + }, + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "./scripts/build", + "test": "./scripts/test", + "verify": "./scripts/verify" + } +} diff --git a/core/core/agents/scripts/build b/core/core/agents/scripts/build new file mode 100755 index 0000000..5de7529 --- /dev/null +++ b/core/core/agents/scripts/build @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +for file in "$ROOT"/src/*.js "$ROOT"/tests/*.js "$ROOT"/examples/*.js ; do + node --no-warnings --check "$file" +done +node --no-warnings -e "require('$ROOT/src')" +printf '%s\n' '{"ok":true,"status":"built"}' diff --git a/core/core/agents/scripts/test b/core/core/agents/scripts/test new file mode 100755 index 0000000..0f2d4e3 --- /dev/null +++ b/core/core/agents/scripts/test @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +node --no-warnings --test "$ROOT"/tests/*.test.js diff --git a/core/core/agents/scripts/verify b/core/core/agents/scripts/verify new file mode 100755 index 0000000..3689152 --- /dev/null +++ b/core/core/agents/scripts/verify @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +"$ROOT/tooling/build" +"$ROOT/tooling/test" +node --no-warnings "$ROOT/examples/two-runtime-agents.js" +node --no-warnings "$ROOT/examples/durable-service.js" diff --git a/core/core/agents/src/client.js b/core/core/agents/src/client.js new file mode 100644 index 0000000..3470d46 --- /dev/null +++ b/core/core/agents/src/client.js @@ -0,0 +1,48 @@ +'use strict'; + +const { success, failure, isResult } = require('../../../shared/contracts/src'); +const { RUNTIME_GATEWAY_ACTIONS } = require('../../../shared/gateway/protocol'); +const { RUNTIME_AGENT_PROTOCOL_VERSION } = require('../../../shared/agent/protocol'); + +function cloneResult(value) { + if (!isResult(value)) throw Object.assign(new Error('runtime_agent_unavailable'), { code: 'runtime_agent_unavailable' }); + return value.ok + ? success(value.status, value.data) + : failure(value.status, { recoverable: value.error.recoverable, data: value.data }); +} + +function createRuntimeAgentDispatchClient({ hub, runtimeKey } = {}) { + if (!hub || typeof hub.invoke !== 'function' || typeof runtimeKey !== 'string') { + throw new TypeError('runtime_agent_client_options_required'); + } + const invoke = async (action, input) => { + try { return cloneResult(await hub.invoke(runtimeKey, action, input)); } + catch (error) { + const code = ['runtime_identity_mismatch', 'runtime_agent_protocol_mismatch'].includes(error?.code) + ? error.code : error?.code === 'invalid_runtime_agent_frame' ? 'invalid_input' : 'runtime_agent_unavailable'; + return failure(code, { recoverable: code === 'runtime_agent_unavailable' }); + } + }; + return Object.freeze({ + plugins: Object.freeze({ invoke: (pluginId, action, input) => invoke('plugins.invoke', { pluginId, action, input }) }), + workforce: Object.freeze({ day: query => invoke('workforce.day', { query }), + employees: (query = {}) => invoke('workforce.employees', { query }), + employee: code => invoke('workforce.employee', { code }) }), + sync: Object.freeze({ + status: id => invoke('sync.status', { id }), + runNow: (id, options = {}) => invoke('sync.run_now', { id, options }), + start: id => invoke('sync.start', { id }), + stop: (id, options = {}) => invoke('sync.stop', { id, options }), + }), + collections: Object.freeze({ health: () => invoke('collections.health', {}) }), + system: Object.freeze({ status: () => invoke('system.status', {}) }), + health: () => invoke('health', {}), + capabilities: () => success('found', { + runtimeAgentProtocolVersion: RUNTIME_AGENT_PROTOCOL_VERSION, + transport: 'outbound_agent', + actions: RUNTIME_GATEWAY_ACTIONS.filter(action => !['runtime.execution', 'health', 'plugins.manage', 'paycom.setup', 'connections.manage', 'diagnostics.seed'].includes(action)), + }), + }); +} + +module.exports = { createRuntimeAgentDispatchClient }; diff --git a/core/core/agents/src/collection-capacity.js b/core/core/agents/src/collection-capacity.js new file mode 100644 index 0000000..f662c21 --- /dev/null +++ b/core/core/agents/src/collection-capacity.js @@ -0,0 +1,70 @@ +'use strict'; +const os = require('node:os'); +const { performance } = require('node:perf_hooks'); +const { LEASE_MS, validateCapacityRequest, validateCapacityResponse } = require('../../../shared/agent/capacity'); + +function capacityDefaults({ cpus = os.availableParallelism(), memoryBytes = os.totalmem() } = {}) { + // A conservative initial ceiling, not a throughput claim. Reserve headroom + // for Core and the OS; operators can tune after running the capacity probe. + return Math.max(1, Math.min(4, Math.floor(cpus / 2), Math.floor((memoryBytes - 2 * 1024 ** 3) / 1024 ** 3))); +} +class CollectionCapacity { + constructor({ workers = capacityDefaults(), clock = () => performance.now(), recoveryMs = LEASE_MS } = {}) { + if (!Number.isInteger(workers) || workers < 1 || workers > 64 + || !Number.isInteger(recoveryMs) || recoveryMs < 0 || recoveryMs > LEASE_MS) throw Error('invalid_collection_capacity'); + this.workers = workers; + this.clock = clock; + this.readyAt = clock() + recoveryMs; + this.active = new Map(); + this.queue = new Map(); + } + sweep() { + const now = this.clock(); + for (const map of [this.active, this.queue]) for (const [key, item] of map) if (item.expiresAt <= now) map.delete(key); + } + request(runtimeKey, input) { + const request = validateCapacityRequest(input); + this.sweep(); + const now = this.clock(); + const respond = (status, workers = 0) => validateCapacityResponse({ type: 'capacity_response', + requestId: request.requestId, status, workers, leaseMs: status === 'granted' ? LEASE_MS : 0 }); + const active = this.active.get(runtimeKey); + if (request.operation === 'release') { + if (active?.jobId === request.jobId) this.active.delete(runtimeKey); + if (this.queue.get(runtimeKey)?.jobId === request.jobId) this.queue.delete(runtimeKey); + return respond('released'); + } + if (active?.jobId === request.jobId) { + active.expiresAt = now + LEASE_MS; + return respond('granted', active.workers); + } + if (request.operation === 'renew') return respond('lost'); + // One queued or active job per DSP prevents queue flooding and gives each + // DSP a turn. Repeated polls retain FIFO position. + if (active) return respond('waiting'); + const queued = this.queue.get(runtimeKey); + if (queued && queued.jobId !== request.jobId) return respond('waiting'); + if (!queued) { + if (this.queue.size >= 128) return respond('waiting'); + this.queue.set(runtimeKey, { jobId: request.jobId, expiresAt: now + LEASE_MS }); + } else queued.expiresAt = now + LEASE_MS; + const available = this.workers - [...this.active.values()].reduce((sum, item) => sum + item.workers, 0); + if (now < this.readyAt || available < 1 || this.queue.keys().next().value !== runtimeKey) return respond('waiting'); + // Reserve room for another DSP when the host budget permits parallel work. + const granted = Math.min(request.workers, available, Math.ceil(this.workers / 2)); + this.queue.delete(runtimeKey); + this.active.set(runtimeKey, { jobId: request.jobId, workers: granted, expiresAt: now + LEASE_MS }); + return respond('granted', granted); + } + disconnect(runtimeKey) { + this.queue.delete(runtimeKey); + // Keep active grants until expiry: the disconnected collector needs time + // to notice renewal failure and terminate its browser work. + } + status() { + this.sweep(); + return { workerLimit: this.workers, activeWorkers: [...this.active.values()].reduce((sum, item) => sum + item.workers, 0), + activeDsps: this.active.size, waitingDsps: this.queue.size, recovering: this.clock() < this.readyAt }; + } +} +module.exports = { CollectionCapacity, capacityDefaults }; diff --git a/core/core/agents/src/control.js b/core/core/agents/src/control.js new file mode 100644 index 0000000..2bf48fa --- /dev/null +++ b/core/core/agents/src/control.js @@ -0,0 +1,167 @@ +'use strict'; + +const fs = require('node:fs'); +const net = require('node:net'); +const path = require('node:path'); +const { + validateGatewayRequest, + validateGatewayResponse, + gatewaySuccess, + gatewayFailure, + RUNTIME_GATEWAY_PROTOCOL_VERSION, +} = require('../../../shared/gateway/protocol'); +const { parseStrictJson } = require('../../../shared/gateway/strict-json'); +const { privateDirectory, socketIdentity, sameIdentity, probeUnixSocket } = require('../../../shared/transport/unix-socket'); + +const CONTROL_SOCKET_BASENAME = 'runtime-agent-control.sock'; +const MAX_CONTROL_FRAME_BYTES = 300 * 1024; +const CONTROL_TIMEOUT_MS = 30_000; +const CONTROL_ACTIONS = new Set([ + 'diagnostics.seed', 'paycom.setup', 'connections.manage', 'health', 'system.status', 'workforce.day', 'workforce.employees', 'workforce.employee', 'sync.status', 'sync.start', 'sync.stop', 'collections.health', +]); + +function coded(code = 'runtime_agent_unavailable') { + return Object.assign(new Error(code), { code }); +} +function validSocketPath(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || path.basename(value) !== CONTROL_SOCKET_BASENAME || Buffer.byteLength(value, 'utf8') > 107) throw coded(); + return value; +} + +class CoreRuntimeAgentControlServer { + constructor(options) { + if (!options || typeof options !== 'object' || Array.isArray(options) + || Object.keys(options).sort().join(',') !== 'hub,socketPath' + || !options.hub || typeof options.hub.invoke !== 'function') throw coded('runtime_boundary_violation'); + this.socketPath = validSocketPath(options.socketPath); + this.hub = options.hub; + this.server = null; + this.connections = new Set(); + this.rootIdentity = null; + this.fileIdentity = null; + } + + async start() { + const root = path.dirname(this.socketPath); + this.rootIdentity = privateDirectory(root); + if (fs.existsSync(this.socketPath)) { + const before = socketIdentity(this.socketPath); + if (await probeUnixSocket(this.socketPath)) throw coded(); + const after = socketIdentity(this.socketPath); + if (!sameIdentity(before, after) || !sameIdentity(this.rootIdentity, privateDirectory(root))) throw coded(); + fs.unlinkSync(this.socketPath); + } + this.server = net.createServer({ allowHalfOpen: true }, socket => this.accept(socket)); + this.server.maxConnections = 32; + await new Promise((resolve, reject) => { + this.server.once('error', reject); + this.server.listen(this.socketPath, () => { this.server.off('error', reject); resolve(); }); + }); + if (!sameIdentity(this.rootIdentity, privateDirectory(root))) throw coded(); + this.fileIdentity = socketIdentity(this.socketPath, { requireMode: false }); + fs.chmodSync(this.socketPath, 0o600); + if (!sameIdentity(this.fileIdentity, socketIdentity(this.socketPath))) throw coded(); + return this; + } + + accept(socket) { + this.connections.add(socket); + let chunks = []; + let size = 0; + let handled = false; + const timer = setTimeout(() => socket.destroy(), CONTROL_TIMEOUT_MS); + const finish = value => { + if (handled || socket.destroyed) return; + handled = true; + socket.end(`${JSON.stringify(value)}\n`); + }; + socket.on('data', chunk => { + if (handled) return socket.destroy(); + size += chunk.length; + if (size > MAX_CONTROL_FRAME_BYTES) return finish(gatewayFailure('invalid_request')); + chunks.push(chunk); + const raw = Buffer.concat(chunks).toString('utf8'); + if (!raw.includes('\n')) return; + chunks = []; + try { + if (!raw.endsWith('\n') || raw.includes('\r') || raw.slice(0, -1).includes('\n')) throw coded('invalid_request'); + const request = validateGatewayRequest(parseStrictJson(raw.slice(0, -1))); + if (!CONTROL_ACTIONS.has(request.action)) throw coded('invalid_request'); + Promise.resolve(this.hub.invoke(request.runtimeKey, request.action, request.input)).then( + result => { try { finish(gatewaySuccess(result)); } catch { finish(gatewayFailure()); } }, + error => finish(gatewayFailure(error?.code)), + ); + } catch (error) { finish(gatewayFailure(error?.code)); } + }); + socket.on('error', () => socket.destroy()); + socket.on('close', () => { + clearTimeout(timer); + this.connections.delete(socket); + }); + } + + async close() { + for (const socket of this.connections) socket.destroy(); + this.connections.clear(); + if (this.server?.listening) await new Promise(resolve => this.server.close(resolve)); + this.server = null; + try { + if (sameIdentity(this.fileIdentity, socketIdentity(this.socketPath))) fs.unlinkSync(this.socketPath); + } catch {} + this.fileIdentity = null; + this.rootIdentity = null; + } +} + +function runtimeAgentControlInvoke(socketPathValue, runtimeKey, action, input, options = {}) { + const socketPath = validSocketPath(socketPathValue); + const timeoutMs = options.timeoutMs || CONTROL_TIMEOUT_MS; + if (!Number.isInteger(timeoutMs) || timeoutMs < 100 || timeoutMs > 120_000) return Promise.reject(coded()); + let before; + try { before = socketIdentity(socketPath); } catch { return Promise.reject(coded()); } + return new Promise((resolve, reject) => { + const socket = net.createConnection(socketPath); + let chunks = []; + let size = 0; + let settled = false; + const done = (error, value) => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + if (error) reject(error); else resolve(value); + }; + const timer = setTimeout(() => done(coded()), timeoutMs); + socket.on('connect', () => { + try { + const request = validateGatewayRequest({ + protocolVersion: RUNTIME_GATEWAY_PROTOCOL_VERSION, runtimeKey, action, input, + }); + if (!CONTROL_ACTIONS.has(request.action)) throw coded('invalid_request'); + socket.write(`${JSON.stringify(request)}\n`); + } catch (error) { done(error); } + }); + socket.on('data', chunk => { + size += chunk.length; + if (size > MAX_CONTROL_FRAME_BYTES) done(coded()); else chunks.push(chunk); + }); + socket.on('error', () => done(coded())); + socket.on('close', () => { if (!settled) done(coded()); }); + socket.on('end', () => { + try { + const raw = Buffer.concat(chunks).toString('utf8'); + if (!raw.endsWith('\n') || raw.includes('\r') || raw.slice(0, -1).includes('\n') + || !sameIdentity(before, socketIdentity(socketPath))) throw coded(); + done(null, validateGatewayResponse(parseStrictJson(raw.slice(0, -1)))); + } catch (error) { done(coded(error?.code)); } + }); + }); +} + +module.exports = { + CONTROL_SOCKET_BASENAME, + CONTROL_ACTIONS: Object.freeze([...CONTROL_ACTIONS]), + CoreRuntimeAgentControlServer, + runtimeAgentControlInvoke, +}; diff --git a/core/core/agents/src/execution-store.js b/core/core/agents/src/execution-store.js new file mode 100644 index 0000000..9207497 --- /dev/null +++ b/core/core/agents/src/execution-store.js @@ -0,0 +1,79 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { openDatabase, transaction } = require('../../../shared/published/database'); +const { validateDspId } = require('../../../shared/paths/platform-paths'); +function fail(code) { throw Object.assign(new Error(code), { code }); } + +class ExecutionStore { + constructor(file) { + this.db = openDatabase(file, { write: true }); + this.db.exec(`CREATE TABLE IF NOT EXISTS dsp_execution( + runtime_key TEXT PRIMARY KEY,organization_id TEXT NOT NULL,mode TEXT NOT NULL DEFAULT 'on_demand', + state TEXT NOT NULL,operation_id TEXT,next_wake_at INTEGER,check_at INTEGER,last_activity INTEGER NOT NULL, + snapshot_ready INTEGER NOT NULL DEFAULT 0,failure_code TEXT,updated_at INTEGER NOT NULL); + CREATE INDEX IF NOT EXISTS execution_due ON dsp_execution(mode,check_at); + CREATE TABLE IF NOT EXISTS dsp_work( + id TEXT PRIMARY KEY,runtime_key TEXT NOT NULL REFERENCES dsp_execution(runtime_key), + action TEXT NOT NULL,input_json TEXT NOT NULL,idempotency_key TEXT NOT NULL, + status TEXT NOT NULL,attempts INTEGER NOT NULL DEFAULT 0,available_at INTEGER NOT NULL, + result_json TEXT,created_at INTEGER NOT NULL,updated_at INTEGER NOT NULL, + UNIQUE(runtime_key,idempotency_key)); + CREATE INDEX IF NOT EXISTS work_queue ON dsp_work(status,available_at,runtime_key); + PRAGMA user_version=1;`); + // Delivery may have succeeded before Core recorded its acknowledgement. + // Replay uses the same downstream idempotency key. + this.db.prepare("UPDATE dsp_work SET status='queued' WHERE status='dispatching'").run(); + } + close() { this.db.close(); } + get(id) { validateDspId(id); return this.db.prepare('SELECT * FROM dsp_execution WHERE runtime_key=?').get(id) || null; } + enroll(id, organizationId, now) { + validateDspId(id); + this.db.prepare(`INSERT INTO dsp_execution(runtime_key,organization_id,state,check_at,last_activity,updated_at) + VALUES(?,?,'adopting',?,?,?) ON CONFLICT(runtime_key) DO NOTHING`).run(id, organizationId, now, now, now); + const row = this.get(id); + if (row.organization_id !== organizationId) fail('runtime_identity_mismatch'); + return row.mode === 'always_on' ? this.update(id, { mode: 'on_demand', state: 'adopting', operation_id: null, + check_at: now, snapshot_ready: 0, failure_code: null }, now) : row; + } + update(id, values, now) { + const allowed = ['mode', 'state', 'operation_id', 'next_wake_at', 'check_at', 'last_activity', 'snapshot_ready', 'failure_code']; + const fields = Object.keys(values); + if (!fields.length || fields.some(field => !allowed.includes(field))) fail('execution_state_invalid'); + this.db.prepare(`UPDATE dsp_execution SET ${fields.map(field => `${field}=?`).join(',')},updated_at=? WHERE runtime_key=?`) + .run(...fields.map(field => values[field]), now, validateDspId(id)); + return this.get(id); + } + enqueue(id, action, input, now) { + if (action !== 'sync.run_now') fail('execution_action_invalid'); + const normalized = require('../../../shared/gateway/protocol').validateActionInput(action, input); + const idempotencyKey = normalized.options.idempotencyKey || crypto.randomUUID(); + const payload = { ...normalized, options: { ...normalized.options, idempotencyKey } }; + return transaction(this.db, () => { + const prior = this.db.prepare('SELECT * FROM dsp_work WHERE runtime_key=? AND idempotency_key=?').get(id, idempotencyKey); + if (prior) { + if (prior.action !== action || prior.input_json !== JSON.stringify(payload)) fail('idempotency_conflict'); + return prior; + } + if (this.db.prepare("SELECT count(*) n FROM dsp_work WHERE status IN ('queued','dispatching')").get().n >= 4096 + || this.db.prepare("SELECT count(*) n FROM dsp_work WHERE runtime_key=? AND status IN ('queued','dispatching')").get(id).n >= 32) fail('execution_queue_full'); + const jobId = `work_${crypto.randomBytes(16).toString('hex')}`; + this.db.prepare("INSERT INTO dsp_work(id,runtime_key,action,input_json,idempotency_key,status,available_at,created_at,updated_at) VALUES(?,?,?,?,?,'queued',?,?,?)") + .run(jobId, id, action, JSON.stringify(payload), idempotencyKey, now, now, now); + this.update(id, { check_at: now }, now); + return this.db.prepare('SELECT * FROM dsp_work WHERE id=?').get(jobId); + }); + } + job(id, now) { return this.db.prepare("SELECT * FROM dsp_work WHERE runtime_key=? AND status='queued' AND available_at<=? ORDER BY created_at,id LIMIT 1").get(id, now); } + pending(id) { return this.db.prepare("SELECT count(*) n FROM dsp_work WHERE runtime_key=? AND status IN ('queued','dispatching')").get(id).n; } + nextJob(id) { return this.db.prepare("SELECT min(available_at) due FROM dsp_work WHERE runtime_key=? AND status='queued'").get(id).due; } + latestJob(id) { return this.db.prepare('SELECT id,status,result_json FROM dsp_work WHERE runtime_key=? ORDER BY created_at DESC,id DESC LIMIT 1').get(id); } + claim(job, now) { this.db.prepare("UPDATE dsp_work SET status='dispatching',attempts=attempts+1,updated_at=? WHERE id=? AND status='queued'").run(now, job.id); } + finish(job, result, now, retry = false) { + this.db.prepare('UPDATE dsp_work SET status=?,result_json=?,available_at=?,updated_at=? WHERE id=?') + .run(retry ? 'queued' : result.ok ? 'delivered' : 'failed', JSON.stringify(result), now + Math.min(60000, 1000 * 2 ** Math.min(job.attempts, 6)), now, job.id); + } + due(now, limit = 20) { return this.db.prepare("SELECT * FROM dsp_execution WHERE mode='on_demand' AND check_at<=? ORDER BY check_at,runtime_key LIMIT ?").all(now, limit); } + occupied() { return this.db.prepare("SELECT count(*) n FROM dsp_execution WHERE mode='on_demand' AND state IN ('starting','running','draining')").get().n; } +} +module.exports = { ExecutionStore }; diff --git a/core/core/agents/src/hub.js b/core/core/agents/src/hub.js new file mode 100644 index 0000000..0f33d81 --- /dev/null +++ b/core/core/agents/src/hub.js @@ -0,0 +1,389 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { CollectionCapacity } = require('./collection-capacity'); +const fs = require('node:fs'); +const net = require('node:net'); +const path = require('node:path'); +const { + RUNTIME_GATEWAY_PROTOCOL_VERSION, + validateGatewayRequest, + validateGatewayResponse, +} = require('../../../shared/gateway/protocol'); +const { + privateDirectory, + socketIdentity, + sameIdentity, + probeUnixSocket, + MAX_UNIX_SOCKET_PATH_BYTES, +} = require('../../../shared/transport/unix-socket'); +const { MAX_AGENT_FRAME_BYTES, encodeFrame, attachFrameReader } = require('../../../shared/agent/framing'); +const { + RUNTIME_AGENT_PROTOCOL_VERSION, + authorityDigest, + validateRegistrationFrame, + registeredFrame, + rejectedFrame, + heartbeatFrame, + validateHeartbeatAckFrame, + requestFrame, + validateResponseFrame, +} = require('../../../shared/agent/protocol'); + +const MAX_AGENT_CONNECTIONS = 256; +const REGISTRATION_TIMEOUT_MS = 5_000; +const DEFAULT_REQUEST_TIMEOUT_MS = 15_000; +const DEFAULT_MAX_PENDING_PER_AGENT = 32; +const DEFAULT_HEARTBEAT_INTERVAL_MS = 5_000; +const DEFAULT_HEARTBEAT_TIMEOUT_MS = 15_000; + +function coded(code = 'runtime_agent_unavailable') { + return Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function checkedAuthority(value) { + if (!plain(value) || Object.keys(value).sort().join(',') !== 'digest,generation' + || !Number.isSafeInteger(value.generation) || value.generation < 1) throw coded(); + return Object.freeze({ digest: authorityDigest(value.digest), generation: value.generation }); +} + +function staticAuthorityCatalog(authoritiesValue) { + if (!plain(authoritiesValue) || Object.keys(authoritiesValue).length < 1) throw coded(); + const authorities = new Map(); + const authorityDigests = new Set(); + for (const [runtimeKey, digest] of Object.entries(authoritiesValue)) { + validateGatewayRequest({ + protocolVersion: RUNTIME_GATEWAY_PROTOCOL_VERSION, + runtimeKey, + action: 'health', + input: {}, + }); + const selectedDigest = authorityDigest(digest); + if (authorityDigests.has(selectedDigest)) throw coded(); + authorityDigests.add(selectedDigest); + authorities.set(runtimeKey, Object.freeze({ digest: selectedDigest, generation: 1 })); + } + return Object.freeze({ + resolve: runtimeKey => authorities.get(runtimeKey) || null, + count: () => authorities.size, + }); +} + +function checkedCatalog(value) { + if (!value || typeof value.resolve !== 'function' || typeof value.count !== 'function') throw coded(); + return value; +} + +function validateOptions(options) { + if (!plain(options) || Object.keys(options).some(key => ![ + 'socketPath', 'authorities', 'authorityCatalog', 'requestTimeoutMs', 'maxPendingPerAgent', + 'heartbeatIntervalMs', 'heartbeatTimeoutMs', 'collectionCapacity', 'maxAgentConnections', + ].includes(key)) || typeof options.socketPath !== 'string' + || !path.isAbsolute(options.socketPath) || path.resolve(options.socketPath) !== options.socketPath + || path.basename(options.socketPath) !== 'runtime-agent-hub.sock' + || Buffer.byteLength(options.socketPath, 'utf8') > MAX_UNIX_SOCKET_PATH_BYTES + || Object.hasOwn(options, 'authorities') === Object.hasOwn(options, 'authorityCatalog')) throw coded(); + const authorityCatalog = Object.hasOwn(options, 'authorities') + ? staticAuthorityCatalog(options.authorities) : checkedCatalog(options.authorityCatalog); + const requestTimeoutMs = options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS; + const maxPendingPerAgent = options.maxPendingPerAgent ?? DEFAULT_MAX_PENDING_PER_AGENT; + const maxAgentConnections = options.maxAgentConnections ?? MAX_AGENT_CONNECTIONS; + const heartbeatIntervalMs = options.heartbeatIntervalMs ?? DEFAULT_HEARTBEAT_INTERVAL_MS; + const heartbeatTimeoutMs = options.heartbeatTimeoutMs ?? DEFAULT_HEARTBEAT_TIMEOUT_MS; + if (!Number.isSafeInteger(maxAgentConnections) || maxAgentConnections < 1 || maxAgentConnections > 4096 + || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 60_000 + || !Number.isSafeInteger(maxPendingPerAgent) || maxPendingPerAgent < 1 || maxPendingPerAgent > 128 + || !Number.isSafeInteger(heartbeatIntervalMs) || heartbeatIntervalMs < 25 || heartbeatIntervalMs > 60_000 + || !Number.isSafeInteger(heartbeatTimeoutMs) || heartbeatTimeoutMs < heartbeatIntervalMs * 2 + || heartbeatTimeoutMs > 120_000) throw coded(); + return Object.freeze({ + socketPath: options.socketPath, + collectionCapacity: options.collectionCapacity || {}, + authorityCatalog, + requestTimeoutMs, + maxPendingPerAgent, + maxAgentConnections, + heartbeatIntervalMs, + heartbeatTimeoutMs, + }); +} + +function digestToken(token) { + return crypto.createHash('sha256').update(token, 'utf8').digest(); +} + +function sameDigest(leftHex, rightHex) { + const left = Buffer.from(leftHex, 'hex'); + const right = Buffer.from(rightHex, 'hex'); + return left.length === right.length && crypto.timingSafeEqual(left, right); +} + +function authorized(expectedHex, token) { + const expected = Buffer.from(expectedHex, 'hex'); + const actual = digestToken(token); + return expected.length === actual.length && crypto.timingSafeEqual(expected, actual); +} + +class CoreRuntimeAgentHub { + constructor(options) { + const selected = validateOptions(options); + this.socketPath = selected.socketPath; + this.authorityCatalog = selected.authorityCatalog; + this.requestTimeoutMs = selected.requestTimeoutMs; + this.maxPendingPerAgent = selected.maxPendingPerAgent; + this.maxAgentConnections = selected.maxAgentConnections; + this.heartbeatIntervalMs = selected.heartbeatIntervalMs; + this.heartbeatTimeoutMs = selected.heartbeatTimeoutMs; + this.collectionCapacity = new CollectionCapacity(selected.collectionCapacity); + this.server = null; + this.rootIdentity = null; + this.socketFileIdentity = null; + this.connections = new Set(); + this.agents = new Map(); + } + + authority(runtimeKey) { + try { + const value = this.authorityCatalog.resolve(runtimeKey); + return value === null ? null : checkedAuthority(value); + } catch { return null; } + } + + authorityCurrent(state) { + const current = state.runtimeKey === null ? null : this.authority(state.runtimeKey); + return Boolean(current && current.generation === state.authorityGeneration + && sameDigest(current.digest, state.authorityDigest)); + } + + send(socket, frame) { + if (!socket.destroyed) socket.write(encodeFrame(frame)); + } + + rejectConnection(socket, code) { + if (!socket.destroyed) socket.end(encodeFrame(rejectedFrame(code))); + } + + beginHeartbeat(state) { + state.heartbeatTimer = setInterval(() => { + if (state.socket.destroyed || !this.authorityCurrent(state)) return state.socket.destroy(); + const now = Date.now(); + if (state.heartbeat !== null) { + if (now - state.heartbeat.sentAt >= this.heartbeatTimeoutMs) state.socket.destroy(); + return; + } + const nonce = crypto.randomBytes(16).toString('hex'); + state.heartbeat = { nonce, sentAt: now }; + try { this.send(state.socket, heartbeatFrame(nonce)); } catch { state.socket.destroy(); } + }, this.heartbeatIntervalMs); + state.heartbeatTimer.unref?.(); + } + + accept(socket) { + const state = { + socket, + runtimeKey: null, + authorityDigest: null, + authorityGeneration: null, + actions: null, + pending: new Map(), + heartbeat: null, + heartbeatTimer: null, + registrationTimer: setTimeout(() => socket.destroy(), REGISTRATION_TIMEOUT_MS), + }; + this.connections.add(state); + const failPending = code => { + for (const pending of state.pending.values()) { + clearTimeout(pending.timer); + pending.reject(coded(code)); + } + state.pending.clear(); + }; + const close = () => { + clearTimeout(state.registrationTimer); + clearInterval(state.heartbeatTimer); + this.connections.delete(state); + if (state.runtimeKey !== null && this.agents.get(state.runtimeKey) === state) { + this.agents.delete(state.runtimeKey); + this.collectionCapacity.disconnect(state.runtimeKey); + } + failPending('runtime_agent_unavailable'); + }; + socket.on('close', close); + socket.on('error', () => socket.destroy()); + attachFrameReader(socket, { + maxFrameBytes: MAX_AGENT_FRAME_BYTES, + onError: () => socket.destroy(), + onFrame: value => { + try { + if (state.runtimeKey === null) { + const registration = validateRegistrationFrame(value); + const expected = this.authority(registration.runtimeKey); + if (!expected || !authorized(expected.digest, registration.registrationToken)) { + return this.rejectConnection(socket, 'runtime_agent_unauthorized'); + } + const existing = this.agents.get(registration.runtimeKey); + if (existing && this.authorityCurrent(existing)) { + return this.rejectConnection(socket, 'runtime_agent_conflict'); + } + if (existing) existing.socket.destroy(); + state.runtimeKey = registration.runtimeKey; + state.authorityDigest = expected.digest; + state.authorityGeneration = expected.generation; + state.actions = new Set(registration.actions); + clearTimeout(state.registrationTimer); + this.agents.set(state.runtimeKey, state); + this.send(socket, registeredFrame()); + this.beginHeartbeat(state); + return; + } + if (!this.authorityCurrent(state)) return socket.destroy(); + if (value?.type === 'heartbeat_ack') { + const heartbeat = validateHeartbeatAckFrame(value); + if (state.heartbeat === null || state.heartbeat.nonce !== heartbeat.nonce) return socket.destroy(); + state.heartbeat = null; + return; + } + if (value?.type === 'capacity_request') { + this.send(socket, this.collectionCapacity.request(state.runtimeKey, value)); + return; + } + const frame = validateResponseFrame(value); + const pending = state.pending.get(frame.requestId); + if (!pending) return socket.destroy(); + state.pending.delete(frame.requestId); + clearTimeout(pending.timer); + try { pending.resolve(validateGatewayResponse(frame.response)); } + catch (error) { pending.reject(coded(error?.code)); } + } catch { + socket.destroy(); + } + }, + }); + } + + async start() { + const root = path.dirname(this.socketPath); + this.rootIdentity = privateDirectory(root); + if (fs.existsSync(this.socketPath)) { + const before = socketIdentity(this.socketPath); + if (await probeUnixSocket(this.socketPath)) throw coded(); + const after = socketIdentity(this.socketPath); + if (!sameIdentity(before, after) || !sameIdentity(this.rootIdentity, privateDirectory(root))) throw coded(); + fs.unlinkSync(this.socketPath); + } + this.server = net.createServer(socket => this.accept(socket)); + this.server.maxConnections = this.maxAgentConnections; + try { + await new Promise((resolve, reject) => { + this.server.once('error', reject); + this.server.listen(this.socketPath, () => { + this.server.off('error', reject); + resolve(); + }); + }); + if (!sameIdentity(this.rootIdentity, privateDirectory(root))) throw coded(); + this.socketFileIdentity = socketIdentity(this.socketPath, { requireMode: false }); + fs.chmodSync(this.socketPath, 0o600); + if (!sameIdentity(this.socketFileIdentity, socketIdentity(this.socketPath))) throw coded(); + return this; + } catch (error) { + await this.close(); + throw coded(error?.code); + } + } + + connected(runtimeKey) { + const state = this.agents.get(runtimeKey); + if (!state || !this.authorityCurrent(state)) { + state?.socket.destroy(); + return false; + } + return true; + } + + status() { + let configured; + try { configured = this.authorityCatalog.count(); } catch { throw coded(); } + if (!Number.isSafeInteger(configured) || configured < 0) throw coded(); + const connected = [...this.agents.values()].filter(state => { + const current = !state.socket.destroyed && this.authorityCurrent(state); + if (!current) state.socket.destroy(); + return current; + }).length; + return Object.freeze({ + protocolVersion: RUNTIME_AGENT_PROTOCOL_VERSION, + configured, + connected, + }); + } + + invoke(runtimeKey, action, input) { + const request = { + protocolVersion: RUNTIME_GATEWAY_PROTOCOL_VERSION, + runtimeKey, + action, + input, + }; + try { + validateGatewayRequest(request); + } catch (error) { + return Promise.reject(coded(error?.code === 'runtime_protocol_mismatch' + ? 'runtime_agent_protocol_mismatch' : 'invalid_runtime_agent_frame')); + } + const state = this.agents.get(runtimeKey); + if (!state || state.socket.destroyed || !this.authorityCurrent(state) + || !state.actions?.has(action) || state.pending.size >= this.maxPendingPerAgent) { + if (state && !this.authorityCurrent(state)) state.socket.destroy(); + return Promise.reject(coded('runtime_agent_unavailable')); + } + const id = crypto.randomBytes(16).toString('hex'); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + state.pending.delete(id); + reject(coded('runtime_agent_unavailable')); + }, this.requestTimeoutMs); + state.pending.set(id, { resolve, reject, timer }); + try { this.send(state.socket, requestFrame(id, request)); } + catch { + clearTimeout(timer); + state.pending.delete(id); + reject(coded('runtime_agent_unavailable')); + } + }); + } + + async close() { + for (const state of this.connections) { + clearInterval(state.heartbeatTimer); + state.socket.destroy(); + } + this.connections.clear(); + this.agents.clear(); + if (this.server) { + if (this.server.listening) await new Promise(resolve => this.server.close(() => resolve())); + this.server = null; + } + try { + const current = socketIdentity(this.socketPath); + if (sameIdentity(current, this.socketFileIdentity)) fs.unlinkSync(this.socketPath); + } catch {} + this.rootIdentity = null; + this.socketFileIdentity = null; + } +} + +module.exports = { + CoreRuntimeAgentHub, + MAX_AGENT_FRAME_BYTES, + MAX_AGENT_CONNECTIONS, + REGISTRATION_TIMEOUT_MS, + DEFAULT_REQUEST_TIMEOUT_MS, + DEFAULT_MAX_PENDING_PER_AGENT, + DEFAULT_HEARTBEAT_INTERVAL_MS, + DEFAULT_HEARTBEAT_TIMEOUT_MS, +}; diff --git a/core/core/agents/src/index.js b/core/core/agents/src/index.js new file mode 100644 index 0000000..d920f5b --- /dev/null +++ b/core/core/agents/src/index.js @@ -0,0 +1,9 @@ +'use strict'; + +const protocol = require('../../../shared/agent/protocol'); +const hub = require('./hub'); +const client = require('./client'); +const credentialFile = require('../../../shared/agent/credential-file'); +const control = require('./control'); + +module.exports = Object.freeze({ ...protocol, ...hub, ...client, ...credentialFile, ...control }); diff --git a/core/core/agents/tests/collection-capacity.test.js b/core/core/agents/tests/collection-capacity.test.js new file mode 100644 index 0000000..ab82d43 --- /dev/null +++ b/core/core/agents/tests/collection-capacity.test.js @@ -0,0 +1,113 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { CollectionCapacity, capacityDefaults } = require('../src/collection-capacity'); +const { LEASE_MS } = require('../../../shared/agent/capacity'); +const job = digit => digit.repeat(32); +const request = (operation = 'acquire', jobId = job('a'), workers = 6) => ({ type: 'capacity_request', requestId: job('f'), operation, jobId, workers }); +test('FIFO grants obey the total worker ceiling and one active job per DSP', () => { + const capacity = new CollectionCapacity({ workers: 1, recoveryMs: 0 }); + assert.equal(capacity.request('one', request()).workers, 1); + assert.equal(capacity.request('two', request()).status, 'waiting'); + assert.equal(capacity.request('three', request()).status, 'waiting'); + assert.equal(capacity.request('one', request('acquire', job('b'))).status, 'waiting'); + capacity.request('one', request('release')); + assert.equal(capacity.request('one', request('acquire', job('b'))).status, 'waiting'); + assert.equal(capacity.request('three', request()).status, 'waiting'); + assert.equal(capacity.request('two', request()).workers, 1); + assert.equal(capacity.status().activeWorkers, 1); + // A different DSP cannot renew or release another DSP's grant. + assert.equal(capacity.request('three', request('renew')).status, 'lost'); + capacity.request('three', request('release')); + assert.equal(capacity.status().activeWorkers, 1); +}); +test('crashed clients retain a grace period; expired grants recover and stale renewals fail', () => { + let now = 0; + const capacity = new CollectionCapacity({ workers: 1, clock: () => now, recoveryMs: 0 }); + capacity.request('one', request()); + capacity.disconnect('one'); + assert.equal(capacity.request('two', request()).status, 'waiting'); + now += LEASE_MS - 1; + assert.equal(capacity.request('two', request()).status, 'waiting'); + now += 1; + assert.equal(capacity.request('two', request()).status, 'granted'); + assert.equal(capacity.request('one', request('renew')).status, 'lost'); +}); +test('Core restart quarantine prevents overlapping surviving grants', () => { + let now = 0; + const capacity = new CollectionCapacity({ workers: 1, clock: () => now }); + assert.equal(capacity.request('one', request()).status, 'waiting'); + now = LEASE_MS; + assert.equal(capacity.request('one', request()).status, 'granted'); +}); +test('renewal is idempotent; expired queue heads do not block live DSPs', () => { + let now = 0; + const capacity = new CollectionCapacity({ workers: 1, clock: () => now, recoveryMs: 0 }); + capacity.request('one', request()); + capacity.request('abandoned', request()); + now = LEASE_MS / 2; + assert.equal(capacity.request('one', request('renew')).workers, 1); + capacity.request('two', request()); + now = LEASE_MS; + capacity.request('one', request('release')); + assert.equal(capacity.request('two', request()).status, 'granted'); +}); +test('configuration and frames fail closed', () => { + for (const workers of [0, 65, NaN, 1.5]) assert.throws(() => new CollectionCapacity({ workers })); + const capacity = new CollectionCapacity({ recoveryMs: 0 }); + assert.throws(() => capacity.request('one', { ...request(), runtimeKey: 'another' })); + assert.throws(() => capacity.request('one', { ...request(), workers: 7 })); + assert.equal(capacityDefaults({ cpus: 8, memoryBytes: 8 * 1024 ** 3 }), 4); + assert.equal(capacityDefaults({ cpus: 2, memoryBytes: 2 * 1024 ** 3 }), 1); +}); + +test('two authenticated agents share the budget through private local sockets', async () => { + const fs = require('node:fs'); + const os = require('node:os'); + const path = require('node:path'); + const crypto = require('node:crypto'); + const { CoreRuntimeAgentHub } = require('../src/hub'); + const { DspRuntimeAgent } = require('dispatch-dsp/runtime/agent/src/agent.js'); + const { RuntimeAgentStatusServer } = require('dispatch-dsp/runtime/agent/src/status.js'); + const { queryCapacity } = require('dispatch-dsp/runtime/collection-manager/src/capacity-runner.js'); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dcap-')); + const resources = []; + const token = crypto.randomBytes(32).toString('base64url'); + const secondToken = crypto.randomBytes(32).toString('base64url'); + const hash = value => crypto.createHash('sha256').update(value).digest('hex'); + const client = { workforce: { day() {} }, sync: { status() {}, runNow() {} }, system: { status() {} } }; + const hub = new CoreRuntimeAgentHub({ socketPath: path.join(root, 'runtime-agent-hub.sock'), + authorities: { 'dsp-one': hash(token), 'dsp-two': hash(secondToken) }, collectionCapacity: { workers: 1, recoveryMs: 0 } }); + try { + await hub.start(); + for (const [runtimeKey, registrationToken] of [['dsp-one', token], ['dsp-two', secondToken]]) { + const directory = path.join(root, runtimeKey); fs.mkdirSync(directory, { mode: 0o700 }); + const agent = new DspRuntimeAgent({ socketPath: hub.socketPath, runtimeKey, registrationToken, client }); + resources.push(agent); await agent.start(); + const status = new RuntimeAgentStatusServer({ socketPath: path.join(directory, 'runtime-agent-status.sock'), agent }); + resources.push(status); await status.start(); + } + const call = (dsp, operation) => queryCapacity(path.join(root, dsp, 'runtime-agent-status.sock'), { operation, jobId: job('a'), workers: 6 }); + assert.equal((await call('dsp-one', 'acquire')).workers, 1); + assert.equal((await call('dsp-two', 'acquire')).status, 'waiting'); + assert.equal((await call('dsp-two', 'renew')).status, 'lost'); + await call('dsp-one', 'release'); + assert.equal((await call('dsp-two', 'acquire')).workers, 1); + await call('dsp-two', 'release'); + } finally { + for (const resource of resources.reverse()) await resource.close(); + await hub.close(); fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('one DSP cannot monopolize a multi-worker budget and concurrent grants stay bounded', () => { + const capacity = new CollectionCapacity({ workers: 3, recoveryMs: 0 }); + assert.equal(capacity.request('one', request()).workers, 2); + assert.equal(capacity.request('two', request()).workers, 1); + assert.equal(capacity.status().activeDsps, 2); + assert.equal(capacity.status().activeWorkers, 3); + assert.equal(capacity.request('three', request()).status, 'waiting'); + capacity.request('one', request('release')); + assert.equal(capacity.request('one', request('acquire', job('b'))).status, 'waiting'); + assert.equal(capacity.request('three', request()).workers, 2); +}); diff --git a/core/core/agents/tests/control.test.js b/core/core/agents/tests/control.test.js new file mode 100644 index 0000000..c1a6b87 --- /dev/null +++ b/core/core/agents/tests/control.test.js @@ -0,0 +1,46 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { success } = require('../../../shared/contracts/src'); +const { CoreRuntimeAgentControlServer, runtimeAgentControlInvoke } = require('../src/control'); + +test('owner-private Runtime Agent control socket exposes only closed lifecycle requests', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-agent-control-')); + fs.chmodSync(root, 0o700); + const socketPath = path.join(root, 'runtime-agent-control.sock'); + const calls = []; + const server = new CoreRuntimeAgentControlServer({ + socketPath, + hub: { invoke: async (runtimeKey, action, input) => { + calls.push({ runtimeKey, action, input }); + return success('found', { desiredState: 'stopped' }); + } }, + }); + try { + await server.start(); + const result = await runtimeAgentControlInvoke(socketPath, 'runtime_control', 'sync.status', { id: 'paycom_hourly' }); + assert.equal(result.ok, true); + assert.deepEqual(calls, [{ runtimeKey: 'runtime_control', action: 'sync.status', input: { id: 'paycom_hourly' } }]); + await assert.rejects( + runtimeAgentControlInvoke(socketPath, 'runtime_control', 'sync.run_now', { id: 'paycom_hourly', options: {} }), + error => error.code === 'invalid_request', + ); + assert.equal(calls.length, 1); + const workforce = await runtimeAgentControlInvoke(socketPath, 'runtime_control', 'workforce.day', + { query: { date: '2026-09-05', limit: 1, offset: 0 } }); + assert.equal(workforce.ok, true); + assert.equal(calls[1].action, 'workforce.day'); + assert.equal(calls[1].input.query.search, null); + + const info = fs.lstatSync(socketPath); + assert.equal(info.uid, process.geteuid()); + assert.equal(info.mode & 0o7777, 0o600); + } finally { + await server.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/core/core/agents/tests/execution.test.js b/core/core/agents/tests/execution.test.js new file mode 100644 index 0000000..f49d141 --- /dev/null +++ b/core/core/agents/tests/execution.test.js @@ -0,0 +1,135 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const { DirectoryExecution } = require('../../../host/controller/execution'); +const { saveStatus } = require('../../../shared/published/status'); +const { success, failure } = require('../../../shared/contracts/src/result'); + +function fixture(t) { + require('../../../shared/plugin-sdk/catalog').configureCatalog(() => [require('../../../tests/fixtures/paycom-plugin.json')]); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'core-execution-')); + const paths = { local: path.join(root, 'local') }; + for (const name of ['local', 'local/state', 'local/config']) fs.mkdirSync(path.join(root, name), { mode: 0o700 }); + const db = new DatabaseSync(':memory:'); + db.exec(`CREATE TABLE installations(runtime_key TEXT PRIMARY KEY,organization_id TEXT,status TEXT,revision INTEGER,backend TEXT); + CREATE TABLE organizations(id TEXT PRIMARY KEY,status TEXT); + CREATE TABLE directory_lifecycle_requests(organization_id TEXT,status TEXT); + CREATE TABLE dsp_removals(organization_id TEXT); + CREATE TABLE dsp_plugins(organization_id TEXT,plugin_id TEXT,desired_state TEXT,applied_state TEXT,revision INTEGER,applied_revision INTEGER);`); + let now = 100000; + const active = new Set(), calls = [], dsps = new Map(), busy = new Set(), paused = new Set(), schedule = new Map(), delivered = new Map(); + let execution; + const manager = { journal: { record: id => dsps.get(id) }, checkedDsp: record => record, + async apply(action, operation, id) { calls.push([action, id]); if (action === 'start') active.add(id); else active.delete(id); } }; + const hub = { connected: id => active.has(id), async invoke(id, action, input) { + assert.equal(active.has(id), true); + if (action === 'runtime.execution') { + if (input.command === 'tick' && schedule.get(id) <= now) schedule.delete(id); + const status = { version: 1, busy: busy.has(id), drained: input.command === 'drain' && !busy.has(id), nextWakeAt: schedule.get(id) ?? null, observedAt: now }; + const data = { id: 'paycom-main-workforce', desiredState: paused.has(id) ? 'stopped' : 'running', activity: 'idle', queuedRunCount: 0, nextDueAt: null }; + saveStatus(path.join(dsps.get(id).root, 'data/published'), { execution: status, 'sync:paycom-main-workforce': success('found', data), + system: success('ready', { components: { auth: {}, collections: { data: { manager: {} } } } }), + connections: success('found', { items: [] }) }, now); + return success('found', status); + } + if (action === 'sync.run_now') { + const key = input.options.idempotencyKey; + if (!delivered.has(key)) delivered.set(key, id); + calls.push(['sync', id, key]); + return success('queued', {}); + } + calls.push([action, id]); return success('accepted', {}); + } }; + const options = { publishedReader: () => ({ workforce: { employees: () => failure('not_initialized') } }), paths, accessStore: { db }, manager, hub, configuration: { version: 1, enabled: true, idleMs: 1000, pollMs: 100, maxActive: 1 }, clock: () => now }; + function open() { execution = new DirectoryExecution(options); execution.wake = () => {}; } + open(); + t.after(async () => { await execution.close(); db.close(); fs.rmSync(root, { recursive: true, force: true }); }); + return { db, calls, active, busy, paused, schedule, delivered, + get execution() { return execution; }, advance: ms => { now += ms; }, + async restart() { await execution.close(); open(); }, + add(n, { running = true } = {}) { + const id = 'dsp_' + n.toString(16).padStart(32, '0'), org = 'org_' + n.toString(16).padStart(32, '0'); + const folder = path.join(root, id); fs.mkdirSync(folder, { mode: 0o700 }); fs.mkdirSync(path.join(folder, 'data'), { mode: 0o700 }); + dsps.set(id, { id, root: folder }); if (running) active.add(id); + db.prepare("INSERT INTO installations VALUES(?,?,'ready',1,'directory_service_v1')").run(id, org); + db.prepare("INSERT INTO organizations VALUES(?,'active')").run(org); + db.prepare("INSERT INTO dsp_plugins VALUES(?,'paycom','enabled','enabled',1,1)").run(org); + return id; + } }; +} + +test('idle workers stop, reads never wake them, and concurrent manual requests share a durable job', async t => { + const c = fixture(t), id = c.add(1); + await c.execution.runPending(); c.advance(1200); await c.execution.runPending(); + assert.equal(c.active.has(id), false); + assert.equal(c.execution.store.get(id).state, 'sleeping'); + assert.match(c.execution.store.get(id).operation_id, /^sleep_[a-f0-9]{32}$/); + const before = c.calls.length; + for (let i = 0; i < 20; i++) assert.equal((await c.execution.invoke(id, 'sync.status', { id: 'paycom-main-workforce' })).ok, true); + assert.equal(c.calls.length, before); + const input = { id: 'paycom-main-workforce', options: { idempotencyKey: 'same-click' } }; + const results = await Promise.all(Array.from({ length: 5 }, () => c.execution.invoke(id, 'sync.run_now', input))); + assert.equal(new Set(results.map(result => result.data.request.id)).size, 1); + await c.restart(); await c.execution.runPending(); + assert.equal(c.delivered.size, 1); + assert.equal(c.calls.filter(call => call[0] === 'start').length, 1); +}); + +test('manual sync wakes a sleeping DSP even when its automatic schedule is paused', async t => { + const c = fixture(t), id = c.add(1); c.paused.add(id); + await c.execution.runPending(); c.advance(1200); await c.execution.runPending(); + assert.equal(c.active.has(id), false); + assert.equal((await c.execution.invoke(id, 'sync.status', { id: 'paycom-main-workforce' })).data.desiredState, 'stopped'); + const result = await c.execution.invoke(id, 'sync.run_now', { id: 'paycom-main-workforce', options: { idempotencyKey: 'paused-manual' } }); + assert.equal(result.ok, true); + await c.execution.runPending(); assert.equal(c.delivered.get('paused-manual'), id); +}); + +test('initial publication keeps existing worker reads available without waking a sleeping DSP', async t => { + const c = fixture(t), id = c.add(1); + await c.execution.runPending(); + assert.equal((await c.execution.invoke(id, 'workforce.employees', { query: {} })).ok, true); + assert.equal(c.calls.filter(call => call[0] === 'workforce.employees').length, 1); + c.advance(1200); await c.execution.runPending(); + const before = c.calls.length; + assert.equal((await c.execution.invoke(id, 'workforce.employees', { query: {} })).status, 'not_initialized'); + assert.equal(c.calls.length, before); +}); + +test('scheduled work survives Core restart while active authentication prevents sleep', async t => { + const c = fixture(t), id = c.add(1); c.schedule.set(id, 110000); + await c.execution.runPending(); c.busy.add(id); c.advance(1200); await c.execution.runPending(); + assert.equal(c.active.has(id), true); + c.busy.delete(id); c.advance(1200); await c.execution.runPending(); + assert.equal(c.active.has(id), false); + await c.restart(); c.advance(10000); await c.execution.runPending(); + assert.equal(c.active.has(id), true); + assert.equal(c.schedule.has(id), false); +}); + +test('suspension and plugin revocation fence queued work; invalid cross-DSP identity fails', async t => { + const c = fixture(t), id = c.add(1); + await c.execution.runPending(); c.advance(1200); await c.execution.runPending(); + await c.execution.invoke(id, 'sync.run_now', { id: 'paycom-main-workforce', options: { idempotencyKey: 'suspended-job' } }); + c.db.prepare("UPDATE installations SET status='suspended'").run(); + await c.execution.runPending(); assert.equal(c.active.has(id), false); assert.equal(c.delivered.size, 0); + assert.equal((await c.execution.invoke('dsp_' + 'f'.repeat(32), 'sync.status', { id: 'paycom-main-workforce' })).ok, false); + c.db.prepare("UPDATE installations SET status='ready'").run(); + c.db.prepare("UPDATE dsp_plugins SET desired_state='disabled'").run(); c.advance(61000); + await c.execution.runPending(); assert.equal(c.delivered.size, 0); + assert.equal(c.execution.store.latestJob(id).status, 'failed'); + assert.equal((await c.execution.invoke(id, 'workforce.employees', { query: {} })).status, 'plugin_disabled'); +}); + +test('worker concurrency is bounded and a waiting DSP proceeds once a slot is released', async t => { + const c = fixture(t), first = c.add(1, { running: false }), second = c.add(2, { running: false }); + await c.execution.runPending(); assert.equal(c.active.size, 1); + c.advance(1200); await c.execution.runPending(); + assert.ok(c.active.size <= 1); + c.advance(150); await c.execution.runPending(); + assert.equal(c.active.size, 1); assert.equal(c.active.has(second), true); assert.equal(c.active.has(first), false); +}); diff --git a/core/core/agents/tests/runtime-agent.test.js b/core/core/agents/tests/runtime-agent.test.js new file mode 100644 index 0000000..b0ffd4c --- /dev/null +++ b/core/core/agents/tests/runtime-agent.test.js @@ -0,0 +1,312 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { spawn } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { success } = require('../../../shared/contracts/src'); +const { + RUNTIME_GATEWAY_PROTOCOL_VERSION, + RUNTIME_GATEWAY_ACTIONS, + gatewaySuccess, +} = require('../../../shared/gateway/protocol'); +const { parseStrictJson } = require('../../../shared/gateway/strict-json'); +const { MAX_AGENT_FRAME_BYTES, encodeFrame } = require('../../../shared/agent/framing'); +const { RUNTIME_AGENT_PROTOCOL_VERSION, validateRegistrationFrame, validateRequestFrame, heartbeatFrame, validateHeartbeatFrame, heartbeatAckFrame, validateHeartbeatAckFrame, responseFrame, CoreRuntimeAgentHub, createRuntimeAgentDispatchClient } = require('../src'); +const { DspRuntimeAgent, RuntimeAgentStatusServer, queryRuntimeAgentStatus } = require('dispatch-dsp/runtime/agent/src/index.js'); + +function isCode(code) { return error => error?.code === code; } +function token() { return crypto.randomBytes(32).toString('base64url'); } +function digest(value) { return crypto.createHash('sha256').update(value).digest('hex'); } +function delay(milliseconds) { return new Promise(resolve => setTimeout(resolve, milliseconds)); } +async function waitFor(predicate, timeoutMs = 2_000) { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) throw new Error('fixture_timeout'); + await delay(10); + } +} + +async function leaveStaleSocket(socketPath) { + const script = [ + "const fs=require('node:fs'),net=require('node:net');", + "const server=net.createServer(()=>{});", + "server.listen(process.argv[1],()=>{fs.chmodSync(process.argv[1],0o600);process.send('ready');});", + ].join(''); + const child = spawn(process.execPath, ['-e', script, socketPath], { + stdio: ['ignore', 'ignore', 'ignore', 'ipc'], + }); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('stale_socket_timeout')), 3_000); + child.once('message', () => { clearTimeout(timer); resolve(); }); + child.once('exit', () => { clearTimeout(timer); reject(new Error('stale_socket_child_exit')); }); + }); + child.kill('SIGKILL'); + await new Promise(resolve => child.once('exit', resolve)); + assert.equal(fs.existsSync(socketPath), true); +} + +function fixtureClient(label, calls = []) { + return { + system: { status: async () => { + calls.push(['system.status']); + return success('ready', { + label, + components: { + auth: { healthy: true, ready: true }, + collections: { + healthy: true, + ready: true, + status: 'ready', + data: { manager: { running: true } }, + }, + }, + }); + } }, + workforce: { day: async query => { + calls.push(['workforce.day', query]); + return success('found', { label, businessDate: query.date, items: [] }); + } }, + sync: { + status: async id => { + calls.push(['sync.status', id]); + return success('found', { label, id, desiredState: 'stopped' }); + }, + runNow: async (id, options) => { + calls.push(['sync.run_now', id, options]); + return success('queued', { label, id, replayed: false }); + }, + }, + }; +} + +function gatewayRequest(runtimeKey, action = 'health', input = {}) { + return { protocolVersion: RUNTIME_GATEWAY_PROTOCOL_VERSION, runtimeKey, action, input }; +} + +function registration(runtimeKey, registrationToken) { + return { + protocolVersion: RUNTIME_AGENT_PROTOCOL_VERSION, + type: 'register', + runtimeKey, + registrationToken, + actions: RUNTIME_GATEWAY_ACTIONS, + }; +} + +test('runtime-agent protocol is closed, strict, versioned, and runtime-bound', () => { + const selectedToken = token(); + assert.equal(validateRegistrationFrame(registration('fixture_alpha', selectedToken)).runtimeKey, 'fixture_alpha'); + const beforeExecution = RUNTIME_GATEWAY_ACTIONS.filter(action => action !== 'runtime.execution'); + assert.deepEqual(validateRegistrationFrame({ ...registration('fixture_alpha', selectedToken), actions: beforeExecution }).actions, beforeExecution, + 'an existing DSP remains connected during a Core-first rolling upgrade'); + assert.throws(() => validateRegistrationFrame({ + ...registration('fixture_alpha', selectedToken), extra: true, + }), isCode('invalid_runtime_agent_frame')); + assert.throws(() => validateRegistrationFrame({ + ...registration('fixture_alpha', selectedToken), protocolVersion: 2, + }), isCode('runtime_agent_protocol_mismatch')); + assert.throws(() => validateRegistrationFrame({ + ...registration('fixture_alpha', selectedToken), actions: ['health'], + }), isCode('invalid_runtime_agent_frame')); + assert.throws(() => parseStrictJson('{"type":"register","type":"response"}'), isCode('invalid_json')); + + const frame = { + protocolVersion: RUNTIME_AGENT_PROTOCOL_VERSION, + type: 'request', + requestId: 'a'.repeat(32), + request: gatewayRequest('fixture_alpha'), + }; + assert.equal(validateRequestFrame(frame, 'fixture_alpha').request.runtimeKey, 'fixture_alpha'); + assert.throws(() => validateRequestFrame(frame, 'fixture_bravo'), isCode('runtime_identity_mismatch')); + assert.throws(() => responseFrame('a'.repeat(32), { + ...gatewaySuccess(success('ready', {})), extra: true, + }), isCode('invalid_runtime_agent_frame')); + const heartbeat = heartbeatFrame('b'.repeat(32)); + assert.equal(validateHeartbeatFrame(heartbeat).nonce, 'b'.repeat(32)); + assert.equal(validateHeartbeatAckFrame(heartbeatAckFrame('b'.repeat(32))).type, 'heartbeat_ack'); + assert.equal(encodeFrame({ type: 'bounded' }).endsWith('\n'), true); + assert.throws(() => encodeFrame({ value: 'x'.repeat(MAX_AGENT_FRAME_BYTES) }), isCode('invalid_runtime_agent_frame')); + assert.throws(() => new CoreRuntimeAgentHub({ + socketPath: '/tmp/runtime-agent-hub.sock', + authorities: { + fixture_alpha: digest(selectedToken), + fixture_bravo: digest(selectedToken), + }, + }), isCode('runtime_agent_unavailable')); +}); + +test('hub and status servers recover owner-private stale sockets after an unclean exit', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-agent-stale-')); + fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const runtimeToken = token(); + const hubSocket = path.join(root, 'runtime-agent-hub.sock'); + await leaveStaleSocket(hubSocket); + const hub = new CoreRuntimeAgentHub({ + socketPath: hubSocket, + authorities: { runtime_stale: digest(runtimeToken) }, + }); + await hub.start(); + await hub.close(); + + const statusSocket = path.join(root, 'runtime-agent-status.sock'); + await leaveStaleSocket(statusSocket); + const statusServer = new RuntimeAgentStatusServer({ + socketPath: statusSocket, + agent: { status: () => ({ registered: false }) }, + }); + await statusServer.start(); + assert.equal((await queryRuntimeAgentStatus(statusSocket)).status, 'runtime_agent_unavailable'); + await statusServer.close(); +}); + +test('one Core hub routes two outbound DSP agents without crossed identity', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-agent-')); + fs.chmodSync(root, 0o700); + const socketPath = path.join(root, 'runtime-agent-hub.sock'); + const alphaToken = token(); + const bravoToken = token(); + const hub = new CoreRuntimeAgentHub({ + socketPath, + authorities: { + fixture_alpha: digest(alphaToken), + fixture_bravo: digest(bravoToken), + }, + requestTimeoutMs: 1_000, + }); + await hub.start(); + const alphaCalls = []; + const bravoCalls = []; + const alphaRuntime = fixtureClient('alpha', alphaCalls); + const alpha = new DspRuntimeAgent({ + socketPath, + runtimeKey: 'fixture_alpha', + registrationToken: alphaToken, + client: alphaRuntime, + }); + const bravo = new DspRuntimeAgent({ + socketPath, + runtimeKey: 'fixture_bravo', + registrationToken: bravoToken, + client: fixtureClient('bravo', bravoCalls), + }); + t.after(async () => { + await Promise.allSettled([alpha.close(), bravo.close()]); + await hub.close(); + fs.rmSync(root, { recursive: true, force: true }); + }); + await Promise.all([alpha.start(), bravo.start()]); + assert.deepEqual(hub.status(), { protocolVersion: 1, configured: 2, connected: 2 }); + + const alphaClient = createRuntimeAgentDispatchClient({ hub, runtimeKey: 'fixture_alpha' }); + const bravoClient = createRuntimeAgentDispatchClient({ hub, runtimeKey: 'fixture_bravo' }); + assert.equal((await alphaClient.health()).data.transport, 'outbound_agent'); + assert.equal((await bravoClient.health()).data.transport, 'outbound_agent'); + assert.equal((await alphaClient.workforce.day({ date: '2026-09-03', limit: 10, offset: 0 })).data.label, 'alpha'); + assert.equal((await bravoClient.workforce.day({ date: '2026-09-03', limit: 10, offset: 0 })).data.label, 'bravo'); + assert.equal(alphaCalls.filter(call => call[0] === 'workforce.day').length, 1); + assert.equal(bravoCalls.filter(call => call[0] === 'workforce.day').length, 1); + + const unauthorized = new DspRuntimeAgent({ + socketPath, + runtimeKey: 'fixture_alpha', + registrationToken: bravoToken, + client: fixtureClient('forged'), + }); + await assert.rejects(unauthorized.start(), isCode('runtime_agent_unauthorized')); + await unauthorized.close(); + assert.equal(hub.status().connected, 2); + + const duplicate = new DspRuntimeAgent({ + socketPath, + runtimeKey: 'fixture_alpha', + registrationToken: alphaToken, + client: fixtureClient('duplicate'), + }); + await assert.rejects(duplicate.start(), isCode('runtime_agent_conflict')); + await duplicate.close(); + assert.equal(hub.status().connected, 2); + + alphaRuntime.system.status = async () => ({ + contractVersion: 1, ok: true, status: 'ready', data: { credential: 'synthetic-invalid-value' }, + }); + const invalid = await alphaClient.system.status(); + assert.equal(invalid.status, 'runtime_agent_unavailable'); + assert.equal(JSON.stringify(invalid).includes('synthetic-invalid-value'), false); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(hub.connected('fixture_alpha'), false); + assert.equal((await bravoClient.system.status()).data.label, 'bravo'); + assert.equal(hub.status().connected, 1); +}); + +test('agent reconnects across authority rotation and hub restart while health follows liveness', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-agent-durable-')); + fs.chmodSync(root, 0o700); + const hubSocket = path.join(root, 'runtime-agent-hub.sock'); + const statusSocket = path.join(root, 'runtime-agent-status.sock'); + let selectedToken = token(); + let authority = { digest: digest(selectedToken), generation: 1 }; + const catalog = { + resolve: runtimeKey => runtimeKey === 'fixture_durable' ? authority : null, + count: () => authority === null ? 0 : 1, + }; + let hub = new CoreRuntimeAgentHub({ + socketPath: hubSocket, + authorityCatalog: catalog, + heartbeatIntervalMs: 25, + heartbeatTimeoutMs: 75, + }); + await hub.start(); + const agent = new DspRuntimeAgent({ + socketPath: hubSocket, + runtimeKey: 'fixture_durable', + registrationTokenProvider: () => selectedToken, + client: fixtureClient('durable'), + reconnectMinMs: 25, + reconnectMaxMs: 100, + }); + const status = new RuntimeAgentStatusServer({ socketPath: statusSocket, agent }); + await status.start(); + t.after(async () => { + await agent.close(); + await status.close(); + await hub.close(); + fs.rmSync(root, { recursive: true, force: true }); + }); + await agent.start(); + assert.equal((await queryRuntimeAgentStatus(statusSocket)).status, 'ready'); + + selectedToken = token(); + authority = { digest: digest(selectedToken), generation: 2 }; + assert.equal(hub.connected('fixture_durable'), false); + // The hub records registration before its acknowledgement reaches the DSP. + // Wait for both ends of the handshake before asserting DSP-side readiness. + await waitFor(() => hub.connected('fixture_durable') && agent.status().registered); + assert.equal(agent.status().registered, true); + assert.equal((await queryRuntimeAgentStatus(statusSocket)).status, 'ready'); + + authority = null; + assert.equal(hub.connected('fixture_durable'), false); + await waitFor(() => agent.status().registered === false); + assert.equal((await queryRuntimeAgentStatus(statusSocket)).status, 'runtime_agent_unavailable'); + + authority = { digest: digest(selectedToken), generation: 2 }; + await waitFor(() => hub.connected('fixture_durable')); + await hub.close(); + await waitFor(() => agent.status().registered === false); + hub = new CoreRuntimeAgentHub({ + socketPath: hubSocket, + authorityCatalog: catalog, + heartbeatIntervalMs: 25, + heartbeatTimeoutMs: 75, + }); + await hub.start(); + await waitFor(() => hub.connected('fixture_durable')); + assert.equal((await createRuntimeAgentDispatchClient({ + hub, runtimeKey: 'fixture_durable', + }).system.status()).data.label, 'durable'); +}); diff --git a/core/core/api/README.md b/core/core/api/README.md new file mode 100644 index 0000000..97608a7 --- /dev/null +++ b/core/core/api/README.md @@ -0,0 +1,79 @@ +# Dispatch API + +`dispatch-api` is the authenticated HTTP service. It owns backend composition, +account/session authorization and DSP coordination. It does not serve dashboard +HTML, and plugin code still executes in DSP workers. The browser manager and auth +broker remain separately supervised Core services; credentials, settings and +business databases remain in their DSP directories. + +From the editable source, the split directory deployment runs: + +```sh +bin/dispatch-api --installation-backend directory_service_v1 --installation-operator --operator --port 4311 +bin/dispatch-dashboard --api-origin http://127.0.0.1:4311 --port 4310 +``` + +Only the API receives `DISPATCH_PLATFORM_CONFIG`. Configure the same canonical +public origin on both services behind the existing reviewed HTTPS proxy. The API +loads its origin from private dashboard settings in directory mode; give the UI +`--public-origin https://dispatch.example.test`. Cookies and browser requests keep +the existing origin and `/api/` paths. The UI forwards cookies, CSRF, signed DSP +views and the original Host without constructing a new identity. It streams a +request once and never automatically retries a mutation. Worker SDK sockets and +local database access remain private and do not make HTTP round trips. + +`host/services/api-units.js` renders `dispatch-api.service` and the existing +`dispatch-platform-local.service` (now the UI). `prepare-startup` prepares both +units for a reviewed migration. It does not activate them. Do not run the legacy +combined controller and the new API against the same platform simultaneously. + +## Source ownership + +- `server.js`: API routing and request policy, with no static-asset dependency. +- `access-http.js`: account, connection, installation and settings endpoints. +- `plugin-operations.js`: automatic manifest-based operation routing, validation, + audit attribution and authorization checks before and after asynchronous work. +- `http.js`: existing request parsing, safe response helpers and protocol policy. +- `directory-platform.js`: directory backend startup and orderly shutdown. +- `main.js`: standalone API command and retained native/OCI startup support. +- `dashboard/server/shell.js`: static UI and bounded-loopback API forwarding. + +The old dashboard module paths are compatibility imports. Starting +`dispatch-dashboard` without `--api-origin` deliberately retains the combined +launcher for existing installations and legacy artifact/recovery consumers. +The new split unit uses `--api-origin` explicitly; the dashboard does not open a +store, start a controller or inherit backend configuration in that mode. +`GET /api/health` checks API database access and returns a minimal service-ready +response. It does not claim that every provider connection or DSP job is healthy. + +## Adding an operation + +Declare an action in a plugin's `dispatch-plugin.json`, including its permission +and input/output schemas. It is available at +`POST /api/plugins//`. DSP identity comes from the authenticated +session or signed owner support view, never a body field. A declaration requests +a permission; it does not create a grant. Existing platform roles continue to +control access. + +An optional `errors` list declares public business error codes. A handler can +throw a matching SDK `DispatchError`; `definePlugin` converts it into the standard +failure result. Undeclared implementation failures remain inside the worker's +normal error boundary. The same error declarations appear in generated docs. + +`dispatch-sdk/operations` validates the definitions and builds convenient clients. +`dispatch-sdk/plugin` wraps worker handlers with the same input/output validation. +The worker host also validates declared contracts independently. Generated +OpenAPI descriptions and TypeScript clients come from those declarations; run +`bin/dispatch plugin generate ` after editing one. Packaging checks for +stale generated contracts. + +This first contract pass covers plugin actions. Existing accounts/settings API +contracts retain their current validators. Paycom action inputs are declared; +its complex result models continue using the workforce contracts. New generated +plugins include explicit input and output schemas. Existing plugin HTTP routes +remain compatible while consumers can adopt the generated action clients. + +The supported schema subset is deliberately closed: objects with explicit +properties/required fields, arrays, strings, numbers, integers, booleans, null, +primitive enums, and bounds. It uses standard JSON Schema keywords and rejects +unsupported keywords rather than silently ignoring validation requirements. diff --git a/core/core/api/access-http.js b/core/core/api/access-http.js new file mode 100644 index 0000000..bcd9e2d --- /dev/null +++ b/core/core/api/access-http.js @@ -0,0 +1,710 @@ +'use strict'; + +const net = require('node:net'); +const { AccessError, exact } = require('../accounts/src'); + +const LOCAL_SESSION_COOKIE = 'dispatch_session'; +const PUBLIC_SESSION_COOKIE = '__Host-dispatch_session'; +const LOGIN_WINDOW_MS = 15 * 60 * 1000; +const LOGIN_ATTEMPTS = 8; +const LOGIN_ADDRESS_ATTEMPTS = 40; +const INVITATION_ADDRESS_ATTEMPTS = 30; + +function cookieHeader(name, value, { maxAge = null, secure = false } = {}) { + const parts = [`${name}=${value}`, 'Path=/', 'HttpOnly', 'SameSite=Strict']; + if (maxAge !== null) parts.push(`Max-Age=${Math.max(0, Math.floor(maxAge))}`); + if (secure) parts.push('Secure'); + return parts.join('; '); +} + +function sessionToken(request, cookieName = LOCAL_SESSION_COOKIE) { + const header = request.headers.cookie; + if (typeof header !== 'string' || header.length > 4096) return null; + const matches = header.split(';').map(value => value.trim()).filter(value => value.startsWith(`${cookieName}=`)); + if (matches.length !== 1) return null; + const value = matches[0].slice(cookieName.length + 1); + return /^[A-Za-z0-9_-]{43}$/.test(value) ? value : null; +} + +function invitePath(token) { return `/#/invitation/${token}`; } + +function createAccessHttp({ + access, secureCookies = false, trustCloudflareAddress = false, invitationDelivery = null, turnstile = null, + paycomSetup = null, connections = null, plugins = null, updates = null, releasePopup = null, backups = null, + platformRuntime = null, requireInvitationDelivery = false, clock = () => new Date(), +}) { + if (!access || typeof access.requireSession !== 'function' || typeof secureCookies !== 'boolean' + || typeof trustCloudflareAddress !== 'boolean' + || (invitationDelivery !== null && typeof invitationDelivery?.send !== 'function') + || (turnstile !== null && (typeof turnstile?.verify !== 'function' || typeof turnstile?.publicConfig?.siteKey !== 'string')) + || typeof requireInvitationDelivery !== 'boolean' || typeof clock !== 'function') { + throw new TypeError('access_http_dependencies_required'); + } + const attempts = new Map(); + const invitationAttempts = new Map(); + const sessionCookie = secureCookies ? PUBLIC_SESSION_COOKIE : LOCAL_SESSION_COOKIE; + + function session(request, { required = true } = {}) { + const token = sessionToken(request, sessionCookie); + const current = token ? access.session(token) : null; + if (!current && required) throw new AccessError('authentication_required', 401); + const viewRef = request.headers['x-dispatch-dsp-view']; + return current && viewRef !== undefined ? access.dspViewSession(current, viewRef) : current; + } + + function requireJson(request) { + const type = String(request.headers['content-type'] || '').split(';')[0].trim().toLowerCase(); + if (type !== 'application/json') throw new AccessError('content_type_required', 415); + } + + function requireMutation(request, current, url) { + if (request.headers['sec-fetch-site'] === 'cross-site') throw new AccessError('request_forbidden', 403); + if (!current || request.headers['x-dispatch-csrf'] !== current.csrfToken) throw new AccessError('csrf_invalid', 403); + requireJson(request); + requireNoQuery(url); + if ((current.dspView || current.user.platformRole !== 'owner') && access.store?.releaseBlocked?.(current.activeOrganizationId)) throw new AccessError('release_busy', 409); + } + + function activeOrganizationId(current) { + if (!current.activeOrganizationId) throw new AccessError('organization_required', 409); + return current.activeOrganizationId; + } + + function requireNoQuery(url) { + if (url.search) throw new AccessError('invalid_request', 400); + } + + function publicSession(current) { + if (!current) return { authenticated: false, bootstrap: access.bootstrapStatus(), turnstile: turnstile?.publicConfig ?? null }; + return { + authenticated: true, + turnstile: turnstile?.publicConfig ?? null, + user: current.user, + platformPermissions: current.platformPermissions, + activeOrganizationId: current.activeOrganizationId, + ...(current.dspView ? { dspView: current.dspView } : {}), + memberships: current.memberships.map(membership => ({ + id: membership.id, + organizationId: membership.organizationId, + roleId: membership.roleId, + roleKey: membership.roleKey, + roleName: membership.roleName, + status: membership.status, + permissions: membership.permissions, + organization: { + id: membership.organization.id, + name: membership.organization.name, + abbreviation: membership.organization.abbreviation, + timezone: membership.organization.timezone, + status: membership.organization.status, + stations: membership.organization.stations, + }, + })), + plugins: current.activeOrganizationId ? (plugins?.listForOrganization + ? plugins.listForOrganization(current.activeOrganizationId) + : require('../accounts/src/plugins').listFor(access.store, current.activeOrganizationId)) : [], + csrfToken: current.csrfToken, + expiresAt: current.expiresAt, + }; + } + + function requestAddress(request) { + const direct = request.socket.remoteAddress || 'unknown'; + const forwarded = request.headers['cf-connecting-ip']; + if (trustCloudflareAddress && ['127.0.0.1', '::1', '::ffff:127.0.0.1'].includes(direct) + && typeof forwarded === 'string' && !forwarded.includes(',') && net.isIP(forwarded.trim())) { + return forwarded.trim(); + } + return direct; + } + + function loginKeys(request, body) { + const email = typeof body.email === 'string' ? body.email.trim().toLowerCase().slice(0, 254) : 'invalid'; + const address = requestAddress(request); + return Object.freeze([ + { key: `${address}\0${email}`, limit: LOGIN_ATTEMPTS }, + { key: `${address}\0*`, limit: LOGIN_ADDRESS_ATTEMPTS }, + ]); + } + + function assertLoginAllowed(keys) { + const timestamp = clock().getTime(); + if (attempts.size > 4096) { + for (const [candidate, value] of attempts) if (value.resetAt <= timestamp) attempts.delete(candidate); + while (attempts.size > 4096) attempts.delete(attempts.keys().next().value); + } + for (const { key, limit } of keys) { + const entry = attempts.get(key); + if (!entry || entry.resetAt <= timestamp) { + attempts.set(key, { count: 0, resetAt: timestamp + LOGIN_WINDOW_MS }); + } else if (entry.count >= limit) throw new AccessError('login_rate_limited', 429); + } + } + + function recordLoginFailure(keys) { + for (const { key } of keys) { + const entry = attempts.get(key); + if (entry) entry.count += 1; + } + } + + function consumeInvitationAttempt(request) { + const timestamp = clock().getTime(); + for (const [candidate, value] of invitationAttempts) { + if (value.resetAt <= timestamp) invitationAttempts.delete(candidate); + } + while (invitationAttempts.size > 2048) invitationAttempts.delete(invitationAttempts.keys().next().value); + const key = requestAddress(request); + const current = invitationAttempts.get(key); + if (!current || current.resetAt <= timestamp) { + invitationAttempts.set(key, { count: 1, resetAt: timestamp + LOGIN_WINDOW_MS }); + return; + } + if (current.count >= INVITATION_ADDRESS_ATTEMPTS) { + throw new AccessError('invitation_rate_limited', 429); + } + current.count += 1; + } + + function requireDeliveryReady() { + if (requireInvitationDelivery && invitationDelivery === null) { + throw new AccessError('invitation_email_unavailable', 503); + } + } + + async function deliverInvitation(result) { + if (result.token === null) { + return { delivery: { status: 'already_processed' }, invitationPath: null }; + } + const path = invitePath(result.token); + if (invitationDelivery === null) { + return { delivery: { status: 'not_configured' }, invitationPath: path }; + } + let status = 'unknown'; + try { + const delivery = await invitationDelivery.send({ ...result.invitation, token: result.token }); + if (['accepted', 'failed', 'unknown'].includes(delivery?.status)) status = delivery.status; + } catch {} + return { + delivery: { status }, + invitationPath: status === 'accepted' ? null : path, + }; + } + + const passwordRecovery = require('./password-recovery-http').createPasswordRecoveryHttp({ + access, delivery: invitationDelivery, turnstile, requestAddress, clock, + }); + + async function route(request, response, url, { readJson, sendJson }) { + // A DSP support context can edit only the selected DSP and the actor's own account. + if (request.headers['x-dispatch-dsp-view'] !== undefined && !['GET', 'HEAD'].includes(request.method) + && !url.pathname.startsWith('/api/organization/') + && !/^\/api\/plugins\/[a-z][a-z0-9-]{0,63}\/[a-z][a-z0-9_.]{0,63}$/.test(url.pathname) + && !['/api/paycom/sync', '/api/auth/logout', '/api/auth/change-password'].includes(url.pathname)) { + throw new AccessError('dsp_view_scope', 403); + } + if (await passwordRecovery.route(request, response, url, { readJson, sendJson })) return true; + if (request.method === 'POST' && url.pathname === '/api/platform/organization/view') { + const current = session(request); + requireMutation(request, current, url); + const viewed = access.beginDspView(current, await readJson(request)); + sendJson(response, 200, { ok: true, status: 'viewing', data: publicSession(viewed), error: null }); + return true; + } + if (request.method === 'GET' && url.pathname === '/api/auth/session') { + sendJson(response, 200, { ok: true, status: 'ready', data: publicSession(session(request, { required: false })), error: null }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/auth/invitation/inspect') { + if (request.headers['sec-fetch-site'] === 'cross-site') throw new AccessError('request_forbidden', 403); + requireNoQuery(url); + requireJson(request); + const body = await readJson(request); + exact(body, ['token']); + consumeInvitationAttempt(request); + sendJson(response, 200, { ok: true, status: 'found', data: access.inspectInvitation(body.token), error: null }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/auth/login') { + if (request.headers['sec-fetch-site'] === 'cross-site') throw new AccessError('request_forbidden', 403); + requireNoQuery(url); + requireJson(request); + const body = await readJson(request); + exact(body, ['email', 'password', ...(turnstile ? ['turnstileToken'] : [])]); + const keys = loginKeys(request, body); + assertLoginAllowed(keys); + if (turnstile) { + try { await turnstile.verify(body.turnstileToken, 'login', requestAddress(request)); } + catch (error) { + if (error instanceof AccessError && error.statusCode < 500) recordLoginFailure(keys); + throw error; + } + } + const { turnstileToken, ...credentials } = body; + try { + const result = await access.signIn(credentials); + attempts.delete(keys[0].key); + sendJson(response, 200, { ok: true, status: 'authenticated', data: publicSession(result.session), error: null }, { + 'Set-Cookie': cookieHeader(sessionCookie, result.token, { maxAge: (result.expiresAt - clock().getTime()) / 1000, secure: secureCookies }), + }); + } catch (error) { + recordLoginFailure(keys); + throw error; + } + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/auth/register') { + if (request.headers['sec-fetch-site'] === 'cross-site') throw new AccessError('request_forbidden', 403); + requireNoQuery(url); + requireJson(request); + const body = await readJson(request); + exact(body, ['token', 'firstName', 'lastName', 'password', 'confirmPassword', ...(turnstile ? ['turnstileToken'] : [])]); + consumeInvitationAttempt(request); + if (turnstile) await turnstile.verify(body.turnstileToken, 'register', requestAddress(request)); + const { turnstileToken, ...registration } = body; + const result = await access.acceptNewUser(registration); + sendJson(response, 201, { ok: true, status: 'authenticated', data: publicSession(result.session), error: null }, { + 'Set-Cookie': cookieHeader(sessionCookie, result.token, { maxAge: (result.expiresAt - clock().getTime()) / 1000, secure: secureCookies }), + }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/auth/accept-invitation') { + const current = session(request); + requireMutation(request, current, url); + const body = await readJson(request); + exact(body, ['token']); + consumeInvitationAttempt(request); + const updated = access.acceptExistingUser(current, body.token); + sendJson(response, 200, { ok: true, status: 'accepted', data: publicSession(updated), error: null }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/auth/change-password') { + const current = session(request); + requireMutation(request, current, url); + const result = await access.changePassword(current, await readJson(request)); + sendJson(response, 200, { ok: true, status: 'password_changed', data: publicSession(result.session), error: null }, { + 'Set-Cookie': cookieHeader(sessionCookie, result.token, { maxAge: (result.expiresAt - clock().getTime()) / 1000, secure: secureCookies }), + }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/auth/select-organization') { + const current = session(request); + requireMutation(request, current, url); + const body = await readJson(request); + exact(body, ['membershipId']); + const updated = access.selectMembership(current, body.membershipId); + sendJson(response, 200, { ok: true, status: 'selected', data: publicSession(updated), error: null }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/auth/logout') { + const current = session(request); + requireMutation(request, current, url); + const body = await readJson(request); + exact(body, []); + access.signOut(current); + sendJson(response, 200, { ok: true, status: 'signed_out', data: null, error: null }, { + 'Set-Cookie': cookieHeader(sessionCookie, '', { maxAge: 0, secure: secureCookies }), + }); + return true; + } + + if (url.pathname === '/api/organization/profile' && ['GET', 'POST'].includes(request.method)) { + const current = session(request); + requireNoQuery(url); + if (request.method === 'POST') requireMutation(request, current, url); + const result = access.organizationProfile(current, request.method === 'POST' ? await readJson(request) : undefined); + sendJson(response, 200, { ok: true, status: 'found', data: result, error: null }); + return true; + } + if (url.pathname === '/api/platform/backups' && ['GET', 'POST'].includes(request.method)) { + const current = session(request); + requireNoQuery(url); + access.requirePlatform(current, 'platform.installations.manage'); + if (!backups) throw new AccessError('installation_operator_disabled', 503); + if (backups.ownerOnly && current.user.platformRole !== 'owner') throw new AccessError('platform_forbidden', 403); + if (request.method === 'POST') { + requireMutation(request, current, url); + backups.command(current, await readJson(request)); + } + sendJson(response, 200, { ok: true, status: 'found', data: backups.view(), error: null }); + return true; + } + if (url.pathname === '/api/updates/popup' && ['GET', 'POST'].includes(request.method)) { + const current = session(request); + requireNoQuery(url); + let result; + if (request.method === 'POST') { + requireMutation(request, current, url); + if (!releasePopup) throw new AccessError('release_popup_unavailable', 409); + result = releasePopup.dismiss(current, await readJson(request)); + } else result = releasePopup?.pending(current) || { release: null }; + sendJson(response, 200, { ok: true, status: 'found', data: result, error: null }); + return true; + } + if (url.pathname === '/api/platform/updates' && ['GET', 'POST'].includes(request.method)) { + const current = session(request); + let releaseId = null; + if (request.method === 'GET' && url.searchParams.has('releaseId')) { + if ([...url.searchParams.keys()].length !== 1 || !/^[a-z][a-z0-9_.-]{2,95}$/.test(url.searchParams.get('releaseId'))) throw new AccessError('invalid_request', 400); + releaseId = url.searchParams.get('releaseId'); + } else requireNoQuery(url); + access.requirePlatform(current, 'platform.installations.manage'); + if (!updates) throw new AccessError('installation_operator_disabled', 503); + if (updates.ownerOnly && (current.user.platformRole !== 'owner' || current.dspView)) throw new AccessError('platform_forbidden', 403); + if (request.method === 'POST') { + requireMutation(request, current, url); + await updates.command(current, await readJson(request)); + } + sendJson(response, 200, { ok: true, status: 'found', data: await updates.view(releaseId), error: null }); + return true; + } + + if (url.pathname === '/api/platform/runtime' && request.method === 'GET') { + const current = session(request); + requireNoQuery(url); + access.requirePlatform(current, 'platform.installations.manage'); + if (current.user.platformRole !== 'owner') throw new AccessError('platform_forbidden', 403); + sendJson(response, 200, { ok: true, status: 'found', data: platformRuntime?.() + || { enabled: false, storageAvailableBytes: null, runtimes: [] }, error: null }); + return true; + } + + if (url.pathname === '/api/platform/diagnostics' && ['GET', 'POST'].includes(request.method)) { + const current = session(request); + requireNoQuery(url); + if (request.method === 'POST') requireMutation(request, current, url); + const result = access.platformDiagnostics(current, request.method === 'POST' ? await readJson(request) : undefined); + sendJson(response, request.method === 'POST' ? 202 : 200, { ok: true, status: 'found', data: result, error: null }); + return true; + } + + if (request.method === 'GET' && url.pathname === '/api/platform/organizations') { + requireNoQuery(url); + const current = session(request); + sendJson(response, 200, { ok: true, status: 'found', data: access.platformOrganizations(current), error: null }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/platform/organizations') { + const current = session(request); + requireMutation(request, current, url); + requireDeliveryReady(); + const result = access.createOrganization(current, await readJson(request)); + const handoff = await deliverInvitation(result); + sendJson(response, result.replayed ? 200 : 201, { + ok: true, + status: result.replayed ? 'replayed' : 'created', + data: { + organization: { + name: result.organization.name, + abbreviation: result.organization.abbreviation, + timezone: result.organization.timezone, + stations: result.organization.stations, + status: result.organization.status, + installation: result.organization.installation, + }, + ownerInvitation: { + email: result.invitation.email, + status: result.invitation.status, + expiresAt: result.invitation.expiresAt, + }, + invitationPath: handoff.invitationPath, + delivery: handoff.delivery, + replayed: result.replayed, + }, + error: null, + }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/platform/organization/status') { + const current = session(request); + requireMutation(request, current, url); + const result = access.setPlatformOrganizationSuspended(current, await readJson(request)); + sendJson(response, 200, { ok: true, status: result.status, data: result, error: null }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/platform/organization/owner-invitation') { + const current = session(request); + requireMutation(request, current, url); + requireDeliveryReady(); + const result = access.createPlatformOwnerInvitation(current, await readJson(request)); + const handoff = await deliverInvitation(result); + sendJson(response, result.replayed ? 200 : 201, { + ok: true, + status: result.replayed ? 'replayed' : 'created', + data: { + ownerInvitation: { + email: result.invitation.email, + status: result.invitation.status, + expiresAt: result.invitation.expiresAt, + }, + invitationPath: handoff.invitationPath, + delivery: handoff.delivery, + replayed: result.replayed, + }, + error: null, + }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/platform/organization/owner-invitation/revoke') { + const current = session(request); + requireMutation(request, current, url); + const result = access.revokePlatformOwnerInvitation(current, await readJson(request)); + sendJson(response, 200, { ok: true, status: result.status, data: result, error: null }); + return true; + } + + if (request.method === 'POST' && ['/api/platform/installation/remove', '/api/platform/installation/delete', '/api/platform/installation/restore'].includes(url.pathname)) { + const current = session(request); + requireMutation(request, current, url); + const body = await readJson(request); + const deleting = url.pathname.endsWith('/delete'); + const keys = deleting ? loginKeys(request, { email: current.user.email }) : null; + if (keys) assertLoginAllowed(keys); + let result; + try { + result = await access.requestPlatformRemoval(current, body, + deleting ? 'destroy' : url.pathname.endsWith('/restore') ? 'resume' : 'decommission'); + } catch (error) { + if (keys && error.code === 'current_password_invalid') recordLoginFailure(keys); + throw error; + } + sendJson(response, 202, { ok: true, status: result.status, data: result, error: null }); + return true; + } + + if (request.method === 'POST' && ['/api/platform/installation/suspend', '/api/platform/installation/resume', + '/api/platform/installation/restart'].includes(url.pathname)) { + const current = session(request); + requireMutation(request, current, url); + const result = access.requestPlatformRuntime(current, await readJson(request), url.pathname.split('/').at(-1)); + sendJson(response, 202, { ok: true, status: result.status, data: result, error: null }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/platform/installation/provision') { + const current = session(request); + requireMutation(request, current, url); + const result = access.requestPlatformInstallationProvisioning(current, await readJson(request)); + sendJson(response, 202, { ok: true, status: result.status, data: result, error: null }); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/platform/installation/retry') { + const current = session(request); + requireMutation(request, current, url); + const result = access.requestPlatformInstallationRetry(current, await readJson(request)); + sendJson(response, 202, { ok: true, status: result.status, data: result, error: null }); + return true; + } + + if (url.pathname === '/api/platform/plugins' && request.method === 'GET') { + requireNoQuery(url); + const current = session(request); access.requirePlatform(current, 'platform.organizations.read'); + if (!plugins) throw new AccessError('plugin_unavailable', 503); + sendJson(response, 200, { ok: true, status: 'found', data: plugins.catalog(), error: null }); + return true; + } + if (url.pathname === '/api/organization/plugins' && request.method === 'GET') { + requireNoQuery(url); + const current = session(request); + if (!plugins) throw new AccessError('plugin_unavailable', 503); + sendJson(response, 200, { ok: true, status: 'found', data: plugins.list(current), error: null }); + return true; + } + const settingsRoute = url.pathname.match(/^\/api\/organization\/plugins\/([a-z][a-z0-9-]{0,63})\/settings(\/options|\/history)?$/); + if (settingsRoute && (request.method === 'GET' || request.method === 'POST' && !settingsRoute[2])) { + let historyInput; + if (settingsRoute[2] === '/history') { + if ([...url.searchParams.keys()].some(key=>key!=='before') || url.searchParams.getAll('before').length>1) throw new AccessError('invalid_input',400); + const before=url.searchParams.get('before'); + if(before!==null && (!/^(0|[1-9][0-9]{0,15})$/.test(before) || !Number.isSafeInteger(Number(before))))throw new AccessError('invalid_input',400); + historyInput={beforeRevision:before===null?null:Number(before)}; + } else requireNoQuery(url); + const current = session(request); + if (!plugins?.settings) throw new AccessError('settings_unavailable',503); + if (request.method === 'POST') requireMutation(request,current,url); + const data = await plugins.settings(current,settingsRoute[1],request.method === 'POST' ? 'update' : settingsRoute[2] === '/history' ? 'history' : settingsRoute[2] ? 'options' : 'get', + request.method === 'POST' ? await readJson(request) : historyInput); + session(request); + sendJson(response,200,{ok:true,status:'found',data,error:null});return true; + } + const pluginRoute = url.pathname.match(/^\/api\/organization\/plugins\/([a-z][a-z0-9-]{0,63})$/); + if (pluginRoute && request.method === 'POST') { + const current = session(request); requireMutation(request, current, url); + if (!plugins) throw new AccessError('plugin_unavailable', 503); + const data = plugins.change(current, pluginRoute[1], await readJson(request)); + sendJson(response, 202, { ok: true, status: 'accepted', data, error: null }); + return true; + } + + if (url.pathname === '/api/organization/connections' && request.method === 'GET') { + requireNoQuery(url); + const current = session(request); + access.requireDspOwner(current); + if (!connections) throw new AccessError('auth_unavailable', 503); + const data = await connections.list(current); + sendJson(response, 200, { ok: true, status: 'found', data: { ...data, services: connections.services.filter(service => data.items.some(item => item.service === service.id)) }, error: null }); + return true; + } + const connectionRoute = url.pathname.match(/^\/api\/organization\/connections\/([a-z][a-z0-9-]{0,63})\/(save|test|disconnect|verify)$/); + if (connectionRoute && request.method === 'POST') { + requireNoQuery(url); + const current = session(request); + requireMutation(request, current, url); + access.requireDspOwner(current); + if (!connections) throw new AccessError('auth_unavailable', 503); + const data = await connections.change(current, connectionRoute[1], connectionRoute[2], + await readJson(request, require('../../shared/contracts/src/connections').CONNECTION_REQUEST_MAX_BYTES)); + sendJson(response, 202, { ok: true, status: 'accepted', data, error: null }); + return true; + } + if (url.pathname === '/api/organization/paycom-setup' && ['GET', 'POST'].includes(request.method)) { + requireNoQuery(url); + const current = session(request); + require('../accounts/src/plugins').requirePlugin(access, current, 'paycom'); + if (!paycomSetup) throw new AccessError('installation_operator_disabled', 503); + if (request.method === 'GET') { + sendJson(response, 200, { ok: true, status: 'found', data: await paycomSetup.status(current), error: null }); + } else { + requireMutation(request, current, url); + const result = await paycomSetup.submit(current, await readJson(request)); + sendJson(response, 202, { ok: true, status: 'accepted', data: result, error: null }); + } + return true; + } + if (url.pathname === '/api/organization/paycom-setup/retry' && request.method === 'POST') { + const current = session(request); + requireMutation(request, current, url); + require('../accounts/src/plugins').requirePlugin(access, current, 'paycom'); + if (!paycomSetup) throw new AccessError('installation_operator_disabled', 503); + const result = await paycomSetup.retry(current, await readJson(request)); + sendJson(response, 200, { ok: true, status: 'accepted', data: result, error: null }); + return true; + } + + if (request.method === 'GET' && url.pathname === '/api/organization/setup') { + requireNoQuery(url); + const current = session(request); + sendJson(response, 200, { ok: true, status: 'found', data: access.organizationSetup(current), error: null }); + return true; + } + + if (request.method === 'GET' && url.pathname === '/api/organization/audit') { + requireNoQuery(url); + const current = session(request); + sendJson(response, 200, { ok: true, status: 'found', data: access.organizationAudit(current, activeOrganizationId(current)), error: null }); + return true; + } + + let match = /^\/api\/organization\/administration$/.exec(url.pathname); + if (request.method === 'GET' && match) { + requireNoQuery(url); + const current = session(request); + sendJson(response, 200, { ok: true, status: 'found', data: access.organizationAdministration(current, activeOrganizationId(current)), error: null }); + return true; + } + + match = /^\/api\/organization\/roles$/.exec(url.pathname); + if (request.method === 'POST' && match) { + const current = session(request); + requireMutation(request, current, url); + const role = access.createRole(current, activeOrganizationId(current), await readJson(request)); + sendJson(response, 201, { ok: true, status: 'created', data: role, error: null }); + return true; + } + + match = /^\/api\/organization\/roles\/([a-z][a-z0-9_-]{2,95})$/.exec(url.pathname); + if (request.method === 'PUT' && match) { + const current = session(request); + requireMutation(request, current, url); + const role = access.updateRole(current, activeOrganizationId(current), match[1], await readJson(request)); + sendJson(response, 200, { ok: true, status: 'updated', data: role, error: null }); + return true; + } + if (request.method === 'DELETE' && match) { + const current = session(request); + requireMutation(request, current, url); + const body = await readJson(request); + exact(body, []); + access.deleteRole(current, activeOrganizationId(current), match[1]); + sendJson(response, 200, { ok: true, status: 'deleted', data: null, error: null }); + return true; + } + + match = /^\/api\/organization\/invitations$/.exec(url.pathname); + if (request.method === 'POST' && match) { + const current = session(request); + requireMutation(request, current, url); + requireDeliveryReady(); + const result = access.createMemberInvitation(current, activeOrganizationId(current), await readJson(request)); + const handoff = await deliverInvitation(result); + sendJson(response, 201, { + ok: true, status: 'created', + data: { + invitation: result.invitation, + invitationPath: handoff.invitationPath, + delivery: handoff.delivery, + }, + error: null, + }); + return true; + } + + match = /^\/api\/organization\/invitations\/([a-z][a-z0-9_-]{2,95})$/.exec(url.pathname); + if (request.method === 'DELETE' && match) { + const current = session(request); + requireMutation(request, current, url); + const body = await readJson(request); + exact(body, []); + access.revokeMemberInvitation(current, activeOrganizationId(current), match[1]); + sendJson(response, 200, { ok: true, status: 'revoked', data: null, error: null }); + return true; + } + + match = /^\/api\/organization\/members\/([a-z][a-z0-9_-]{2,95})\/role$/.exec(url.pathname); + if (request.method === 'PUT' && match) { + const current = session(request); + requireMutation(request, current, url); + const body = await readJson(request); + exact(body, ['roleId']); + access.updateMemberRole(current, activeOrganizationId(current), match[1], body.roleId); + sendJson(response, 200, { ok: true, status: 'updated', data: null, error: null }); + return true; + } + + match = /^\/api\/organization\/members\/([a-z][a-z0-9_-]{2,95})$/.exec(url.pathname); + if (request.method === 'DELETE' && match) { + const current = session(request); + requireMutation(request, current, url); + const body = await readJson(request); + exact(body, []); + access.removeMember(current, activeOrganizationId(current), match[1]); + sendJson(response, 200, { ok: true, status: 'deleted', data: null, error: null }); + return true; + } + + return false; + } + + return { route, session, requireMutation, publicSession }; +} + +module.exports = { + SESSION_COOKIE: LOCAL_SESSION_COOKIE, + PUBLIC_SESSION_COOKIE, + cookieHeader, + sessionToken, + invitePath, + createAccessHttp, +}; diff --git a/core/core/api/compatibility-paycom.js b/core/core/api/compatibility-paycom.js new file mode 100644 index 0000000..5a31818 --- /dev/null +++ b/core/core/api/compatibility-paycom.js @@ -0,0 +1,89 @@ +'use strict'; +const { AccessError } = require('../accounts/src'); +function createHandler({ runtimeContext, access, accessHttp, config, now, readJson, + sendJson: send, dailyQuery, publicSdkFailure, publicSyncView, publicSdkResult, integerParameter, IDEMPOTENCY_RE }) { + return async function handle(request, response, url) { + const sendJson = (response, status, value) => { + runtimeContext(request, 'workforce.read'); + return send(response, status, value); + }; + if (request.method === 'GET' && url.pathname === '/api/paycom/daily') { + const { runtime } = runtimeContext(request, 'workforce.read'); + const query = dailyQuery(url.searchParams); + const [workforce, syncResult] = await Promise.all([ + runtime.workforce.day(query), + runtime.sync.status(config.syncId), + ]); + const workforceError = workforce.ok ? null : publicSdkFailure(workforce, 'workforce_unavailable'); + const statusCode = workforce.ok ? 200 : workforceError.code === 'invalid_input' ? 400 : 503; + sendJson(response, statusCode, { + ok: workforce.ok, + status: workforce.ok ? workforce.status : workforceError.code, + data: workforce.ok ? { + day: workforce.data, + sync: publicSyncView(syncResult), + generatedAt: now().toISOString(), + } : null, + error: workforceError, + }); + return true; + } + + if (request.method === 'GET' && (url.pathname === '/api/paycom/employees' || url.pathname.startsWith('/api/paycom/employees/'))) { + const { runtime } = runtimeContext(request, 'workforce.read'); + const { workforceQuery, workforceEmployeeCode } = require('../../shared/contracts/src/workforce'); + let result; + if (url.pathname === '/api/paycom/employees') { + if ([...url.searchParams.keys()].some(key => !['limit', 'offset'].includes(key) || url.searchParams.getAll(key).length !== 1)) throw new AccessError('invalid_input', 400); + const query = workforceQuery({ + limit: integerParameter(url.searchParams.get('limit'), 100, { minimum: 1, maximum: 100 }), + offset: integerParameter(url.searchParams.get('offset'), 0), + }); + result = typeof runtime.workforce.employees === 'function' ? await runtime.workforce.employees(query) : null; + } else { + if (url.search) throw new AccessError('invalid_input', 400); + let code; + try { code = workforceEmployeeCode(url.pathname.slice('/api/paycom/employees/'.length)); } + catch { throw new AccessError('invalid_input', 400); } + result = typeof runtime.workforce.employee === 'function' ? await runtime.workforce.employee(code) : null; + } + if (!result) throw new AccessError('workforce_unavailable', 503); + const error = result.ok ? null : publicSdkFailure(result, 'workforce_unavailable'); + sendJson(response, result.ok ? 200 : error.code === 'employee_not_found' ? 404 : error.code === 'invalid_input' ? 400 : 503, { + ok: result.ok, status: result.status, data: result.ok ? result.data : null, error, + }); + return true; + } + + if (request.method === 'GET' && url.pathname === '/api/paycom/sync') { + if (url.search) throw Object.assign(new Error('invalid_request'), { statusCode: 400 }); + const { runtime } = runtimeContext(request, 'workforce.read'); + const result = publicSyncView(await runtime.sync.status(config.syncId)); + sendJson(response, result.ok ? 200 : 503, result); + return true; + } + + if (request.method === 'POST' && url.pathname === '/api/paycom/sync') { + const context = runtimeContext(request, 'sync.run'); + accessHttp.requireMutation(request, context.session, url); + const body = await readJson(request); + if (Object.keys(body).sort().join(',') !== 'idempotencyKey' + || typeof body.idempotencyKey !== 'string' || !IDEMPOTENCY_RE.test(body.idempotencyKey)) { + throw Object.assign(new Error('invalid_input'), { statusCode: 400 }); + } + const result = await context.runtime.sync.runNow(config.syncId, { + idempotencyKey: body.idempotencyKey, + }); + access.audit({ + actorUserId: context.session.user.id, organizationId: context.organization.id, + action: 'sync.run.request', targetType: 'sync', targetId: config.syncId, + result: result.ok ? 'succeeded' : 'denied', + }); + sendJson(response, result.ok ? 202 : 409, publicSdkResult(result, 'sync_unavailable')); + return true; + } + + return false; + }; +} +module.exports = { createHandler }; diff --git a/core/core/api/core-maintenance.js b/core/core/api/core-maintenance.js new file mode 100644 index 0000000..8858666 --- /dev/null +++ b/core/core/api/core-maintenance.js @@ -0,0 +1,20 @@ +'use strict'; +const path = require('node:path'); +const crypto = require('node:crypto'); +const { privateJson } = require('../installations/src/release-delivery-files'); +function createCoreMaintenance(localRoot) { + return () => { + if (!localRoot) return null; + try { + const value = privateJson(path.join(localRoot, 'config/core-maintenance.json'), process.geteuid(), true); + if (!value) return null; + if (!/^rollout_[a-f0-9]{32}$/.test(value.rolloutId) || !/^[a-f0-9]{64}$/.test(value.nonce)) throw Error(); + return value; + } catch { return { nonce: null }; } // Invalid state must never open traffic. + }; +} +function probeAllowed(state, supplied) { + return typeof supplied === 'string' && /^[a-f0-9]{64}$/.test(supplied) && typeof state?.nonce === 'string' && /^[a-f0-9]{64}$/.test(state.nonce) + && crypto.timingSafeEqual(Buffer.from(supplied), Buffer.from(state.nonce)); +} +module.exports = { createCoreMaintenance, probeAllowed }; diff --git a/core/core/api/directory-platform.js b/core/core/api/directory-platform.js new file mode 100644 index 0000000..db1833c --- /dev/null +++ b/core/core/api/directory-platform.js @@ -0,0 +1,149 @@ +'use strict'; + +const path = require('node:path'); +const { AccessStore } = require('../accounts/src/store'); +const { AccessControlService } = require('../accounts/src/service'); +const { loadPlatformPaths, platformPaths } = require('../../shared/paths/platform-paths'); +const { loadInstallation } = require('../../host/services/installation'); +const { privateDirectory } = require('../../host/controller/operations'); +const { DirectoryJournal } = require('../../host/controller/journal'); +const { openDirectoryRuntime } = require('../../host/controller/runtime'); +const { directoryAccessAuthority, BACKEND } = require('../../host/controller/access-authority'); +const { DirectoryProvisioningWorker } = require('../../host/controller/provisioning'); +const { createRuntimeAgentDispatchClient } = require('../agents/src/client'); +const { createApiServer } = require('./server'); +const { dashboardConfig } = require('./http'); +const { createInstallationRuntimeResolver } = require('./runtime-router'); +const { createOwnerPaycomSetup } = require('../accounts/src/owner-paycom-setup'); +const { createOwnerConnections } = require('../accounts/src/owner-connections'); +const { createOwnerOnboardingWorker } = require('../installations/src/owner-onboarding'); +const { DirectoryLifecycleWorker } = require('../../host/controller/lifecycle'); +const { createDirectoryMonitor } = require('../../host/capacity/monitor'); +const { createDirectoryDiagnostics } = require('../../host/controller/diagnostics'); +const { loadDashboardSettings } = require('../../host/controller/dashboard-settings'); +const { ManualBackups, interruptedRestore } = require('../../host/storage/manual-backups'); +const { invitationDeliveryFromEnvironment } = require('./invitation-email'); +const { DirectoryExecution } = require('../../host/controller/execution'); + +async function startDirectoryApi({ paths, installation, host, port = 4310, address = '127.0.0.1', + installationOperator = false, operator = false, publicOrigin = null, secureCookies = false, + onError = () => {}, runtimeFactory = openDirectoryRuntime, + environment = process.env, invitationFetchImpl, executionConfiguration, serverFactory = createApiServer } = {}) { + paths = platformPaths(paths.platformRoot); + if (!['127.0.0.1', '::1', 'localhost'].includes(address) || !Number.isInteger(port) || port < 0 || port > 65535) { + throw new Error('directory_dashboard_invalid'); + } + let store, runtime, worker, execution, server, closing, updateControl; + const close = () => closing ||= (async () => { + if (server?.listening) await new Promise(resolve => server.close(resolve)); + await updateControl?.close(); + await worker?.close(); + await execution?.close(); + try { await runtime?.close(); } finally { store?.close(); } + })(); + try { + if (interruptedRestore(paths)) throw new Error('directory_restore_incomplete'); + const invitationDelivery = invitationDeliveryFromEnvironment({ environment, + paths: { secretsRoot: path.join(paths.local, 'secrets') }, publicOrigin, + fetchImpl: invitationFetchImpl }); + // Make approved declarations available before one-time legacy adoption. + if (require('../plugins/package-catalog').packageCatalog(paths)) { + const definitions = () => require('../plugins/package-catalog').packageCatalog(paths)?.definitions() || []; + require('../../shared/plugin-sdk/catalog').configureCatalog(definitions); + require('dispatch-protocol/plugin-sdk/catalog').configureCatalog(definitions); + } + const databaseRoot = privateDirectory(path.join(paths.local, 'state/access-control')); + store = new AccessStore({ databaseRoot, database: path.join(databaseRoot, 'access-control.sqlite3') }); + // Directory work never wakes the legacy user services, including when this + // process inherits an old deployment's environment. + store.wakeWorkers = workers => { if (workers.includes('reconcile')) worker?.wake(); }; + const access = new AccessControlService(store, { installationBackend: BACKEND, installationOperatorEnabled: installationOperator }); + const journal = new DirectoryJournal(paths); + const authority = directoryAccessAuthority({ paths, store, journal }); + runtime = await runtimeFactory({ paths, installation, journal, host, ...authority, backgroundRecovery: true, onError }); + execution = new DirectoryExecution({ paths, accessStore: store, manager: runtime.manager, hub: runtime.hub, + configuration: executionConfiguration, onError }); + const invoke = (key, action, input) => execution.invoke(key, action, input); + if (runtime.manager.pluginBackend) store.pluginMetadataFor = (organizationId, pluginId) => { + const row = store.db.prepare('SELECT runtime_key FROM installations WHERE organization_id=?').get(organizationId); + const root = runtime.manager.checkedDsp(runtime.manager.journal.record(row.runtime_key)).root; + const { installationReceipt, installedPackage } = require('../../host/plugins/install'); + const receipt = installationReceipt(root, pluginId, true); + return receipt && receipt.state !== 'uninstalled' ? require('../../shared/plugin-sdk/package-files').verifyPackage(path.join(root,'plugins',pluginId,'versions',receipt.version),receipt.digest).plugin : null; + }; + if (runtime.manager.pluginBackend) { + const definitions = () => require('../plugins/package-catalog').packageCatalog(paths)?.definitions() || []; + require('../../shared/plugin-sdk/catalog').configureCatalog(definitions); + require('dispatch-protocol/plugin-sdk/catalog').configureCatalog(definitions); + } + const installationCoordinator = runtime.manager.pluginBackend + ? require('../../host/plugins/directory-lifecycle').createDirectoryInstallation({ paths, manager: runtime.manager, execution, store }) : null; + const plugins = require('../accounts/src/plugins').createPluginService({ store, access, invoke, installationCoordinator, + settingsPort: runtime.manager.pluginBackend ? (id,pluginId,request) => runtime.manager.pluginBackend.request(id,'plugin.settings',{pluginId,request}) : null }); + const verification = runtime.manager.pluginBackend ? require('../../host/controller/paycom-verification') + .createPaycomVerification({ backend: runtime.manager.pluginBackend }) : null; + const paycomSetup = createOwnerPaycomSetup({ store, access, invoke, + ...(runtime.manager.pluginBackend ? { enroll: require('../../host/controller/paycom-enrollment') + .createPaycomEnrollment({ backend: runtime.manager.pluginBackend, verification }), + beginVerification: verification.start, readReadiness: verification.readiness } : {}) }); + const connections = createOwnerConnections({ store, access, invoke, paycomSetup }); + const onboarding = createOwnerOnboardingWorker({ store, invoke, backends: [BACKEND], testProvider: verification?.poll }); + const backups = new ManualBackups({ paths, store, access }); + const deletions = new (require('../../host/controller/deletion').DirectoryDeletion)({ paths, store, + manager: runtime.manager, backups, execution, onError }); + access.directoryDeletion = deletions; + store.directoryDeletion = deletions; + const lifecycle = new DirectoryLifecycleWorker({ store, manager: runtime.manager, onError, + onChanged: id => execution.changed(id) }); + const diagnostics = createDirectoryDiagnostics({ store, invoke }); + worker = new DirectoryProvisioningWorker({ store, manager: runtime.manager, onError, + afterProvisioning: async () => { + await deletions.runPending(); + const results = await Promise.allSettled([lifecycle.runPending(), plugins.runPending(), diagnostics.runPending(), onboarding.runPending('directory_onboarding')]); + for (const result of results) if (result.status === 'rejected') onError(result.reason); + } }); + const client = createRuntimeAgentDispatchClient({ runtimeKey: 'unassigned', hub: runtime.hub }); + // Configuration opts into independent updates; activation belongs to the + // external worker and this controller's scoped DSP lifecycle. + updateControl = await require('../updates/directory').directoryUpdates({ paths, store, manager: runtime.manager, execution }); + const updates = updateControl.service; + const config = dashboardConfig({}); + const pluginAssets = require('../plugins/assets').createPluginAssets({ dspRoot: id => runtime.manager.checkedDsp(runtime.manager.journal.record(id)).root }); + const installedCore = require('../installations/src/release-delivery-files').privateJson(require('../../host/releases/core').receiptFile(paths), process.geteuid(), true); + const coreMaintenance = () => { + const state = require('../installations/src/release-delivery-files').privateJson(path.join(paths.local, 'state/updates/releases.json'), process.geteuid(), true); + return state?.operation?.product === 'core' ? { phase: 'updating', nonce: state.operation.preparation?.nonce } : null; + }; + server = serverFactory({ dashboards: require('../updates/dashboard').dashboardProvider({paths,store}), coreIdentity: installedCore, coreMaintenance, access, client, config, operator, paycomSetup, connections, plugins, pluginAssets, publicOrigin, secureCookies, updates, backups, + // Public installations require email setup before creating invitations. + // Loopback-only development can still hand off invitation links manually. + invitationDelivery, + platformRuntime: createDirectoryMonitor({ store, manager: runtime.manager, paths, execution }), + runtimeResolver: createInstallationRuntimeResolver({ localClient: client, + localOrganizationId: config.organization.id, runtimeAgentHub: { invoke } }) }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, address, () => { server.off('error', reject); resolve(); }); + }); + if (installationOperator) Promise.resolve(runtime.recovery).then(() => { if (!closing) { worker.start(); execution.wake(); } }); + return { server, store, access, runtime, worker, execution, lifecycle, deletions, backups, close }; + } catch (error) { await close(); throw error; } +} + +async function mainDirectory(options, dependencies = {}) { + const paths = loadPlatformPaths(); + const settings = loadDashboardSettings(paths); + const publicOrigin = options.publicOrigin ?? settings?.publicOrigin ?? null; + const secureCookies = Boolean(publicOrigin) || options.secureCookies; + const app = await startDirectoryApi({ paths, installation: loadInstallation(paths), + port: dependencies.compatibility ? settings?.port ?? options.port : options.port, address: options.host, operator: options.operator, publicOrigin, secureCookies, + installationOperator: options.installationOperator, serverFactory: dependencies.serverFactory, + onError: () => process.stderr.write('{"ok":false,"status":"directory_reconciliation_deferred"}\n') }); + const close = () => { app.close().catch(() => { process.exitCode = 1; }); }; + process.once('SIGTERM', close); process.once('SIGINT', close); + process.stdout.write(JSON.stringify({ ok: true, status: 'ready', url: publicOrigin || `http://${options.host === '::1' ? '[::1]' : options.host}:${app.server.address().port}`, + authentication: 'required', installationOperator: options.installationOperator }) + '\n'); + return 0; +} + +module.exports = { startDirectoryApi, mainDirectory }; diff --git a/core/core/api/http.js b/core/core/api/http.js new file mode 100644 index 0000000..46b6b72 --- /dev/null +++ b/core/core/api/http.js @@ -0,0 +1,226 @@ +'use strict'; +const { AccessError } = require('../accounts/src/validation'); +const SERVER_OPTIONS = Object.freeze({ maxHeaderSize: 16 * 1024, requestTimeout: 30_000, + headersTimeout: 10_000, keepAliveTimeout: 5_000 }); +const DEFAULT_SYNC_ID = 'paycom-main-workforce'; +const MAX_BODY_BYTES = 8192; +const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; +const PUBLIC_STATUS_RE = /^[a-z][a-z0-9_]{0,63}$/; +const IDEMPOTENCY_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{15,127}$/; +const SAFE_METHODS = new Set(['GET', 'HEAD']); +const LOCAL_REQUEST_ERRORS = new Set(['invalid_input', 'invalid_json', 'request_too_large']); + +function plain(value) { + return value && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function checkedPublicOrigin(value) { + if (value === null) return null; + if (typeof value !== 'string') throw new TypeError('dashboard_dependencies_required'); + let selected; + try { selected = new URL(value); } catch { throw new TypeError('dashboard_dependencies_required'); } + if (selected.protocol !== 'https:' || selected.origin !== value + || selected.pathname !== '/' || selected.username || selected.password || selected.search || selected.hash) { + throw new TypeError('dashboard_dependencies_required'); + } + return selected; +} + +function requirePublicRequest(request, publicOrigin) { + if (!publicOrigin) return; + const host = request.headers.host; + if (typeof host !== 'string' || host !== publicOrigin.host) { + throw new AccessError('request_forbidden', 403); + } + let visitor; + try { visitor = JSON.parse(request.headers['cf-visitor']); } catch { visitor = null; } + if (!plain(visitor) || Object.keys(visitor).length !== 1 || !['http', 'https'].includes(visitor.scheme)) { + throw new AccessError('request_forbidden', 403); + } + if (visitor.scheme === 'http') { + if (!SAFE_METHODS.has(request.method)) throw new AccessError('request_forbidden', 403); + const redirect = new URL(request.url, publicOrigin); + if (redirect.origin !== publicOrigin.origin) throw new AccessError('request_forbidden', 403); + return redirect.href; + } + if (!SAFE_METHODS.has(request.method) && request.headers.origin !== publicOrigin.origin) { + throw new AccessError('request_forbidden', 403); + } + return null; +} + +function boundedText(value, fallback, maximum = 120) { + const selected = value === undefined ? fallback : value; + if (typeof selected !== 'string' || selected.length < 1 || selected.length > maximum || /[\0\r\n]/.test(selected)) { + throw new TypeError('dashboard_config_invalid'); + } + return selected; +} + +function dashboardConfig(environment = process.env) { + const timezone = boundedText(environment.DISPATCH_DASHBOARD_TIMEZONE, 'America/Los_Angeles', 64); + try { new Intl.DateTimeFormat('en-US', { timeZone: timezone }).format(); } + catch { throw new TypeError('dashboard_config_invalid'); } + return Object.freeze({ + organization: Object.freeze({ + id: 'local-dsp', + name: boundedText(environment.DISPATCH_DASHBOARD_DSP_NAME, 'Example Delivery LLC'), + }), + site: Object.freeze({ + id: 'local-site', + code: boundedText(environment.DISPATCH_DASHBOARD_STATION, 'TST1', 32), + }), + timezone, + syncId: boundedText(environment.DISPATCH_DASHBOARD_SYNC_ID, DEFAULT_SYNC_ID, 64), + }); +} + +function sourceDate(now, timezone) { + const parts = new Intl.DateTimeFormat('en-CA', { + timeZone: timezone, year: 'numeric', month: '2-digit', day: '2-digit', + }).formatToParts(now); + const values = Object.fromEntries(parts.filter(part => part.type !== 'literal').map(part => [part.type, part.value])); + return `${values.year}-${values.month}-${values.day}`; +} + +function securityHeaders(contentType = null) { + return { + ...(contentType ? { 'Content-Type': contentType } : {}), + 'Content-Security-Policy': "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; connect-src 'self'; font-src 'self'; base-uri 'none'; frame-ancestors 'none'; form-action 'self'", + 'Cross-Origin-Opener-Policy': 'same-origin', + 'Cross-Origin-Resource-Policy': 'same-origin', + 'Permissions-Policy': 'camera=(), microphone=(), geolocation=(), payment=(), usb=()', + 'Referrer-Policy': 'no-referrer', + 'Strict-Transport-Security': 'max-age=31536000', + 'X-Content-Type-Options': 'nosniff', + 'X-Frame-Options': 'DENY', + }; +} + +function sendJson(response, statusCode, value, headers = {}) { + const bytes = Buffer.from(`${JSON.stringify(value)}\n`); + response.writeHead(statusCode, { + ...securityHeaders('application/json; charset=utf-8'), + 'Cache-Control': 'no-store', + 'Content-Length': bytes.length, + ...headers, + }); + response.end(bytes); +} + +async function readJson(request, maximumBytes = MAX_BODY_BYTES) { + const chunks = []; + let size = 0; + for await (const chunk of request) { + size += chunk.length; + if (size > maximumBytes) throw Object.assign(new Error('request_too_large'), { statusCode: 413 }); + chunks.push(chunk); + } + if (size === 0) return {}; + let value; + try { value = JSON.parse(Buffer.concat(chunks).toString('utf8')); } + catch { throw Object.assign(new Error('invalid_json'), { statusCode: 400 }); } + if (!plain(value)) throw Object.assign(new Error('invalid_json'), { statusCode: 400 }); + return value; +} + +function integerParameter(value, fallback, { minimum = 0, maximum = Number.MAX_SAFE_INTEGER } = {}) { + if (value === null) return fallback; + if (!/^\d+$/.test(value)) throw Object.assign(new Error('invalid_input'), { statusCode: 400 }); + const result = Number(value); + if (!Number.isSafeInteger(result) || result < minimum || result > maximum) { + throw Object.assign(new Error('invalid_input'), { statusCode: 400 }); + } + return result; +} + +function dailyQuery(searchParams) { + const allowed = new Set(['date', 'search', 'attention', 'lifecycleStatus', 'limit', 'offset', 'sort', 'direction', 'department', 'station']); + if ([...searchParams.keys()].some(key => !allowed.has(key)) || searchParams.getAll('date').length !== 1) { + throw Object.assign(new Error('invalid_input'), { statusCode: 400 }); + } + const date = searchParams.get('date'); + if (!DATE_RE.test(date || '')) throw Object.assign(new Error('invalid_input'), { statusCode: 400 }); + const optional = key => { + const values = searchParams.getAll(key); + if (values.length > 1) throw Object.assign(new Error('invalid_input'), { statusCode: 400 }); + return values.length === 0 || values[0] === '' ? undefined : values[0]; + }; + const query = { + date, + ...(optional('department') === undefined ? {} : {department:optional('department')}), + ...(optional('station') === undefined ? {} : {station:optional('station')}), + ...(optional('sort') === undefined ? {} : { sort: optional('sort') }), + ...(optional('direction') === undefined ? {} : { direction: optional('direction') }), + ...(optional('search') === undefined ? {} : { search: optional('search') }), + ...(optional('attention') === undefined ? {} : { attention: optional('attention') }), + ...(optional('lifecycleStatus') === undefined ? {} : { lifecycleStatus: optional('lifecycleStatus') }), + limit: integerParameter(optional('limit') ?? null, 100, { minimum: 1, maximum: 100 }), + offset: integerParameter(optional('offset') ?? null, 0), + }; + try { require('../../shared/contracts/src/workforce').workforceDayQuery(query); } + catch { throw new AccessError('invalid_input', 400); } + return query; +} + +function publicSdkFailure(result, fallback) { + const status = typeof result?.status === 'string' && PUBLIC_STATUS_RE.test(result.status) + ? result.status : fallback; + if (!plain(result?.error) || result.error.code !== status) { + return { code: fallback, recoverable: false }; + } + return { code: status, recoverable: result.error.recoverable === true }; +} + +function publicSdkResult(result, fallback) { + if (result?.ok === true) return result; + const error = publicSdkFailure(result, fallback); + return { contractVersion: 1, ok: false, status: error.code, error, data: null }; +} + +function publicHttpFailure(error) { + const claimed = Number.isInteger(error?.statusCode) ? error.statusCode : 500; + const statusCode = claimed >= 400 && claimed <= 599 ? claimed : 500; + // These expected availability failures contain no private diagnostic data. + // Keep every other server failure opaque, including untrusted lookalike errors. + if (statusCode === 503 && error instanceof AccessError + && ['release_worker_unavailable', 'installation_operator_disabled', 'invitation_email_unavailable', 'turnstile_unavailable', 'password_recovery_unavailable', 'password_recovery_busy'].includes(error.code)) { + return { statusCode, code: error.code }; + } + if (statusCode >= 500) return { statusCode, code: 'dashboard_unavailable' }; + if (error instanceof AccessError && typeof error.code === 'string' && PUBLIC_STATUS_RE.test(error.code)) { + return { statusCode, code: error.code }; + } + const code = typeof error?.message === 'string' && LOCAL_REQUEST_ERRORS.has(error.message) + ? error.message : 'invalid_input'; + return { statusCode, code }; +} + +function publicSyncView(result) { + if (!result?.ok || !plain(result.data)) { + const error = publicSdkFailure(result, 'sync_unavailable'); + return { ok: false, status: error.code, error, data: null }; + } + return { + ok: true, + status: result.status, + error: null, + data: { + id: result.data.id, + desiredState: result.data.desiredState, + activity: result.data.activity, + lastSucceededAt: result.data.lastSucceededAt, + nextDueAt: result.data.nextDueAt, + lastError: result.data.lastError, + businessContext: result.data.businessContext, + alerts: result.data.alerts, + activeRun: result.data.activeRun, + queuedRunCount: result.data.queuedRunCount, + queuedRequest: result.data.queuedRequest || null, + }, + }; +} + + +module.exports = { SERVER_OPTIONS, DEFAULT_SYNC_ID, IDEMPOTENCY_RE, dashboardConfig, sourceDate, dailyQuery, publicSyncView, publicSdkFailure, publicSdkResult, publicHttpFailure, integerParameter, readJson, sendJson, securityHeaders, checkedPublicOrigin, requirePublicRequest }; diff --git a/core/core/api/invitation-email.js b/core/core/api/invitation-email.js new file mode 100644 index 0000000..5dc117b --- /dev/null +++ b/core/core/api/invitation-email.js @@ -0,0 +1,345 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); + +const CLOUDFLARE_API_ORIGIN = 'https://api.cloudflare.com'; +const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/; +const ACCOUNT_ID_RE = /^[a-f0-9]{32}$/; +const TOKEN_RE = /^[A-Za-z0-9_-]{40,128}$/; +const INVITATION_TOKEN_RE = /^[A-Za-z0-9_-]{43}$/; + +function fail() { + throw Object.assign(new Error('invitation_email_config_invalid'), { + code: 'invitation_email_config_invalid', + }); +} + +function exactHttpsOrigin(value) { + if (typeof value !== 'string') fail(); + let selected; + try { selected = new URL(value); } catch { return fail(); } + if (selected.protocol !== 'https:' || selected.origin !== value + || selected.pathname !== '/' || selected.username || selected.password || selected.search || selected.hash) fail(); + return selected.origin; +} + +function readPrivateApiToken(file) { + if (typeof file !== 'string' || !path.isAbsolute(file) || path.resolve(file) !== file + || /[\0\r\n]/.test(file)) fail(); + const parent = path.dirname(file); + let parentInfo; + let before; + try { + parentInfo = fs.lstatSync(parent); + before = fs.lstatSync(file); + } catch { return fail(); } + if (!parentInfo.isDirectory() || parentInfo.isSymbolicLink() || parentInfo.uid !== process.geteuid() + || (parentInfo.mode & 0o7777) !== 0o700 || fs.realpathSync(parent) !== parent + || !before.isFile() || before.isSymbolicLink() || before.uid !== process.geteuid() + || before.nlink !== 1 || (before.mode & 0o7777) !== 0o600 || before.size < 40 || before.size > 256 + || fs.realpathSync(file) !== file) fail(); + + let descriptor; + let value; + try { + descriptor = fs.openSync(file, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)); + const opened = fs.fstatSync(descriptor); + if (!opened.isFile() || opened.uid !== process.geteuid() || opened.nlink !== 1 + || (opened.mode & 0o7777) !== 0o600 || opened.dev !== before.dev || opened.ino !== before.ino + || opened.size !== before.size) fail(); + value = fs.readFileSync(descriptor, 'utf8').replace(/\r?\n$/, ''); + const after = fs.fstatSync(descriptor); + if (after.dev !== opened.dev || after.ino !== opened.ino || after.size !== opened.size + || after.mtimeMs !== opened.mtimeMs || after.ctimeMs !== opened.ctimeMs) fail(); + } catch (error) { + if (error?.code === 'invitation_email_config_invalid') throw error; + return fail(); + } finally { + if (descriptor !== undefined) fs.closeSync(descriptor); + } + if (!TOKEN_RE.test(value)) fail(); + return value; +} + +function escapeHtml(value) { + return String(value) + .replaceAll('&', '&') + .replaceAll('<', '<') + .replaceAll('>', '>') + .replaceAll('"', '"') + .replaceAll("'", '''); +} + +function ownerInvitationMessage({ invitationUrl, expiresAt }) { + const expiry = new Intl.DateTimeFormat('en-US', { + month: 'long', day: 'numeric', year: 'numeric', hour: 'numeric', minute: '2-digit', + timeZone: 'UTC', timeZoneName: 'short', + }).format(new Date(expiresAt)); + const subject = "You're invited to Dispatch"; + const intro = 'Get started with Dispatch to set up and manage your DSP.'; + const guidance = 'Accept your invitation, then create an account or sign in. You’ll enter your DSP details during setup.'; + const expiration = `This one-time invitation expires ${expiry}.`; + const unexpected = 'Didn’t expect this invitation? You can safely ignore this email.'; + const security = 'Dispatch will never ask you to send provider credentials or passwords by email.'; + const text = [subject, '', 'DSP OWNER INVITATION', '', intro, '', guidance, '', + 'Accept invitation:', invitationUrl, '', expiration, '', unexpected, security].join('\n'); + const html = ` + + + + + ${escapeHtml(subject)} + + + +
Accept your DSP owner invitation. You’ll enter your DSP details during setup.
+ + +
+ +`; + return Object.freeze({ subject, text, html, invitationUrl }); +} + +function teamInvitationMessage({ organizationName, roleName, invitationUrl, expiresAt }) { + const date = new Date(expiresAt); + const expiryDate = new Intl.DateTimeFormat('en-US', { + month: 'short', day: 'numeric', year: 'numeric', timeZone: 'UTC', + }).format(date); + const expiryTime = new Intl.DateTimeFormat('en-US', { + hour: 'numeric', minute: '2-digit', timeZone: 'UTC', timeZoneName: 'short', + }).format(date); + const subject = "You're invited to Dispatch"; + const guidance = 'Create an account or sign in to join.'; + const expiration = `Expires ${expiryDate} · ${expiryTime}`; + const unexpected = 'Not expecting this? You can ignore this email.'; + const text = [subject, '', 'TEAM INVITATION', '', `DSP: ${organizationName}`, `Your role: ${roleName}`, + '', 'Accept invitation:', invitationUrl, '', guidance, '', expiration, '', unexpected].join('\n'); + const html = ` + + + + + ${escapeHtml(subject)} + + + +
Your team invitation is here. ${escapeHtml(guidance)}
+ + +
+ + + + +
+

Dispatch

+
 
+
+ + +
+

TEAM INVITATION

+

You’re invited.

+ + +
+

DSP

+

${escapeHtml(organizationName)}

+ + +
YOUR ROLE${escapeHtml(roleName)}
+
+
 
+ Accept invitation +
+

${escapeHtml(guidance)}

+
+

${escapeHtml(expiration)}

+

${escapeHtml(unexpected)}

+
+
+
+ +
+ +`; + return Object.freeze({ subject, text, html, invitationUrl }); +} + +function invitationMessage({ kind, recipient, organizationName, roleName, expiresAt, token, publicOrigin }) { + const owner = kind === 'organization_owner'; + if (typeof recipient !== 'string' || !EMAIL_RE.test(recipient) || recipient.length > 254 + || (!owner && (typeof organizationName !== 'string' || organizationName.length < 1 || organizationName.length > 120 + || typeof roleName !== 'string' || roleName.length < 1 || roleName.length > 64)) + || typeof expiresAt !== 'string' || !Number.isFinite(Date.parse(expiresAt)) + || typeof token !== 'string' || !INVITATION_TOKEN_RE.test(token)) fail(); + const origin = exactHttpsOrigin(publicOrigin); + const invitationUrl = `${origin}/#/invitation/${token}`; + if (owner) return ownerInvitationMessage({ invitationUrl, expiresAt }); + if (kind === 'organization_member') return teamInvitationMessage({ organizationName, roleName, invitationUrl, expiresAt }); + const expiry = new Date(expiresAt).toUTCString(); + const text = [ + "You're invited to Dispatch", + '', + `You have been invited to join ${organizationName} as ${roleName}.`, + '', + 'Accept your invitation:', + invitationUrl, + '', + `This one-time invitation expires ${expiry}.`, + 'If you were not expecting this invitation, you can safely ignore this email.', + 'Dispatch will never ask you to send provider credentials or passwords by email.', + ].join('\n'); + const html = ` + + + + +
+ + +
+

Dispatch

+

You're invited

+

You have been invited to join ${escapeHtml(organizationName)} as ${escapeHtml(roleName)}.

+

Accept invitation

+

This one-time invitation expires ${escapeHtml(expiry)}.

+

If you were not expecting this invitation, you can safely ignore this email.

+

Dispatch will never ask you to send provider credentials or passwords by email.

+
+
+ +`; + return Object.freeze({ subject: "You're invited to Dispatch", text, html, invitationUrl }); +} + +class CloudflareInvitationDelivery { + constructor({ accountId, apiToken, publicOrigin, senderAddress, fetchImpl = globalThis.fetch, timeoutMs = 10_000 } = {}) { + if (typeof accountId !== 'string' || !ACCOUNT_ID_RE.test(accountId) + || typeof apiToken !== 'string' || !TOKEN_RE.test(apiToken) + || typeof fetchImpl !== 'function' || !Number.isInteger(timeoutMs) || timeoutMs < 1_000 || timeoutMs > 30_000) fail(); + this.accountId = accountId; + this.apiToken = apiToken; + this.publicOrigin = exactHttpsOrigin(publicOrigin); + const address = senderAddress ?? `invites@${new URL(this.publicOrigin).hostname}`; + if (typeof address !== 'string' || address.length > 254 || !EMAIL_RE.test(address)) fail(); + this.from = Object.freeze({ address, name: 'Dispatch' }); + this.fetch = fetchImpl; + this.timeoutMs = timeoutMs; + } + + async send(invitation) { + const message = invitationMessage({ + kind: invitation?.kind, + recipient: invitation?.email, + organizationName: invitation?.organizationName, + roleName: invitation?.roleName, + expiresAt: invitation?.expiresAt, + token: invitation?.token, + publicOrigin: this.publicOrigin, + }); + return this.deliver(invitation.email, message); + } + + sendPasswordReset(reset) { + return this.deliver(reset.email, require('./password-recovery-email').passwordRecoveryMessage({ + token: reset.token, publicOrigin: this.publicOrigin, + })); + } + + sendPasswordResetConfirmation(reset) { + return this.deliver(reset.email, require('./password-recovery-email').passwordRecoveryMessage({ + publicOrigin: this.publicOrigin, confirmation: true, + })); + } + + async deliver(recipient, message) { + if (typeof recipient !== 'string' || !EMAIL_RE.test(recipient) || recipient.length > 254) fail(); + const payload = { + to: recipient, + from: this.from, + subject: message.subject, + html: message.html, + text: message.text, + }; + let response; + try { + response = await this.fetch( + `${CLOUDFLARE_API_ORIGIN}/client/v4/accounts/${this.accountId}/email/sending/send`, + { + method: 'POST', + redirect: 'error', + headers: { + Authorization: `Bearer ${this.apiToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + signal: AbortSignal.timeout(this.timeoutMs), + }, + ); + } catch { + return Object.freeze({ status: 'unknown' }); + } + + let body; + try { body = await response.json(); } catch { body = null; } + if (!response.ok) { + return Object.freeze({ status: response.status >= 400 && response.status < 500 ? 'failed' : 'unknown' }); + } + if (body?.success !== true || !body.result || typeof body.result !== 'object') { + return Object.freeze({ status: 'failed' }); + } + const normalized = recipient.toLowerCase(); + const recipients = key => Array.isArray(body.result[key]) + ? body.result[key].filter(value => typeof value === 'string').map(value => value.toLowerCase()) : []; + if (recipients('delivered').includes(normalized) || recipients('queued').includes(normalized)) { + return Object.freeze({ status: 'accepted' }); + } + if (recipients('permanent_bounces').includes(normalized) + || recipients('suppressed_recipients').includes(normalized)) { + return Object.freeze({ status: 'failed' }); + } + return Object.freeze({ status: 'unknown' }); + } +} + +function invitationDeliveryFromEnvironment({ environment = process.env, paths, publicOrigin, fetchImpl } = {}) { + const accountId = environment?.DISPATCH_EMAIL_ACCOUNT_ID; + if (accountId === undefined) return null; + if (!paths || typeof paths.secretsRoot !== 'string') fail(); + const apiToken = readPrivateApiToken(path.join(paths.secretsRoot, 'email', 'cloudflare-api-token')); + return new CloudflareInvitationDelivery({ accountId, apiToken, publicOrigin, + senderAddress: environment.DISPATCH_EMAIL_FROM_ADDRESS, fetchImpl }); +} + +module.exports = { + CLOUDFLARE_API_ORIGIN, + exactHttpsOrigin, + readPrivateApiToken, + invitationMessage, + CloudflareInvitationDelivery, + invitationDeliveryFromEnvironment, +}; diff --git a/core/core/api/main.js b/core/core/api/main.js new file mode 100644 index 0000000..749a542 --- /dev/null +++ b/core/core/api/main.js @@ -0,0 +1,237 @@ +'use strict'; + +const path = require('node:path'); +const fs = require('node:fs'); +const { createRuntimeAgentDispatchClient } = require('../agents/src/client'); +const { resolveLocalRuntimePaths } = require('../../shared/paths/runtime-paths'); +const { + AccessStore, + AccessControlService, + createAccessRuntimeAgentAuthorityCatalog, +} = require('../accounts/src'); +const { CoreRuntimeAgentHub, CoreRuntimeAgentControlServer } = require('../agents/src'); +const { createApiServer } = require('./server'); +const { dashboardConfig } = require('./http'); +const { createInstallationRuntimeResolver } = require('./runtime-router'); +const { invitationDeliveryFromEnvironment } = require('./invitation-email'); +const { turnstileFromEnvironment } = require('./turnstile'); + +function parseArguments(argv = process.argv.slice(2)) { + const result = { + host: '127.0.0.1', port: 4311, operator: false, installationOperator: false, + installationBackend: 'native_service_v1', secureCookies: false, publicOrigin: null, + }; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === '--operator') result.operator = true; + else if (argument === '--installation-operator') result.installationOperator = true; + else if (argument === '--installation-backend') result.installationBackend = argv[++index]; + else if (argument === '--secure-cookies') result.secureCookies = true; + else if (argument === '--public-origin') result.publicOrigin = argv[++index]; + else if (argument === '--host') result.host = argv[++index]; + else if (argument === '--port') result.port = Number(argv[++index]); + else if (argument === '--help') result.help = true; + else throw new TypeError('dashboard_argument_invalid'); + } + if (!['127.0.0.1', '::1', 'localhost'].includes(result.host) + || !['native_service_v1', 'oci_container_v1', 'directory_service_v1'].includes(result.installationBackend) + || !Number.isInteger(result.port) || result.port < 1 || result.port > 65535) { + throw new TypeError('dashboard_argument_invalid'); + } + if (result.publicOrigin !== null) { + let origin; + try { origin = new URL(result.publicOrigin); } catch { throw new TypeError('dashboard_argument_invalid'); } + if (origin.protocol !== 'https:' + || origin.origin !== result.publicOrigin || origin.username || origin.password + || origin.pathname !== '/' || origin.search || origin.hash || !result.secureCookies) { + throw new TypeError('dashboard_argument_invalid'); + } + } + return result; +} + +function usage() { + return [ + 'Usage: ./bin/dispatch-api [--port 4311] [--operator] [--installation-operator] [--installation-backend native_service_v1] [--secure-cookies] [--public-origin https://host]', + '', + 'The API binds to loopback only. Human login and DSP membership checks are always enabled.', + '--operator enables permission-gated Sync now.', + '--installation-operator enables platform provisioning requests; directory mode runs its worker with the API.', + '--installation-backend directory_service_v1 uses the local directory worker and requires private DISPATCH_PLATFORM_CONFIG.', + 'Use --secure-cookies and an exact --public-origin behind a reviewed HTTPS reverse proxy.', + ].join('\n'); +} + +async function main(argv = process.argv.slice(2), dependencies = {}) { + let options; + try { options = parseArguments(argv); } + catch { + process.stderr.write(`${usage()}\n`); + return 2; + } + if (options.help) { + process.stdout.write(`${usage()}\n`); + return 0; + } + if (options.installationBackend === 'directory_service_v1') return require('./directory-platform').mainDirectory(options, dependencies); + const client = dependencies.client || createRuntimeAgentDispatchClient({ + runtimeKey: 'unassigned', hub: { invoke: async () => { throw new Error('runtime_agent_unavailable'); } }, + }); + const config = dependencies.config || dashboardConfig(); + const paths = dependencies.paths || resolveLocalRuntimePaths(); + let accessStore = dependencies.accessStore || null; + let access = dependencies.access || null; + if (!access) { + accessStore = accessStore || new AccessStore(paths.accessControl); + access = new AccessControlService(accessStore, { + installationOperatorEnabled: options.installationOperator, + installationBackend: options.installationBackend, + }); + } + const turnstile = dependencies.turnstile === undefined + ? turnstileFromEnvironment({ paths, publicOrigin: options.publicOrigin }) + : dependencies.turnstile; + const invitationDelivery = dependencies.invitationDelivery === undefined + ? invitationDeliveryFromEnvironment({ paths, publicOrigin: options.publicOrigin }) + : dependencies.invitationDelivery; + const installationsRoot = dependencies.installationsRoot === undefined + ? (Object.hasOwn(process.env, 'DISPATCH_INSTALLATIONS_ROOT') + ? process.env.DISPATCH_INSTALLATIONS_ROOT : null) + : dependencies.installationsRoot; + if (installationsRoot !== null && (typeof installationsRoot !== 'string' + || !path.isAbsolute(installationsRoot) || installationsRoot.includes('\0'))) { + throw new TypeError('dashboard_dependencies_required'); + } + let runtimeAgentHub = dependencies.runtimeAgentHub === undefined ? null : dependencies.runtimeAgentHub; + let ownsRuntimeAgentHub = false; + if (runtimeAgentHub === null && installationsRoot !== null) { + if (!accessStore) throw new TypeError('runtime_agent_authority_store_required'); + runtimeAgentHub = new CoreRuntimeAgentHub({ + socketPath: dependencies.runtimeAgentHubSocket + || process.env.DISPATCH_RUNTIME_AGENT_HUB_SOCKET + || `${paths.runtimeRoot}/runtime-agent-hub.sock`, + authorityCatalog: createAccessRuntimeAgentAuthorityCatalog({ store: accessStore }), + collectionCapacity: process.env.DISPATCH_COLLECTION_WORKER_LIMIT === undefined ? {} : { + workers: Number(process.env.DISPATCH_COLLECTION_WORKER_LIMIT), + }, + }); + await runtimeAgentHub.start(); + ownsRuntimeAgentHub = true; + } + let runtimeAgentControl = dependencies.runtimeAgentControl === undefined ? null : dependencies.runtimeAgentControl; + let ownsRuntimeAgentControl = false; + const runtimeAgentControlSocket = dependencies.runtimeAgentControlSocket + || process.env.DISPATCH_RUNTIME_AGENT_CONTROL_SOCKET || null; + if (runtimeAgentControl === null && runtimeAgentControlSocket !== null) { + if (!runtimeAgentHub) throw new TypeError('runtime_agent_control_hub_required'); + runtimeAgentControl = new CoreRuntimeAgentControlServer({ + socketPath: runtimeAgentControlSocket, + hub: runtimeAgentHub, + }); + await runtimeAgentControl.start(); + ownsRuntimeAgentControl = true; + } + const runtimeResolver = dependencies.runtimeResolver || createInstallationRuntimeResolver({ + localClient: client, + localOrganizationId: config.organization.id, + runtimeAgentHub, + }); + const plugins = runtimeAgentHub && accessStore + ? require('../accounts/src/plugins').createPluginService({ store: accessStore, access, + backends: ['oci_container_v1', 'native_service_v1'], invoke: (key, action, input) => runtimeAgentHub.invoke(key, action, input) }) : null; + const paycomSetup = runtimeAgentHub && accessStore + ? require('../accounts/src/owner-paycom-setup').createOwnerPaycomSetup({ + store: accessStore, access, invoke: (runtimeKey, action, input) => runtimeAgentHub.invoke(runtimeKey, action, input), + }) : null; + const connections = runtimeAgentHub && accessStore + ? require('../accounts/src/owner-connections').createOwnerConnections({ + store: accessStore, access, paycomSetup, invoke: (runtimeKey, action, input) => runtimeAgentHub.invoke(runtimeKey, action, input), + }) : null; + const loadCatalogs = () => { + const releases = require('../installations/src/release-catalog') + .loadPrivateOciReleaseCatalog(process.env.DISPATCH_OCI_RELEASE_CATALOG_FILE); + const platformReleases = require('../installations/src/platform-release-catalog') + .loadPlatformReleaseCatalog(process.env.DISPATCH_PLATFORM_RELEASE_CATALOG_FILE, releases); + return { releases, platformReleases }; + }; + const updates = accessStore ? require('../accounts/src/platform-updates').createPlatformUpdates({ + store: accessStore, ...loadCatalogs(), loadCatalogs, enabled: options.installationOperator, + delivery: require('./release-delivery').createReleaseDelivery(paths.localRoot), + canaryVerifier: require('../accounts/src/rollout-canary').createCanaryVerifier(runtimeAgentHub), + }) : null; + let coreIdentity = null; + const backups = accessStore ? require('../accounts/src/platform-backups').createPlatformBackups({ + store: accessStore, enabled: options.installationOperator, + archive: require('../installations/src/backup-archive-status').backupArchiveStatus, + }) : null; + const codeRoot = path.resolve(__dirname, "../.."); + const identityFile = path.join(codeRoot, '..', 'deployment.json'); + if (/^\/opt\/dispatch-platform\/releases\/[a-z][a-z0-9_.-]{2,95}\/core-artifact\/code$/.test(codeRoot)) { + const identity = JSON.parse(fs.readFileSync(identityFile, 'utf8')); + if (codeRoot !== `/opt/dispatch-platform/releases/${identity.releaseId}/core-artifact/code` + || !/^[a-f0-9]{40}$/.test(identity.sourceCommit) || typeof identity.version !== 'string') throw new Error('core_identity_invalid'); + coreIdentity = { releaseId: identity.releaseId, version: identity.version, sourceCommit: identity.sourceCommit }; + } + const server = (dependencies.serverFactory || createApiServer)({ + client, + access, + config, + operator: options.operator, + secureCookies: options.secureCookies, + publicOrigin: options.publicOrigin, + invitationDelivery, + turnstile, + paycomSetup, + connections, + plugins, + updates, + backups, + runtimeResolver, + coreIdentity, + releasePopup: accessStore ? require('../accounts/src/release-popup').createReleasePopup({ + store: accessStore, + release: require('../accounts/src/release-popup').loadPopup(path.join(codeRoot, 'dashboard/release-popup.json'), coreIdentity), + }) : null, + coreMaintenance: require('./core-maintenance').createCoreMaintenance(paths.localRoot), + }); + const pluginTimer = plugins ? setInterval(() => plugins.runPending().catch(() => {}), 2000) : null; + pluginTimer?.unref(); + const controller = new AbortController(); + const close = () => { + if (controller.signal.aborted) return; + controller.abort(); + clearInterval(pluginTimer); + server.close(async () => { + if (ownsRuntimeAgentControl) try { await runtimeAgentControl.close(); } catch {} + if (ownsRuntimeAgentHub) try { await runtimeAgentHub.close(); } catch {} + try { await plugins?.runPending(); } catch {} + try { accessStore?.close(); } catch {} + }); + }; + process.once('SIGINT', close); + process.once('SIGTERM', close); + await new Promise((resolve, reject) => { + server.once('error', error => { + if (ownsRuntimeAgentControl) runtimeAgentControl.close().catch(() => {}); + if (ownsRuntimeAgentHub) runtimeAgentHub.close().catch(() => {}); + reject(error); + }); + server.listen(options.port, options.host, () => { + const address = server.address(); + process.stdout.write(`${JSON.stringify({ + ok: true, + status: 'ready', + url: `http://${options.host === '::1' ? '[::1]' : options.host}:${address.port}`, + operator: options.operator, + installationOperator: options.installationOperator, + authentication: 'required', + secureCookies: options.secureCookies, + publicOrigin: options.publicOrigin, + })}\n`); + resolve(); + }); + }); + return 0; +} + +module.exports = { parseArguments, usage, main }; diff --git a/core/core/api/package.json b/core/core/api/package.json new file mode 100644 index 0000000..12b629a --- /dev/null +++ b/core/core/api/package.json @@ -0,0 +1,10 @@ +{ + "name": "dispatch-api", + "version": "0.1.0", + "private": true, + "description": "Authenticated Dispatch HTTP API and backend service composition", + "type": "commonjs", + "main": "server.js", + "engines": { "node": ">=22" }, + "scripts": { "start": "../../bin/dispatch-api", "test": "node --no-warnings --test tests/*.test.js" } +} diff --git a/core/core/api/password-recovery-email.js b/core/core/api/password-recovery-email.js new file mode 100644 index 0000000..df7f4b7 --- /dev/null +++ b/core/core/api/password-recovery-email.js @@ -0,0 +1,36 @@ +'use strict'; +const { exactHttpsOrigin } = require('./invitation-email'); + +function passwordRecoveryMessage({ token, publicOrigin, confirmation = false }) { + if (publicOrigin !== exactHttpsOrigin(publicOrigin) || (!confirmation && (typeof token !== 'string' || !/^[A-Za-z0-9_-]{43}$/.test(token)))) { + throw new Error('password_recovery_email_invalid'); + } + const subject = confirmation ? 'Your Dispatch password was reset' : 'Reset your Dispatch password'; + const intro = confirmation + ? 'Your Dispatch password has been changed. All existing sessions have been signed out.' + : 'We received a request to reset your Dispatch password.'; + const guidance = confirmation + ? 'If you did not make this change, reset your password immediately and contact your Dispatch administrator.' + : 'This link can be used once and expires in 30 minutes. If you did not request a reset, you can ignore this email. Your password has not changed.'; + // Fragments are not sent in HTTP requests or Referer headers. The app sends + // this secret only in the reset POST body, never to a third-party widget. + const url = confirmation ? `${publicOrigin}/#/forgot-password` : `${publicOrigin}/#/reset-password/${token}`; + const label = confirmation ? 'Secure your account' : 'Reset password'; + return { + subject, + text: [subject, '', intro, '', label + ':', url, '', guidance, + '', 'Dispatch will never ask you to send your password by email.'].join('\n'), + html: `${subject} + +
+
+

Dispatch

+

${subject}

+

${intro}

+

${label}

+

${guidance}

+

Dispatch will never ask you to send your password by email.

+
`, + }; +} +module.exports = { passwordRecoveryMessage }; diff --git a/core/core/api/password-recovery-http.js b/core/core/api/password-recovery-http.js new file mode 100644 index 0000000..59f9cc4 --- /dev/null +++ b/core/core/api/password-recovery-http.js @@ -0,0 +1,79 @@ +'use strict'; +const { AccessError, exact, email } = require('../accounts/src'); +const { consumeRecoveryLimits } = require('../accounts/src/password-recovery'); +const GENERIC_MESSAGE = 'If an account exists for that email, we’ll send a password reset link.'; + +function createPasswordRecoveryHttp({ access, delivery, turnstile, requestAddress, clock }) { + let pending = 0; + let resetting = 0; + function ready() { + if (typeof delivery?.sendPasswordReset !== 'function' || typeof delivery?.sendPasswordResetConfirmation !== 'function') { + throw new AccessError('password_recovery_unavailable', 503); + } + if (pending >= 16) throw new AccessError('password_recovery_busy', 503); + } + function limit(request, kind) { + const window = 15 * 60 * 1000; + if (!consumeRecoveryLimits(access.store, [ + { key: `${kind}:ip:${requestAddress(request)}`, count: kind === 'request' ? 20 : 30, window }, + { key: `${kind}:global`, count: kind === 'request' ? 200 : 100, window }, + ], clock().getTime())) throw new AccessError('password_recovery_rate_limited', 429); + } + function schedule(work, reserved = false) { + if (!reserved) pending += 1; + // The HTTP response is written before any account lookup or email work. + // Bound queued/in-flight work; raw tokens only exist in process memory. + setImmediate(async () => { + try { await work(); } catch { + // Do not log email addresses, passwords, reset tokens or provider errors. + } finally { pending -= 1; } + }); + } + async function deliver(method, message) { + let status = 'unknown'; + try { status = (await delivery[method](message))?.status || 'unknown'; } catch {} + access.audit({ action: `account.password.reset.${method === 'sendPasswordReset' ? 'email' : 'notification'}.${status === 'accepted' ? 'accepted' : 'failed'}`, + targetType: 'user', targetId: message.userId }); + } + return { + async route(request, response, url, { readJson, sendJson }) { + if (!['/api/auth/forgot-password', '/api/auth/reset-password'].includes(url.pathname)) return false; + if (request.method !== 'POST') throw new AccessError('method_not_allowed', 405); + if (url.search) throw new AccessError('invalid_request', 400); + if (request.headers['sec-fetch-site'] === 'cross-site') throw new AccessError('request_forbidden', 403); + if (String(request.headers['content-type'] || '').split(';')[0].trim().toLowerCase() !== 'application/json') { + throw new AccessError('content_type_required', 415); + } + const requesting = url.pathname === '/api/auth/forgot-password'; + limit(request, requesting ? 'request' : 'reset'); + ready(); + const body = await readJson(request); + if (requesting) { + exact(body, ['email', ...(turnstile ? ['turnstileToken'] : [])]); + const selectedEmail = email(body.email); + if (turnstile) await turnstile.verify(body.turnstileToken, 'forgot_password', requestAddress(request)); + ready(); // Recheck after asynchronous verification. + sendJson(response, 202, { ok: true, status: 'accepted', data: { message: GENERIC_MESSAGE }, error: null }); + schedule(async () => { + const message = access.requestPasswordReset({ email: selectedEmail }); + if (message) await deliver('sendPasswordReset', message); + }); + } else { + // Limit expensive scrypt work even for an attacker holding a valid token. + if (resetting >= 2) throw new AccessError('password_recovery_busy', 503); + ready(); + // Reserve notification capacity before hashing yields to other requests. + pending += 1; + resetting += 1; + let result; + try { result = await access.resetPassword(body); } + catch (error) { pending -= 1; throw error; } + finally { resetting -= 1; } + schedule(() => deliver('sendPasswordResetConfirmation', result), true); + sendJson(response, 200, { ok: true, status: 'complete', data: { message: 'Your password has been reset. Sign in with your new password.' }, error: null }); + } + return true; + }, + }; +} +module.exports = { GENERIC_MESSAGE, createPasswordRecoveryHttp }; diff --git a/core/core/api/plugin-operations.js b/core/core/api/plugin-operations.js new file mode 100644 index 0000000..a0f7752 --- /dev/null +++ b/core/core/api/plugin-operations.js @@ -0,0 +1,51 @@ +'use strict'; +const { AccessError } = require('../accounts/src'); +const { requirePlugin } = require('../accounts/src/plugins'); +const { plugin } = require('../../shared/plugin-sdk/catalog'); +const { operationInput, operationOutput } = require('../../sdk/src/operations'); +const { isResult } = require('../../shared/contracts/src/result'); +const { readJson, sendJson, publicSdkResult } = require('./http'); + +function createPluginOperations({ access, accessHttp, runtimeContext, findPlugin = plugin }) { + return async (request, response, url) => { + const match = /^\/api\/plugins\/([a-z][a-z0-9-]{0,63})\/([a-z][a-z0-9_.]{0,63})$/.exec(url.pathname); + if (!match || request.method !== 'POST') return false; + const session = accessHttp.session(request); + const selected = access.organizationFor(session, 'dashboard.view'); + const definition = access.store.pluginMetadataFor ? access.store.pluginMetadataFor(selected.organization.id, match[1]) : findPlugin(match[1]); + const operation = definition?.actions.find(item => item.id === match[2]); + if (!operation) throw new AccessError('invalid_input', 400); + const context = runtimeContext(request, operation.permission); + accessHttp.requireMutation(request, context.session, url); + requirePlugin(access, context.session, definition.id); + const revision = () => access.store.db.prepare('SELECT revision FROM dsp_plugins WHERE organization_id=? AND plugin_id=?') + .get(context.organization.id, definition.id)?.revision; + const before = revision(); + const revalidate = () => { + const current = accessHttp.session(request); + const after = access.runtimeFor(current, operation.permission); + const organization = requirePlugin(access, current, definition.id); + if (organization.id !== context.organization.id) throw new AccessError('organization_forbidden', 403); + if (before !== revision() || after.installation.runtimeKey !== context.installation.runtimeKey + || after.installation.revision !== context.installation.revision) throw new AccessError('plugin_unavailable', 409); + }; + let input; + try { input = operationInput(operation, await readJson(request)); } + catch (error) { + if (error.code === 'invalid_input' || error.code === 'invalid_request') throw new AccessError('invalid_input', 400); + throw error; + } + revalidate(); + if (!context.runtime.plugins) throw new AccessError('plugin_unavailable', 503); + const result = await context.runtime.plugins.invoke(definition.id, operation.id, input); + revalidate(); + if (!isResult(result)) throw new AccessError('plugin_unavailable', 503); + if (result.ok) operationOutput(operation, result.data); + access.audit({ actorUserId: context.session.user.id, organizationId: context.organization.id, + action: 'plugin.action.request', targetType: 'plugin', targetId: definition.id, + result: result.ok ? 'succeeded' : 'denied' }); + sendJson(response, result.ok ? 200 : 409, publicSdkResult(result, 'plugin_unavailable')); + return true; + }; +} +module.exports = { createPluginOperations }; diff --git a/core/core/api/release-delivery.js b/core/core/api/release-delivery.js new file mode 100644 index 0000000..332a099 --- /dev/null +++ b/core/core/api/release-delivery.js @@ -0,0 +1,35 @@ +'use strict'; +const path=require('node:path'); +const crypto=require('node:crypto'); +const {AccessError}=require('../accounts/src/validation'); +const {privateJson,atomic}=require('../installations/src/release-delivery-files'); +const {VERSION}=require('../installations/src/release-delivery-contract'); +const {releaseNotes,loadReleaseNotes}=require('../installations/src/release-notes'); +const {loadReleaseHistory}=require('../installations/src/release-history'); +function createReleaseDelivery(localRoot) { + if(!localRoot)return null; + const root=path.join(localRoot,'config'); + function view(){ + try { + const value=privateJson(path.join(root,'release-delivery-status.json'),process.geteuid(),true); + if(!value)return null; + if(!['idle','preparing','failed','ready'].includes(value.state)||(value.version!==null&&(typeof value.version!=='string'||!VERSION.test(value.version))) + ||!Array.isArray(value.changelog)||value.changelog.length>100||typeof value.retryable!=='boolean')throw Error(); + const changelog=value.changelog.map(item=>{ + if(!['added','improved','fixed','removed','changed'].includes(item.kind)||typeof item.title!=='string'||item.title.length>160 + ||typeof item.description!=='string'||item.description.length>600)throw Error(); + return {kind:item.kind,title:item.title,description:item.description}; + }); + // Keep only closed, human-facing state; never pass daemon diagnostics or URLs through. + let notes=null; + if(value.notes)try{notes=releaseNotes(value.notes,{releaseId:`dispatch_${value.version.replace('+','_')}`,sourceCommit:value.notes.sourceCommit,changelog});}catch{} + return {state:value.state,version:value.version,changelog,...(notes?{notes}:{}),retryable:value.retryable, + message:value.state==='failed'?(value.retryable?'This update could not be prepared. Retrying automatically.':'This update needs operator review before it can be prepared.'):value.state==='preparing'?'Downloading and verifying this update…':null}; + }catch{return {state:'failed',version:null,changelog:[],retryable:false,message:'Update discovery is unavailable.'};} + } + return {view,history:()=>loadReleaseHistory(localRoot),notes:(id,release)=>loadReleaseNotes(localRoot,id,release),retry(){ + if(!view()?.retryable)throw new AccessError('update_unavailable',409); + atomic(path.join(root,'release-delivery-retry.json'),{nonce:crypto.randomBytes(16).toString('hex')}); + }}; +} +module.exports={createReleaseDelivery}; diff --git a/core/core/api/runtime-router.js b/core/core/api/runtime-router.js new file mode 100644 index 0000000..823aba3 --- /dev/null +++ b/core/core/api/runtime-router.js @@ -0,0 +1,64 @@ +'use strict'; + +const { INSTALLATION_IDENTIFIER_RE } = require('../../shared/contracts/src'); +const { createRuntimeAgentDispatchClient } = require('../agents/src'); + +function fail(code = 'runtime_boundary_violation') { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function runtimeClient(value) { + return Boolean(value?.workforce) && typeof value.workforce.day === 'function' + && Boolean(value.sync) && typeof value.sync.status === 'function' && typeof value.sync.runNow === 'function' + && Boolean(value.system) && typeof value.system.status === 'function'; +} + +function createInstallationRuntimeResolver(options = {}) { + if (!plain(options) || Object.keys(options).some(key => ![ + 'localClient', 'localOrganizationId', 'runtimeAgentHub', 'runtimeAgentClientFactory', + ].includes(key))) throw new TypeError('runtime_resolver_dependencies_required'); + const { + localClient, + localOrganizationId = 'local-dsp', + runtimeAgentHub = null, + runtimeAgentClientFactory = createRuntimeAgentDispatchClient, + } = options; + if (!runtimeClient(localClient) || typeof runtimeAgentClientFactory !== 'function' + || (runtimeAgentHub !== null && typeof runtimeAgentHub?.invoke !== 'function') + || typeof localOrganizationId !== 'string' || !INSTALLATION_IDENTIFIER_RE.test(localOrganizationId)) { + throw new TypeError('runtime_resolver_dependencies_required'); + } + const clients = new Map(); + return Object.freeze((installation, organization) => { + if (!plain(installation) || Object.keys(installation).sort().join(',') !== 'organizationId,runtimeKey,status' + || !plain(organization) || typeof organization.id !== 'string' + || installation.organizationId !== organization.id || installation.status !== 'ready' + || typeof installation.runtimeKey !== 'string' + || !INSTALLATION_IDENTIFIER_RE.test(installation.runtimeKey)) fail('runtime_identity_mismatch'); + if (installation.runtimeKey === 'local') { + if (organization.id !== localOrganizationId) fail('runtime_identity_mismatch'); + return localClient; + } + if (runtimeAgentHub === null) return null; + const existing = clients.get(installation.runtimeKey); + if (existing) { + clients.delete(installation.runtimeKey); clients.set(installation.runtimeKey, existing); + return existing; + } + const client = runtimeAgentClientFactory({ hub: runtimeAgentHub, runtimeKey: installation.runtimeKey }); + if (!runtimeClient(client)) fail(); + if (clients.size >= 64) clients.delete(clients.keys().next().value); + clients.set(installation.runtimeKey, client); + return client; + }); +} + +module.exports = { + createInstallationRuntimeResolver, + runtimeClient, +}; diff --git a/core/core/api/server.js b/core/core/api/server.js new file mode 100644 index 0000000..52684d2 --- /dev/null +++ b/core/core/api/server.js @@ -0,0 +1,213 @@ +'use strict'; +const http = require('node:http'); +const { AccessError } = require('../accounts/src'); +const { createAccessHttp } = require('./access-http'); +const { createInstallationRuntimeResolver } = require('./runtime-router'); +const { SERVER_OPTIONS, IDEMPOTENCY_RE, dashboardConfig, sourceDate, dailyQuery, publicSyncView, + publicSdkFailure, publicSdkResult, publicHttpFailure, integerParameter, readJson, + sendJson, securityHeaders, checkedPublicOrigin, requirePublicRequest } = require('./http'); +function createApiHandler({ + client, + access, + config = dashboardConfig(), + fallback = null, + operator = false, + secureCookies = false, + publicOrigin = null, + requireInvitationDelivery = Boolean(publicOrigin), + invitationDelivery = null, + turnstile = null, + paycomSetup = null, + connections = null, + updates = null, + releasePopup = null, + backups = null, + platformRuntime = null, + plugins = null, + pluginAssets = null, + dashboards = null, + runtimeResolver = null, + coreIdentity = null, coreMaintenance = () => null, + now = () => new Date(), +} = {}) { + if (!client?.workforce || typeof client.workforce.day !== 'function' + || !client?.sync || typeof client.sync.status !== 'function' || typeof client.sync.runNow !== 'function' + || !client?.system || typeof client.system.status !== 'function' + || !access || typeof access.session !== 'function' || typeof access.runtimeFor !== 'function' + || typeof coreMaintenance !== 'function' || typeof operator !== 'boolean' || typeof secureCookies !== 'boolean' + || (invitationDelivery !== null && typeof invitationDelivery?.send !== 'function') + || (runtimeResolver !== null && typeof runtimeResolver !== 'function') || typeof now !== 'function') { + throw new TypeError('dashboard_dependencies_required'); + } + const checkedOrigin = checkedPublicOrigin(publicOrigin); + if (checkedOrigin && !secureCookies) throw new TypeError('dashboard_dependencies_required'); + const accessHttp = createAccessHttp({ + access, + secureCookies, + trustCloudflareAddress: Boolean(checkedOrigin), + invitationDelivery, + turnstile, + paycomSetup, + connections, + updates, + releasePopup, + backups, + platformRuntime, + plugins, + requireInvitationDelivery, + clock: now, + }); + const resolveRuntime = runtimeResolver || createInstallationRuntimeResolver({ + localClient: client, + localOrganizationId: config.organization.id, + }); + const runtimeContext = (request, permission) => { + const session = accessHttp.session(request); + const context = access.runtimeFor(session, permission); + const runtime = resolveRuntime(context.installation, context.organization); + if (!runtime) throw new AccessError('installation_not_ready', 409); + return { ...context, runtime, session }; + }; + + const pluginOperations = require('./plugin-operations').createPluginOperations({ access, accessHttp, runtimeContext }); + + // Legacy public URLs translate into scoped SDK operations. Core never loads + // executable HTTP handlers from another repository or an installed plugin. + const pluginHandlers = [{ id: 'paycom', httpPrefixes: ['/api/paycom'] }].map(definition => { + const handler = require('./compatibility-paycom').createHandler({ + runtimeContext: (request, permission) => { + const current = accessHttp.session(request); + require('../accounts/src/plugins').requirePlugin(access, current, definition.id); + return runtimeContext(request, permission); + }, + access, accessHttp, config, now, readJson, sendJson, dailyQuery, publicSdkFailure, + publicSyncView, publicSdkResult, integerParameter, IDEMPOTENCY_RE, + }); + return { definition, handler }; + }); + + return async (request, response) => { + try { + if (typeof request.url !== 'string' || !/^\/(?!\/)[^\0\r\n\\]*$/.test(request.url)) { + throw new AccessError('request_forbidden', 403); + } + const redirect = requirePublicRequest(request, checkedOrigin); + if (redirect) { + response.writeHead(308, { + ...securityHeaders('text/plain; charset=utf-8'), + 'Cache-Control': 'no-store', + 'Content-Length': 0, + Location: redirect, + }); + response.end(); + return; + } + const url = new URL(request.url, 'http://127.0.0.1'); + if (request.method === 'GET' && url.pathname === '/api/platform/core-health' && !url.search) { + if (!coreIdentity) throw new AccessError('core_identity_unavailable', 503); + access.store.db.prepare('SELECT count(*) FROM users').get(); + const state = coreMaintenance(); + const probe = require('./core-maintenance').probeAllowed(state, request.headers['x-dispatch-recovery-probe']); + if (probe) require('../installations/src/core-database-probe').verifyCoreDatabase(access.store.db); + sendJson(response, 200, { ok: true, data: { ...coreIdentity, ...(probe ? { recoveryProbe: 'passed' } : {}) }, error: null }); + return; + } + if (coreMaintenance()) { + response.writeHead(503, { ...securityHeaders('text/plain; charset=utf-8'), 'Cache-Control': 'no-store', 'Retry-After': '10' }); + response.end('Dispatch is verifying an update. Please try again shortly.'); + return; + } + if (request.method === 'GET' && url.pathname === '/api/health' && !url.search) { + access.store.db.prepare('SELECT 1 FROM users LIMIT 1').get(); + sendJson(response, 200, { ok: true, status: 'ready', data: { service: 'dispatch-api' }, error: null }); + return; + } + if (request.method === 'GET' && url.pathname === '/api/dashboard') { + if (!dashboards || (url.search && url.search !== '?identity=1')) throw new AccessError('release_dashboard_unavailable', 503); + const current=accessHttp.session(request,{required:false}); + sendJson(response,200,{ok:true,data:dashboards(current,url.search === '?identity=1'),error:null});return; + } + if (dashboards && !['GET','HEAD','OPTIONS'].includes(request.method) && request.headers['x-dispatch-dashboard']) { + const current=accessHttp.session(request,{required:false}); + if(dashboards(current,true).digest!==request.headers['x-dispatch-dashboard']) throw new AccessError('dashboard_changed',409); + } + const pluginAsset = /^\/api\/plugin-assets\/([a-z][a-z0-9-]{0,63})\/([1-9][0-9]{0,14})$/.exec(url.pathname); + if (pluginAsset && request.method === 'GET' && !url.search) { + const session = accessHttp.session(request); + const organization = require('../accounts/src/plugins').requirePlugin(access, session, pluginAsset[1]); + const installation = access.store.installation(organization.id); + const revision = Number(pluginAsset[2]); + const current = access.store.db.prepare('SELECT revision FROM dsp_plugins WHERE organization_id=? AND plugin_id=?').get(organization.id, pluginAsset[1]); + if (current?.revision !== revision || !installation || typeof pluginAssets !== 'function') throw new AccessError('plugin_unavailable', 409); + const data = await pluginAssets({ runtimeKey: installation.runtimeKey, pluginId: pluginAsset[1], revision }); + const currentSession = accessHttp.session(request); + const after = require('../accounts/src/plugins').requirePlugin(access, currentSession, pluginAsset[1]); + if (after.id !== organization.id || access.store.installation(after.id)?.runtimeKey !== installation.runtimeKey || access.store.db.prepare('SELECT revision FROM dsp_plugins WHERE organization_id=? AND plugin_id=?').get(organization.id, pluginAsset[1])?.revision !== revision) throw new AccessError('plugin_unavailable', 409); + sendJson(response, 200, { ok: true, status: 'found', data, error: null }); return; + } + if (await accessHttp.route(request, response, url, { readJson, sendJson })) return; + if (await pluginOperations(request, response, url)) return; + for (const { definition, handler } of pluginHandlers) { + if (definition.httpPrefixes.some(prefix => url.pathname === prefix || url.pathname.startsWith(prefix + '/')) + && await handler(request, response, url)) return; + } + + if (request.method === 'GET' && url.pathname === '/api/bootstrap') { + const { runtime, organization, membership, session } = runtimeContext(request, null); + const syncResult = require('../accounts/src/plugins').available(access.store, organization.id, 'paycom') + ? await runtime.sync.status(config.syncId) : null; + const sync = publicSyncView(syncResult); + const timezone = sync.data?.businessContext?.timezone || organization.timezone; + sendJson(response, 200, { + ok: true, + status: 'ready', + data: { + organization: { id: organization.id, name: organization.name, abbreviation: organization.abbreviation }, + site: { id: `${organization.id}:${organization.stations[0].code}`, code: organization.stations[0].code }, + user: { name: session.user.name, role: membership.roleName }, + timezone, + today: sourceDate(now(), timezone), + operatorActions: operator && membership.permissions.includes('sync.run'), + csrfToken: session.csrfToken, + }, + }); + return; + } + + if (request.method === 'GET' && url.pathname === '/api/integrations') { + const { runtime, organization } = runtimeContext(request, 'integrations.read'); + const [system, syncResult] = await Promise.all([ + runtime.system.status(), + require('../accounts/src/plugins').available(access.store, organization.id, 'paycom') + ? runtime.sync.status(config.syncId) : null, + ]); + const systemError = system.ok ? null : publicSdkFailure(system, 'system_unavailable'); + sendJson(response, system.ok ? 200 : 503, { + ok: system.ok, + status: system.ok ? system.status : systemError.code, + data: system.ok ? { system: system.data, paycomSync: publicSyncView(syncResult) } : null, + error: systemError, + }); + return; + } + + if (!['GET', 'HEAD'].includes(request.method)) { + sendJson(response, 405, { ok: false, status: 'method_not_allowed', data: null, error: { code: 'method_not_allowed' } }); + return; + } + if (url.pathname.startsWith('/api/')) { + sendJson(response, 404, { ok: false, status: 'not_found', data: null, error: { code: 'not_found' } }); + return; + } + if (!fallback || !await fallback(request, response, url)) { + sendJson(response, 404, { ok: false, status: 'not_found', data: null, error: { code: 'not_found' } }); + } + } catch (error) { + const { statusCode, code } = publicHttpFailure(error); + sendJson(response, statusCode, { ok: false, status: code, data: null, error: { code } }); + } + }; +} + +function createApiServer(options) { return http.createServer(SERVER_OPTIONS, createApiHandler(options)); } +module.exports = { createApiHandler, createApiServer, SERVER_OPTIONS }; diff --git a/core/core/api/tests/fixtures/shell.cjs b/core/core/api/tests/fixtures/shell.cjs new file mode 100644 index 0000000..0ded948 --- /dev/null +++ b/core/core/api/tests/fixtures/shell.cjs @@ -0,0 +1,2 @@ +'use strict'; +require('../../../../tooling/plugin-development-shell'); diff --git a/core/core/api/tests/service.test.js b/core/core/api/tests/service.test.js new file mode 100644 index 0000000..411ac5e --- /dev/null +++ b/core/core/api/tests/service.test.js @@ -0,0 +1,93 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { fork } = require('node:child_process'); +const { once } = require('node:events'); +const path = require('node:path'); +const { createApiServer } = require('../server'); +const { fixture, enableFixturePlugin } = require('../../accounts/tests/plugin-fixture'); +const { success } = require('../../../shared/contracts/src/result'); +const { renderApiUnits } = require('../../../host/services/api-units'); + +test('dashboard restarts independently, API scope survives, and API failure leaves the UI available', async t => { + const f = await fixture(t); + for (const dsp of f.dsps) enableFixturePlugin(f.store, dsp.id); + const client = { workforce: { day() {} }, sync: { status() {}, runNow() {} }, system: { status() {} } }; + const api = createApiServer({ access: f.access, client, plugins: f.plugins, runtimeResolver: (_, org) => ({ ...client, + plugins: { invoke: async () => success('found', { organization: org.id }) } }) }); + await new Promise(resolve => api.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => api.close(resolve))); + const apiOrigin = `http://127.0.0.1:${api.address().port}`; + assert.equal((await fetch(apiOrigin + '/')).status, 404); + assert.equal((await (await fetch(apiOrigin + '/api/health')).json()).data.service, 'dispatch-api'); + const shell = async () => { + const child = fork(path.join(__dirname, 'fixtures/shell.cjs'), [apiOrigin], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }); + const ready = await Promise.race([once(child, 'message').then(([message]) => message), once(child, 'exit').then(() => { throw new Error('shell_start_failed'); })]); + assert.notEqual(ready.pid, process.pid); + const close = async () => { if (child.exitCode !== null || child.signalCode !== null) return; const exited = once(child, 'exit'); child.kill('SIGTERM'); await exited; }; + t.after(close); return { url: `http://127.0.0.1:${ready.port}`, close }; + }; + const first = await shell(); + const request = (base, dsp, body, csrf = true) => fetch(base + '/api/plugins/paycom/workforce.day', { + method: 'POST', headers: { 'content-type': 'application/json', cookie: `dispatch_session=${dsp.token}`, + ...(csrf ? { 'x-dispatch-csrf': dsp.owner.csrfToken } : {}) }, body: JSON.stringify(body), + }); + assert.equal((await fetch(first.url + '/')).status, 200); + assert.equal((await (await fetch(first.url + '/api/auth/session')).json()).data.authenticated, false); + assert.equal((await request(first.url, { token: 'invalid', owner: { csrfToken: 'invalid' } }, { query: {} })).status, 401); + assert.equal((await request(first.url, f.dsps[0], { query: {} }, false)).status, 403); + assert.equal((await request(first.url, f.dsps[0], { query: {}, dspId: f.dsps[1].runtimeKey })).status, 400); + for (const dsp of f.dsps) { + const response = await request(first.url, dsp, { query: {} }); + assert.equal(response.status, 200); assert.equal((await response.json()).data.organization, dsp.id); + } + await first.close(); + assert.equal((await request(apiOrigin, f.dsps[0], { query: {} })).status, 200); + const second = await shell(); + assert.equal((await request(second.url, f.dsps[1], { query: {} })).status, 200); + await new Promise(resolve => { api.close(resolve); api.closeAllConnections(); }); + assert.equal((await fetch(second.url + '/')).status, 200); + const unavailable = await request(second.url, f.dsps[0], { query: {} }); + assert.equal(unavailable.status, 502); assert.equal((await unavailable.json()).error.code, 'api_unavailable'); +}); + +test('split units assign private backend configuration and controller ownership only to the API', () => { + const units = renderApiUnits({ source: '/srv/dispatch/live', node: '/srv/dispatch/local/tools/node', + config: '/srv/dispatch/local/config/platform.json', uid: 1000, gid: 1000, port: 4310, apiPort: 4311, + publicOrigin: 'https://dispatch.example.test' }); + assert.match(units['dispatch-api.service'], /dispatch-api .*--installation-operator/); + const dashboard = units['dispatch-platform-local.service']; + assert.match(dashboard, /--api-origin http:\/\/127.0.0.1:4311/); + assert.doesNotMatch(dashboard, /DISPATCH_PLATFORM_CONFIG|--installation-operator|Requires=|PartOf=/); +}); + +test('revocation while reading a request prevents execution and rejected actions retain their error and audit record', async t => { + const f = await fixture(t), dsp = f.dsps[0]; + enableFixturePlugin(f.store, dsp.id); + let calls = 0, resolveContext; + const checked = new Promise(resolve => { resolveContext = resolve; }); + const runtimeFor = f.access.runtimeFor.bind(f.access); + f.access.runtimeFor = (...args) => { const value = runtimeFor(...args); resolveContext(); return value; }; + const client = { workforce: { day() {} }, sync: { status() {}, runNow() {} }, system: { status() {} }, + plugins: { invoke: async () => { calls++; return require('../../../shared/contracts/src/result').failure('entries_paused'); } } }; + const api = createApiServer({ access: f.access, client, runtimeResolver: () => client }); + await new Promise(resolve => api.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => api.close(resolve))); + const body = JSON.stringify({ query: {} }), url = `http://127.0.0.1:${api.address().port}/api/plugins/paycom/workforce.day`; + const headers = { cookie: `dispatch_session=${dsp.token}`, 'x-dispatch-csrf': dsp.owner.csrfToken, 'content-type': 'application/json', 'content-length': Buffer.byteLength(body) }; + let incoming; + const response = new Promise((resolve, reject) => { + incoming = require('node:http').request(url, { method: 'POST', headers }, result => { + result.resume(); result.once('end', () => resolve(result.statusCode)); + }); + incoming.once('error', reject); incoming.write(body.slice(0, 1)); + }); + await checked; + f.store.db.prepare("UPDATE dsp_plugins SET desired_state='disabled',revision=2 WHERE organization_id=?").run(dsp.id); + incoming.end(body.slice(1)); + assert.equal(await response, 409); assert.equal(calls, 0); + enableFixturePlugin(f.store, dsp.id); + const denied = await fetch(url, { method: 'POST', headers, body }); + assert.equal(denied.status, 409); assert.equal((await denied.json()).error.code, 'entries_paused'); + assert.equal(f.store.db.prepare("SELECT result FROM audit_events WHERE action='plugin.action.request' ORDER BY rowid DESC LIMIT 1").get().result, 'denied'); +}); diff --git a/core/core/api/turnstile.js b/core/core/api/turnstile.js new file mode 100644 index 0000000..327fe61 --- /dev/null +++ b/core/core/api/turnstile.js @@ -0,0 +1,76 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { AccessError } = require('../accounts/src'); +const { exactHttpsOrigin } = require('./invitation-email'); + +const SITEVERIFY_URL = 'https://challenges.cloudflare.com/turnstile/v0/siteverify'; +const KEY_RE = /^[A-Za-z0-9_-]{20,128}$/; + +function invalidConfig() { throw new Error('turnstile_config_invalid'); } + +function readSecret(file) { + let descriptor; + try { + const parent = path.dirname(file), directory = fs.lstatSync(parent), before = fs.lstatSync(file); + if (!directory.isDirectory() || directory.uid !== process.geteuid() + || (directory.mode & 0o7777) !== 0o700 || fs.realpathSync(parent) !== parent + || !before.isFile() || before.isSymbolicLink() || before.uid !== process.geteuid() + || before.nlink !== 1 || (before.mode & 0o7777) !== 0o600 || before.size > 256) invalidConfig(); + descriptor = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + const opened = fs.fstatSync(descriptor); + if (opened.dev !== before.dev || opened.ino !== before.ino || opened.size !== before.size + || opened.mode !== before.mode || opened.uid !== before.uid || opened.nlink !== 1) invalidConfig(); + const secret = fs.readFileSync(descriptor, 'utf8').replace(/\r?\n$/, ''); + const after = fs.fstatSync(descriptor); + if (after.size !== opened.size || after.mtimeMs !== opened.mtimeMs || after.ctimeMs !== opened.ctimeMs + || !KEY_RE.test(secret)) invalidConfig(); + return secret; + } catch { invalidConfig(); } + finally { if (descriptor !== undefined) fs.closeSync(descriptor); } +} + +function createTurnstile({ siteKey, secret, hostname, fetchImpl = globalThis.fetch, timeoutMs = 8000 }) { + if (typeof siteKey !== 'string' || !KEY_RE.test(siteKey) || typeof secret !== 'string' || !KEY_RE.test(secret) + || typeof hostname !== 'string' || !/^[a-z0-9.-]+$/.test(hostname) + || typeof fetchImpl !== 'function' || !Number.isInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 15000) invalidConfig(); + return Object.freeze({ + publicConfig: Object.freeze({ siteKey }), + async verify(token, action, remoteip) { + if (!['login', 'register', 'forgot_password'].includes(action)) invalidConfig(); + if (typeof token !== 'string' || token.length < 1 || token.length > 2048 || /\s/.test(token)) { + throw new AccessError('turnstile_required', 400); + } + let result; + try { + const response = await fetchImpl(SITEVERIFY_URL, { + method: 'POST', redirect: 'error', signal: AbortSignal.timeout(timeoutMs), + headers: { 'Content-Type': 'application/json' }, + // Never send account credentials, invitation tokens, or form contents. + body: JSON.stringify({ secret, response: token, ...(remoteip ? { remoteip } : {}) }), + }); + if (!response.ok) throw new Error('siteverify_unavailable'); + result = await response.json(); + if (typeof result?.success !== 'boolean') throw new Error('siteverify_invalid_response'); + } catch { throw new AccessError('turnstile_unavailable', 503); } + if (!result.success || result.hostname !== hostname || result.action !== action) { + throw new AccessError('turnstile_invalid', 403); + } + }, + }); +} + +function turnstileFromEnvironment({ environment = process.env, paths, publicOrigin, fetchImpl } = {}) { + const siteKey = environment.DISPATCH_TURNSTILE_SITE_KEY; + if (siteKey === undefined) return null; + try { exactHttpsOrigin(publicOrigin); } catch { invalidConfig(); } + if (!paths || typeof paths.secretsRoot !== 'string' || !path.isAbsolute(paths.secretsRoot) + || path.resolve(paths.secretsRoot) !== paths.secretsRoot) invalidConfig(); + const secret = readSecret(path.join(paths.secretsRoot, 'turnstile', 'secret-key')); + // Cloudflare's dummy keys must never activate a public installation. + if (/^[123]x/.test(siteKey) || /^[123]x/.test(secret)) invalidConfig(); + return createTurnstile({ siteKey, secret, hostname: new URL(publicOrigin).hostname, fetchImpl }); +} + +module.exports = { SITEVERIFY_URL, createTurnstile, turnstileFromEnvironment }; diff --git a/core/core/auth-broker/README.md b/core/core/auth-broker/README.md new file mode 100644 index 0000000..ad55283 --- /dev/null +++ b/core/core/auth-broker/README.md @@ -0,0 +1,38 @@ +# Core auth broker + +`server.js` runs the supervised Core plugin backend. `coordinator.js` owns DSP +worker admission, connection authorization, package generations and temporary +browser sessions. `service.js` remains the small injectable SDK facade. + +Credentials retain the existing encrypted DSP vault and key format. The vault, +key, profiles and attempt guards remain below `dsps//`; no credential database +is created in Core. `runtime/workers/authentication-server.js` mounts only the +selected DSP's auth storage and approved installed login adapters. Ordinary +plugin workers receive authenticated browser access without vault access. Auth +workers cannot read the DSP's business databases or agent registration token. +The privileged host controller remains a trusted platform authority. + +Owner Settings, connection tests and signed platform-owner DSP views use the +same connection administration protocol through the SDK framework transport. +Each request revalidates Core DSP authority. Plugin connection grants additionally +pin the DSP's acknowledged package digest and revision; declaring a service in a +manifest does not grant it automatically. Core may relay owner credentials in +memory to that DSP's worker, but does not persist them or log them. + +When another DSP is queued, idle authentication workers yield their slots even +if status polling keeps them warm. Active requests, provider verification and +plugin browser leases retain their workers. Directory DSPs enroll Paycom through +the vault worker without waking the full collection runtime; subsequent setup +waits in its existing onboarding queue when runtime capacity is occupied. + +The existing provider session, attempt-guard, native Chrome and assistance code +is reused. Paycom's adapter comes from the DSP's installed package. Cortex stays +a built-in automatic sign-in and owner-entered email-code connection. A plugin +update does not interrupt an in-progress Cortex verification: metadata and its +verification code retain the existing worker, and new logins wait for an idle +worker before switching package generations. There are no Cortex collectors. + +Disabling a plugin revokes its jobs and sessions; DSP suspension revokes all of +that DSP's workers. Credentials, guards and saved profiles survive installs, +upgrades and uninstall. Credential removal remains a separate owner operation. +See [runtime and migration](../../docs/plugin-runtime.md). diff --git a/core/core/auth-broker/coordinator.js b/core/core/auth-broker/coordinator.js new file mode 100644 index 0000000..8bc4b13 --- /dev/null +++ b/core/core/auth-broker/coordinator.js @@ -0,0 +1,201 @@ +'use strict'; +const crypto = require('node:crypto'); +const { DispatchError, boundedJson } = require('../../sdk/src/protocol'); +const { service } = require('../../shared/contracts/src/connections'); +const { validateRequest } = require('../../shared/contracts/src/auth-request'); +const fail = code => { throw new DispatchError(code, { recoverable: true }); }; +const same = (left, right) => ['dspId', 'pluginId', 'installationRevision', 'jobId'].every(key => left[key] === right[key]); + +// Core retains worker/lease references only. Vault decryption, profile state and +// provider code stay in the admitted DSP-scoped authentication worker. +class AuthenticationCoordinator { + constructor({ manager, workers, contextFor, authorizeRequest, authorizePlugin, relay, manualRetryFor = () => false, generationFor = () => '', clock = Date.now, idleMs = 5000 }) { + if (!manager || !workers || [contextFor, authorizeRequest, authorizePlugin, relay].some(value => typeof value !== 'function')) throw new TypeError('auth_coordinator_dependencies_required'); + Object.assign(this, { manager, workers, contextFor, authorizeRequest, authorizePlugin, relay, manualRetryFor, generationFor, clock, idleMs }); + this.dsps = new Map(); this.sessions = new Map(); this.closing = false; this.polling = null; + this.timer = setInterval(() => this.poll().catch(() => {}), 2000); this.timer.unref(); + } + async ensure(dspId, retainGeneration = false, signal) { + if (this.closing) fail('service_unavailable'); + if (signal?.aborted) fail('cancelled'); + let entry = this.dsps.get(dspId); + if (entry?.closing) { await entry.closing; entry = null; } + const generation = await this.generationFor(dspId); + if (entry && entry.generation !== generation && !retainGeneration) { + await entry.ready; + if (entry.requests || [...this.sessions.values()].some(session => session.entry === entry)) fail('session_busy'); + const activity = await this.workers.request(entry.row, { action: 'activity' }); + if (!activity.ok || activity.busy || entry.requests) fail('session_busy'); + await this.closeEntry(dspId, entry); + return this.ensure(dspId, retainGeneration, signal); + } + if (!entry) { + const context = await this.contextFor(dspId); + if (this.closing) fail('service_unavailable'); + if (this.dsps.has(dspId)) return this.ensure(dspId, retainGeneration, signal); + entry = { context, generation, row: null, lease: null, requests: 0, waiters: 0, controller: new AbortController(), lastUsed: this.clock(), closing: null, ready: null }; + this.dsps.set(dspId, entry); + entry.ready = this.manager.acquire(context, { connection: 'dsp-authentication', ttlMs: 300000, tabs: 6 }, { signal: entry.controller.signal }) + .then(lease => { entry.lease = lease; entry.row = this.manager.store.get(lease.leaseId); return entry; }) + .catch(error => { if (this.dsps.get(dspId) === entry) this.dsps.delete(dspId); throw error; }); + } + entry.waiters++; + let cancel; + try { + await (signal ? Promise.race([entry.ready, new Promise((_, reject) => { + cancel = () => { + if (entry.waiters === 1 && !entry.row) entry.controller.abort(); + reject(new DispatchError('cancelled')); + }; + signal.addEventListener('abort', cancel, { once: true }); if (signal.aborted) cancel(); + })]) : entry.ready); + } finally { + entry.waiters--; if (cancel) signal.removeEventListener('abort', cancel); + if (signal?.aborted && entry.waiters === 0 && !entry.row) entry.controller.abort(); + } + if (this.closing || this.manager.store.get(entry.lease.leaseId)?.state !== 'active') fail('service_unavailable'); + return entry; + } + async request(dspId, value, { signal } = {}) { + const request = validateRequest(boundedJson(value)); + if (!await this.authorizeRequest(dspId, request)) fail('permission_denied'); + if (signal?.aborted) fail('cancelled'); + // An installed adapter update must not destroy a pending Cortex email + // verification. Metadata and its owner-entered code use that existing worker; + // new logins wait until it is idle, then receive the new package generation. + const retain = ['health', 'activity', 'list', 'status', 'profile-readiness', 'providers'].includes(request.action) + || request.action === 'connections' && (request.input?.command === 'list' + || request.input?.command === 'verify' && request.input.service === 'cortex'); + const entry = await this.ensure(dspId, retain, signal); + if (entry.closing) fail('service_unavailable'); + entry.requests++; entry.lastUsed = this.clock(); + try { + if (signal?.aborted || !await this.authorizeRequest(dspId, request)) fail('permission_denied'); + const response = await this.workers.request(entry.row, request, { signal }); + if (!await this.authorizeRequest(dspId, request) || signal?.aborted) { + if (request.action === 'acquire-browser' && response?.session?.lease) await this.workers.request(entry.row, + { action: 'release-browser', lease: response.session.lease }).catch(() => {}); + fail(signal?.aborted ? 'cancelled' : 'permission_denied'); + } + return response; + } finally { entry.requests--; entry.lastUsed = this.clock(); } + } + async connectionStatus(context, connection, options) { + if (!await this.authorizePlugin(context, connection)) fail('permission_denied'); + const response = await this.request(context.dspId, { action: 'connections', input: { command: 'list' } }, options); + const value = response.items?.find(item => item.service === connection); + if (!response.ok || !value || !await this.authorizePlugin(context, connection)) fail('permission_denied'); + const states = { not_connected: 'unconfigured', connected: 'ready', checking: 'checking', + verification_required: 'verification_required', credentials_rejected: 'rejected', not_verified: 'unavailable', temporarily_unavailable: 'unavailable' }; + if (!states[value.state]) fail('invalid_response'); + return { connection, configured: value.configured, state: states[value.state] }; + } + async acquire(context, { connection, ttlMs }, options) { + if (!await this.authorizePlugin(context, connection)) fail('permission_denied'); + const selected = service(connection); + // Only Core's persisted collection job can authorize an interactive retry. + // The plugin cannot request it through connections.acquire input. + const manualRetry = connection === 'paycom' && await this.manualRetryFor(context) === true; + const response = await this.request(context.dspId, { action: 'acquire-browser', profile: selected.profile, + collector: context.pluginId, runId: context.jobId, ttlSeconds: Math.ceil(ttlMs / 1000), + ...(manualRetry ? { manualRetry: true } : {}) }, options); + if (!response.ok) fail(response.status); + const entry = this.dsps.get(context.dspId); + const leaseId = 'connection_' + crypto.randomBytes(16).toString('hex'); + let relay; + try { + if (!entry?.row || !await this.authorizePlugin(context, connection) || options?.signal?.aborted) fail('permission_denied'); + relay = await this.relay(context, entry.row, response.session.browser); + this.sessions.set(leaseId, { context: Object.freeze({ ...context }), connection, lease: response.session.lease, + entry, relay, ttlMs }); + return { leaseId, connection, ttlMs, protocol: 'cdp', endpoint: relay.endpoint, access: response.session.browser.access }; + } catch (error) { + await relay?.close(); + if (entry?.row) await this.workers.request(entry.row, { action: 'release-browser', lease: response.session.lease }); + throw error; + } + } + owned(context, id) { + const session = this.sessions.get(id); + if (!session || !same(context, session.context)) fail('lease_not_found'); + return session; + } + async renew(context, id) { + const session = this.owned(context, id); + if (!await this.authorizePlugin(context, session.connection)) { await this.release(context, id); fail('permission_denied'); } + await this.manager.renew(session.entry.context, session.entry.lease.leaseId); + const response = await this.workers.request(session.entry.row, { action: 'renew-browser', lease: session.lease, ttlSeconds: Math.ceil(session.ttlMs / 1000) }); + if (!response.ok || !await this.authorizePlugin(context, session.connection)) { await this.release(context, id); fail('lease_lost'); } + return { renewed: true, ttlMs: session.ttlMs }; + } + async release(context, id) { + const session = this.owned(context, id); + return session.releasing ||= (async () => { + await session.relay.close(); + const response = await this.workers.request(session.entry.row, { action: 'release-browser', lease: session.lease }); + if (!response.ok && response.status !== 'lease_not_found') fail('browser_cleanup_failed'); + this.sessions.delete(id); + return { released: true }; + })().catch(error => { session.releasing = null; throw error; }); + } + async closeEntry(dspId, entry) { + if (entry.closing) return entry.closing; + entry.closing = (async () => { + entry.controller.abort(); + await entry.ready.catch(() => {}); + for (const [id, session] of this.sessions) if (session.entry === entry) { + await session.relay.close(); this.sessions.delete(id); + } + if (entry.lease) await this.manager.release(entry.context, entry.lease.leaseId); + if (this.dsps.get(dspId) === entry) this.dsps.delete(dspId); + })().catch(error => { entry.closing = null; throw error; }); + return entry.closing; + } + poll() { + if (this.polling) return this.polling; + this.polling = (async () => { + for (const [dspId, entry] of [...this.dsps]) { + if (!entry.row || entry.requests || entry.waiters || entry.closing) continue; + try { + const response = await this.workers.request(entry.row, { action: 'activity' }); + if (!response.ok) throw new Error(); + if (entry.requests || entry.waiters) continue; + const leased = [...this.sessions.values()].some(session => session.entry === entry); + // Status polling can keep an otherwise idle worker warm forever. + // Give queued DSPs its slot after the worker confirms it is idle; + // in-progress sign-ins, verification and plugin leases stay intact. + const waiting = this.manager.status().queued > 0; + if (!response.busy && !leased && (waiting || this.clock() - entry.lastUsed >= this.idleMs)) { + await this.closeEntry(dspId, entry); + // Admit the queued DSP before considering another warm worker. + await this.manager.pump(); + } else await this.manager.renew(entry.context, entry.lease.leaseId); + } catch { await this.closeEntry(dspId, entry); } + } + })().finally(() => { this.polling = null; }); + return this.polling; + } + async revoke(dspId) { + const entry = this.dsps.get(dspId); + if (entry) await this.closeEntry(dspId, entry); + await this.manager.revoke(dspId); + } + async revokePlugin(dspId, pluginId) { + for (const [id, session] of this.sessions) if (session.context.dspId === dspId && session.context.pluginId === pluginId) { + await this.release(session.context, id); + } + } + handlers() { + return { + 'connections.status': (context, { connection }, options) => this.connectionStatus(context, connection, options), + 'connections.acquire': (context, input, options) => this.acquire(context, input, options), + 'connections.renew': (context, { leaseId }) => this.renew(context, leaseId), + 'connections.release': (context, { leaseId }) => this.release(context, leaseId), + }; + } + async close() { + this.closing = true; clearInterval(this.timer); await this.polling; + await Promise.all([...this.dsps].map(([id, entry]) => this.closeEntry(id, entry))); + } +} +module.exports = { AuthenticationCoordinator }; diff --git a/core/core/auth-broker/server.js b/core/core/auth-broker/server.js new file mode 100644 index 0000000..3a6bc01 --- /dev/null +++ b/core/core/auth-broker/server.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node +'use strict'; +const path = require('node:path'); +const fs = require('node:fs'); +const { loadPlatformPaths } = require('../../shared/paths/platform-paths'); +const { AccessStore } = require('../accounts/src/store'); +const { DirectoryJournal } = require('../../host/controller/journal'); +const { directoryAccessAuthority } = require('../../host/controller/access-authority'); +const { inspectDsp } = require('../../host/storage/storage'); +const { loadInstallation } = require('../../host/services/installation'); +const { privateDirectory, acquireLock } = require('../../host/controller/operations'); +const { openDatabase } = require('../../shared/published/database'); +const { openPluginBackend } = require('../plugins/backend'); +const { serveBackend } = require('../plugins/transport'); + +async function main() { + process.umask(0o077); + const paths = loadPlatformPaths(), lock = acquireLock(paths, 'plugin-backend'); + const definitions = () => require('../plugins/package-catalog').packageCatalog(paths)?.definitions() || []; + require('../../shared/plugin-sdk/catalog').configureCatalog(definitions); + require('dispatch-protocol/plugin-sdk/catalog').configureCatalog(definitions); + const databaseRoot = privateDirectory(path.join(paths.local, 'state/access-control')); + const store = new AccessStore({ databaseRoot, database: path.join(databaseRoot, 'access-control.sqlite3') }); + const journal = new DirectoryJournal(paths), authority = directoryAccessAuthority({ paths, store, journal }); + const permitted = id => { + try { + const selected = authority.context(id); + return ['active', 'pending_owner', 'setup_required'].includes(selected.organization.status) + && ['provisioning', 'waiting_for_owner', 'waiting_for_provider_auth', 'verifying', 'ready'].includes(selected.installation.status) + && !store.activeLifecycleJob(selected.organization.id) + && !store.db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(selected.organization.id); + } catch { return false; } + }; + const dspRoot = id => { + const record = journal.record(id), dsp = inspectDsp(paths, id); + if (!record || record.creationId !== dsp.creationId) throw new Error('directory_identity_mismatch'); + return dsp.root; + }; + const configuration = require('../../host/browser-assistance/runner').loadConfiguration(paths); + const queue = configuration && new (require('../../host/browser-assistance/queue').AssistanceQueue)(configuration); + let backend, server, closing; + const close = () => closing ||= (async () => { + await server?.close(); await backend?.close(); await queue?.close(); store.close(); fs.closeSync(lock); + })(); + try { + if (configuration) await require('../../host/browser-assistance/runner').reapSessions(configuration); + backend = await openPluginBackend({ paths, installation: loadInstallation(paths), store, dspRoot, permitted, + timezoneFor: id => authority.context(id).organization.timezone, + networkPolicy: require('../../host/networking/network-policy').loadNetworkPolicy(paths), + assistance: queue ? { queue, configuration } : null, + wake: id => { + const file = path.join(paths.local, 'state/execution/execution.sqlite3'); + if (!fs.existsSync(file)) return; + const db = openDatabase(file, { write: true }); + try { db.prepare('UPDATE dsp_execution SET next_wake_at=?,check_at=? WHERE runtime_key=?').run(Date.now(), Date.now(), id); } + finally { db.close(); } + } }); + server = await serveBackend({ paths, backend, dspRoot, permitted }); + for (const record of journal.all()) if (permitted(record.id)) { + try { await server.ensure(record.id); } catch { /* Unmounted DSPs are prepared by their lifecycle controller. */ } + } + process.once('SIGTERM', () => close().catch(() => { process.exitCode = 1; })); + process.once('SIGINT', () => close().catch(() => { process.exitCode = 1; })); + return { backend, server, close }; + } catch (error) { await close(); throw error; } +} +if (require.main === module) main().catch(() => { process.stderr.write('plugin_backend_start_failed\n'); process.exitCode = 1; }); +module.exports = { main }; diff --git a/core/core/auth-broker/service.js b/core/core/auth-broker/service.js new file mode 100644 index 0000000..8a2c789 --- /dev/null +++ b/core/core/auth-broker/service.js @@ -0,0 +1,22 @@ +'use strict'; +const { DispatchError, identifier, boundedJson } = require('../../sdk/src/protocol'); + +function createAuthBroker({ browserManager, connections }) { + if (!browserManager || typeof connections?.status !== 'function') throw new TypeError('auth_broker_dependencies_required'); + return Object.freeze({ handlers: Object.freeze({ + 'connections.status': async (context, { connection }, { signal }) => { + identifier(connection); + const status = await connections.status(context, connection, { signal }); + // Keep persistent coordination responses separate from credential-bearing + // adapter responses. Only these bounded fields cross the plugin API. + if (!status || typeof status.configured !== 'boolean' || !['unconfigured', 'ready', 'checking', 'verification_required', 'rejected', 'unavailable'].includes(status.state)) { + throw new DispatchError('invalid_response'); + } + return boundedJson({ connection, configured: status.configured, state: status.state }); + }, + 'connections.acquire': (context, input, options) => browserManager.acquire(context, input, options), + 'connections.renew': (context, { leaseId }) => browserManager.renew(context, leaseId), + 'connections.release': (context, { leaseId }) => browserManager.release(context, leaseId), + }) }); +} +module.exports = { createAuthBroker }; diff --git a/core/core/auth-broker/tests/coordinator.test.js b/core/core/auth-broker/tests/coordinator.test.js new file mode 100644 index 0000000..83aecc4 --- /dev/null +++ b/core/core/auth-broker/tests/coordinator.test.js @@ -0,0 +1,146 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { BrowserStore } = require('../../browser-manager/store'); +const { BrowserManager } = require('../../browser-manager/manager'); +const { AuthenticationCoordinator } = require('../coordinator'); + +test('one admitted auth worker serves a DSP; SDK leases remain bound to the originating plugin job', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-coordinator-')); + const file = path.join(root, 'state/browser.sqlite3'); + const store = new BrowserStore(file); + const started = [], stopped = [], calls = [], requests = [], relays = []; + let granted = true, generation = 'first', busy = false, manual = false; + const workers = { start: async row => { started.push(row.id); return { protocol: 'worker', endpoint: 'worker://' + row.id, access: row.id }; }, + close: async row => { stopped.push(row.id); return true; }, + request: async (_row, input) => { + calls.push(input.action); requests.push(input); + if (input.action === 'connections') return { ok: true, items: [{ service: 'paycom', configured: true, state: 'connected' }] }; + if (input.action === 'acquire-browser') return { ok: true, session: { lease: 'private-lease', browser: { protocol: 'cdp', endpoint: 'private-browser', access: 'full' } } }; + if (input.action === 'renew-browser') return { ok: true }; + if (input.action === 'release-browser') return { ok: true }; + if (input.action === 'enroll-paycom') return { ok: true, status: 'configured' }; + if (input.action === 'activity') return { ok: true, busy }; + throw new Error('unexpected worker operation'); + } }; + const manager = new BrowserManager({ store, workers, authorize: () => true, limits: { sessions: 1, tabs: 6, perDsp: 1 } }); await manager.start(); + const coordinator = new AuthenticationCoordinator({ manager, workers, idleMs: 0, generationFor: () => generation, + contextFor: async dspId => ({ dspId, pluginId: 'auth-broker', installationRevision: 1, jobId: 'auth' }), + authorizeRequest: () => true, authorizePlugin: () => granted, manualRetryFor: () => manual, + relay: async () => { const value = { endpoint: 'job-private-browser', closed: false, async close() { this.closed = true; } }; relays.push(value); return value; }, + }); + t.after(async () => { await coordinator.close(); await manager.close(); store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const context = { dspId: 'dsp_' + 'a'.repeat(32), pluginId: 'sample', installationRevision: 1, jobId: 'job-a' }; + const statuses = await Promise.all([coordinator.connectionStatus(context, 'paycom'), coordinator.connectionStatus(context, 'paycom')]); + assert.equal(started.length, 1); assert.equal(statuses[0].state, 'ready'); + await coordinator.request(context.dspId, { action: 'enroll-paycom', credentials: { password: 'never-persist-this' }, intent: 'create' }); + assert.equal(fs.readFileSync(file).includes('never-persist-this'), false); + const lease = await coordinator.acquire(context, { connection: 'paycom', ttlMs: 90000 }); + assert.equal(lease.endpoint, 'job-private-browser'); + assert.equal(requests.findLast(input => input.action === 'acquire-browser').manualRetry, undefined); + manual = true; + const manualLease = await coordinator.acquire(context, { connection: 'paycom', ttlMs: 90000 }); + assert.equal(requests.findLast(input => input.action === 'acquire-browser').manualRetry, true); + await coordinator.release(context, manualLease.leaseId); + await assert.rejects(coordinator.release({ ...context, jobId: 'job-b' }, lease.leaseId), { code: 'lease_not_found' }); + assert.equal((await coordinator.renew(context, lease.leaseId)).renewed, true); + granted = false; + await assert.rejects(coordinator.renew(context, lease.leaseId), { code: 'permission_denied' }); + assert.equal(relays[0].closed, true); + assert.ok(calls.includes('release-browser')); + generation = 'updated'; busy = true; + await coordinator.request(context.dspId, { action: 'connections', input: { command: 'verify', service: 'cortex', + verificationId: 'v'.repeat(22), code: '123456', expiresAt: Date.now() + 60000 } }); + assert.equal(started.length, 1); + await assert.rejects(coordinator.request(context.dspId, { action: 'enroll-paycom', credentials: { password: 'synthetic' }, intent: 'create' }), { code: 'session_busy' }); + busy = false; + await coordinator.request(context.dspId, { action: 'enroll-paycom', credentials: { password: 'synthetic' }, intent: 'create' }); + assert.equal(started.length, 2); assert.equal(stopped.length, 1); + const cancellation = new AbortController(); + const queued = coordinator.request('dsp_' + 'b'.repeat(32), { action: 'connections', input: { command: 'list' } }, { signal: cancellation.signal }); + const rejected = assert.rejects(queued, { code: 'cancelled' }); + await new Promise(resolve => setImmediate(resolve)); cancellation.abort(); await rejected; + await new Promise(resolve => setImmediate(resolve)); + assert.equal(manager.pending.size, 0); assert.equal(started.length, 2); + await coordinator.poll(); + assert.equal(stopped.length, 2); assert.equal(manager.status().sessions, 0); +}); + +for (const signingIn of [false, true]) test(`a third DSP can save while status polling keeps idle authentication workers warm (sign-in: ${signingIn})`, async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-fairness-')); + const store = new BrowserStore(path.join(root, 'state/browser.sqlite3')); + const ids = ['a', 'b', 'c'].map(letter => 'dsp_' + letter.repeat(32)); + const busy = new Set(signingIn ? [ids[0]] : []), stopped = [], saved = []; + const workers = { + start: async row => ({ protocol: 'worker', endpoint: 'worker://' + row.id, access: row.id }), + close: async row => { stopped.push(row.dsp_id); return true; }, + request: async (row, request) => { + if (request.action === 'activity') return { ok: true, busy: busy.has(row.dsp_id) }; + if (request.action === 'connections') return { ok: true, items: [] }; + assert.equal(request.action, 'enroll-paycom'); + saved.push(row.dsp_id); return { ok: true, status: 'configured' }; + }, + }; + const manager = new BrowserManager({ store, workers, authorize: () => true, limits: { sessions: 2, tabs: 12 } }); + await manager.start(); + const coordinator = new AuthenticationCoordinator({ manager, workers, idleMs: 60000, + contextFor: dspId => ({ dspId, pluginId: 'core-auth', installationRevision: 1, jobId: 'auth' }), + authorizeRequest: () => true, authorizePlugin: () => true, relay: () => { throw new Error('no browser needed'); }, + }); + t.after(async () => { await coordinator.close(); await manager.close(); store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + for (const id of ids.slice(0, 2)) await coordinator.request(id, { action: 'connections', input: { command: 'list' } }); + const pending = coordinator.request(ids[2], { action: 'enroll-paycom', intent: 'create', credentials: { password: 'synthetic-fair-save' } }, + { signal: AbortSignal.timeout(3000) }); + // Attach immediately so a regression's timeout is an ordinary test failure. + const outcome = pending.then(value => ({ value }), error => ({ error })); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(manager.status().queued, 1); + await coordinator.poll(); + assert.deepEqual(stopped, [ids[signingIn ? 1 : 0]], 'yield only the one idle worker needed by the queue'); + await manager.pump(); + const result = await outcome; + assert.equal(result.error, undefined); + assert.equal(result.value.status, 'configured'); + assert.deepEqual(saved, [ids[2]]); + assert.equal(manager.status().sessions, 2); + assert.equal(fs.readFileSync(path.join(root, 'state/browser.sqlite3')).includes('synthetic-fair-save'), false); +}); + +test('queued DSPs do not evict an in-flight request or an outstanding plugin browser lease', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-contention-')); + const store = new BrowserStore(path.join(root, 'state/browser.sqlite3')); + const ids = ['a', 'b'].map(letter => 'dsp_' + letter.repeat(32)), stopped = []; + let unblock, hold = false; + const workers = { + start: async row => ({ protocol: 'worker', endpoint: 'worker://' + row.id, access: row.id }), + close: async row => { stopped.push(row.dsp_id); return true; }, + request: async (_row, request) => { + if (request.action === 'activity') return { ok: true, busy: false }; + if (hold) await new Promise(resolve => { unblock = resolve; }); + return { ok: true, items: [] }; + }, + }; + const manager = new BrowserManager({ store, workers, authorize: () => true, limits: { sessions: 1, tabs: 6 } }); + await manager.start(); + const coordinator = new AuthenticationCoordinator({ manager, workers, idleMs: 60000, + contextFor: dspId => ({ dspId, pluginId: 'core-auth', installationRevision: 1, jobId: 'auth' }), + authorizeRequest: () => true, authorizePlugin: () => true, relay: () => {}, + }); + t.after(async () => { unblock?.(); coordinator.sessions.clear(); await coordinator.close(); await manager.close(); store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const list = id => coordinator.request(id, { action: 'connections', input: { command: 'list' } }); + await list(ids[0]); hold = true; + const reading = list(ids[0]); + await new Promise(resolve => setImmediate(resolve)); + const cancel = new AbortController(); + const queued = coordinator.request(ids[1], { action: 'connections', input: { command: 'list' } }, { signal: cancel.signal }); + const cancelled = assert.rejects(queued, { code: 'cancelled' }); + await new Promise(resolve => setImmediate(resolve)); + await coordinator.poll(); assert.deepEqual(stopped, []); + hold = false; unblock(); await reading; + coordinator.sessions.set('retained', { entry: coordinator.dsps.get(ids[0]) }); + await coordinator.poll(); assert.deepEqual(stopped, []); + coordinator.sessions.clear(); cancel.abort(); await cancelled; +}); diff --git a/core/core/auth-broker/tests/enrollment.test.js b/core/core/auth-broker/tests/enrollment.test.js new file mode 100644 index 0000000..3f1248b --- /dev/null +++ b/core/core/auth-broker/tests/enrollment.test.js @@ -0,0 +1,47 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { createPaycomEnrollment } = require('../../../host/controller/paycom-enrollment'); +const dsp = 'dsp_' + 'a'.repeat(32); +const input = () => ({ command: 'enroll', requestId: 'setup_' + 'b'.repeat(32), expiresAt: Date.now() + 30000, + intent: 'create', credentials: { clientCode: 'synthetic', username: 'synthetic', password: 'synthetic-secret', + pin1: 'one', pin2: 'two', pin3: 'three', pin4: 'four', pin5: 'five' } }); + +test('direct enrollment forwards only to the selected DSP and confirms the vault acknowledgement', async () => { + const calls = []; + const enroll = createPaycomEnrollment({ backend: { request: async (...args) => { + calls.push(args); return args[2].action === 'enroll-paycom' ? { ok: true, status: 'configured' } + : { ok: true, status: 'accepted', connection: { service: 'paycom', configured: true, state: 'checking', checkedAt: null, reason: null, retryAt: null } }; + } } }); + assert.deepEqual(await enroll(dsp, input()), { contractVersion: 1, ok: true, status: 'succeeded', data: { configured: true } }); + assert.equal(calls[0][0], dsp); assert.equal(calls[0][1], 'auth.request'); + assert.equal(calls[0][2].action, 'enroll-paycom'); assert.equal(calls[0][2].intent, 'create'); + assert.ok(calls[0][3].signal instanceof AbortSignal); + assert.equal((await enroll(dsp, { ...input(), expiresAt: Date.now() - 1 })).status, 'invalid_input'); + assert.equal(calls.length, 2); + assert.equal(calls[1][0], dsp); + assert.deepEqual(calls[1][2], { action: 'connections', input: { command: 'test', service: 'paycom' } }); +}); + +test('only an explicit missing profile permits create fallback for replacement', async () => { + let status = 'profile_not_configured'; const intents = []; + const enroll = createPaycomEnrollment({ backend: { request: async (_id, _op, request) => { + if (request.action !== 'enroll-paycom') throw new Error('check transport unavailable'); + intents.push(request.intent); + return request.intent === 'create' ? { ok: true, status: 'configured' } : { ok: false, status }; + } } }); + assert.equal((await enroll(dsp, { ...input(), intent: 'replace' })).ok, true); + assert.deepEqual(intents, ['replace', 'create']); + status = 'profile_locked'; intents.length = 0; + assert.equal((await enroll(dsp, { ...input(), intent: 'replace' })).status, 'profile_locked'); + assert.deepEqual(intents, ['replace']); +}); + +test('lost or invalid enrollment acknowledgements report an unconfirmed save without replay', async () => { + for (const respond of [() => { throw new Error('lost response'); }, () => ({ ok: true, status: 'unexpected' })]) { + let calls = 0; + const enroll = createPaycomEnrollment({ backend: { request: async () => { calls++; return respond(); } } }); + await assert.rejects(enroll(dsp, input()), { code: 'auth_unavailable', statusCode: 503 }); + assert.equal(calls, 1); + } +}); diff --git a/core/core/auth-broker/tests/paycom-verification.test.js b/core/core/auth-broker/tests/paycom-verification.test.js new file mode 100644 index 0000000..99b4aba --- /dev/null +++ b/core/core/auth-broker/tests/paycom-verification.test.js @@ -0,0 +1,72 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { createPaycomVerification } = require('../../../host/controller/paycom-verification'); +const { createPaycomEnrollment } = require('../../../host/controller/paycom-enrollment'); +const dsp = 'dsp_' + 'a'.repeat(32); +const blank = service => ({ service, configured: false, state: 'not_connected', checkedAt: null, reason: null, retryAt: null }); +function fixture() { + const calls = []; + let paycom = { ...blank('paycom'), configured: true, state: 'not_verified' }; + const backend = { request: async (id, operation, request) => { + assert.equal(id, dsp); assert.equal(operation, 'auth.request'); calls.push(request); + if (request.action === 'enroll-paycom') return { ok: true, status: 'configured' }; + if (request.input.command === 'list') return { ok: true, status: 'found', items: [blank('cortex'), paycom] }; + paycom = { ...paycom, state: 'checking', reason: null }; + return { ok: true, status: 'accepted', connection: paycom }; + } }; + return { calls, backend, verification: createPaycomVerification({ backend }), + set: value => { paycom = { ...paycom, ...value }; } }; +} + +test('onboarding starts an unsent check, polls it without duplicate tests, and consumes fresh evidence', async () => { + const f = fixture(); + for (let i = 0; i < 3; i++) assert.equal((await f.verification.poll(dsp)).status, 'running'); + const checkedAt = new Date().toISOString(); + f.set({ state: 'connected', checkedAt }); + assert.deepEqual((await f.verification.poll(dsp)).data, + { profileId: 'paycom-main', provider: 'paycom', status: 'authenticated', testedAt: checkedAt }); + assert.equal(f.calls.filter(call => call.input?.command === 'test').length, 1); + assert.equal(JSON.stringify(f.calls).includes('credentials'), false); +}); + +test('interrupted and stale checks recover, but rejected credentials do not trigger login loops', async () => { + for (const state of [ + { state: 'temporarily_unavailable', reason: 'check_interrupted' }, + { state: 'connected', checkedAt: '2000-01-01T00:00:00.000Z' }, + ]) { + const f = fixture(); f.set(state); + assert.equal((await f.verification.poll(dsp)).status, 'running'); + assert.equal(f.calls.filter(call => call.input?.command === 'test').length, 1); + } + const f = fixture(); f.set({ state: 'credentials_rejected', reason: 'invalid_credentials' }); + for (let i = 0; i < 3; i++) assert.equal((await f.verification.poll(dsp)).status, 'invalid_credentials'); + assert.ok(f.calls.every(call => call.input.command === 'list')); +}); + +test('a busy broker check is observed and malformed or unavailable responses never report success', async () => { + const f = fixture(); f.set({ state: 'checking' }); + const original = f.backend.request; + f.backend.request = (...args) => args[2].input.command === 'test' + ? { ok: false, status: 'session_busy' } : original(...args); + assert.equal((await f.verification.start(dsp)).state, 'checking'); + for (const response of [{ ok: false, status: 'service_unavailable' }, { ok: true, status: 'found', items: [] }]) { + f.backend.request = async () => response; + assert.equal((await f.verification.poll(dsp)).ok, false); + } +}); + +test('a lost check acknowledgement preserves the confirmed save and onboarding joins the running check', async () => { + const f = fixture(); const original = f.backend.request; + f.backend.request = async (...args) => { + const response = await original(...args); + if (args[2].input?.command === 'test') throw new Error('lost check response'); + return response; + }; + const enroll = createPaycomEnrollment({ backend: f.backend, verification: f.verification }); + assert.equal((await enroll(dsp, { command: 'enroll', requestId: 'setup_' + 'b'.repeat(32), expiresAt: Date.now() + 30000, + intent: 'create', credentials: { clientCode: 'synthetic', username: 'synthetic', password: 'synthetic-secret', + pin1: 'one', pin2: 'two', pin3: 'three', pin4: 'four', pin5: 'five' } })).ok, true); + assert.equal((await f.verification.poll(dsp)).status, 'running'); + assert.equal(f.calls.filter(call => call.input?.command === 'test').length, 1); +}); diff --git a/core/core/browser-manager/README.md b/core/core/browser-manager/README.md new file mode 100644 index 0000000..1091277 --- /dev/null +++ b/core/core/browser-manager/README.md @@ -0,0 +1,25 @@ +# Core browser manager + +The independently supervised Core plugin backend owns browser admission through +`manager.js` and `store.js`. It starts DSP-scoped authentication workers through +`host/services/authentication-worker.js`; plugins use `dispatch-sdk` connection +sessions. Browser creation, owner login checks and collection share this budget. + +The default single-host budget is two authentication workers, one per DSP, with +six page tabs per worker and twelve tabs overall. Admission is queued and bounded. +A lease occupies capacity until its worker cgroup is confirmed stopped. The +private SQLite ledger and exclusive kernel lock permit restart recovery before +new work is admitted. Idle workers exit; browser profiles remain in DSP storage. +The private CDP bridge enforces the page limit, including observed popup targets. + +Normal plugins cannot launch Chrome, select host paths, access provider passwords, +or reach another DSP. A plugin receives a temporary job-scoped Unix browser +endpoint, renewed and released by the SDK. Plugins still own navigation and +collection code. Each process has separate filesystem, PID and network namespaces +and explicit CPU, memory and task limits. No per-DSP Linux accounts are created. + +The daemon is launched by `host/services/plugin-backend.js`, independently of the +dashboard/controller. Stopping the dashboard does not stop the daemon. Core holds +references and policy; browser processes run in separate limited service cgroups. +These defaults bound resource consumption; they are not a fleet-capacity estimate +or a multi-host scheduler. See [runtime and migration](../../docs/plugin-runtime.md). diff --git a/core/core/browser-manager/manager.js b/core/core/browser-manager/manager.js new file mode 100644 index 0000000..bc9692e --- /dev/null +++ b/core/core/browser-manager/manager.js @@ -0,0 +1,177 @@ +'use strict'; +const crypto = require('node:crypto'); +const { performance } = require('node:perf_hooks'); +const { DispatchError, identifier, key } = require('../../sdk/src/protocol'); + +function contextOf(row) { + return Object.freeze({ dspId: row.dsp_id, pluginId: row.plugin_id, installationRevision: row.revision, jobId: row.job_id }); +} +function matches(row, context) { + return row && row.dsp_id === context.dspId && row.plugin_id === context.pluginId + && row.revision === context.installationRevision && row.job_id === context.jobId; +} +function error(code) { return new DispatchError(code, { recoverable: true }); } + +class BrowserManager { + constructor({ store, workers, authorize, clock = Date.now, monotonicClock = () => performance.now(), limits = {} }) { + if (!store || typeof workers?.start !== 'function' || typeof workers?.close !== 'function' + || typeof authorize !== 'function') throw new TypeError('browser_manager_dependencies_required'); + this.limits = { sessions: 2, tabs: 6, perDsp: 1, queue: 128, queueMs: 30000, startMs: 300000, ...limits }; + if (Object.keys(this.limits).sort().join(',') !== 'perDsp,queue,queueMs,sessions,startMs,tabs' + || Object.values(this.limits).some(value => !Number.isSafeInteger(value) || value < 1) + || this.limits.sessions > 64 || this.limits.tabs > 256 || this.limits.queue > 1024 + || this.limits.queueMs > 300000 || this.limits.startMs > 300000) throw new TypeError('browser_limits_invalid'); + this.store = store; this.workers = workers; this.authorize = authorize; this.clock = clock; this.monotonicClock = monotonicClock; + this.deadlines = new Map(); + this.pending = new Map(); this.handles = new Map(); this.launches = new Map(); this.closing = new Map(); + this.pumping = null; this.ready = false; this.stopped = false; this.timer = null; + } + async start() { + if (this.ready || this.stopped) throw error('service_unavailable'); + // Only an exclusively supervised owner may open this service. After restart + // no in-memory browser capability survives: reap persisted worker identities + // before new admission. A failed close continues consuming capacity. + for (const row of this.store.rows()) await this.finish(row.id, error('lease_lost')); + this.ready = true; + this.timer = setInterval(() => { this.pump().catch(() => {}); }, 1000); this.timer.unref(); + } + async acquire(context, { connection, ttlMs, tabs = 1 }, { signal } = {}) { + identifier(connection); identifier(context.pluginId); key(context.jobId); + if (!/^dsp_[a-f0-9]{32}$/.test(context.dspId) || !Number.isSafeInteger(context.installationRevision) + || context.installationRevision < 1 || !Number.isInteger(ttlMs) || ttlMs < 30000 || ttlMs > 300000 + || !Number.isInteger(tabs) || tabs < 1 || tabs > this.limits.tabs) throw error('invalid_request'); + if (!this.ready || this.stopped) throw error('service_unavailable'); + if (signal?.aborted) throw error('cancelled'); + if (!await this.authorize(context, connection)) throw error('permission_denied'); + if (signal?.aborted || this.stopped) throw error('cancelled'); + const id = `browser_${crypto.randomBytes(24).toString('hex')}`; + const now = this.clock(); + const conflict = this.store.enqueue(id, context, { connection, ttlMs, tabs }, now, now + this.limits.queueMs, this.limits.queue); + if (conflict) throw error(conflict); + this.deadlines.set(id, this.monotonicClock() + this.limits.queueMs); + let resolve, reject; + const promise = new Promise((yes, no) => { resolve = yes; reject = no; }); + const cancel = () => { this.finish(id, error('cancelled')).catch(() => {}); }; + this.pending.set(id, { resolve, reject, detach: () => signal?.removeEventListener('abort', cancel) }); + signal?.addEventListener('abort', cancel, { once: true }); + if (signal?.aborted) cancel(); + this.pump().catch(() => { this.finish(id, error('service_unavailable')).catch(() => {}); }); + return promise; + } + async pump() { + if (this.stopped || !this.ready) return; + if (this.pumping) return this.pumping; + this.pumping = (async () => { + for (const row of this.store.rows()) { + if (this.expired(row) || row.state === 'closing') { + // Cleanup may be slow; other DSPs can use unrelated remaining slots. + this.finish(row.id, error(row.state === 'queued' ? 'queue_timeout' : 'lease_lost')).catch(() => {}); + } + } + for (const row of this.store.rows().filter(item => item.state === 'queued')) { + if (this.stopped) break; + if (!await this.authorize(contextOf(row), row.connection)) { await this.finish(row.id, error('permission_denied')); continue; } + if (!this.store.claim(row.id, this.limits, this.clock())) continue; + this.deadlines.set(row.id, this.monotonicClock() + this.limits.startMs); + const controller = new AbortController(); + const launch = { controller, promise: null }; + // Install the record before dispatching asynchronous worker startup. + this.launches.set(row.id, launch); + launch.promise = Promise.resolve().then(() => this.launch(row.id, controller.signal)); + } + this.store.prune(this.clock()); + })().finally(() => { this.pumping = null; }); + return this.pumping; + } + async launch(id, signal) { + try { + const row = this.store.get(id); + const handle = await this.workers.start(row, { signal }); + if (!['cdp', 'worker'].includes(handle?.protocol) || typeof handle.endpoint !== 'string' || typeof handle.access !== 'string') throw error('invalid_response'); + this.handles.set(id, handle); + if (signal.aborted || this.stopped || this.store.get(id)?.state !== 'starting' + || !await this.authorize(contextOf(row), row.connection)) throw error('permission_denied'); + this.store.state(id, 'active', this.clock() + row.ttl_ms); + this.deadlines.set(id, this.monotonicClock() + row.ttl_ms); + const pending = this.pending.get(id); + pending?.detach(); this.pending.delete(id); + pending?.resolve({ leaseId: id, connection: row.connection, ttlMs: row.ttl_ms, + protocol: handle.protocol, endpoint: handle.endpoint, access: handle.access }); + } catch (cause) { + this.launches.delete(id); + this.finish(id, cause instanceof DispatchError ? cause : error('authentication_failed')).catch(() => {}); + } finally { this.launches.delete(id); } + } + async finish(id, reason = null) { + if (this.closing.has(id)) return this.closing.get(id); + const row = this.store.get(id); + if (!row || row.state === 'closed') return { released: true }; + const pending = this.pending.get(id); + if (pending) { pending.detach(); pending.reject(reason || error('cancelled')); this.pending.delete(id); } + this.store.state(id, 'closing'); + const launch = this.launches.get(id); launch?.controller.abort(reason); + const closing = (async () => { + // Host close is idempotent and keyed by the persisted lease id, so it can + // reap a process even if startup never returned a browser endpoint. + if (row.state !== 'queued') { + // Cancel startup before waiting, then close again after it settles. A + // late startup result must never outlive an already-reassigned slot. + if (launch) { + await this.workers.close(row); + await launch.promise; + } + const stopped = await this.workers.close(this.store.get(id)); + if (stopped !== true) throw error('browser_cleanup_failed'); + } + this.handles.delete(id); this.deadlines.delete(id); this.store.state(id, 'closed'); + return { released: true }; + })().catch(() => { throw error('browser_cleanup_failed'); }).finally(() => { this.closing.delete(id); }); + this.closing.set(id, closing); + return closing; + } + owned(context, id) { + key(id); const row = this.store.get(id); + if (!matches(row, context)) throw error('lease_not_found'); + return row; + } + expired(row) { + const deadline = this.deadlines.get(row.id); + return deadline === undefined ? row.expires_at <= this.clock() : deadline <= this.monotonicClock(); + } + async renew(context, id) { + const row = this.owned(context, id); + if (this.stopped || row.state !== 'active' || this.expired(row)) { + await this.finish(id, error('lease_lost')); throw error('lease_lost'); + } + if (!await this.authorize(context, row.connection)) { await this.finish(id, error('permission_denied')); throw error('permission_denied'); } + if (this.workers.renew) { + try { await this.workers.renew(row); } + catch { await this.finish(id, error('lease_lost')); throw error('lease_lost'); } + } + if (this.store.get(id)?.state !== 'active') throw error('lease_lost'); + if (this.expired(row) || !await this.authorize(context, row.connection)) { + await this.finish(id, error('lease_lost')); throw error('lease_lost'); + } + this.store.renew(id, this.clock()); this.deadlines.set(id, this.monotonicClock() + row.ttl_ms); + return { renewed: true, ttlMs: row.ttl_ms }; + } + async release(context, id) { this.owned(context, id); return this.finish(id); } + async revoke(dspId, pluginId = null) { + for (const row of this.store.rows().filter(item => item.dsp_id === dspId && (!pluginId || item.plugin_id === pluginId))) { + await this.finish(row.id, error('permission_denied')); + } + } + status() { + const rows = this.store.rows(); + return { sessions: rows.filter(row => ['starting', 'active', 'closing'].includes(row.state)).length, + queued: rows.filter(row => row.state === 'queued').length, closing: rows.filter(row => row.state === 'closing').length }; + } + async close() { + this.stopped = true; clearInterval(this.timer); + if (this.pumping) await this.pumping; + const results = await Promise.allSettled(this.store.rows().map(row => this.finish(row.id, error('service_unavailable')))); + await Promise.allSettled([...this.launches.values()].map(item => item.promise)); + if (results.some(item => item.status === 'rejected')) throw error('browser_cleanup_failed'); + } +} +module.exports = { BrowserManager, contextOf }; diff --git a/core/core/browser-manager/store.js b/core/core/browser-manager/store.js new file mode 100644 index 0000000..aee94fa --- /dev/null +++ b/core/core/browser-manager/store.js @@ -0,0 +1,54 @@ +'use strict'; +const { openDatabase, transaction } = require('../../shared/published/database'); +const { exclusiveLock } = require('../../shared/published/lock'); + +class BrowserStore { + constructor(file) { + this.unlock = exclusiveLock(file + '.lock'); + try { + this.db = openDatabase(file, { write: true }); + if (![0, 1].includes(this.db.prepare('PRAGMA user_version').get().user_version)) throw new Error('browser_schema_incompatible'); + this.db.exec(`CREATE TABLE IF NOT EXISTS browser_leases( + sequence INTEGER PRIMARY KEY AUTOINCREMENT,id TEXT UNIQUE NOT NULL, + dsp_id TEXT NOT NULL,plugin_id TEXT NOT NULL,revision INTEGER NOT NULL,job_id TEXT NOT NULL, + connection TEXT NOT NULL,tabs INTEGER NOT NULL,ttl_ms INTEGER NOT NULL, + state TEXT NOT NULL CHECK(state IN ('queued','starting','active','closing','closed')), + expires_at INTEGER NOT NULL,created_at INTEGER NOT NULL); + CREATE UNIQUE INDEX IF NOT EXISTS browser_profile_owner ON browser_leases(dsp_id,connection) + WHERE state<>'closed'; + CREATE INDEX IF NOT EXISTS browser_queue ON browser_leases(state,sequence); + PRAGMA user_version=1;`); + } catch (error) { this.db?.close(); this.unlock(); throw error; } + } + get(id) { return this.db.prepare('SELECT * FROM browser_leases WHERE id=?').get(id) || null; } + rows() { return this.db.prepare("SELECT * FROM browser_leases WHERE state<>'closed' ORDER BY sequence").all(); } + enqueue(id, context, input, now, expiresAt, maximum) { + return transaction(this.db, () => { + const rows = this.rows(); + if (rows.filter(row => row.state === 'queued').length >= maximum) return 'queue_full'; + if (rows.some(row => row.dsp_id === context.dspId && (row.connection === input.connection || row.state === 'queued'))) return 'session_busy'; + this.db.prepare(`INSERT INTO browser_leases(id,dsp_id,plugin_id,revision,job_id,connection,tabs,ttl_ms,state,expires_at,created_at) + VALUES(?,?,?,?,?,?,?,?,'queued',?,?)`).run(id, context.dspId, context.pluginId, context.installationRevision, + context.jobId, input.connection, input.tabs, input.ttlMs, expiresAt, now); + return null; + }); + } + claim(id, limits, now) { + return transaction(this.db, () => { + const row = this.get(id); + if (row?.state !== 'queued') return false; + const occupied = this.rows().filter(item => ['starting', 'active', 'closing'].includes(item.state)); + if (occupied.length >= limits.sessions || occupied.reduce((sum, item) => sum + item.tabs, 0) + row.tabs > limits.tabs + || occupied.filter(item => item.dsp_id === row.dsp_id).length >= limits.perDsp) return false; + return this.db.prepare("UPDATE browser_leases SET state='starting',expires_at=? WHERE id=? AND state='queued'") + .run(now + limits.startMs, id).changes === 1; + }); + } + state(id, state, expiresAt = null) { + this.db.prepare('UPDATE browser_leases SET state=?,expires_at=COALESCE(?,expires_at) WHERE id=?').run(state, expiresAt, id); + } + renew(id, now) { this.db.prepare("UPDATE browser_leases SET expires_at=?+ttl_ms WHERE id=? AND state='active'").run(now, id); } + prune(now) { this.db.prepare("DELETE FROM browser_leases WHERE state='closed' AND created_at ({ dspId: 'dsp_' + letter.repeat(32), pluginId: 'sample', installationRevision: 1, jobId }); +const request = { connection: 'sample', ttlMs: 30000 }; +const handle = { endpoint: 'ws://127.0.0.1/example', access: 'full', protocol: 'cdp' }; +const turn = () => new Promise(resolve => setImmediate(resolve)); +async function fixture(t, options = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-browser-manager-')); + let now = 100000, allowed = true; + const store = new BrowserStore(path.join(root, 'state/browser.sqlite3')); + const starts = [], stops = []; + const workers = options.workers || { async start(row) { starts.push(row.dsp_id); return handle; }, async close(row) { stops.push(row.id); return true; } }; + const manager = new BrowserManager({ store, workers, authorize: () => allowed, clock: () => now, monotonicClock: () => now, + limits: { sessions: 1, tabs: 2, ...options.limits } }); + t.after(async () => { await manager.close().catch(() => {}); store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + await manager.start(); + return { manager, store, workers, starts, stops, time: value => { now = value; }, allow: value => { allowed = value; }, root }; +} +test('separate DSPs queue fairly and released capacity becomes available', async t => { + const { manager, starts } = await fixture(t); + const a = await manager.acquire(owner('a'), request); + const b = manager.acquire(owner('b'), request); + const c = manager.acquire(owner('c'), request); + await turn(); assert.deepEqual(manager.status(), { sessions: 1, queued: 2, closing: 0 }); + await manager.release(owner('a'), a.leaseId); await manager.pump(); + const second = await b; + assert.deepEqual(starts, [owner('a').dspId, owner('b').dspId]); + await manager.release(owner('b'), second.leaseId); await manager.pump(); + await c; + assert.deepEqual(starts, [owner('a').dspId, owner('b').dspId, owner('c').dspId]); +}); +test('an SDK session runs through the bound service, auth broker and browser manager', async t => { + const { manager, stops } = await fixture(t); + const auth = createAuthBroker({ browserManager: manager, connections: { status: async () => ({ configured: true, state: 'ready', password: 'never forwarded' }) } }); + const service = createPluginService({ authorize: () => true, handlers: auth.handlers }); + const client = createDispatchClient({ transport: service.bind(owner('a')) }); + assert.deepEqual(await client.connections.status('sample'), { connection: 'sample', configured: true, state: 'ready' }); + await client.connections.withSession(request, async browser => assert.equal(browser.endpoint, handle.endpoint)); + assert.equal(manager.status().sessions, 0); assert.equal(stops.length, 1); +}); +test('leases cannot be renewed or released by another DSP, plugin, job or installation revision', async t => { + const { manager } = await fixture(t); + const lease = await manager.acquire(owner('a'), request); + for (const other of [owner('b'), { ...owner('a'), pluginId: 'other' }, owner('a', 'job_2'), { ...owner('a'), installationRevision: 2 }]) { + await assert.rejects(manager.renew(other, lease.leaseId), { code: 'lease_not_found' }); + await assert.rejects(manager.release(other, lease.leaseId), { code: 'lease_not_found' }); + } + assert.equal(manager.status().sessions, 1); +}); +test('failed browser cleanup retains capacity until cleanup succeeds', async t => { + let canClose = false; + const { manager } = await fixture(t, { workers: { start: async () => handle, close: async () => canClose } }); + const lease = await manager.acquire(owner('a'), request); + await assert.rejects(manager.release(owner('a'), lease.leaseId), { code: 'browser_cleanup_failed' }); + const waiting = manager.acquire(owner('b'), request); await turn(); + assert.deepEqual(manager.status(), { sessions: 1, queued: 1, closing: 1 }); + canClose = true; await manager.release(owner('a'), lease.leaseId); await manager.pump(); + await waiting; +}); +test('cancelled startup cannot release capacity before a late launch has stopped', async t => { + let finishStart, didStart; + const started = new Promise(resolve => { didStart = resolve; }); + const stops = []; + const { manager } = await fixture(t, { workers: { + start: () => { didStart(); return new Promise(resolve => { finishStart = resolve; }); }, + close: async row => { stops.push(row.id); return true; }, + } }); + const controller = new AbortController(); + const pending = manager.acquire(owner('a'), request, { signal: controller.signal }); + const rejected = assert.rejects(pending, { code: 'cancelled' }); + await started; controller.abort(); await rejected; await turn(); + assert.equal(manager.status().closing, 1); + finishStart(handle); await turn(); await turn(); + assert.equal(manager.status().sessions, 0); assert.ok(stops.length >= 2); +}); +test('expiry and permission revocation close browsers before further use', async t => { + const { manager, time, allow } = await fixture(t); + const a = await manager.acquire(owner('a'), request); + time(130001); + await assert.rejects(manager.renew(owner('a'), a.leaseId), { code: 'lease_lost' }); + assert.equal(manager.status().sessions, 0); + const b = await manager.acquire(owner('b'), request); allow(false); + await assert.rejects(manager.renew(owner('b'), b.leaseId), { code: 'permission_denied' }); + assert.equal(manager.status().sessions, 0); +}); +test('restart recovery closes persisted workers before admitting another request', async t => { + const { manager, store, stops } = await fixture(t); + const lease = await manager.acquire(owner('a'), request); + // Simulate lost coordinator memory while retaining its durable lease record. + clearInterval(manager.timer); manager.stopped = true; + const replacement = new BrowserManager({ store, authorize: () => true, workers: { + start: async () => handle, close: async row => { stops.push(row.id); return true; }, + } }); + await replacement.start(); + assert.ok(stops.includes(lease.leaseId)); + assert.equal(store.get(lease.leaseId).state, 'closed'); + assert.equal(replacement.status().sessions, 0); + await replacement.close(); +}); +test('a second coordinator cannot open the same lease store until the owner closes it', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-browser-lock-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const file = path.join(root, 'state/browser.sqlite3'); + const first = new BrowserStore(file); + try { assert.throws(() => new BrowserStore(file), { code: 'service_already_running' }); } + finally { first.close(); } + const replacement = new BrowserStore(file); replacement.close(); +}); +test('wall-clock rollback does not extend a browser lease', async t => { + const { manager, time } = await fixture(t); + let monotonic = 0; manager.monotonicClock = () => monotonic; + const lease = await manager.acquire(owner('a'), request); + time(1); monotonic = 30001; + await assert.rejects(manager.renew(owner('a'), lease.leaseId), { code: 'lease_lost' }); + assert.equal(manager.status().sessions, 0); +}); +test('one DSP cannot fill the queue or use a second browser for the same connection', async t => { + const { manager } = await fixture(t); + await manager.acquire(owner('a'), request); + await assert.rejects(manager.acquire(owner('a', 'job_2'), request), { code: 'session_busy' }); + const controller = new AbortController(); + const waiting = manager.acquire(owner('b'), request, { signal: controller.signal }); + const rejected = assert.rejects(waiting, { code: 'cancelled' }); + await turn(); + await assert.rejects(manager.acquire(owner('b', 'job_2'), { ...request, connection: 'other' }), { code: 'session_busy' }); + controller.abort(); await rejected; +}); diff --git a/core/core/installations/COLLECTION-CAPACITY.md b/core/core/installations/COLLECTION-CAPACITY.md new file mode 100644 index 0000000..abb834d --- /dev/null +++ b/core/core/installations/COLLECTION-CAPACITY.md @@ -0,0 +1,77 @@ +# Collection capacity and staged verification + +Every native DSP retains its own account, credentials, browser profile, database, +collection locks and runtime. Core coordinates scheduling metadata only: an opaque +job ID and a requested worker count. No Paycom credentials or collected records +enter the capacity queue. + +## Capacity control + +Native Paycom jobs acquire a grant through their local Runtime Agent status socket +and existing authenticated agent bridge before launching a collector. Core allows +one active grant and one queue position per DSP. FIFO ordering prevents a DSP +from repeatedly jumping ahead of waiting DSPs. A grant can reduce the requested +Paycom timecard concurrency (1–6) to the available worker budget. It never increases +it. Each DSP can use at most half the total budget (rounded up), leaving room +for another DSP when the budget is greater than one. Authentication/setup outside collection jobs is not governed by this budget. + +The automatic initial worker ceiling is the minimum of four, half the available +CPUs (rounded down), and one worker per GiB after reserving 2 GiB for Core and the +OS, with a minimum of one. This is a conservative starting configuration, not a +measured throughput guarantee. Set `DISPATCH_COLLECTION_WORKER_LIMIT` (integer +1–64) in the Core service environment after measuring the host. Invalid overrides +prevent Core from starting. Per-DSP systemd CPU/memory limits remain in effect. + +Clients poll while waiting and renew active grants every ten seconds. Grants and +abandoned queue entries expire after two minutes. A failed renewal cancels the +collector; ordinary release waits for child termination. Disconnects retain active +grants until expiry. After Core restarts, a two-minute recovery interval allows +surviving clients to stop before any new grants are issued. Capacity errors never +fall back to uncoordinated collection. During a mixed-version rollout only upgraded +DSPs participate in this budget. + +The existing 15-minute Paycom interval and up-to-60-second variation remain. The +variation now includes the DSP runtime identity, spreading otherwise identical +schedules. Explicit manual sync requests remain immediate requests to the queue. +The Paycom page displays waiting, collecting, retry, paused and authentication +states plus the last successful sync time. Waiting for capacity is not a collector +execution timeout and does not produce a false overdue-run alert. + +## Measure before tuning + +Run the bounded local browser probe from a trusted checkout on the host to size: + +```sh +./runtime/supervisor/scripts/measure-collection-capacity --dsps 1,2 --workers 2 --seconds 5 +``` + +It launches separate temporary browser profiles and private pipes, generates and +reads synthetic tables, and removes its processes and files afterward. It does +not use Paycom credentials, contact Paycom, or touch DSP data. JSON reports startup +latency, operation P50/P95, failures, aggregate browser RSS, minimum host free memory +and host CPU utilization. RSS sums include shared pages; host measurements can +include unrelated load. These results measure local browser capacity only. Measure +representative live collection duration/failures on designated test DSPs before +raising the production limit. Repeat with increasing DSP counts while retaining +headroom for Core, backups and authentication. + +## Verify a test DSP before the fleet + +For a user-requested release, add `--canary-organization ORGANIZATION_ID` to the +existing `dispatch-access-admin rollout-start` command. Choose an explicitly +identified disposable test DSP that is active, native, ready, connected to Paycom, +and has its workforce sync running. This option does not create a DSP or enable +its sync. The selected DSP is persisted at the front of the rollout; the existing +Core backup/update/verification stages still run first. + +After that DSP's normal lifecycle upgrade verifies its runtime and publication, +Core requests a fresh, idempotent workforce sync and requires a successful +collection after the verification start plus healthy collection storage. Other +DSPs remain queued until this check passes. Success is recorded durably in the +existing audit store and survives Core restarts. Failure, unavailable verification, +or a 30-minute timeout pauses the rollout; resolve the test DSP and use the existing +resume command to retry. Removing the selected test DSP blocks the gate. Rollouts +without a selected test DSP retain the existing sequential verification behavior. + +Adding this capability does not authorize a production rollout. Release version, +publication and rollout remain user initiated. diff --git a/core/core/installations/NATIVE-DSPS.md b/core/core/installations/NATIVE-DSPS.md new file mode 100644 index 0000000..5a3e939 --- /dev/null +++ b/core/core/installations/NATIVE-DSPS.md @@ -0,0 +1,50 @@ +# Native DSP services + +Dispatch uses one shared release of built-in DSP code. Each DSP has a separate Linux account, private data and secrets, and a systemd service. No container image, container engine, per-DSP application checkout or per-DSP dependency install is created. DSPs cannot install plugins or run custom code. + +## Creating and operating a DSP + +The dashboard invitation flow creates the DSP and owner invitation, provisions infrastructure, accepts the invitation, and collects DSP details. Those steps complete DSP onboarding. The owner can optionally connect Paycom later from the Paycom page; authentication or import failures do not disable the workspace. Core owns shared login, users, memberships, invitations, administration and the dashboard. + +Creating infrastructure allocates a `dsp-` account, private directories under `/var/lib/dispatch/tenants/`, a registration credential, one DSP service and its authenticated Runtime Agent relay. The service mounts the shared immutable package at `/opt/dispatch` and only that DSP's private files at `/var/lib/dispatch/`. CPU, memory, tasks and temporary storage are bounded. Browsers use private Unix sockets and Chrome's debugging pipe, with no tenant debugging TCP port. + +The shared Core/dashboard package lives at `/opt/dispatch-platform/releases//core-artifact/code`. DSP code, Node and Chrome live at `/opt/dispatch-runtime/releases//runtime-artifact`. The separately supervised updater continues working when Core restarts. The existing `runtime/` source directory name remains for compatibility; it does not select the deployment backend. + +Remove DSP revokes tenant sessions and access immediately, pauses queued work and backup scheduling, and stops and disables its services. The retained DSP stays in Removed with its data, completed backups, registration credentials, service definitions, and required release artifacts preserved. Backups are protected from expiration while removed. Restore DSP verifies the runtime and publication before reopening access and restoring its previous collection schedule; ordinary backup scheduling and retention resume afterward. Missed scheduled runs are not replayed. Users belong to one DSP, and platform support uses the separate viewing context. + +Permanently delete DSP is available only after removal and requires the acting Platform Owner's password. It erases local files and remote backups, removes the account and services, revokes registration credentials, and erases DSP metadata and user accounts. Platform owner accounts are preserved. Final root coordination metadata is swept only after account, files and Core identity are gone. Failures remain retryable and visible. + +Historical full-platform backups contain shared identity records and can contain the deleted DSP's data. Explicit DSP deletion erases Core/full-platform backups containing that DSP as well as its individual backups. Complete root-verified organization inventories preserve Core backups that do not contain it; older incomplete inventories remain conservative. Other DSPs' individual archives remain. See [recovery](RECOVERY.md). + +See [collection capacity and staged verification](COLLECTION-CAPACITY.md) for shared collection limits, the browser sizing probe, and optional test-DSP rollout verification. + +## One platform version + +A rollout backs up and verifies Core first, updates Core, then updates every DSP sequentially. Native DSPs still waiting for their owner or DSP details retain their setup state. DSPs without Paycom can be ready and receive updates without publication checks; connected DSPs continue to verify publication continuity. Suspended DSPs receive new code and remain stopped. A pending DSP without infrastructure receives the target release assignment. Newly created DSPs join the rollout; active provisioning/onboarding is allowed to settle before its update. + +A failed update pauses progress and uses the in-flight safety snapshot and old code for compensation. These temporary copies exist only while needed to complete or recover the rollout. The rollout is not marked complete until the root cleanup worker attests current service definitions, confirms recovery proofs, checks that no process still uses old releases, and removes obsolete Core/runtime/helper/updater/watcher releases and local recovery copies. Current code and a newer already-prepared candidate can remain. Historical rollback uses Cloudflare backups. + +## Transition from the container deployment + +Native releases require the updated release watcher. Once this branch is merged and a user-requested release is being prepared, run the [release-delivery bootstrap](RELEASES.md) from that clean merged checkout using the existing private host configuration and GitHub token. This updates discovery only; it does not select or start a rollout. The next normal rollout installs the native Core and host helpers. Existing OCI DSPs must be explicitly migrated or removed before the native fleet can complete; the current test DSP was authorized for removal rather than migration. + +The root backup configuration and decryption key must be present before rollout. The candidate's fixed `prepare-backup` entrypoint starts verified backup service setup before Core replacement. It can initialize an empty repository following the requested backup wipe. Export a private recovery kit and keep it off the VPS before relying on disaster recovery. + +Starting a native rollout rejects non-native DSPs that have not finished decommissioning before it queues the Core update. Decommissioned history does not block the transition. + +## Verification and local builds + +Use clearly named disposable test DSPs in the existing live setup for operational checks. Verify the installed release and backend first: a live host still running the container backend cannot validate native provisioning. Keep checks scoped to the test DSPs and protect existing DSPs and shared services. Do not run full-host loss, reboot or destructive recovery drills on the live host. + +CI runs repository, frontend browser and native service/package verification. It does not create a testing VM. The old [VM lab](tests/native-lab/README.md) is retained only as historical recovery-test source and is not part of the current verification workflow. + +Run `./tooling/verify` for repository tests. A native package build requires a clean checkout, Node, Python, `patchelf` and Chrome at `/opt/google/chrome`: + +```sh +./runtime/tooling/build /absolute/new-output-directory +./runtime/tooling/verify +``` + +The second command creates a temporary package and exercises two disposable service accounts, private Chrome, independent stopping and restoration of code/data/accounts/services. It requires passwordless sudo and cleans its fixtures. Legacy OCI examples are retained for compatibility investigation and are not invoked by normal build, verification or release publication. + +Native services retry startup at a fixed interval without a permanent systemd start-limit latch, so delayed Core availability is recoverable. diff --git a/core/core/installations/OVERVIEW.md b/core/core/installations/OVERVIEW.md new file mode 100644 index 0000000..c3803e3 --- /dev/null +++ b/core/core/installations/OVERVIEW.md @@ -0,0 +1,152 @@ +--- +title: Dispatch Provisioner +status: current +last_verified: 2026-09-03 +--- + +# Dispatch Provisioner + +`core/provisioner` owns the internal server-side capabilities that materialize, supervise, configure, activate, back up, restore, upgrade, suspend, and retire one isolated Dispatch runtime per DSP. Version `0.6.0` implements layout version `1`, durable provisioning job schema version `3`, service-plan version `3`, Access Control-authorized live jobs and Runtime Agent credentials, managed Paycom activation, and the Access Control-fenced lifecycle controller. It is not exposed through the public SDK, Runtime Gateway, Runtime Agent protocol, or browser API. + +Access Control is the lifecycle and organization/runtime-binding authority. Its durable provisioning request is an outbox record; the Provisioner first creates an idempotent but unrunnable job, Access Control records that exact job as current, and only then may the Provisioner persist a live-job authorization. A crash at any boundary leaves the request replayable and the installation non-ready. A successful live service job reconciles to `waiting_for_owner` or the legacy `waiting_for_provider_auth` state. For isolated DSPs, workspace reconciliation promotes the installation to `ready` and the organization to `active` once the owner is active and DSP details have been applied. Paycom authentication and first publication are optional integration work, performed later without changing workspace readiness. + +## Runtime layout + +`src/layout.js` consumes a manifest already bound to separately loaded server authority. It supports only reviewed template `isolated_dsp_v1` and derives every path beneath one preconfigured owner-private installations root. A caller cannot select a filesystem path. + +Each runtime receives distinct private `config`, component data, secrets, state, runtime socket, provider staging, logs, and backup roots. `secrets/runtime-agent` holds only that runtime's exact-mode registration token. Access Control is deliberately absent because human identity and installation routing are control-plane responsibilities. Internal path/environment projection maps the layout into the exact Auth Broker, Collection Manager, Runtime Agent, Paycom, and CDF settings without exporting managed paths through the public SDK or emitting `DISPATCH_LOCAL_ROOT` or an Access Control database root. + +The installations root must already exist as a canonical owner-owned exact-mode `0700` directory outside and not above the source checkout. Existing structural directories must also be canonical owner-owned exact-mode `0700` directories on the expected filesystem. Symlinks, mount/device changes, wrong types, special mode bits, unknown structural entries, source overlap, root replacement, authority disagreement, and unsupported templates fail closed before a partial tree is extended. + +Creation is idempotent and every mutation is read back. Layout cleanup remains a separate, non-recursive fixture capability that requires trusted failed/no-retention authority and refuses files, unknown entries, or nonempty leaves. + +## Durable jobs + +`src/job-store.js` owns schema version `3` in fixed file `provisioner.sqlite3` beneath a dedicated pre-created control-state root. The root is canonical, owner-owned, exact mode `0700`, external to the checkout, and separate from installations and service-unit roots. Database files and SQLite sidecars are owner-owned regular single-link files with exact mode `0600`. + +Startup uses WAL, full synchronization, foreign keys, bounded storage, integrity checks, complete stored-schema comparison, and root/database identity pinning. An owned exact-mode zero-byte database left before SQLite initialization is recovered transactionally. Schema version `1` is validated and migrated to version `2`; version `2` is then migrated to version `3`, which adds the Access Control acknowledgement authorization table without weakening retained fixture jobs. Unsupported or drifted schemas fail closed. + +Two immutable pipelines are readable: + +```text +installation_layout_v1 + runtime_layout_materialize + runtime_layout_verify + +installation_services_v1 + runtime_layout_materialize + runtime_layout_verify + runtime_service_render + runtime_service_validate + runtime_service_install + runtime_service_start + runtime_service_verify +``` + +New jobs use the pipeline selected by trusted server composition. Persisted jobs retain their exact pipeline ID, version, and stage snapshot. Progress derives its total from that persisted snapshot rather than the current default. + +Request creation transactionally enforces the authority-bound installation, `platform.installations.manage`, explicit operator enablement, lifecycle and revision rules, authority-scoped idempotency, and one active job per installation. A same-key/same-payload request returns the original sanitized job; a changed payload returns `idempotency_conflict`. + +Workers claim with bounded leases, a fixed forward-attempt limit, and increasing fences. Every filesystem or supervisor mutation is guarded by the current installation generation, worker, fence, status, and unexpired lease. Short unit-file operations and nonblocking systemd requests run while the claim transaction prevents a replacement fence. Blocking readiness waits happen outside that transaction; the next mutation or checkpoint revalidates authority. Checkpoint and terminal SQL repeat the claim predicates. + +Completed layout and service checkpoints are revalidated by later stages, and complete layout plus service health is read again immediately before success. A stale worker cannot render, install, reload, enable, start, stop, restore, checkpoint, cancel, fail, or succeed after reclaim. + +## Managed service plan + +`src/services.js` derives three base units and, when central Agent transport is configured, one fourth unit from the authority-bound runtime key: + +- one isolated Auth Broker; +- one isolated Collection Manager, ordered after its matching broker; +- one isolated Runtime Gateway, ordered after both matching infrastructure services. +- one outbound Runtime Agent, ordered after the matching Runtime Gateway and connected only to the server-configured central hub. + +Paycom and CDF remain Collection Manager child processes. The central dashboard and Access Control are never rendered per DSP. + +The service-unit root is trusted server configuration, not a manifest or request field. It must be a pre-created canonical owner-owned exact-mode `0700` directory, external to the checkout and the installation/control roots, and its device/inode identity is pinned. Runtime keys, unit names, executable paths, working directories, health commands, environment keys, arguments, and systemd operations are all code-owned. Issued plans are internally branded so the supervisor adapter rejects caller-fabricated plan objects. + +The renderer: + +- writes a complete candidate bundle under the runtime's private configuration root; +- permits ordinary path spaces through deterministic systemd escaping; +- rejects reserved characters and an Auth Broker Unix-socket path longer than the Linux pathname limit; +- pins executable and working-directory identity plus executable content digest; +- writes and fsyncs exact-mode `0600` files atomically; +- rejects unknown candidate entries and unsafe destination files; +- runs the fixed root-owned `systemd-analyze verify` command with bounded output and timeout; +- emits only `{servicePlanVersion,status,serviceCount,changed}` receipts. + +Units use `Restart=always`, bounded start limits, `KillMode=control-group`, `UMask=0077`, `NoNewPrivileges`, `RestrictSUIDSGID`, `LockPersonality`, native syscall architecture, and the reviewed address-family/namespace settings. A fixed root-owned `env --ignore-environment` launcher discards the user manager's inherited environment before executing the component with only the closed rendered values. The units also unset legacy local-root, Access Control, `NODE_OPTIONS`, and dynamic-loader overrides as defense in depth. No managed unit reads an environment file. + +## Installation, health, and rollback + +A private exact-mode `0600` transaction journal is written under the trusted unit root before any unit is replaced. Its opaque filename is derived from a digest rather than exposing the runtime key. The journal records the complete prior unit bytes, active state, and persistent-versus-runtime enablement scope. Candidate unit files are promoted atomically one at a time under the durable journal, followed by daemon reload, runtime enablement for the fixture gate, ordered nonblocking starts, and read-back verification. + +The user-systemd adapter accepts only a plan issued by `src/services.js` and fixed operations: snapshot, reload, enable/disable, start/stop, failed-state reset, restore, inspect, health, and bounded restart evidence. It validates the installed fragment path, exact process argument, effective UID, cgroup, environment, restart policy, kill mode, umask, enabled/active state, Auth Broker, Runtime Gateway, and Runtime Agent status-socket type/mode/path/owning file descriptor, broker/gateway/Agent health, and Collection Manager lease/status health. After positive read-back, the journal is durably marked `verified` before the job transitions to success, allowing a post-commit cleanup interruption to be distinguished from an unverified installation. A successful `systemctl start` alone is never accepted. + +Any service-stage error or cancellation enters durable compensation before rollback begins. Compensation survives worker death, uses a higher fence on reclaim, and has a separate code-owned bound of three attempts. Rollback stops and disables candidate units in reverse dependency order, restores prior unit bytes, reloads systemd, restores prior enabled/active state, and compares the resulting state with the journal. Cancellation additionally removes only the exact candidate files. Rollback never recursively removes runtime data or unknown service-root entries. + +Service-plan version `3` adds the optional supervised Runtime Agent to the version-`2` Auth Broker/Collection Manager/Runtime Gateway set. Startup and every fenced mutation validate the complete contiguous persisted checkpoint prefix; service stages also validate the current journal before any supervisor action. Retained earlier-version service job/checkpoint or journal state is rejected before a version-`3` unit mutation rather than being reinterpreted. No live managed earlier-version installation was adopted; old temporary fixtures require the explicit authorized fixture teardown before reuse. + +Forward-attempt exhaustion after service work also enters compensation before terminal failure. If rollback itself cannot be completed within its bound, the job fails as `service_installation_failed`; it never becomes ready. Terminal success retains a `verified` journal and successful compensation retains a `restored` journal as private reconciliation evidence outside the DSP layout. The next fenced render finalizes that settled journal before changing any candidate; the controller performs no unfenced post-terminal filesystem mutation. The explicit fixture teardown removes settled journals after proving no competing job exists. No failed candidate replaces prior unit bytes. + +## Access Control reconciliation boundary + +Fixture registration still requires separate server-loaded pending-fixture/no-retention authority and the `fixture_` runtime-key namespace. Live registration instead requires an Access Control-derived manifest and closed registration authority. A live job is deliberately not claimable when it is first inserted. The implemented durable handshake is: + +1. Access Control atomically validates platform authority, idempotency, revision, and lifecycle, moves the installation to `provisioning`, and writes a pending outbox request. +2. The reconciler creates or reuses one private per-runtime Agent token, stores only its digest/generation in Access Control, and never returns the token. +3. The reconciler registers the server-derived live manifest and creates or replays the matching Provisioner job. +4. Access Control atomically records that exact Provisioner job as the installation's current job and marks the outbox request `dispatched`. +5. The Provisioner persists a `live_job_authorizations` acknowledgement bound to the exact job, organization, and runtime; only then may a worker claim it. +6. Before every live host mutation, the worker reopens the Access Control authority transaction, checks the exact manifest revision/current job/runtime/organization status, executes the bounded mutation while that authority transaction is held, and checks the same authority again. +7. A terminal Provisioner job is reconciled back into Access Control. Success becomes ready after the owner and DSP details are complete; otherwise it waits for those steps. Failure remains non-ready with a sanitized code. + +`src/installation-provisioning.js` owns the Access Control side of that protocol. Replaying after a crash at any numbered boundary converges on the same request and job. The Provisioner database remains an executor journal rather than a competing installation registry. No browser or public SDK route can call the reconciler or choose its manifest, runtime, roots, units, or worker identity. + +## Provider activation + +`src/managed-paycom.js` renders the reviewed Paycom Collection Manager definition from the authoritative DSP timezone and fixed project executable. Its profile, source, plans, dependency graph, sync definition, and first request (`paycom-main`, `current`, `full`) are code-owned. The project-root ownership/write chain and code-owned Paycom release-tree, specification, and executable digests bind activation to the reviewed release. The definition contains no credentials, and activation requires exact read-back with no unknown configured collector/source/plan/sync entries. + +`src/managed-auth-setup.js` binds the existing Auth Broker setup workflow to one authoritative managed layout and exact configured service plan, including the Runtime Agent when central transport is enabled. It requires literal `create` or `replace` intent and acquires a durable Access Control setup lease before mutation. That lease blocks activation and concurrent setup; every fixed service, vault, and credential-ingress mutation revalidates and renews it, while an expired worker cannot continue. The workflow stops and reads back that runtime's service set, invokes the existing `/dev/tty` credential helper with explicit managed component roots, then restarts and verifies the same service set before releasing the lease. It does not test the provider; `activate` owns the single bounded Auth Broker authentication test. Credentials never enter an argument, environment variable, Access Control, Provisioner state, event, receipt, or routine output. CAPTCHA, MFA, lockout, unknown layouts, ambiguous submissions, and terminal provider states remain human-review stops enforced by the Auth Broker/provider adapter. + +`src/activation.js`, `src/managed-activation-runtime.js`, and `src/managed-activation-evidence.js` then perform one idempotent activation. They recheck layout, exact services, Auth Broker, Collection Manager, gateway identity, and provider evidence; apply and attest the fixed definition; run/reuse the job-bound `paycom-periods` baseline; enqueue/reuse the manager-owned current/full five-plan batch; and heartbeat the Access Control fence while polling. A stale worker exits without cancelling shared work. Only the current lease holder cancels/drains exact work at its deadline, leaving activation `verifying` if terminal drain cannot be proven. A fixed catalog-hashed Paycom evidence helper opens the manager and Paycom stores read-only; the Core adapter invokes it through a bounded no-shell stdin/stdout contract. The helper binds the current preparatory run, exact batch verification runs, immutable publication-origin runs, active publication IDs, and content digests while proving the selected target is present in the current pay-period baseline. Access Control persists the closed evidence bundle and its digest atomically with activation-job success, installation `ready`, and organization `active`. A thrown error, failed publication, expired claim, stale worker, or mismatched identity cannot commit readiness. + +The server-owner-local entrypoints are `dispatch-managed-activation`, `dispatch-installation-reconcile`, `dispatch-installation-lifecycle`, and `dispatch-runtime-agent-authority`. They require fixed absolute control roots in server/operator configuration and accept no secret, command, endpoint, runtime key, unit name, or provider response from a browser. The private platform console can append fixed provisioning intent and change the organization access overlay; the reconciler derives any resulting fixed suspend/resume work. HTTP never invokes these entrypoints or receives Provisioner identity. + +## Managed lifecycle + +`src/backups.js` snapshots only the fixed data/state roots while that runtime's services are stopped. It leaves the Auth Broker master key in the separate secrets root, rejects unsafe filesystem entries, writes a private relative-entry/hash manifest, and atomically promotes one server-generated backup directory. Suspended-only restore creates a safety backup and swaps the fixed roots with interruption rollback. `src/lifecycle.js` sequences these adapters through the current Access Control lifecycle lease/fence. + +Ready-runtime backup, upgrade, suspension, and decommission first record the managed sync intent, stop it with drain enabled, and prove the manager queue quiescent. Upgrade resolves a different release from an owner-private server catalog, takes a backup, uses the unit transaction journal, verifies target service health and unchanged publication evidence, restores prior sync intent, then advances Access Control's release and manifest revision. Failure restores and verifies prior unit/service/schedule state unless organization suspension now requires the runtime to remain stopped. Resume revalidates infrastructure/publication evidence without authentication, collection, or publication before restoring prior sync intent. Decommission stops and disables units while retaining service definitions, runtime data, and existing backups; it creates no final backup. Removed DSPs can resume their saved configuration and schedules after runtime verification. Permanent destruction requires prior removal and the acting administrator’s password in the platform console, or an approved local command, with failed-attempt retry allowed only through the linked destruction job and strict safe-tree removal. + +The entrypoint `dispatch-installation-lifecycle` is server-owner-local. It accepts only closed operation arguments and derives runtime, roots, units, releases, backup binding, and job stages from Access Control/server configuration. `dispatch-installation-reconcile` also resumes queued/expired lifecycle jobs and converges organization suspension/resumption intent. + +## Boundary and receipts + +Provisioning-request projections contain only request status, job ID, installation/manifest revisions, replay state, and an allowlisted failure. Public-style job and activation projections are likewise closed and aggregate. Manifests, organization/runtime identities, paths, unit names, commands, environments, PIDs, cgroups, sockets, journals, workers, fences, checkpoint bodies, SQLite details, publication IDs/targets, provider responses, stdout/stderr, and exceptions remain internal. + +Directory ownership and a user service manager isolate normal operation from other Unix users, not malicious code running as the same account. Stronger per-runtime operating-system isolation remains required before a production second-DSP pilot. + +## Verification + +Credential-free package verification: + +```bash +./core/installations/scripts/verify +``` + +This builds the package and runs credential-free layout/job/service/activation/lifecycle checks. One compact lifecycle integration covers backup/restore, suspension/resumption evidence, failed-upgrade rollback, successful release advancement, retained decommissioning, and separately approved destruction under temporary roots. Authentication evidence remains synthetic; no credentials or network are used, so this is not live-provider acceptance or a production release-upgrade claim. + +The explicit Linux user-systemd fixture gate is: + +```bash +./core/installations/scripts/verify-systemd-fixture +``` + +It drives one fixture through the real seven-stage durable job pipeline and installs a second under a rollback journal, using uniquely named runtime-only units for both. It starts the real Auth Broker, Collection Manager, Runtime Gateway, and outbound Runtime Agent processes, validates health and socket/process/cgroup identity, verifies both Agents through one temporary central hub, rejects crossed runtime targeting, executes gateway-backed status and sync-now calls, completes credential-free collector workers in each isolated manager, proves one healthy automatic restart, forces repeated fixture-manager terminations until the configured start limit suppresses further restart, rolls both fixtures back or removes the committed fixture bundle, verifies failure isolation, removes fixture units/journals, and confirms the existing local reference services were not restarted. It does not use provider credentials, open EXMP state, add public routing, or perform a live-provider `ready` transition. + +For isolated DSPs, `src/owner-onboarding.js` executes an optional owner connection request through the fixed private `paycom.setup` protocol. `optional-paycom-activation.js` records and verifies integration publication evidence independently of workspace readiness, then starts and reads back recurrence. Connection failures remain retryable without disabling the DSP. Existing legacy activation requests keep their original recovery path. Backups, upgrades, suspension and restoration of provisioned DSPs that have never completed Paycom activation verify infrastructure without requiring a Paycom schedule or publication. After a connection succeeds, lifecycle publication continuity checks apply as before. The shared runtime-only Paycom definition and activation implementation live under `runtime-container/src`; the legacy provisioner modules preserve their existing exports. The OCI image contains neither Access Control nor host provisioning code. See DSP acceptance. + +## Update recovery + +[Update recovery and Cloudflare R2 backups](RECOVERY.md) describes Core rollback, recovery after interruption, DSP compensation replay, encrypted off-server restore verification, setup, and recovery limits. diff --git a/core/core/installations/RECOVERY.md b/core/core/installations/RECOVERY.md new file mode 100644 index 0000000..a85024d --- /dev/null +++ b/core/core/installations/RECOVERY.md @@ -0,0 +1,247 @@ +# Update recovery and Cloudflare R2 backups + +A rollout still updates Core first, then every DSP one at a time. A failure pauses the rollout. DSPs already verified on the new release stay there; the failed DSP returns to its prior release when recovery succeeds. Remaining DSPs stay on their working release until the owner resumes the rollout. + +## Core safety + +Before stopping anything, the candidate verifies the installed release, both fixed host-switch permissions, free disk space, database integrity, schema compatibility, and runtime protocol compatibility. It trials its database startup against a copy. Schema transitions must appear in the candidate’s explicitly reviewed migration list; unknown transitions fail preflight. + +The updater arms an independent user-systemd recovery timer, stops reconciliation, drains any active reconciliation job, stops Core, and uses SQLite's backup API to capture a consistent database. It restores that backup to a temporary database and checks it before proceeding. In-flight snapshot attempts live separately under `LOCAL_ROOT/backups/platform-core/ROLLOUT_ID/attempt-N`. + +The candidate blocks normal HTTP traffic while verification runs. An unpredictable private probe checks identity, database integrity, foreign keys, required tables, and a write/read transaction which is rolled back. Health must remain good during a 15-second observation period. Only then does Core reopen traffic and reconciliation. + +The recovery journal lives outside the database being restored. Before promotion, recovery stops the candidate, restores the database in place using SQLite's backup API, restores the previous immutable host helper and service units, and verifies the prior Core. Once the old service can accept writes, replay never restores that database again. After promotion, recovery only finishes opening traffic; it never rewinds newly accepted work. + +The separate recovery timer uses the updater's existing process lock. It cannot interfere with an active update, and takes over after an interrupted or timed-out updater exits. It persists a paused rollout after recovery and then disarms itself. Corrupt backups or failed recovery leave the rollout stopped for investigation; they do not authorize a destructive best-effort restore. This is recovery protection, not a zero-downtime guarantee: Core restarts and verification cause a brief maintenance interval. + +For inspection or manual recovery, run as the dashboard service account from a trusted checkout containing this change: + +```bash +./core/installations/bin/dispatch-core-recover status /absolute/local/root rollout_ID +./core/installations/bin/dispatch-core-recover recover /absolute/local/root rollout_ID +``` + +Use the actual `rollout_` plus 32 hexadecimal characters from the backup directory. The command verifies the immutable candidate artifact and shares the updater lock. Its output contains state and release identities, never passwords or the private probe token. Do not manually copy a database file over a running SQLite database or delete its WAL files. + +## DSP safety + +Existing lifecycle protections quiesce scheduled collection, stop the old runtime, snapshot its data/state, runtime configuration and credential-encryption material, install the pinned native package, verify infrastructure and publication data, and restore the prior code/data on failure. Human runtime requests remain unavailable while the installation is upgrading. Backups now reserve room for a full restore before copying data. + +A durable compensation checkpoint is written before the restored runtime and its background work start again. Replaying an interrupted compensation skips the data restore after this checkpoint, preserving new records collected by the restored runtime. Lease fencing still applies to recovery mutations. If recovery itself fails, the DSP remains unavailable rather than being reported healthy. + +## Cloudflare R2 setup + +R2 is the selected off-server destination. The implementation uses restic's encrypted S3 backend. R2 credentials and the repository password are root-only and never passed into a DSP or browser. Until the destination is configured, local recovery works but off-server protection is **not active**. + +Use a dedicated private R2 Standard bucket. Disable public access. Create an R2 S3 access key with object read/write permissions restricted to that bucket. The bucket-scoped S3 key is different from a Cloudflare management API token. Keep a copy of the restic password in a password manager outside this server; R2 access alone cannot decrypt a backup. + +Legacy backups use **retain all backups**. Automatic retention never runs `forget` or `prune` against that shared repository. An explicitly confirmed **Delete DSP** request is the exception described below. New dashboard-managed snapshots use the independent archives described below. For protection against deletion by the backup key, configure indefinite R2 bucket locks on these prefixes (substitute the configured repository prefix): + +- `dispatch/config` +- `dispatch/keys/` +- `dispatch/data/` +- `dispatch/index/` +- `dispatch/snapshots/` + +Leave `dispatch/locks/` unlocked: restic must remove its temporary repository locks. Do not lock the entire bucket. Keep the original legacy locks in place. The root exporter also uses a separate R2 management token to verify and create the new archive retention rules; it preserves unrelated bucket locks. The legacy Remove operation retains backups; **Delete DSP** explicitly erases them. + +On the host, install restic 0.16 or newer at `/usr/bin/restic`. Using a protected terminal, create the following root-owned files with mode `0600`. Do not put real credentials in Git, chat, command arguments, or shell history. + +`/etc/dispatch/offsite-backup.json`: + +```json +{ + "schemaVersion": 1, + "accountId": "YOUR_32_CHARACTER_CLOUDFLARE_ACCOUNT_ID", + "bucket": "dispatch-backups", + "prefix": "dispatch", + "localRoot": "/absolute/dispatch/local/root", + "coreUid": 1001, + "retention": "retain-all" +} +``` + +`/etc/dispatch/offsite-backup-credentials.json`: + +```json +{ + "accessKeyId": "R2_S3_ACCESS_KEY_ID", + "secretAccessKey": "R2_S3_SECRET_ACCESS_KEY" +} +``` + +`/etc/dispatch/offsite-backup-password`: a randomly generated password of at least 32 characters, saved separately outside this host as well. Use the actual service UID and local root from the host deployment configuration. + +After a release containing this code has been prepared by release delivery, invoke its verified immutable backup entrypoint as root: + +```bash +sudo /opt/dispatch-platform/releases/RELEASE_ID/core-artifact/code/core/installations/bin/dispatch-offsite-backup init +sudo /opt/dispatch-platform/releases/RELEASE_ID/core-artifact/code/core/installations/bin/dispatch-offsite-backup enable +``` + +`init` is only for a new repository; it does not replace an existing one. `enable` exports existing completed snapshots, proves a new canary can be encrypted, uploaded, downloaded and restored, installs a root-owned timer pinned to this immutable artifact, and only then enables the required-backup policy. It does not start a platform rollout. + +The timer scans completed Core snapshots and DSP backup manifests every 15 seconds after its previous run. It copies eligible snapshots to a private staging directory, validates the manifest, encrypts/uploads, downloads/restores, and compares every restored file to the original. Only a successful restore produces a root-owned verification receipt. Each Core replacement and DSP upgrade then waits for its matching receipt. Upload/download/validation failure prevents that replacement and returns the working service through normal recovery. Native replacements require a full recovery proof and allow up to one hour for upload and restore verification. Failure leaves the update recoverable; the runtime is not replaced without the matching proof. + +Status and a full read/check of repository data: + +```bash +sudo /opt/dispatch-platform/releases/RELEASE_ID/core-artifact/code/core/installations/bin/dispatch-offsite-backup status +sudo /opt/dispatch-platform/releases/RELEASE_ID/core-artifact/code/core/installations/bin/dispatch-offsite-backup check +systemctl status dispatch-offsite-backup.timer dispatch-offsite-backup.service +``` + +Every new export includes an actual restore drill. `check` can also be run to detect later remote corruption; it does not overwrite any production data. No stale receipt is reused for a different snapshot digest. + +Native recovery archives include application code, exact Node/Chrome dependencies, databases and files, configuration, encryption material, registration credentials, Linux identities, service definitions and startup settings for their scope. A dashboard Core recovery capsule contains only Core; a full-system recovery set combines it with independent DSP capsules. Internal rollout safety capsules retain their existing full-platform format. Individual DSP archives include their completed readiness and suspension evidence so subsequent lifecycle operations can verify continuity after a fresh-host restore. The dashboard’s individual restore restores data and DSP metadata into the existing compatible runtime. + +This is a Dispatch application recovery system for Ubuntu 24.04 amd64, not a disk image of unrelated VPS applications. Recovery installs the required Ubuntu packages and restores Dispatch’s exact private runtime dependencies. Records created after the chosen snapshot are outside that snapshot. A compatible fresh host, sufficient disk/RAM, network access to R2/package repositories, and a separately retained recovery kit are required. Existing conflicting Dispatch paths or account IDs cause restore to stop. + +### Full platform restoration + +Export the private kit as root from the installed release: + +```sh +sudo /opt/dispatch-platform/releases/RELEASE_ID/core-artifact/code/core/installations/bin/dispatch-recovery-kit export /absolute/new-private-kit-directory +``` + +Move that directory off the VPS into protected storage. It contains storage access and decryption secrets, the recovery program, Node and its dependencies, and restic. Keep the kit current when storage credentials/password change. Losing both the VPS and its only decryption key cannot be repaired by an R2 archive. + +On a fresh Ubuntu 24.04 amd64 VPS, restore as root: + +```sh +./restore list +./restore restore BACKUP_ID all +# For a pre-update full Core snapshot in the shared repository: +./restore restore platform-core SNAPSHOT_ID +``` + +Use the exact backup ID and retention tier from the list. The kit downloads and authenticates the encrypted archive, verifies the complete file inventory, installs prerequisites, recreates accounts, restores ownership/code/data/secrets, restores startup services, and verifies Core identity and the health API of each active DSP. Suspended DSPs remain disabled. Interrupted rollouts are paused so recovery does not immediately update the restored version again. A free-space check runs before accounts or installed files change. A failed host restore is not reported as successful; investigate the error on the replacement host before retrying, since partially installed files are not overwritten automatically. + +Successful native rollouts prune local safety snapshots and obsolete release trees only after the fleet has settled and root confirms full recovery proofs. Remote backups remain the historical version store. Active restoration and compensation snapshots stay pinned until those operations settle. + +## Verification + +Tests exercise real SQLite backups/restores with an open supervisor connection, actual SIGKILL at update boundaries, failure before candidate promotion, failed off-server verification, corrupt backups, and replay after service restart. HTTP tests prove maintenance blocks both public and authenticated requests without changing sessions. DSP tests prove the recovery checkpoint is durable and fenced. A real restic repository test proves encrypted upload/download/restore, password rejection, and absence of plaintext business records in repository files. CI installs restic to run this test rather than skipping it. + +Cloudflare R2 setup still requires a real bucket and credentials. Local restic tests do not substitute for the R2 canary and bucket-lock checks on the selected account. + +References: [R2 S3 credentials](https://developers.cloudflare.com/r2/api/tokens/), [R2 bucket locks](https://developers.cloudflare.com/r2/buckets/bucket-locks/), [restic S3 backend](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html), [restic restore](https://restic.readthedocs.io/en/stable/050_restore.html). + + +## Independent Core, DSP and full-system backups + +The platform owner's Backups page provides three scopes. A Core operation affects +only Core; a DSP operation affects only the selected DSP. A full-system backup +creates a separate Core archive and one independent archive for every eligible +DSP, together with an encrypted manifest referencing those archives. A full-system +restore or deletion explicitly selects that complete set. Removed DSPs remain +stopped and are excluded from new backups; their previous archives remain held. + +Every DSP, Core, and the full system has its own hourly, daily, or weekly schedule, +time zone, time/day, and retention policy. Every new schedule starts disabled, +including when upgrading from the old shared policy. Only the platform owner can +change schedules. Enabling the full-system schedule does not enable individual +schedules. Each full-system run uses its own retention policy for its components. + +Core archives contain platform-owner accounts, Core's schedule, platform +configuration, and platform credentials. The copied Access Control database is +scrubbed and vacuumed before export: DSP users, memberships, organization data, +installation coordination, DSP schedules, runtime registration credentials, and +backup catalogs are excluded. Core recovery capsules exclude DSP runtime roots, +DSP services, and the tenant provisioner database. Core-only restore preserves +live DSP records and DSP/full-system schedules, revokes platform-owner sessions, +creates a verified safety backup, applies Core settings, restarts only the dashboard, +and checks Core health. Failed verification attempts recovery from the safety copy. +Current account passwords and disablement are preserved during in-place restores. + +DSP snapshots contain runtime data, state, configuration, auth-broker secrets, +organization details, roles, permissions, users, and that DSP's individual +schedule. Private host identity metadata supports full-system disaster recovery. +In-place DSP restores preserve host identities, registration credentials and the +current runtime release; they use the existing fenced stop, safety-backup, +restore, verification and recovery sequence. They do not revive old invitations +or sessions or grant platform roles. + +Full-system sets remain incomplete until every component and their encrypted +manifest verify. A busy/unavailable DSP prevents a new full-system set from being +queued rather than being silently skipped. In-place full-system restore requires +the same DSP inventory, validates all selected archives before queuing any work, +restores Core first, and stops subsequent components after a failure. Progress +remains visible per component. The full-system schedule is restored only after +all components succeed. A partial restore is never reported as complete. + +### Independent archive retention and deletion + +The **Storage** tab shows measured encrypted R2 bytes for Core, each DSP, and +removed DSPs' retained archives, with a measurement timestamp. The overall total +counts each archive once. Full-system set totals include their component archives +and manifest; those totals are not added a second time. Shared repository/rollout +safety storage, manifests, and unassigned archive objects are shown separately. +Local working copies are outside these R2 totals. Expired or deleting archives +continue to consume storage until their objects are actually removed. + +The root exporter uses read-only, paginated [S3 object-size listings](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html). +Measurements refresh after archive changes and approximately every five minutes; +they include archives beyond the dashboard's recent-history limit. Failed scans +show the last measurement as stale, or unavailable when there is no measurement. +Usage failures do not block backup or recovery operations. + +Each component snapshot has its own encrypted restic repository at +`archives/TIER/BACKUP_ID/`. Tiers are `all`, `7`, `30`, `90`, or `365` days. +Full-system manifests live separately at `sets/SET_ID/`; they contain archive +references and the full-system schedule, not duplicate DSP data. Every upload is +verified by downloading it and checking integrity before being shown as ready. + +Deleting a Core or DSP backup removes only that selected archive. Deleting a +full-system set deletes all of its component archives and its manifest, with +completion withheld until storage confirms removal. Deleting an individual +component makes a set referencing it incomplete; other component archives remain +independently usable. No archive-deletion action deletes live Core or DSP data. +Storage failures remain visible and retryable. Automatic retention cannot expire +archives held for removed DSPs or active restores. + +Root serializes storage mutations under the exporter lock. Explicit archive +deletion temporarily lifts only the configured archive-tier retention locks and +restores them from a durable journal. Broader administrator locks block deletion. +The storage credential and lock-management token never enter the dashboard. + +### Recovery on a replacement host + +Export and store the private recovery kit outside the VPS. On a clean Ubuntu +24.04 amd64 replacement host, run `./restore list`, then +`./restore restore-system SET_ID`. The kit downloads and authenticates all +components, assembles their scoped metadata into a verified recovery capsule, +and only then installs accounts, code, secrets, services, and data. A missing or +corrupt component aborts before installed host paths change. Existing Dispatch +installations are never overwritten by the replacement-host command. + +`./restore restore CORE_BACKUP_ID TIER` recovers only Core on a clean host. +Legacy full-platform safety snapshots retain their separate recovery command; +ordinary Core/DSP archives are not mixed with those internal rollout checkpoints. + +### Activation and inspection + +When a verified Core release switches the host helpers and an offsite configuration exists, it also starts an independent `dispatch-backup-enable-RELEASE_ID.service`. That service performs the encryption/restore canary and enables the root export timer and mandatory offsite policy. This occurs within the normal Core-first rollout; a merge does not modify the live dashboard. Initial setup requires the root-only storage credentials/password. An empty legacy repository is initialized during enable. Automatic scheduling is configured on the Backups page after the service is connected. + +`dispatch-offsite-backup check` checks both the legacy repository and all non-expired independent archives. A backup operation that cannot obtain verification or download within one hour fails visibly. A native restore safety upload requires a full recovery proof with a one-hour lease-renewing deadline. Inspect the backup worker and reconciler journals for a failed operation; do not delete a safety snapshot or manually reopen a failed DSP to bypass recovery. + +### Permanent DSP deletion + +Active DSPs expose Remove DSP. Removal stops services and backup creation, +revokes DSP sessions, blocks login, and retains data and existing backups. +Removed DSPs expose Restore DSP and Permanently delete DSP after shutdown +completes. Permanent deletion requires the acting platform owner's password. +The worker removes the DSP's services, verifies remote archive erasure, and then +removes local data, backup files, host account, and scoped Core metadata. Failed +deletion remains visible and retryable. Newly isolated Core backups and other +DSPs' archives survive DSP deletion; historical mixed safety snapshots are +handled conservatively as described below. + +Under the root exporter's existing flock, explicit deletion purges every known DSP backup ID in every independent archive tier, including incomplete uploads and expired records. Legacy snapshots are selected by their server-derived source tags, forgotten, and pruned with `--max-unused 0` from the shared encrypted repository, so unused target data is not left in mixed packs. Remaining snapshot IDs are compared before/after and restic checks the remaining repository data. Core/full-platform snapshots containing the DSP are erased. Root-verified complete organization inventories preserve unrelated snapshots, including those made before the DSP existed. Older incomplete inventories cannot prove absence and remain subject to erasure. Other DSPs’ individual snapshots are preserved. No new backup is exported for a DSP whose current lifecycle job is destruction. + +Deletion temporarily lifts the exact configured archive-tier and legacy data/index/snapshot locks needed by that request. This makes those shared prefixes writable during the serialized purge; automatic retention remains unchanged. The config/key locks and unrelated administrator locks are preserved, and broader locks block deletion. Removed rules are saved to a root-only journal before changing R2, restored in `finally`, and restored before any subsequent scan after interruption. A failed restoration leaves the journal in place and blocks success. Do not run another storage writer or change these rules outside the serialized root worker during a purge. + +Only after R2 objects are absent, legacy pruning/checks finish, and lock restoration succeeds does root publish an opaque receipt bound to the lifecycle job, DSP, and runtime key. The lifecycle worker requires that receipt whenever offsite protection is enabled. A one-hour receipt timeout or storage failure keeps the deletion incomplete; a retry is idempotent. The dashboard backup catalog is marked deleted only after local destruction also verifies. + +Full-host capture supports the current Ubuntu 26.04 VPS and Ubuntu 24.04. The verified replacement-host target is Ubuntu 24.04 amd64. The capsule carries private Node libraries, including its loader and libc, to preserve the captured application runtime across those host versions. diff --git a/core/core/installations/RELEASES.md b/core/core/installations/RELEASES.md new file mode 100644 index 0000000..860e88d --- /dev/null +++ b/core/core/installations/RELEASES.md @@ -0,0 +1,528 @@ +> Historical native/OCI recovery reference. The monolithic release commands below +> are retired. Follow the repository root RELEASES.md for current development. + +# Automated release delivery + +A user-requested release and its chosen version authorize publication followed by an agent-operated rollout. Updates is a read-only changelog viewer; the platform owner does not need to start installation there. Publication and server preparation still precede rollout, preserving the existing verification gates. + +## Publish + +All Dispatch changelog copy and revisions must be authored by a **Luna subagent at +Max reasoning** (`model: "gpt-5.6-luna"`, `reasoning_effort: "max"`, +`fork_turns: "none"`). This applies everywhere: GitHub summaries and entries, +platform Updates, dashboard popups, and changelog examples or previews. Supply +verified release changes and the current authoring schema; label synthetic examples +as fictional. The coordinating agent verifies facts and structure and returns prose +corrections to Luna. If Luna Max is unavailable, report that limitation instead of +substituting another author. + +After the user selects a version, run `.github/workflows/release.yml` on `main` with `version` and a `changelog` JSON array. Each change has `kind` (`added`, `improved`, `fixed`, `removed`, or `changed`), `title`, and `description`. Pass inputs through a JSON file/stdin when using `gh workflow run`; do not interpolate changelog text into shell code. + +The workflow verifies the selected commit, builds the native DSP package with its runtime dependencies, verifies two isolated service accounts, and builds portable Core/bridge bundles. It uploads the packages, `dispatch-release.json`, and checksums into a draft, checks GitHub's asset digests, then publishes. It never replaces a published version. Failed uploads leave a draft; a retry can resume only a matching draft and matching asset bytes. + +Core bundles contain application files and the helper manifest. Packaging builds the dashboard JavaScript and CSS from the selected Git commit in disposable storage using the committed npm lockfile; it requires npm on the build machine and never uses ignored checkout bundles. Retained verified components already include those assets, so publication can reuse them without rebuilding. Installed hosts need no frontend build tools. They contain no host paths, service credentials, databases or deployment configuration. The host renders its own units and deployment manifest after verifying the published package. + +## One-time host setup + +Use the clean merged checkout to create a portable bootstrap bundle outside the repository: + +```sh +./core/installations/bin/dispatch-release-build --core-only /tmp/dispatch-bootstrap-core.json +``` + +Create a private JSON file with the existing platform service's `uid`, `gid`, `localRoot`, `unitRoot`, `publicOrigin` (`https://dispatch.example.test`) and `port`. Existing private catalogs must be at `/config/oci-releases.json` and `platform-releases.json`, matching the platform's provisioning environment. The local and unit directories must belong to that service account. + +Create a repository-scoped GitHub credential with read-only Contents access to `example-organization/dispatch-platform`. Supply it through protected stdin, never a command argument: + +```sh +sudo ./core/installations/bin/dispatch-release-delivery-install /path/to/private-host-config.json /tmp/dispatch-bootstrap-core.json < /path/to/private-token-file +``` + +This installs sealed watcher code separately from Core, root-only configuration/token files under `/etc/dispatch/`, durable state under `/var/lib/dispatch-release-delivery/`, and a system timer. The token stays on the server and is never forwarded to asset redirect hosts or DSPs. Re-running the bootstrap with the same verified bundle can update the token/configuration; newer watcher code is installed at a new immutable commit path. + +The root watcher downloads only from the fixed private repository and GitHub's release-asset hosts. It checks asset hashes, the published tag's commit and its ancestry on main, bundle paths, file manifests and matching runtime identity. It installs only immutable release directories and the verified release's argument-free host-switch permission. Catalog/status writes run as the platform service account. Neither the watcher nor the bootstrap changes active Core/helper pointers or creates rollout records. + +## Recovery and status + +The timer checks approximately every 45 seconds. Finished downloads survive retries; incomplete downloads resume when the server honors validated byte ranges. Preparation and catalog registration are idempotent across interruption. A release appears in the catalog only after preparation succeeds. Published identities and prepared directories are never overwritten. Preparation failures use bounded exponential backoff. The release operator investigates failures before retrying. + +The release status adapter reads the credential-free `release-delivery-status.json`. The retained operator API can write a bounded retry request after platform-owner authorization; the changelog viewer does not expose this control. Internal errors and download URLs stay out of the UI. Root daemon logs contain closed error codes. Inspect `journalctl -u dispatch-release-watch.service` and the root-only state file when intervention is needed. + +Legacy releases without `dispatch-release.json` remain usable in existing catalogs but are not automatically imported. The automation applies to releases published by the new workflow. The independently pinned Core updater remains in place throughout discovery, preparation and rollout. + +## Local retention + +After Core promotion, discovery and the independent updater use the current Core artifact. Completed native fleet rollouts remove obsolete local release directories after root verification and process checks. GitHub publication history and Cloudflare recovery archives remain remote. The download worker removes superseded download staging directories. See [native DSPs](NATIVE-DSPS.md) and [recovery](RECOVERY.md). + +## Grouped release notes and history + +The `changelog` workflow input also accepts a rich object. Read this section before +preparing grouped notes. Legacy arrays remain supported and render under “What’s new”. + +```json +{ + "groups": [{"id":"backups","title":"Backups & recovery","icon":"database"}], + "changelog": [{ + "kind":"added", + "title":"Independent backups", + "description":"Back up Core and DSPs separately.", + "group":"backups", + "icon":"copy", + "details":"Optional longer explanation, shown when View details is expanded." + }], + "afterUpdating": [{"title":"Backup schedules","description":"Enable the backup schedules you want to run."}] +} +``` + +These are illustrative entries, not instructions applicable to every release. Write +short, factual descriptions and include after-update actions only when the release +requires them. Each action description must make sense on its own. An empty `details` +string omits expanded text; an empty `afterUpdating` list omits the notice. Group order +and entry order are authored; group and category counts are always calculated. + +Groups have unique lowercase slug IDs (up to 40 characters), titles up to 80 characters, +and at least one change. Up to 20 groups and 100 changes are supported. Change titles +are limited to 160 characters, descriptions to 600, and details to 4,000 (plain text, +with newlines allowed). Up to 10 after-update actions may contain a 160-character title +and 600-character description. The presentation plus its plain-changelog compatibility +copy must fit within 254 KiB, leaving room for metadata in preparation-status receipts. +Approved icon identifiers are `database`, `users`, `user-plus`, `copy`, `calendar-clock`, +`chart-column`, `shield`, `check-circle`, `trash`, `lock`, `send`, `refresh-cw`, `plus`, +`pencil`, `trending-up`, and `info`. No remote icons, HTML, or executable content are used. + +The builder derives the original `{kind,title,description}` array from these entries +and leaves the version-1 `dispatch-release.json` contract unchanged. It also produces +`dispatch-release-notes.json` with schema version 1, the release ID, source commit, +and rich notes. GitHub's release body is generated from the same input, including +expanded explanations and after-update instructions. Both attachment digests are +verified during publication. Editing the GitHub body alone does not change the UI. + +### GitHub release notes + +**Only the GitHub release body** uses the approved inline-attribution design (A): +a short summary, then **New**, **Improved**, **Fixed**, **Removed**, and +**Maintenance**, omitting empty sections. +`added` appears under New; `changed` and `improved` share Improved. Each compact +bullet starts with the change title in bold, then an em dash and concise description. +Finish the same paragraph with `by @author · PR #number`, linking the author profile +and PR separately. For changes spanning multiple PRs, pair each PR with its own +author. Attribution stays visible outside any Details disclosure. The release +page supplies the version heading; the generated body does not repeat it. Extended +explanations stay in per-entry Details disclosures. Actual after-update instructions +appear before the change sections. +There is no separate Highlights section. A Full changelog comparison link closes +the body when a previous release exists. + +Rich authoring accepts optional GitHub-only fields: + +- Top-level `github`: optional `summary` (nonempty plain text, up to 600 characters) + and `previousTag` (the verified previous GitHub release tag, up to 160 characters). + Supply a summary for new releases and omit `previousTag` for the first release. +- Per-entry `github`: `pullRequests` accepts up to 20 distinct PR references from + `example-organization/dispatch-platform`, each shaped as `{"number": 55, "author": "example-organization"}`. + `number` is a positive integer; `author` is the verified PR author's GitHub login + without `@` (bot logins ending in `[bot]` are supported). Older integer-only + references still produce labeled PR links without inventing an author. New + release authoring must supply the object form for each associated PR. + `maintenance` is an optional boolean. + Set `maintenance: true` only for internal maintenance, refactoring, build, or + test changes; this changes its GitHub section without changing its shared `kind`. + +Luna Max authors the summary, entry copy, and classifications from the verified +changes. The coordinating agent retrieves each associated merged PR's number and +`author.login` from GitHub and verifies the previous tag before passing metadata to +Luna. Use the PR author, not the release publisher or merge operator. Include all +associated PR references on new entries; omit unavailable references instead of +guessing them. Changes without an associated PR remain readable without attribution. +The renderer creates the PR and comparison URLs and escapes plain-text copy for +Markdown. The fictional fixture +[`examples/github-changelog.json`](examples/github-changelog.json) demonstrates +this input alongside audience classification and concise popup overrides. + +The builder consumes `github` fields only when writing `CHANGELOG.md`, which is +passed to GitHub's release body. It strips them before creating the unchanged v1 +installation manifest, rich Updates sidecar, or popup data. The Updates page keeps +its feature groups, icons, counts, and expanded details; the dashboard popup keeps +its existing concise, audience-filtered sections. Old input without GitHub metadata +remains supported. Published releases and historical notes are not rewritten. + +### Dashboard update popup + +For releases that should announce changes on dashboard entry, add `audience` to +**every** rich changelog entry and after-update action. Use `platform` for changes +exclusive to platform administration, or `dsp` for changes relevant to DSPs and their +dashboards. Platform owners see both; DSP owners see only `dsp` entries. Classify +shared changes as `dsp`. Audience labels are never displayed in the popup. + +Each changelog entry can also include `popup: {"title":"Short title","description":"Concise explanation."}`. +This overrides only its popup copy; the existing title, description and details remain +in the GitHub release and platform Updates page. Without an override, the popup uses +the original title and description. Required after-update actions use the same copy +across surfaces and must have an explicit audience. The synthetic example is +[`dashboard/examples/popup-changelog.json`](../../dashboard/examples/popup-changelog.json). + +The builder strips these authoring-only fields from the existing v1 notes attachment +and installation manifest. It embeds the curated copy in +`code/dashboard/release-popup.json` within the checksummed Core bundle, +with the release ID, version and source commit. Existing watchers can therefore +prepare the release unchanged. The file is private to the server and is not a static +web asset. Legacy authoring without audience fields remains valid and produces no +popup; missing classification must never expose legacy platform notes to a DSP. + +`GET /api/updates/popup` returns only the authenticated owner's visible changelog for +the running Core release, after its platform rollout has completed and Core verification +has succeeded. A DSP owner also needs an active DSP with a ready installation on that +release. Staff and platform-owner DSP viewing sessions receive no popup. A DSP with no +relevant entries receives none. The API removes audience and source metadata. + +The centered dialog shows New, Improved (including Changed), Fixed, and Removed +sections as needed, plus actual after-update actions. It checks once on dashboard +entry and does not interrupt navigation with newly discovered updates. Got it, X and +Escape all use the same authenticated, CSRF-protected POST with `{ "releaseId": "..." }`. +Dismissals are idempotent records in `release_popup_dismissals`, keyed by user and +release, surviving new sessions and devices. The dialog closes after persistence +succeeds; a failed save leaves a retry action. Background clicks do not dismiss it. +Only the current release is announced, so returning users do not receive a backlog +of dialogs. Existing platform Updates history remains unchanged. + +### Compatibility transition + +Older watchers ignore the optional notes attachment and prepare the unchanged v1 +manifest normally. Older dashboards continue reading their unchanged installation +catalog. The new dashboard renders legacy notes without requiring rich metadata. +During a release-driven rollout, Core promotion updates the independently supervised +watcher to the new Core artifact (see `core-systemd-deployment.js`). On its next tick, +the new watcher imports the rich attachment even if the older watcher already marked +that release ready; it does not download the runtime again. Installing the new watcher +with the bootstrap above also enables this capability before Core promotion. No extra +release, version choice, or rollout is implicit in this transition. + +The notes attachment must match the manifest's release ID, commit, and complete ordered +legacy changelog. Its GitHub digest is checked and pinned. Later changes or removal +are rejected once observed. Verified notes are stored as private per-release files in +`/config/release-notes/`, keeping extra fields out of legacy catalogs. + +### Past changelogs + +The Updates release selector offers published release history, marks the installed +release, and allows reading past notes without offering a rollback. Installed notes +remain visible after completion. Rollout is operated through the local command below; +only the newest available release can start a new rollout. + +The watcher snapshots local catalog metadata to `/config/release-history/`. +After current-release preparation it also backfills GitHub releases that have verified +v1 manifests: at most five per timer tick, downloading only manifests and optional notes. +It checks digests, tag/commit identity, and ancestry on main, pins accepted fingerprints, +and retries failures with bounded backoff. History failures do not hide the current +update. Releases published before structured manifests can be shown if a local catalog +or rollout snapshot still contains their notes; missing legacy text is not invented. + +These small metadata records survive installation-package retention. The API returns +history summaries plus the selected release's notes through +`GET /api/platform/updates?releaseId=`, under the same platform-owner authorization. +Invalid or unknown IDs fail; historical metadata never populates installation catalogs. +Missing or unreadable rich notes fall back to the verified plain changelog. + +## Efficient update coordination and recovery artifacts + +Updates still require verified backups, verify Core first, and upgrade DSPs one +at a time. Scoped Core and DSP exports consume sealed snapshots without stopping +the dashboard, reconciliation worker, or DSP services. Legacy whole-host capture +retains its writer freeze and active-job guards. + +Queue creation wakes the independent user service after the outer database +transaction commits. Productive reconciliation passes drain immediately; waits +and healthy pending work exit successfully. A fixed `backup-ready` notification +file wakes the root exporter through a systemd path unit. One-minute fallback +timers recover missed notifications. The separate updater survives Core restarts. + +Immutable installed release trees are captured into shared, encrypted Restic +repositories at `recovery-artifacts/`. A per-release file lock +serializes first capture, and a full readback verifies each new shared artifact +before publishing references. Subsequent archives contain the authenticated file +inventory and exact artifact snapshot references instead of the runtime bytes. +Recovery hydrates and verifies those references before ordinary capsule +validation and restore. Missing or altered artifacts stop recovery before install. +The local cache is disposable; backup deletion and local release cleanup never +remove the remote shared artifacts. Their separate indefinite R2 lock deliberately +retains even currently unreferenced release artifacts. Automatic remote artifact +garbage collection is not enabled. Storage totals include shared artifact bytes +once, separately from per-backup bytes. + +Use a recovery kit exported from this implementation for backups using shared +artifacts; older kits cannot hydrate these references. New kits still restore +older self-contained backups. The full repository check verifies referenced +shared repositories as well as the individual backup repositories. + +Lifecycle and Core stage start/end times, attempts, durations, safe failure codes, +and dependency waits are stored in `operation_stage_timings` in the private +access-control database. A running interval without an end records interruption; +a retry creates a new interval. Export receipts separately record snapshot-copy, +recovery-capture, repository-preparation and encrypted-upload timings. These are +operational diagnostics, not authentication or business payload logs. + + +## Release-driven rollout + +After publication, the release operator runs the following **on the platform host as the Core service account**, using the confirmed live local root, user-selected version and verified release commit. The protected local command uses filesystem authority, records the existing active platform owner as the rollout actor, and creates no browser session. It prints only release identity and public rollout status, never login credentials or backup payloads. + +```sh +./bin/dispatch-access-admin rollout-status --local-root /path/to/live/local --version VERSION --commit SOURCE_COMMIT +./bin/dispatch-access-admin rollout-start --local-root /path/to/live/local --version VERSION --commit SOURCE_COMMIT +``` + +Use the installed Core command, or the verified release checkout when upgrading from a version that predates this CLI. Do not use a development worktree's default database. Every command requires the exact version and 40-character commit. Status reports `not_prepared` until matching Core/DSP catalogs exist, `ready` before start, then the rollout's actual status and backup/Core/DSP progress. A conflicting commit is rejected. Wait for `ready` before starting; investigate preparation errors in the independently supervised release watcher. + +Poll `rollout-status` until `completed`, verifying Core and every remaining DSP against the selected release. Start is idempotent for an existing target, including paused or completed rollouts. A different active rollout is rejected. Backups must verify before Core switches, Core must verify before DSP updates, and DSPs update sequentially through existing supervised workers. The command queues work; its successful exit alone does not mean installation completed. + +If a rollout pauses, investigate the failed service or backup, resolve the cause, and then run `rollout-resume` with the same flags. `rollout-pause` is available to the operator with the same target guard. Neither status polling nor repeated start resumes failed work. Preserve the existing rollback, backup verification and recovery checks. Report completion only after verified rollout completion; otherwise report the concrete blocker. + +The Updates page always exposes published release history and changelogs, including installed and older releases. Refresh, version search, release selection and expanded notes are read-only; operational controls remain outside this page. + +## Split release packages (manifest v2) + +The release builder accepts `--format legacy|split`. The publication workflow +exposes the same choice and defaults to `legacy` during migration. Legacy output +remains compatible with installed v1 watchers. Split output uploads exactly: + +- `dispatch-release.json`: release identity, component hashes and expanded sizes, + runtime compatibility, pinned dependency versions, and embedded structured notes. +- `dispatch-app.tar.gz`: Core/dashboard/helper files, bridge, and DSP application code. +- `dispatch-dependencies.tar.gz`: pinned Node/Chrome and the runtime libraries + captured with Node. Dependency contents have no application commit or build timestamp. + +The v2 runtime identity hashes the ordered application/dependency digest pair. +The assembled runtime retains its original full file inventory and installed +layout. Archive extraction rejects links, traversal, unexpected roots, missing +files, duplicate entries, oversized inventories and checksum mismatches. Expanded +sizes are checked against the authenticated inventories and used for disk preflight. + +The watcher reads both manifest versions. It caches the latest verified dependency +archive by SHA-256 and reuses it across releases. Each GitHub release remains +self-contained: its dependency asset is uploaded and verified even when another +release has identical bytes. Installed releases and recovery backups contain full +runtime files and do not depend on the download cache. Successful preparation +removes superseded dependency downloads; it does not alter recovery retention. + +Interrupted downloads resume against the same GitHub asset ID, size and digest. +A server that ignores byte ranges causes a clean restart. Every resumed file is +hashed in full before use. Completed packages survive preparation retries. + +`runtime-dependencies.json` pins Node and Chrome. The publication workflow installs +that exact Node version and downloads Chrome for Testing from Google's versioned +archive, checking the committed archive SHA-256. Split builds reject version drift. +Shared-library changes still change the dependency digest and require a download. +`build-metrics.json` records local build duration and package sizes; it is not a +release attachment. Runtime preparation measurements live in the root-only +`/var/lib/dispatch-release-delivery/preparation-progress.json`, including stage +elapsed times, current download bytes, disk requirements and dependency reuse. + +### Migration order + +Deploy v2 reader support using a legacy-format package or rerun the verified +watcher bootstrap on each host. Confirm the installed watcher code supports both +manifest versions before selecting `split` for publication. Do not replace an +existing published manifest or remove its assets. Old releases and their sidecar +notes remain readable. Publication alone does not initiate an update. + +### Unified operator entry point + +Use `core/installations/bin/dispatch-install` from the trusted checkout or +verified installed Core code, as root. This command handles host release delivery; +DSP onboarding still uses the existing provisioning flow. + +```sh +# Inspect prerequisites without changing the host. +sudo dispatch-install check --config /root/dispatch-host-config.json + +# Configure release delivery for a new or existing host. Supply the existing +# bootstrap Core bundle built with --core-only; keep the token on protected stdin. +sudo dispatch-install setup --config /root/dispatch-host-config.json --core /root/dispatch-bootstrap-core.json < /root/github-token + +# Prepare an exact release through the same engine used by automatic discovery. +sudo dispatch-install prepare --version VERSION --commit COMMIT + +# Prepare, then start the existing backup-gated coordinator for that exact release. +sudo dispatch-install update --version VERSION --commit COMMIT + +# Inspect preparation, rollout phase, DSP status and bounded operation timings. +sudo dispatch-install status --version VERSION --commit COMMIT +``` + +Setup requires the configured Linux service account and host prerequisites. It +creates missing service directories, empty catalogs and provisioning environment, +preserving existing files. Account creation, initial host-control authority, owner +setup, external origin routing and backup credentials remain explicit host setup +steps. Preparation installs verified artifacts and registers catalogs without +activating Core. Update requires the existing configured platform and active owner; +it uses the original Core-first, sequential-DSP coordinator, backup proofs and +recovery checkpoints. Repeating update never resumes a paused rollout. Use the +existing explicit rollout-resume command after diagnosis. + +## Faster release preparation and supervised handoff + +Prepare one immutable `.changelog/.json` fragment in each PR, using the +rich authoring schema above (including audience and optional popup copy). Luna Max +authors these entries under the Dispatch workflow. CI validates the fragments and +requires a new fragment for PR changes; fragments already merged to main are not +edited. `examples/fictional-fragment.json` is an explicitly fictional example. +Internal changes use per-entry `github.maintenance`. Fragment files do not contain +release versions or previous tags. After-update instructions describe only actual +required user actions. + +From a clean checkout, aggregate fragments added since the actual last release: + +```sh +node tooling/release-notes.js collect PREVIOUS_TAG COMMIT > /tmp/dispatch-notes.json +``` + +Review the aggregate against every merged change since that tag. When adopting +fragments for the first time, include any older changes without fragments in an +explicit Luna-authored notes file; aggregation cannot recover unrecorded changes. +Conflicting group definitions fail validation. GitHub, Updates, and popup data +continue to derive from the same entries. Optional PR links must be verified. + +After the user requests a release and selects its version, run this on the actual +host as the Core account, from the clean selected main checkout: + +```sh +python3 tooling/release.py start --local-root /path/to/live/local --version VERSION --commit COMMIT --notes /tmp/dispatch-notes.json +python3 tooling/release.py status --local-root /path/to/live/local --version VERSION --commit COMMIT +``` + +Omitting `--notes` aggregates fragments since GitHub's latest published release. +Preflight checks the selected commit's main ancestry, existing tags/releases, +changelog schema, live database readiness, configured Turnstile secret permissions, +free space, and updater supervision. No provider secrets enter workflow inputs, +logs, or the service definition. This is configuration validation, not a claim that +a real user completed an external provider's browser challenge. + +The command saves one private request and a copy of its worker under +`LOCAL_ROOT/releases/`, then enables a separate user systemd service. The worker +survives terminal disconnects, Core restarts, and user-manager restarts. Existing +host configuration must keep the Core user's systemd manager running at boot. +`status` reports publication, server preparation, rollout progress, and completion. +The original explicit version and commit remain fixed throughout the operation. + +Publication accepts successful verification only from the exact commit's newest +main push run, with all three existing jobs successful. If that run is still +active, selection waits up to five minutes for it before choosing fallback +verification. Its retained artifact must +have an unexpired matching identity and a verified GitHub archive digest. Otherwise +the release workflow invokes all three verification jobs, including browser checks. +Artifacts are retained for seven days. The builder validates component hashes and +commit identities again, reuses Core/bridge/runtime components, and generates fresh +release metadata and popup copy for the selected version. The host independently +verifies the published tag, manifests and downloaded artifacts before installation. + +Publication signals the fixed release watcher path, with its timer retained as a +fallback. Once the catalogs match, the worker starts the existing backup-gated +rollout, verifies Core first, then each DSP sequentially. A completed workflow is +not a completed rollout. Completion requires all backups and all remaining services +to verify the target, followed by a successful check of the public authentication +session endpoint. The Updates page remains read-only. + +A failed publication or paused rollout stops for diagnosis. After resolving the +cause, use `tooling/release.py resume` with the same identity flags. The command +does not rerun failed publication workflows; inspect and recover the original run +first. An uncertain workflow dispatch is reconciled by its unique request ID and +never blindly sent again. `rollout-status` uses a deferred read-only transaction, +so observation does not compete with rollout workers for SQLite's writer lock. + +The root `dispatch-recovery-prewarm.service` prepares immutable release recovery +payloads after release staging and periodically in the background. It retains the +existing encrypted upload and readback proof before caching a shared dependency. +It does not substitute for fresh data/configuration/secret snapshots. Its per-root +locks coordinate with backups, and a maintenance lock prevents cache pruning +while preparation runs. Failure leaves ordinary backup verification mandatory. + +Backup exports refresh their eligible queue while other uploads run, up to the +existing limit of three workers. New snapshots can use idle slots immediately; +removal and deletion intent is checked again before export. Active uploads drain +before releasing the parent deletion lock, including when discovery fails. + + +CI retains separate, digest-checked legacy and split component archives from the +same verified commit. Publication selects the requested format and rebuilds only +release-specific metadata and popup copy; split assembly preserves the dependency +archive bytes. Local build metrics are excluded from retained component archives. +The supervised release command accepts `--format split` on `start` after the +reader-first migration; omission keeps `legacy`. The selected format is persisted +with the request and checked again against the published asset set. + +## Reproducible verification builds + +The versioned `release-formats.json` defines native legacy and split asset names, +manifest schema versions, sidecar rules, and CI artifact suffixes. JavaScript +validation/build/publication and Python CI/publication checks share that contract. +Detached release workers snapshot their publication requirements in the immutable +request so they do not depend on a later checkout change. + +For CI, `dispatch-release-build VERSION NOTES /absolute/new/output --format both` +creates `legacy/` and `split/` from one Core/bridge build and one native dependency +stage. Each format still has its own manifests, checksums, inventory validation, +and retained-component reuse checks. `both` is a verification build option; +publication still selects `legacy` or `split`, with the existing reader-first +migration requirement. No runtime deployment or backup checks are skipped. + +Choose an output parent on a disk with room for expanded dependencies and both +archives; avoid a small tmpfs such as `/tmp`. The builder estimates space on that +filesystem, keeps its staging directories there, refuses existing output, and +removes its own output on handled build failures. Disk-full, archive timeout and +child interruption have distinct error codes. Disk estimates cannot reserve +space against unrelated processes, and abrupt host termination can leave scratch; +inspect and remove only the failed operation's directory before retrying. + +CI stages only the contract's component files before uploading; build metrics and +scratch are excluded. Artifact downloads verify GitHub's digest before bounded +extraction and clean partial extraction after failures or interruption. See +`DEVELOPMENT.md` at the repository root for pinned local tooling and compact PR/CI +commands. + +## Release readiness and recovery diagnostics + +Run `python3 tooling/release.py check --local-root /path/to/live/local` during +preparation, before choosing or publishing a version. It runs GitHub access and +host checks in a fresh user-systemd service using resolved absolute executable +paths. `start` repeats this check and stores the paths in its private immutable +request; the detached worker therefore does not depend on an interactive PATH. +The check does not publish a release or start a rollout. + +On hosts with the readiness service installed, the check signals an independent +root metadata scan and waits up to 45 seconds for a fresh, host-bound receipt. +Every intended native DSP must be present. The scan checks ownership, directory +and file permissions, unsupported links/files, backup size limits and free space +without reading payload contents or stopping services. Live changes or browser +artifacts can require waiting for a DSP to settle and checking again. The scan +never replaces the stopped-service snapshot or encrypted restore verification. +Older installed releases report `installed_release_predates_backup_readiness`; +their ordinary backup gates remain mandatory. A supported scanner that is missing, +stale or reports a problem stops publication. The independent readiness timer and +path service are installed by verified backup enablement alongside the existing +backup services; they continue working while exports are busy. + +The same preparation signal starts background recovery prewarming for immutable +release files. Root-owned readiness and prewarming receipts contain bounded codes, +counts and timings, without credentials or private file contents. Prewarming keeps +its existing locks, encrypted upload and readback proof. Fresh data, configuration +and secret snapshots are still captured for each rollout. No elapsed-time reduction +is assumed until measured on subsequent releases. + +Use `tooling/release.py status` with the existing identity flags and `--compact` +for a concise view of the release phase, backup members, Core and DSP progress. +The ordinary status also retains operation-stage timings. Phase history records +execution, attention and resumed intervals separately across worker restarts. + +Command failures are persisted as safe codes with their failed phase. Deterministic +failures stop for diagnosis instead of restarting indefinitely. After resolving the +cause, use the same `resume` command and release identity. A failure to execute the +initial GitHub CLI proves that no dispatch was sent and allows explicit retry; +network failures and other uncertain dispatch outcomes resume workflow observation +without another publication request. Recovery writes are serialized with the worker. +Published assets, verification gates and the Core-first sequential rollout remain +unchanged. + +Inactive browser cleanup additionally restricts only the recognized `.cache` and +`.cache/fontconfig` directories from 0755 to 0700. It retains cache contents and +rejects active profiles, symlinked directories, foreign ownership and unexpected +permissions. Existing stale PulseAudio runtime-link cleanup continues to apply. diff --git a/core/core/installations/bin/dispatch-backup-expire b/core/core/installations/bin/dispatch-backup-expire new file mode 100755 index 0000000..ff3aea5 --- /dev/null +++ b/core/core/installations/bin/dispatch-backup-expire @@ -0,0 +1,34 @@ +#!/usr/bin/node --no-warnings +'use strict'; +// Root has already verified archive retention. Delete the local copy only as +// its owning service account; never recursively remove tenant paths as root. +process.umask(0o077); +const fs = require('node:fs'); +const path = require('node:path'); +const { readHelperRequest } = require('../src/oci-helper-input'); +const { verifySnapshot } = require('../src/offsite-backup'); +try { + if (process.geteuid() === 0 || process.argv.length !== 2) throw Error(); + const value = readHelperRequest(0, 16384); + if ( + Object.keys(value).sort().join(',') !== 'digest,source' || + !/^[a-f0-9]{64}$/.test(value.digest) || + !path.isAbsolute(value.source) || + !/^(backup|breq)_[a-f0-9]{32}$/.test(path.basename(value.source)) + ) + throw Error(); + const parent = path.dirname(value.source); + if (!['backups', 'scheduled-core'].includes(path.basename(parent))) throw Error(); + if (verifySnapshot(value.source, process.geteuid()).digest !== value.digest) throw Error(); + fs.rmSync(value.source, { recursive: true }); + const fd = fs.openSync(parent, 'r'); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + process.stdout.write('{"ok":true}\n'); +} catch { + process.stderr.write('backup_expiration_failed\n'); + process.exitCode = 1; +} diff --git a/core/core/installations/bin/dispatch-backup-import b/core/core/installations/bin/dispatch-backup-import new file mode 100755 index 0000000..b733c7e --- /dev/null +++ b/core/core/installations/bin/dispatch-backup-import @@ -0,0 +1,67 @@ +#!/usr/bin/node --no-warnings +'use strict'; +// Called by the root archive worker as the target DSP's OS user. Root never +// writes into a tenant-controlled directory or follows tenant-owned parents. +process.umask(0o077); +const fs = require('node:fs'), + path = require('node:path'); +const { readHelperRequest } = require('../src/oci-helper-input'); +const { tree, verifySnapshot } = require('../src/offsite-backup'); +const { HOST_TENANT_ROOT, opaqueRuntimeSuffix } = require('../../runtime-host-identity'); +try { + if (process.geteuid() === 0 || process.argv.length !== 2) throw Error(); + const value = readHelperRequest(0, 16384); + if ( + Object.keys(value).sort().join(',') !== 'digest,id,runtimeKey,source' || + !/^backup_[a-f0-9]{32}$/.test(value.id) || + !/^[a-f0-9]{64}$/.test(value.digest) || + !/^\/var\/lib\/dispatch-restore-staging\/import-[a-zA-Z0-9]+\/snapshot$/.test(value.source) + ) + throw Error(); + const installation = path.join( + HOST_TENANT_ROOT, + opaqueRuntimeSuffix(value.runtimeKey), + 'runtime', + value.runtimeKey, + ), + backups = path.join(installation, 'backups'); + for (const p of [installation, backups]) { + const s = fs.lstatSync(p); + if ( + !s.isDirectory() || + s.uid !== process.geteuid() || + s.mode & 0o077 || + fs.realpathSync(p) !== p + ) + throw Error(); + } + if (verifySnapshot(value.source, process.geteuid()).digest !== value.digest) throw Error(); + const target = path.join(backups, value.id), + temporary = path.join(backups, `.download-${value.id}`); + if (!fs.existsSync(target)) { + if (fs.existsSync(temporary)) { + const s = fs.lstatSync(temporary); + if ( + !s.isDirectory() || + s.uid !== process.geteuid() || + fs.realpathSync(temporary) !== temporary + ) + throw Error(); + fs.rmSync(temporary, { recursive: true }); + } + tree(value.source, process.geteuid(), temporary); + if (verifySnapshot(temporary, process.geteuid()).digest !== value.digest) throw Error(); + fs.renameSync(temporary, target); + const fd = fs.openSync(backups, 'r'); + try { + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + } + if (verifySnapshot(target, process.geteuid()).digest !== value.digest) throw Error(); + process.stdout.write('{"ok":true}\n'); +} catch { + process.stderr.write('backup_import_failed\n'); + process.exitCode = 1; +} diff --git a/core/core/installations/bin/dispatch-backup-readiness b/core/core/installations/bin/dispatch-backup-readiness new file mode 100755 index 0000000..7c0e052 --- /dev/null +++ b/core/core/installations/bin/dispatch-backup-readiness @@ -0,0 +1,18 @@ +#!/usr/bin/node --no-warnings +'use strict'; +process.umask(0o077); +const path = require('node:path'); +const { atomic } = require('../src/release-delivery-files'); +const { FILE, inspectHost, localIdentity } = require('../src/backup-readiness'); +try { + if (process.geteuid() !== 0 || process.argv.length !== 2) throw Error(); + const artifact = path.resolve(__dirname, "../../../.."); + const id = path.basename(path.dirname(artifact)); + if (require('../src/core-recovery-host').rootArtifact(id).root !== artifact) throw Error(); + const config = require('../src/offsite-backup').loadConfig(); + let result; + try { result = inspectHost(config); } + catch { result = { schemaVersion: 1, localRootHash: localIdentity(config.localRoot), checkedAt: Date.now(), status: 'attention', members: [], error: 'backup_inspection_failed' }; } + atomic(FILE, result, 0o644); + process.stdout.write(JSON.stringify({ status: result.status, checkedAt: result.checkedAt, installations: result.members.length }) + '\n'); +} catch { process.stderr.write('{"status":"backup_readiness_unavailable"}\n'); process.exitCode = 1; } diff --git a/core/core/installations/bin/dispatch-core-backup-import b/core/core/installations/bin/dispatch-core-backup-import new file mode 100755 index 0000000..4b22786 --- /dev/null +++ b/core/core/installations/bin/dispatch-core-backup-import @@ -0,0 +1,17 @@ +#!/usr/bin/node --no-warnings +'use strict'; +process.umask(0o077); +const fs=require('node:fs'),path=require('node:path'); +try{ +if(process.geteuid()===0||process.argv.length!==2)throw Error(); +const input=require('../src/oci-helper-input').readHelperRequest(0,16384); +if(Object.keys(input).sort().join(',')!=='digest,id,localRoot,source'||!/^breq_[a-f0-9]{32}$/.test(input.id)||!/^\/var\/lib\/dispatch-restore-staging\/import-[a-zA-Z0-9]+\/snapshot$/.test(input.source)||!path.isAbsolute(input.localRoot))throw Error(); +const root=path.join(input.localRoot,'backups/scheduled-core'); +for(const dir of [input.localRoot,path.dirname(root),root]){const s=fs.lstatSync(dir);if(!s.isDirectory()||s.uid!==process.geteuid()||s.mode&0o077||fs.realpathSync(dir)!==dir)throw Error();} +const {verifySnapshot,tree}=require('../src/offsite-backup'); +if(verifySnapshot(input.source,process.geteuid()).digest!==input.digest)throw Error(); +const target=path.join(root,input.id),temp=path.join(root,'.import-'+input.id); +if(!fs.existsSync(target)){fs.rmSync(temp,{recursive:true,force:true});tree(input.source,process.geteuid(),temp);fs.renameSync(temp,target);} +if(verifySnapshot(target,process.geteuid()).digest!==input.digest)throw Error(); +process.stdout.write('{"ok":true}\n'); +}catch{process.stderr.write('core_backup_import_failed\n');process.exitCode=1;} diff --git a/core/core/installations/bin/dispatch-core-recover b/core/core/installations/bin/dispatch-core-recover new file mode 100755 index 0000000..9086a78 --- /dev/null +++ b/core/core/installations/bin/dispatch-core-recover @@ -0,0 +1,55 @@ +#!/usr/bin/node --no-warnings +'use strict'; +// Run as the dashboard service account, from a trusted release checkout. +process.umask(0o077); +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { privateJson } = require('../src/release-delivery-files'); +async function main() { + const [action, localRoot, rolloutId, locked] = process.argv.slice(2); + if (!['status', 'recover', 'watch'].includes(action) || !localRoot || !path.isAbsolute(localRoot) + || fs.realpathSync(localRoot) !== localRoot || !/^rollout_[a-f0-9]{32}$/.test(rolloutId) + || process.argv.length > 6 || (locked && locked !== '--locked') || process.geteuid() === 0) throw Error(); + const lock = path.join(localRoot, 'data/access-control/platform-update.lock'); + const fd = fs.openSync(lock, fs.constants.O_CREAT | fs.constants.O_RDWR | fs.constants.O_NOFOLLOW, 0o600); + const stat = fs.fstatSync(fd); + if (!stat.isFile() || stat.uid !== process.geteuid() || stat.nlink !== 1 || (stat.mode & 0o7777) !== 0o600 + || fs.realpathSync(lock) !== lock) throw Error(); + if (!locked) { + const result = spawnSync('/usr/bin/flock', ['--nonblock', '--conflict-exit-code', '75', lock, + '/usr/bin/node', '--no-warnings', __filename, action, localRoot, rolloutId, '--locked'], + { stdio: ['ignore', 'inherit', 'inherit', fd] }); + fs.closeSync(fd); + if (result.status === 75 && action !== 'watch') process.stderr.write('{"status":"updater_busy"}\n'); + process.exitCode = result.status === 75 && action === 'watch' ? 0 : result.status ?? 1; return; + } + fs.closeSync(fd); + const journal = privateJson(path.join(localRoot, 'backups/platform-core', rolloutId, 'recovery.json'), process.geteuid()); + const { rootArtifact, createHostRecovery } = require('../src/core-recovery-host'); + const { root, config } = rootArtifact(journal.releaseId); + if (config.localRoot !== localRoot) throw Error(); + const recovery = require('../src/core-recovery').createCoreRecovery({ localRoot, + context: { rolloutId, releaseId: journal.releaseId, attempt: journal.attempt }, adapter: createHostRecovery(config, root) }); + if (action !== 'status') { + if (recovery.view().phase === 'promoted') await recovery.verify(); + else await recovery.recover(); + // A restore also restores the rollout tables. Reconcile them under the same + // updater lock so the older supervisor cannot immediately restart the update. + const store = require('../src/platform-update-store').openPlatformUpdateStore(path.join(localRoot, 'data/access-control')); + try { + store.transaction(() => { + const promoted = recovery.view().phase === 'promoted'; + store.db.prepare('UPDATE platform_rollout_core SET status=?,failure_code=?,updated_at=? WHERE rollout_id=?') + .run(promoted ? 'succeeded' : 'failed', promoted ? null : 'core_apply_failed', Date.now(), rolloutId); + if (!promoted) store.db.prepare("UPDATE platform_rollouts SET status='paused',updated_at=? WHERE id=? AND status!='completed'").run(Date.now(), rolloutId); + }); + } finally { store.close(); } + await createHostRecovery(config, root).disarmRecovery(); + } + const state = recovery.view(); + process.stdout.write(JSON.stringify({ status: state.phase, rolloutId, releaseId: state.releaseId, + priorReleaseId: state.prior.releaseId, attempt: state.attempt }) + '\n'); +} +main().catch(error => { process.stderr.write(JSON.stringify({ status: ['core_recovery_required', 'core_backup_invalid', + 'core_health_failed', 'core_service_failed'].includes(error?.code) ? error.code : 'core_recovery_unavailable' }) + '\n'); process.exitCode = 1; }); diff --git a/core/core/installations/bin/dispatch-install b/core/core/installations/bin/dispatch-install new file mode 100755 index 0000000..17e5847 --- /dev/null +++ b/core/core/installations/bin/dispatch-install @@ -0,0 +1,5 @@ +#!/usr/bin/node --no-warnings +'use strict'; +require('../src/install-command').main(process.argv.slice(2)).catch(error => { + process.stderr.write(`${error.message}\n`); process.exitCode = 1; +}); diff --git a/core/core/installations/bin/dispatch-installation-lifecycle b/core/core/installations/bin/dispatch-installation-lifecycle new file mode 100755 index 0000000..21939f7 --- /dev/null +++ b/core/core/installations/bin/dispatch-installation-lifecycle @@ -0,0 +1,150 @@ +#!/usr/bin/env node +'use strict'; + +const crypto = require('node:crypto'); +const path = require('node:path'); +const { installationFailure } = require('../../../shared/contracts/src'); +const { AccessStore } = require('../../accounts/src/store'); +const { createAccessInstallationLifecycleAuthority } = require('../../accounts/src/installation-lifecycle'); +const { loadPrivateReleaseCatalog, loadPrivateOciReleaseCatalog } = require('../src/release-catalog'); + +const { createProtectedOciHostClient } = require('../src/oci-protected-client'); +const { createOciContainerAdapter } = require('../src/oci-adapter'); +const { createOciRuntimeAgentCredentialPort } = require('../src/oci-runtime-agent-credential'); +const { createOciInstallationLifecycle } = require('../src/oci-lifecycle'); +const { createOciRuntimeLifecyclePort } = require('../src/oci-runtime-lifecycle-port'); +const { createRuntimeAgentDispatchClient, runtimeAgentControlInvoke } = require('../../agents/src'); + +function fail(code = 'invalid_input') { throw Object.assign(new Error(code), { code }); } +function absoluteEnvironment(name) { + const value = process.env[name]; + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) fail('runtime_boundary_violation'); + return value; +} +function parse(argv) { + const [command, organizationId, argument, extra] = argv; + if (!['status', 'backups', 'backup', 'restore', 'upgrade', 'suspend', 'resume', 'decommission', 'destroy'] + .includes(command) || !/^[a-z][a-z0-9_-]{2,95}$/.test(organizationId || '')) fail(); + if (['status', 'backups', 'backup', 'suspend', 'resume', 'decommission'].includes(command) + && (argument !== undefined || extra !== undefined)) fail(); + if (command === 'restore' && (!/^[a-z][a-z0-9_-]{2,95}$/.test(argument || '') || extra !== undefined)) fail(); + if (command === 'upgrade' && (!/^[a-z][a-z0-9_.-]{2,95}$/.test(argument || '') || extra !== undefined)) fail(); + if (command === 'destroy' && (argument !== '--approve-permanent-destruction' || extra !== undefined)) fail(); + return { command, organizationId, argument }; +} + +function idempotencyKey(input, revision) { + const digest = crypto.createHash('sha256') + .update(`${input.organizationId}\0${input.command}\0${input.argument || ''}\0${revision}`) + .digest('hex').slice(0, 32); + return `lifecycle:${input.command}:${digest}`; +} +function activeRequest(row) { + let receipts; + let request; + try { + receipts = JSON.parse(row.stage_receipts_json); + request = JSON.parse(receipts.__request); + } catch { fail('installation_operation_failed'); } + if (!request || typeof request !== 'object' || Array.isArray(request) + || request.operation !== row.operation) fail('installation_operation_failed'); + return request; +} + +async function main() { + const input = parse(process.argv.slice(2)); + const accessRoot = absoluteEnvironment('DISPATCH_ACCESS_CONTROL_DATABASE_ROOT'); + const store = new AccessStore({ + databaseRoot: accessRoot, + database: path.join(accessRoot, 'access-control.sqlite3'), + }); + try { + const control = store.installationControl(input.organizationId); + if (!control) fail('installation_not_found'); + const oci = ['oci_container_v1', 'native_service_v1'].includes(store.installationBackend(input.organizationId)); + const catalog = oci ? loadPrivateOciReleaseCatalog(absoluteEnvironment('DISPATCH_OCI_RELEASE_CATALOG_FILE')) + : loadPrivateReleaseCatalog(process.env.DISPATCH_RELEASE_CATALOG_FILE); + + const authority = createAccessInstallationLifecycleAuthority({ + store, + organizationId: input.organizationId, + authorityScope: 'platform_lifecycle', + releaseCatalog: Object.keys(catalog), + destructionEnabled: input.command === 'destroy', + }); + if (input.command === 'status') { + const active = store.activeLifecycleJob(input.organizationId); + process.stdout.write(`${JSON.stringify({ ok: true, status: control.status, + revision: control.revision, manifestRevision: control.manifestRevision, + releaseId: control.releaseId, + lifecycle: active ? authority.inspect(active.id) : null })}\n`); + return; + } + if (input.command === 'backups') { + process.stdout.write(`${JSON.stringify({ ok: true, status: 'found', items: authority.backups() })}\n`); + return; + } + let lifecycle; + if (oci) { + const socket = absoluteEnvironment('DISPATCH_RUNTIME_AGENT_CONTROL_SOCKET'); + const credentials = createOciRuntimeAgentCredentialPort({ + credentialRoot: absoluteEnvironment('DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT'), + }); + const host = createProtectedOciHostClient({ dispatchRequest: (request, dispatch) => authority.dispatchHostRequest(request, dispatch) }); + const adapter = createOciContainerAdapter({ hostRegistry: host.hostRegistry, hostExecutor: host.hostExecutor, + credentialPort: credentials, releaseResolver: id => { if (!catalog[id]) fail('release_not_found'); return catalog[id]; } }); + lifecycle = createOciInstallationLifecycle({ authority, adapter, hostExecutor: host.hostExecutor, + backupManagerFactory: (plan, claim) => host.createBackupManager(plan, claim), + runtimeFactory: plan => createOciRuntimeLifecyclePort({ client: createRuntimeAgentDispatchClient({ + runtimeKey: plan.runtimeKey, hub: { invoke: (key, action, value) => runtimeAgentControlInvoke(socket, key, action, value) }, + }) }), + }); + } else { + fail('installation_operation_not_allowed'); + } + const active = store.activeLifecycleJob(input.organizationId); + if (active && active.status === 'running' && active.attempt >= active.max_attempts + && active.lease_expires_at <= Date.now()) { + authority.retryExhausted(active.id); + } + let jobId; + if (active) { + if (active.operation !== input.command) fail('installation_operation_in_progress'); + const request = activeRequest(active); + if (input.command === 'restore' && request.backupId !== input.argument + || input.command === 'upgrade' && request.releaseId !== input.argument) { + fail('idempotency_conflict'); + } + jobId = active.id; + } else { + const operation = { + operation: input.command, + idempotencyKey: idempotencyKey(input, control.revision), + expectedRevision: control.revision, + }; + if (input.command === 'restore') operation.backupId = input.argument; + if (input.command === 'upgrade') operation.releaseId = input.argument; + jobId = authority.request(operation).id; + } + const result = await lifecycle.run(jobId, `worker_${crypto.randomUUID().replaceAll('-', '')}`); + const backups = authority.backups(); + if (oci && result.status === 'succeeded' && ['decommission', 'destroy'].includes(result.operation)) { + require('../src/retire-oci-credentials').retireOciCredentials({ store, credentialPort: createOciRuntimeAgentCredentialPort({ + credentialRoot: absoluteEnvironment('DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT') }) }); + } + process.stdout.write(`${JSON.stringify({ ok: result.status === 'succeeded', status: result.status, + operation: result.operation, installationState: result.installationState, + installationRevision: result.installationRevision, manifestRevision: result.manifestRevision, + failure: result.failure, backups })}\n`); + if (result.status !== 'succeeded') process.exitCode = 1; + } finally { store.close(); } +} + +main().catch(error => { + const selected = error?.code === 'invalid_input' + ? { code: 'invalid_input', category: 'request', recoverable: false } + : installationFailure(error); + process.stdout.write(`${JSON.stringify({ ok: false, status: selected.code, failure: selected })}\n`); + process.exitCode = 1; +}); diff --git a/core/core/installations/bin/dispatch-installation-reconcile b/core/core/installations/bin/dispatch-installation-reconcile new file mode 100755 index 0000000..554d649 --- /dev/null +++ b/core/core/installations/bin/dispatch-installation-reconcile @@ -0,0 +1,220 @@ +#!/usr/bin/env node +'use strict'; + +const crypto = require('node:crypto'); +const path = require('node:path'); +const { installationFailure } = require('../../../shared/contracts/src'); +const { PROJECT_ROOT } = require('../../../shared/paths/runtime-paths'); +const { AccessStore } = require('../../accounts/src/store'); +const { + createAccessControlLiveAuthorityResolver, + createInstallationProvisioningReconciler, +} = require('../../accounts/src/installation-provisioning'); +const { createAccessInstallationLifecycleAuthority } = require('../../accounts/src/installation-lifecycle'); +const { createDurableInstallationProvisioner } = require('../src/jobs'); +const { createInstallationLifecycleReconciler } = require('../src/lifecycle-reconcile'); +const { loadPrivateReleaseCatalog, loadPrivateOciReleaseCatalog } = require('../src/release-catalog'); +const { createRuntimeAgentCredentialManager } = require('../src/runtime-agent-credential'); +const { createOciRuntimeAgentCredentialPort } = require('../src/oci-runtime-agent-credential'); +const { createProtectedOciHostClient } = require('../src/oci-protected-client'); +const { createOciContainerAdapter } = require('../src/oci-adapter'); +const { createOciInstallationLifecycle } = require('../src/oci-lifecycle'); +const { createOciRuntimeLifecyclePort } = require('../src/oci-runtime-lifecycle-port'); +const { createRuntimeAgentDispatchClient, runtimeAgentControlInvoke } = require('../../agents/src'); + +function absoluteEnvironment(name) { + const value = process.env[name]; + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); + return value; +} + +function parse(argv) { + if (argv.length > 1) throw Object.assign(new Error('invalid_input'), { code: 'invalid_input' }); + if (argv.length === 0) return 20; + const match = /^--limit=([1-9]|[1-4][0-9]|50)$/.exec(argv[0]); + if (!match) throw Object.assign(new Error('invalid_input'), { code: 'invalid_input' }); + return Number.parseInt(match[1], 10); +} + +async function main() { + const limit = parse(process.argv.slice(2)); + const accessRoot = absoluteEnvironment('DISPATCH_ACCESS_CONTROL_DATABASE_ROOT'); + const stateRoot = absoluteEnvironment('DISPATCH_PROVISIONER_STATE_ROOT'); + const installationsRoot = absoluteEnvironment('DISPATCH_INSTALLATIONS_ROOT'); + const unitRoot = absoluteEnvironment('DISPATCH_SYSTEMD_UNIT_ROOT'); + const runtimeAgentHubSocket = absoluteEnvironment('DISPATCH_RUNTIME_AGENT_HUB_SOCKET'); + const releaseCatalog = loadPrivateReleaseCatalog(process.env.DISPATCH_RELEASE_CATALOG_FILE); + const ociReleaseCatalog = loadPrivateOciReleaseCatalog(process.env.DISPATCH_OCI_RELEASE_CATALOG_FILE); + const ociEnabled = process.env.DISPATCH_OCI_RELEASE_CATALOG_FILE !== undefined + || process.env.DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT !== undefined; + if (ociEnabled && (process.env.DISPATCH_OCI_RELEASE_CATALOG_FILE === undefined + || process.env.DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT === undefined)) { + throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); + } + if (!ociEnabled) throw Object.assign(new Error('installation_operation_not_allowed'), { code: 'installation_operation_not_allowed' }); + const runtimeAgentCredentials = createRuntimeAgentCredentialManager({ installationsRoot }); + let provisioner; + const ociRuntimeAgentCredentials = createOciRuntimeAgentCredentialPort({ + credentialRoot: absoluteEnvironment('DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT'), + }); + const runtimeAgentControlSocket = absoluteEnvironment('DISPATCH_RUNTIME_AGENT_CONTROL_SOCKET'); + const ociHost = createProtectedOciHostClient({ dispatchRequest: (request, dispatch) => { + const authority = Object.hasOwn(request.claim, 'generation') ? provisioner : null; + if (!authority) throw new Error('runtime_boundary_violation'); + return authority.dispatchHostRequest(request, dispatch); + } }); + const ociAdapter = createOciContainerAdapter({ + hostRegistry: ociHost.hostRegistry, + hostExecutor: ociHost.hostExecutor, + releaseResolver: (releaseId, fixture) => { + const selected = ociReleaseCatalog[releaseId]; + if (fixture || !selected) { + throw Object.assign(new Error('release_not_found'), { code: 'release_not_found' }); + } + return selected; + }, + credentialPort: ociRuntimeAgentCredentials, + }); + const store = new AccessStore({ + databaseRoot: accessRoot, + database: path.join(accessRoot, 'access-control.sqlite3'), + }); + provisioner = createDurableInstallationProvisioner({ + stateRoot, + installationsRoot, + unitRoot, + projectRoot: PROJECT_ROOT, + liveAuthorityResolver: createAccessControlLiveAuthorityResolver({ store }), + runtimeAgentHubSocket, + ociAdapter, + }); + try { + const platformReleases = require('../src/platform-release-catalog') + .loadPlatformReleaseCatalog(process.env.DISPATCH_PLATFORM_RELEASE_CATALOG_FILE, ociReleaseCatalog); + const rolloutCoordinator = require('../../accounts/src/platform-updates').createPlatformUpdates({ + store, releases: ociReleaseCatalog, platformReleases, enabled: true, + }); + const before = require('../src/drain-worker').progressKey(store.db); + rolloutCoordinator.tick(); + const corePending = store.db.prepare(`SELECT 1 FROM platform_rollouts r + LEFT JOIN platform_rollout_core c ON c.rollout_id=r.id + WHERE r.status!='completed' AND (c.status IS NULL OR c.status!='succeeded') LIMIT 1`).get(); + const backupPhase = require('../../accounts/src/rollout-backups').rolloutBackupProgress(store.db, + store.db.prepare("SELECT id FROM platform_rollouts WHERE status!='completed' LIMIT 1").get()?.id || 'none'); + if (corePending && (!backupPhase || backupPhase.status !== 'running')) { + process.stdout.write(`${JSON.stringify({ ok: true, status: 'waiting_for_core', pending: 1 })}\n`); + require('../../accounts/src/worker-wakeup').wake(['core'], { databaseRoot: accessRoot }); + return { progressed: false, pending: 1, failed: 0 }; + } + const backupWorker = require('../src/platform-backup-worker').createPlatformBackupWorker({ + store, localRoot: path.resolve(accessRoot, '../..'), + archive: require('../src/backup-archive-status').backupArchiveStatus, + restartCore: async () => { + const fs=require('node:fs'); + const deployment=JSON.parse(fs.readFileSync(path.resolve(PROJECT_ROOT,'../deployment.json'))); + const result=require('node:child_process').spawnSync('/usr/bin/systemctl',['--user','restart','dispatch-dashboard.service'],{encoding:'utf8',timeout:60000,maxBuffer:4096}); + if(result.status!==0)throw Error('core_service_failed'); + const {readHealth}=require('../src/core-recovery-host'); + for(let attempt=0;attempt<30;attempt++) { + try {const health=await readHealth(deployment);if(health.releaseId===deployment.releaseId&&health.sourceCommit===deployment.sourceCommit)return;}catch{} + await new Promise(resolve=>setTimeout(resolve,1000)); + } + throw Error('core_health_failed'); + }, + }); + await backupWorker.tick(); + const workerId = `worker_${crypto.randomUUID().replaceAll('-', '')}`; + const result = corePending ? { processed: 0, completed: 0, failed: 0, pending: 0 } : createInstallationProvisioningReconciler({ + store, + provisioner, + runtimeAgentCredentialFactory: backend => ['oci_container_v1', 'native_service_v1'].includes(backend) + ? ociRuntimeAgentCredentials : runtimeAgentCredentials, + }) + .runPending(workerId, limit); + require('../../accounts/src/organization-profile').applyOrganizationProfiles(store); + const authorityFactory = (organizationId, authorityScope) => + createAccessInstallationLifecycleAuthority({ + store, + organizationId, + authorityScope, + releaseCatalog: [...new Set([...Object.keys(releaseCatalog), ...Object.keys(ociReleaseCatalog)])], + }); + const lifecycle = await createInstallationLifecycleReconciler({ + store, + authorityFactory, + concurrency: corePending ? 3 : 1, + backupOnly: Boolean(corePending), + runtimeFactory: (organizationId, authority) => { + // Every concurrent job owns its host dispatcher and fenced authority. + const lifecycleHost = createProtectedOciHostClient({ dispatchRequest: (request, dispatch) => authority.dispatchHostRequest(request, dispatch) }); + const lifecycleAdapter = createOciContainerAdapter({ hostRegistry: lifecycleHost.hostRegistry, + hostExecutor: lifecycleHost.hostExecutor, credentialPort: ociRuntimeAgentCredentials, + releaseResolver: (releaseId, fixture) => { + if (fixture || !ociReleaseCatalog[releaseId]) throw Error('release_not_found'); + return ociReleaseCatalog[releaseId]; + } }); + const backend = store.installationBackend(organizationId); + if (!['oci_container_v1', 'native_service_v1'].includes(backend)) throw Object.assign(new Error('installation_operation_not_allowed'), { code: 'installation_operation_not_allowed' }); + return createOciInstallationLifecycle({ + authority, + adapter: lifecycleAdapter, + hostExecutor: lifecycleHost.hostExecutor, + backupManagerFactory: (plan, claim) => lifecycleHost.createBackupManager(plan, claim), + runtimeFactory: plan => createOciRuntimeLifecyclePort({ + client: createRuntimeAgentDispatchClient({ + runtimeKey: plan.runtimeKey, + hub: { + invoke: (runtimeKey, action, input) => + runtimeAgentControlInvoke(runtimeAgentControlSocket, runtimeKey, action, input), + }, + }), + }), + }); + }, + }).runPending(`lifecycle_${crypto.randomUUID().replaceAll('-', '')}`, limit); + await backupWorker.tick(); + rolloutCoordinator.tick(); + if (corePending) { + process.stdout.write(JSON.stringify({ ok: true, status: 'backing_up', lifecycle }) + '\n'); + require('../../accounts/src/worker-wakeup').wake(['core'], { databaseRoot: accessRoot }); + return { progressed: before !== require('../src/drain-worker').progressKey(store.db), pending: true, failed: lifecycle.failed }; + } + const credentialsRetired = require('../src/retire-oci-credentials') + .retireOciCredentials({ store, credentialPort: ociRuntimeAgentCredentials }); + const onboarding = await require('../src/owner-onboarding').createOwnerOnboardingWorker({ + store, projectRoot: PROJECT_ROOT, + invoke: (runtimeKey, action, input) => runtimeAgentControlInvoke(runtimeAgentControlSocket, runtimeKey, action, input), + }).runPending(`onboard_${crypto.randomUUID().replaceAll('-', '')}`, limit); + const diagnostics = await require('../src/diagnostics-worker').createDiagnosticsWorker({ + store, invoke: (runtimeKey, action, input) => runtimeAgentControlInvoke(runtimeAgentControlSocket, runtimeKey, action, input), + }).runPending(`diagnostics_${crypto.randomUUID().replaceAll('-', '')}`, limit); + const failed = result.failed + lifecycle.failed + onboarding.failed + diagnostics.failed; + const rollout = rolloutCoordinator.view().rollout; + const pending = result.pending + (lifecycle.pending ? 1 : 0) + (rollout?.status === 'running' ? 1 : 0); + process.stdout.write(`${JSON.stringify({ + ok: true, + status: pending ? 'pending' : failed ? 'failed' : 'settled', + ...result, + failed, + pending, + lifecycle, + rollout: rollout ? { status: rollout.status, total: rollout.total, updated: rollout.updated } : null, + onboarding, + diagnostics, + credentialsRetired, + })}\n`); + if (failed) process.exitCode = 1; + if (lifecycle.completed) require('../src/worker-notify').exportReady(); + return { pending, failed, progressed: before !== require('../src/drain-worker').progressKey(store.db) || result.processed > 0 }; + } finally { + provisioner.close(); + store.close(); + } +} + +require('../src/drain-worker').drain(main).catch(error => { + const selected = installationFailure(error); + process.stdout.write(`${JSON.stringify({ ok: false, status: selected.code, failure: selected })}\n`); + process.exitCode = 1; +}); diff --git a/core/core/installations/bin/dispatch-oci-host-helper b/core/core/installations/bin/dispatch-oci-host-helper new file mode 100755 index 0000000..f60d671 --- /dev/null +++ b/core/core/installations/bin/dispatch-oci-host-helper @@ -0,0 +1,31 @@ +#!/usr/bin/node +'use strict'; + +const { readHelperRequest } = require('../src/oci-helper-input'); +const { installationFailure } = require('../../../shared/contracts/src/installation'); +const { createOciHostHelper } = require('../src/oci-host-helper'); + +function response(value) { + process.stdout.write(`${JSON.stringify(value)}\n`); +} + +function main() { + process.umask(0o077); + let helper; + try { + if (process.argv.length !== 2) throw new Error('runtime_boundary_violation'); + const request = readHelperRequest(0, 256 * 1024); + helper = createOciHostHelper(); + const result = helper.execute(request); + response({ ok: true, result }); + return 0; + } catch (error) { + const failure = installationFailure(error); + response({ ok: false, status: failure.code, ...(['image_inspect', 'image_load', 'loaded_image_inspect', 'manifest_attestation'].includes(error.hostStep) ? { step: error.hostStep } : {}) }); + return 1; + } finally { + try { helper?.close(); } catch {} + } +} + +process.exitCode = main(); diff --git a/core/core/installations/bin/dispatch-oci-host-issuer b/core/core/installations/bin/dispatch-oci-host-issuer new file mode 100755 index 0000000..01077dc --- /dev/null +++ b/core/core/installations/bin/dispatch-oci-host-issuer @@ -0,0 +1,16 @@ +#!/usr/bin/node +'use strict'; + +const { readHelperRequest } = require('../src/oci-helper-input'); +const { dispatchAuthorized } = require('../src/oci-host-issuer'); +const { installationFailure } = require('../../../shared/contracts/src/installation'); +process.umask(0o077); +try { + if (process.argv.length !== 2) throw new Error('runtime_boundary_violation'); + const response = dispatchAuthorized(readHelperRequest(0, 256 * 1024)); + process.stdout.write(`${JSON.stringify(response)}\n`); + process.exitCode = response.ok ? 0 : 1; +} catch (error) { + process.stdout.write(`${JSON.stringify({ ok: false, status: installationFailure(error).code })}\n`); + process.exitCode = 1; +} diff --git a/core/core/installations/bin/dispatch-oci-tenant-backup-helper b/core/core/installations/bin/dispatch-oci-tenant-backup-helper new file mode 100755 index 0000000..53bf455 --- /dev/null +++ b/core/core/installations/bin/dispatch-oci-tenant-backup-helper @@ -0,0 +1,48 @@ +#!/usr/bin/node +'use strict'; + +const { readHelperRequest } = require('../src/oci-helper-input'); +const { installationFailure } = require('../../../shared/contracts/src/installation'); +const { createInstallationBackupManager } = require('../src/backups'); + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} +function exact(value, keys) { + if (!plain(value) || Object.keys(value).sort().join(',') !== [...keys].sort().join(',')) { + throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); + } +} +function main() { + process.umask(0o077); + try { + if (process.geteuid() === 0 || process.argv.length !== 2) throw new Error('runtime_boundary_violation'); + const request = readHelperRequest(0, 128 * 1024); + exact(request, ['version', 'operation', 'layout', 'payload']); + if (request.version !== 1 || !plain(request.payload)) throw new Error('runtime_boundary_violation'); + const manager = createInstallationBackupManager({ layout: request.layout, full: true }); + let result; + if (request.operation === 'snapshot') { + exact(request.payload, ['spec']); result = manager.snapshot(request.payload.spec, callback => callback()); + } else if (request.operation === 'inspect') { + exact(request.payload, ['spec']); result = manager.inspect(request.payload.spec); + } else if (request.operation === 'restore') { + exact(request.payload, ['source', 'operationId']); + result = manager.restore(request.payload.source, request.payload.operationId, callback => callback()); + } else if (request.operation === 'inspect_restored') { + exact(request.payload, ['source']); result = manager.inspectRestored(request.payload.source); + } else if (request.operation === 'destroy') { + exact(request.payload, ['authority']); result = manager.destroy(request.payload.authority, callback => callback()); + } else if (request.operation === 'verify_destroyed') { + exact(request.payload, []); result = manager.verifyDestroyed(); + } else throw new Error('runtime_boundary_violation'); + process.stdout.write(`${JSON.stringify({ ok: true, result })}\n`); + return 0; + } catch (error) { + const failure = installationFailure(error); + process.stdout.write(`${JSON.stringify({ ok: false, status: failure.code })}\n`); + return 1; + } +} +process.exitCode = main(); diff --git a/core/core/installations/bin/dispatch-offsite-backup b/core/core/installations/bin/dispatch-offsite-backup new file mode 100755 index 0000000..d23de1f --- /dev/null +++ b/core/core/installations/bin/dispatch-offsite-backup @@ -0,0 +1,133 @@ +#!/usr/bin/node --no-warnings +'use strict'; +process.umask(0o077); +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { atomic } = require('../src/release-delivery-files'); +const { loadConfig, createRestic, exportSnapshot, WORK } = require('../src/offsite-backup'); +const { RECEIPTS, POLICY } = require('../src/offsite-policy'); +const { HOST_TENANT_ROOT, opaqueRuntimeSuffix } = require('../../runtime-host-identity'); +function directories(root, pattern) { + try { return fs.readdirSync(root, { withFileTypes: true }).filter(e => e.isDirectory() && pattern.test(e.name)).map(e => e.name); } + catch (error) { if (error.code === 'ENOENT') return []; throw error; } +} +function sources(config, excludedRuntimes = new Set()) { + const result = []; + const root = path.join(config.localRoot, 'backups/platform-core'); + for (const rollout of directories(root, /^rollout_[a-f0-9]{32}$/)) { + for (const attempt of directories(path.join(root, rollout), /^attempt-[1-9][0-9]*$/)) { + const source = path.join(root, rollout, attempt); + if (fs.existsSync(path.join(source, 'manifest.json')) && !JSON.parse(fs.readFileSync(path.join(source, 'manifest.json'))).localOnly) result.push({ source, uid: config.coreUid }); + } + } + for (const suffix of directories(HOST_TENANT_ROOT, /^[a-f0-9]{20}$/)) { + const runtimeRoot = path.join(HOST_TENANT_ROOT, suffix, 'runtime'); + for (const runtime of directories(runtimeRoot, /^[a-z][a-z0-9_-]{2,95}$/)) { + if (excludedRuntimes.has(runtime)) continue; + if (opaqueRuntimeSuffix(runtime) !== suffix) throw Error(); + const installation = path.join(runtimeRoot, runtime); + const uid = fs.lstatSync(installation).uid; + if (uid === 0 || uid === config.coreUid || fs.realpathSync(installation) !== installation) throw Error(); + for (const name of directories(path.join(installation, 'backups'), /^[a-z][a-z0-9_-]{2,95}$/)) { + const source = path.join(installation, 'backups', name); + if (fs.existsSync(path.join(source, 'manifest.json'))) result.push({ source, uid }); + } + } + } + return result; +} +async function main() { + const [action, locked] = process.argv.slice(2); + if (process.geteuid() !== 0 || !['init', 'scan', 'check', 'status', 'enable', 'prewarm'].includes(action) + || (locked && locked !== '--locked') || process.argv.length > 4) throw Error(); + // The timer stays pinned to verified immutable code, independent of Core restarts. + const artifact = path.resolve(__dirname, "../../../.."); + const { rootArtifact } = require('../src/core-recovery-host'); + const id = path.basename(path.dirname(artifact)); + if (rootArtifact(id).root !== artifact) throw Error(); + fs.mkdirSync(WORK, { recursive: true, mode: 0o700 }); + fs.mkdirSync(RECEIPTS, { recursive: true, mode: 0o755 }); fs.chmodSync(RECEIPTS, 0o755); + for (const [dir, mode] of [[WORK, 0o700], [RECEIPTS, 0o755]]) { + const stat = fs.lstatSync(dir); + if (stat.uid !== 0 || !stat.isDirectory() || (stat.mode & 0o7777) !== mode || fs.realpathSync(dir) !== dir) throw Error(); + } + if (action === 'prewarm') { + const result = require('../src/recovery-prewarm').prewarm({ + report: status => atomic(path.join(RECEIPTS, 'recovery-prewarm.json'), status, 0o644), + }); + process.stdout.write(JSON.stringify(result) + '\n'); return; + } + if (!locked) { + const lockOptions = action === 'enable' ? ['--wait', '3300'] : ['--nonblock', '--conflict-exit-code', '75']; + const result = spawnSync('/usr/bin/flock', [...lockOptions, path.join(WORK, 'worker.lock'), + '/usr/bin/node', '--no-warnings', __filename, action, '--locked'], { stdio: ['ignore', 'inherit', 'inherit'] }); + process.exitCode = result.status === 75 ? 0 : result.status ?? 1; return; + } + if (action === 'status') { + const value = require('../src/offsite-policy').publicRootJson(path.join(RECEIPTS, 'status.json'), true); + process.stdout.write(JSON.stringify(value || { status: 'not_verified' }) + '\n'); return; + } + require('../src/backup-scratch').cleanupBackupScratch(WORK); + require('../src/recovery-artifacts').pruneLocalCache(); + require('../src/backup-scratch').cleanupRestoreStaging(); + const config = loadConfig(); const run = createRestic(config.environment); + if (action === 'init') run(['init', '--repository-version', '2']); + if (action === 'enable') { + try { run(['cat', 'config']); } catch { run(['init', '--repository-version', '2']); } + } + if (action === 'check') { + run(['check', '--read-data']); + require('../src/backup-archives').createBackupArchives(config).check(); + process.stdout.write('{"status":"repository_checked"}\n'); return; + } + let count = 0, failed = 0; + try { require('../src/retired-dsp-metadata').purgeRetiredMetadata(config); } catch { failed++; } + const archiveWorker = require('../src/backup-archives').createBackupArchives(config); + let archives; + for (let pass = 0; pass < 5; pass++) { + const before = require('../src/drain-worker').backupQueueKey(config); + archives = await archiveWorker.scan(); + if (before === require('../src/drain-worker').backupQueueKey(config)) break; + } + count += archives.verified; failed += archives.failed; + const managedIds = new Set(archives.managedIds); + // Newest snapshots first so an active rollout is not stuck behind old archives. + const pending = sources(config, new Set([...(archives.deletedRuntimeKeys || []), ...(archives.heldRuntimes || [])])).sort((a, b) => fs.statSync(b.source).mtimeMs - fs.statSync(a.source).mtimeMs); + for (const source of pending) { + if (managedIds.has(path.basename(source.source)) + || (archives.deletedRuntimeKeys || []).includes(path.basename(path.dirname(path.dirname(source.source))))) continue; + try { exportSnapshot({ ...source, run, config }); count++; } + catch { failed++; } + } + try { await require('../src/release-retention').pruneReleases(config); } catch { failed++; } + const status = { schemaVersion: 1, status: failed ? 'backup_failed' : 'verified', checkedAt: Date.now(), verifiedSnapshots: count, failedSnapshots: failed }; + atomic(path.join(RECEIPTS, 'status.json'), status, 0o644); fs.chmodSync(path.join(RECEIPTS, 'status.json'), 0o644); + if (action === 'enable') { + // Confirm encrypted uploads even on a fresh server with no snapshots. + const { DatabaseSync } = require('node:sqlite'); + const canary = fs.mkdtempSync(path.join(WORK, 'canary-')); + try { + const file = path.join(canary, 'access-control-before.sqlite3'); + const db = new DatabaseSync(file); db.exec("CREATE TABLE recovery_canary(value TEXT); INSERT INTO recovery_canary VALUES('dispatch upload confirmed')"); db.close(); fs.chmodSync(file, 0o600); + atomic(path.join(canary, 'manifest.json'), { version: 1, kind: 'core', sha256: require('../src/release-delivery-files').hashFileSync(file), size: fs.statSync(file).size }); + exportSnapshot({ source: canary, uid: 0, run }); + } finally { fs.rmSync(canary, { recursive: true, force: true }); } + if (failed) throw Error(); + const unit = `[Unit]\nDescription=Encrypt and verify Dispatch off-server backups\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUMask=0077\nExecStart=/usr/bin/node --no-warnings ${__filename} scan\nTimeoutStartSec=1h\n`; + const timer = '[Unit]\nDescription=Export completed Dispatch backups\n\n[Timer]\nOnBootSec=15s\nOnUnitInactiveSec=60s\nAccuracySec=1s\n\n[Install]\nWantedBy=timers.target\n'; + atomic('/etc/systemd/system/dispatch-offsite-backup.service', unit, 0o644); + atomic('/etc/systemd/system/dispatch-offsite-backup.timer', timer, 0o644); + atomic('/etc/systemd/system/dispatch-offsite-backup.path', `[Unit]\nDescription=Export newly completed Dispatch snapshots\n\n[Path]\nPathChanged=${config.localRoot}/run/backup-ready\nUnit=dispatch-offsite-backup.service\n\n[Install]\nWantedBy=multi-user.target\n`, 0o644); + require('../src/recovery-prewarm').install(__filename, config.localRoot); + require('../src/backup-readiness').install(path.join(__dirname, "./dispatch-backup-readiness"), config.localRoot); + for (const args of [['daemon-reload'], ['enable', '--now', 'dispatch-offsite-backup.timer', 'dispatch-offsite-backup.path', 'dispatch-recovery-prewarm.timer', 'dispatch-recovery-prewarm.path', 'dispatch-backup-readiness.timer', 'dispatch-backup-readiness.path'], + ['start', '--no-block', 'dispatch-recovery-prewarm.service', 'dispatch-backup-readiness.service']]) { + if (spawnSync('/usr/bin/systemctl', args, { timeout: 30000 }).status !== 0) throw Error(); + } + atomic(POLICY, { schemaVersion: 1, required: true }, 0o644); fs.chmodSync(POLICY, 0o644); + } + if (require('../src/drain-worker').workPending(config)) require('../src/worker-notify').userWorkers(config); + process.stdout.write(JSON.stringify(status) + '\n'); if (failed) process.exitCode = 1; +} +main().catch(() => { process.stderr.write('{"status":"offsite_backup_unavailable"}\n'); process.exitCode = 1; }); diff --git a/core/core/installations/bin/dispatch-platform-update b/core/core/installations/bin/dispatch-platform-update new file mode 100755 index 0000000..18351ff --- /dev/null +++ b/core/core/installations/bin/dispatch-platform-update @@ -0,0 +1,41 @@ +#!/usr/bin/node --no-warnings +'use strict'; + +// Keep this supervisor pinned outside the Core release that it replaces. +process.umask(0o077); +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { openPlatformUpdateStore } = require('../src/platform-update-store'); +const { loadPrivateOciReleaseCatalog } = require('../src/release-catalog'); +const { loadPlatformReleaseCatalog } = require('../src/platform-release-catalog'); +const { createPlatformCoreUpdater } = require('../src/platform-core-update'); +async function main() { + const root = process.env.DISPATCH_ACCESS_CONTROL_DATABASE_ROOT; + if (typeof root !== 'string' || !path.isAbsolute(root) || fs.realpathSync(root) !== root) throw new Error(); + const info = fs.lstatSync(root); + if (!info.isDirectory() || info.uid !== process.geteuid() || (info.mode & 0o7777) !== 0o700) throw new Error(); + const lock = path.join(root, 'platform-update.lock'); + const fd = fs.openSync(lock, fs.constants.O_CREAT | fs.constants.O_RDWR | fs.constants.O_NOFOLLOW, 0o600); + const stat = fs.fstatSync(fd); + if (!stat.isFile() || stat.uid !== process.geteuid() || stat.nlink !== 1 || (stat.mode & 0o7777) !== 0o600) throw new Error(); + if (process.argv.length === 2) { + const result = spawnSync('/usr/bin/flock', ['--nonblock', '--conflict-exit-code', '75', lock, + '/usr/bin/node', '--no-warnings', __filename, '--locked'], { stdio: ['ignore', 'inherit', 'inherit', fd] }); + fs.closeSync(fd); + process.exitCode = result.status === 75 ? 0 : result.status ?? 1; + return; + } + fs.closeSync(fd); + if (process.argv.length !== 3 || process.argv[2] !== '--locked') throw new Error(); + const runtimes = loadPrivateOciReleaseCatalog(process.env.DISPATCH_OCI_RELEASE_CATALOG_FILE); + const platformReleases = loadPlatformReleaseCatalog(process.env.DISPATCH_PLATFORM_RELEASE_CATALOG_FILE, runtimes); + const store = openPlatformUpdateStore(root); + try { + const result = await createPlatformCoreUpdater({ store, platformReleases }).run(); + process.stdout.write(JSON.stringify(result) + '\n'); + if (result.status === 'core_verified') require('../../accounts/src/worker-wakeup').wake(['reconcile'], { databaseRoot: root }); + if (result.status === 'core_update_failed') process.exitCode = 1; + } finally { store.close(); } +} +main().catch(() => { process.stderr.write('{"status":"platform_updater_unavailable"}\n'); process.exitCode = 1; }); diff --git a/core/core/installations/bin/dispatch-recovery-kit b/core/core/installations/bin/dispatch-recovery-kit new file mode 100755 index 0000000..1785526 --- /dev/null +++ b/core/core/installations/bin/dispatch-recovery-kit @@ -0,0 +1,39 @@ +#!/usr/bin/node --no-warnings +'use strict'; +process.umask(0o077); +const fs = require('node:fs'), path = require('node:path'); +try { + const [action, destination] = process.argv.slice(2); + if (process.geteuid() !== 0 || action !== 'export' || process.argv.length !== 4 + || !path.isAbsolute(destination) || path.resolve(destination) !== destination || fs.existsSync(destination)) throw Error(); + const config = require('../src/offsite-backup').loadConfig(); + fs.mkdirSync(destination, { mode: 0o700 }); + const root = path.resolve(__dirname, "../../.."); + const files = ['core/installations/src/recovery-artifacts.js', 'core/installations/src/assemble-system-recovery.js', 'core/accounts/src/core-backup.js', 'core/installations/src/recovery-kit-runner.js', 'core/installations/src/recovery-capsule.js', + 'core/installations/src/host-recovery-bundle.js', 'core/installations/src/r2-backup-storage.js', + 'core/installations/src/release-delivery-files.js', 'core/runtime-host-identity.js', + 'shared/contracts/src/installation.js', 'shared/contracts/src/input.js', 'shared/paths/runtime-paths.js']; + for (const relative of files) { + const target = path.join(destination, 'code', relative); + fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 }); + fs.copyFileSync(path.join(root, relative), target); fs.chmodSync(target, 0o600); + } + const nodeStage = path.join(destination, 'node-runtime'); + require('../src/portable-node').bundleNode('/usr/bin/node', nodeStage); + fs.renameSync(path.join(nodeStage, 'node'), path.join(destination, 'node')); + fs.renameSync(path.join(nodeStage, 'lib'), path.join(destination, 'lib')); + fs.renameSync(path.join(nodeStage, 'host-files'), path.join(destination, 'host-files')); + fs.rmdirSync(nodeStage); + for (const binary of ['restic']) { + fs.copyFileSync(fs.realpathSync(`/usr/bin/${binary}`), path.join(destination, binary)); + fs.chmodSync(path.join(destination, binary), 0o700); + } + fs.copyFileSync('/etc/dispatch/offsite-backup-password', path.join(destination, 'password')); + fs.chmodSync(path.join(destination, 'password'), 0o600); + fs.writeFileSync(path.join(destination, 'storage.json'), JSON.stringify({ accountId: config.accountId, bucket: config.bucket, prefix: config.prefix, + credential: JSON.parse(fs.readFileSync('/etc/dispatch/offsite-backup-credentials.json')) }), { mode: 0o600 }); + fs.writeFileSync(path.join(destination, 'run.js'), `require(require('node:path').join(__dirname,'code','core','installations','src','recovery-kit-runner')).main(__dirname,process.argv.slice(2)).then(result=>console.log(JSON.stringify(result,null,2))).catch(e=>{console.error(e.message);process.exitCode=1});\n`, { mode: 0o600 }); + fs.writeFileSync(path.join(destination, 'restore'), '#!/bin/sh\nset -eu\nKIT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)"\nif test -d "$KIT_DIR/host-files/usr/share/nodejs"; then\n mkdir -p /usr/share/nodejs\n cp -R "$KIT_DIR/host-files/usr/share/nodejs/." /usr/share/nodejs/\nfi\nexec "$KIT_DIR/lib/ld-linux-x86-64.so.2" --library-path "$KIT_DIR/lib" "$KIT_DIR/node" --no-warnings "$KIT_DIR/run.js" "$@"\n', { mode: 0o700 }); + fs.writeFileSync(path.join(destination, 'README.txt'), 'Keep this private recovery kit outside the VPS. It contains the decryption key and storage credentials.\nOn a clean Ubuntu 24.04 amd64 VPS, run as root:\n ./restore list\n ./restore restore-system FULL_SYSTEM_BACKUP_ID\nFor Core-only recovery: ./restore restore CORE_BACKUP_ID all\nFor a legacy pre-update full-platform backup use: ./restore restore platform-core SNAPSHOT_ID\nThe restore downloads and verifies the backup, installs prerequisites, restores code, accounts, secrets, configuration and data, then starts recorded services. Existing Dispatch data is never overwritten.\n', { mode: 0o600 }); + console.log(JSON.stringify({ status: 'recovery_kit_exported', destination })); +} catch { console.error('recovery_kit_export_failed'); process.exitCode = 1; } diff --git a/core/core/installations/bin/dispatch-release-build b/core/core/installations/bin/dispatch-release-build new file mode 100755 index 0000000..d4d5a2c --- /dev/null +++ b/core/core/installations/bin/dispatch-release-build @@ -0,0 +1,22 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +const {build,portableCore}=require('../src/release-delivery-build'); +(async()=>{ + const args=process.argv.slice(2); + if(args.length===2&&args[0]==='--core-only'){portableCore(args[1]);return;} + const [version, notes, output, ...options] = args; + let format = 'legacy', prepared = null; + const seen = new Set(); + for (let i = 0; i < options.length; i++) { + const option = options[i]; + if (['--format', '--prepared'].includes(option) && !seen.has(option) && options[i + 1]) { + seen.add(option); + if (option === '--format') format = options[++i]; else prepared = options[++i]; + } else if (i === 0 && options.length === 1 && require('node:path').isAbsolute(option)) { + prepared = option; // Preserve the original legacy assembly command. + } else throw Error('invalid_release_build_options'); + } + if (!version || !notes || !output) throw Error('usage: dispatch-release-build VERSION NOTES OUTPUT [--format legacy|split|both] [--prepared DIRECTORY]'); + const value=await build(version,notes,output,format,prepared); + console.log(JSON.stringify({status:'release_built',version:value.version,sourceCommit:value.sourceCommit})); +})().catch(error=>{console.error(JSON.stringify({code:error.message,requiredBytes:error.requiredBytes,availableBytes:error.availableBytes}));process.exitCode=1;}); diff --git a/core/core/installations/bin/dispatch-release-delivery-install b/core/core/installations/bin/dispatch-release-delivery-install new file mode 100755 index 0000000..67d4963 --- /dev/null +++ b/core/core/installations/bin/dispatch-release-delivery-install @@ -0,0 +1,37 @@ +#!/usr/bin/node --no-warnings +'use strict'; +// One-time host bootstrap. No release is selected or rolled out by this command. +const fs=require('node:fs'); +const path=require('node:path'); +const {spawnSync}=require('node:child_process'); +const {bundle,COMMIT}=require('../src/release-delivery-contract'); +const {configuration,CONFIG,TOKEN,STATE}=require('../src/release-delivery-config'); +const {rootParents,atomic}=require('../src/release-delivery-files'); +const {writeBundle,installTree,removeStage}=require('../src/release-delivery-install'); +async function main(){ + process.umask(0o022); + if(process.geteuid()!==0||process.argv.length!==4)throw Error('usage: sudo dispatch-release-delivery-install CONFIG_JSON PORTABLE_CORE_JSON < TOKEN_FILE'); + const config=configuration(JSON.parse(fs.readFileSync(process.argv[2],'utf8'))); + const portable=JSON.parse(fs.readFileSync(process.argv[3],'utf8'));bundle(portable,'core',portable.sourceCommit); + if(!COMMIT.test(portable.sourceCommit))throw Error(); + let token='';process.stdin.setEncoding('utf8');for await(const chunk of process.stdin){token+=chunk;if(token.length>4096)throw Error('invalid_token');} + token=token.trim();if(!token||/\s/.test(token))throw Error('invalid_token'); + rootParents('/opt');rootParents('/etc');rootParents('/var/lib'); + fs.mkdirSync(path.dirname(CONFIG),{recursive:true,mode:0o755});rootParents(path.dirname(CONFIG)); + fs.mkdirSync(STATE,{recursive:true,mode:0o700});rootParents(STATE); + require('../src/install-layout').initializeLayout(config); + const base='/opt/dispatch-release-delivery/releases';fs.mkdirSync(base,{recursive:true,mode:0o755});rootParents(base); + const stage=path.join(STATE,'bootstrap');removeStage(stage);fs.mkdirSync(stage,{mode:0o700}); + try{ + writeBundle(portable,stage); + const target=path.join(base,portable.sourceCommit);installTree(stage,target); + atomic(CONFIG,config);atomic(TOKEN,token+'\n');token=''; + const service=`[Unit]\nDescription=Discover and prepare Dispatch releases\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/bin/node --no-warnings ${target}/code/core/installations/bin/dispatch-release-watch\nEnvironment=PATH=/usr/bin:/bin\nUMask=0022\nTimeoutStartSec=45min\nTimeoutStopSec=30s\nKillMode=control-group\nPrivateTmp=true\nProtectSystem=full\nReadWritePaths=/etc/sudoers.d /etc/apparmor.d\nProtectKernelTunables=true\nProtectControlGroups=true\n\n`; + const timer='[Unit]\nDescription=Check for published Dispatch releases\n\n[Timer]\nOnBootSec=30s\nOnUnitInactiveSec=45s\nAccuracySec=1s\nUnit=dispatch-release-watch.service\n\n[Install]\nWantedBy=timers.target\n'; + rootParents('/etc/systemd/system');atomic('/etc/systemd/system/dispatch-release-watch.service',service,0o644);atomic('/etc/systemd/system/dispatch-release-watch.timer',timer,0o644); + require('../src/release-ready-notify').install(config.localRoot); + for(const args of [['daemon-reload'],['enable','--now','dispatch-release-watch.timer','dispatch-release-watch.path']])if(spawnSync('/usr/bin/systemctl',args,{stdio:'inherit'}).status!==0)throw Error('watcher_service_failed'); + console.log('Release discovery enabled. Published updates will be prepared automatically; rollouts remain user-started.'); + }finally{removeStage(stage);} +} +main().catch(error=>{console.error(error.message);process.exitCode=1;}); diff --git a/core/core/installations/bin/dispatch-release-watch b/core/core/installations/bin/dispatch-release-watch new file mode 100755 index 0000000..99085ad --- /dev/null +++ b/core/core/installations/bin/dispatch-release-watch @@ -0,0 +1,55 @@ +#!/usr/bin/node --no-warnings +'use strict'; +const fs=require('node:fs'); +const path=require('node:path'); +const {spawnSync}=require('node:child_process'); +const {configuration,CONFIG,TOKEN,STATE}=require('../src/release-delivery-config'); +const {privateJson,rootParents}=require('../src/release-delivery-files'); +const {createGitHubReleaseSource}=require('../src/release-delivery-github'); +const {createReleaseWatcher}=require('../src/release-delivery-watch'); +const {prepareRelease}=require('../src/release-delivery-install'); +async function main(){ + process.umask(0o022); + if(process.geteuid()!==0)throw Error(); + rootParents(path.dirname(CONFIG)); + const config=configuration(privateJson(CONFIG,0)); + fs.mkdirSync(STATE,{recursive:true,mode:0o700});rootParents(STATE); + const args = process.argv.slice(2); + const locked = args[0] === '--locked'; if (locked) args.shift(); + const request = args.length ? require('../src/install-command').parse(['prepare', ...args]) : null; + const target = request ? { version: request.version, sourceCommit: request.sourceCommit } : null; + const lock=path.join(STATE,'watch.lock'); + const fd=fs.openSync(lock,fs.constants.O_CREAT|fs.constants.O_RDWR|fs.constants.O_NOFOLLOW,0o600); + const st=fs.fstatSync(fd);if(!st.isFile()||st.uid!==0||st.nlink!==1||(st.mode&0o7777)!==0o600)throw Error(); + if(!locked){ + const result=spawnSync('/usr/bin/flock',['--nonblock','--conflict-exit-code','75',lock,process.execPath,'--no-warnings',__filename,'--locked',...args],{stdio:['ignore','inherit','inherit',fd]}); + fs.closeSync(fd);if (result.status === 75) process.stdout.write('{"status":"busy"}\n'); + process.exitCode=result.status===75?0:result.status??1;return; + } + fs.closeSync(fd); + const secret=fs.lstatSync(TOKEN);if(!secret.isFile()||secret.isSymbolicLink()||secret.uid!==0||secret.nlink!==1||(secret.mode&0o7777)!==0o600||secret.size>4096)throw Error(); + const publish=input=>{ + const result=spawnSync('/usr/bin/setpriv',[`--reuid=${config.uid}`,`--regid=${config.gid}`,'--clear-groups',process.execPath,'--no-warnings',path.resolve(__dirname, "../src/release-delivery-publish.js")],{ + input:JSON.stringify({config,input}),encoding:'utf8',timeout:30_000,env:{PATH:'/usr/bin:/bin'},maxBuffer:4096}); + if(result.status!==0)throw Object.assign(Error('release_catalog_publish_failed'),{code:'release_catalog_publish_failed'}); + }; + let localHistory='ready'; + try{publish({action:'history'});}catch{localHistory='unavailable';} + const source=createGitHubReleaseSource({token:fs.readFileSync(TOKEN,'utf8').trim()}); + const result=await createReleaseWatcher({root:STATE,source,target, + publishNotes:notes=>publish({action:'notes',notes}), + prepare:input=>prepareRelease({...input,config}),publish:input=>publish({action:'publish',...input}), + status:status=>publish({action:'status',status}),retryRequest:()=>{try{return privateJson(path.join(config.localRoot,'config/release-delivery-retry.json'),config.uid,true);}catch{return null;}}, + }).run(); + if (result.status === 'release_ready') { + spawnSync('/usr/bin/systemctl', ['start', '--no-block', 'dispatch-recovery-prewarm.service'], {stdio:'ignore', timeout:5000}); + } + // History errors must not change the availability of the current update. + try { + result.history=await require('../src/release-history-sync').createReleaseHistorySync({root:STATE,source, + publish:input=>publish({action:'history_entry',...input})}).run(); + } catch { result.history={status:'unavailable'}; } + result.history.localCatalog=localHistory; + process.stdout.write(JSON.stringify(result)+'\n'); +} +main().catch(()=>{process.stderr.write('{"status":"release_watcher_unavailable"}\n');process.exitCode=1;}); diff --git a/core/core/installations/bin/dispatch-runtime-agent-authority b/core/core/installations/bin/dispatch-runtime-agent-authority new file mode 100755 index 0000000..596a37c --- /dev/null +++ b/core/core/installations/bin/dispatch-runtime-agent-authority @@ -0,0 +1,115 @@ +#!/usr/bin/env node +'use strict'; + +const path = require('node:path'); +const { AccessStore } = require('../../accounts/src/store'); +const { createRuntimeAgentCredentialManager } = require('../src/runtime-agent-credential'); + +function fail(code = 'runtime_boundary_violation') { + throw Object.assign(new Error(code), { code }); +} + +function absoluteEnvironment(name) { + const value = process.env[name]; + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) fail(); + return value; +} + +function parse(argv) { + if (argv.length !== 2 || !['issue', 'rotate', 'revoke'].includes(argv[0]) + || !/^[a-z][a-z0-9_-]{2,95}$/.test(argv[1])) fail('invalid_input'); + return { operation: argv[0], organizationId: argv[1] }; +} + +function main() { + const { operation, organizationId } = parse(process.argv.slice(2)); + const accessRoot = absoluteEnvironment('DISPATCH_ACCESS_CONTROL_DATABASE_ROOT'); + const installationsRoot = absoluteEnvironment('DISPATCH_INSTALLATIONS_ROOT'); + const store = new AccessStore({ + databaseRoot: accessRoot, + database: path.join(accessRoot, 'access-control.sqlite3'), + }); + try { + const control = store.installationControl(organizationId); + if (!control || control.runtimeKey === 'local') fail('installation_not_found'); + const credentials = createRuntimeAgentCredentialManager({ installationsRoot }); + const prior = store.runtimeAgentAuthority(control.runtimeKey); + let generation = prior?.generation ?? null; + let changed = false; + if (operation === 'revoke') { + if (prior?.status === 'active') { + store.transaction(() => store.revokeRuntimeAgentAuthority({ + organizationId, + runtimeKey: control.runtimeKey, + expectedGeneration: prior.generation, + timestamp: Date.now(), + })); + generation = prior.generation + 1; + changed = true; + } + changed = credentials.revoke(control.runtimeKey, prior?.token_hash ?? null) || changed; + } else { + if (operation === 'rotate' && prior?.status !== 'active') fail('runtime_agent_unauthorized'); + let credential = null; + try { + const authority = store.transaction(() => { + const current = store.runtimeAgentAuthority(control.runtimeKey); + if (operation === 'rotate' && (current?.status !== 'active' + || current.generation !== prior.generation)) fail('runtime_agent_authority_conflict'); + if (operation === 'issue' && prior && (current?.status !== prior.status + || current.generation !== prior.generation)) fail('runtime_agent_authority_conflict'); + credential = credentials.issue(control.runtimeKey, { + rotate: operation === 'rotate' || current?.status === 'revoked', + }); + if (!current) return store.recordRuntimeAgentAuthority({ + organizationId, + runtimeKey: control.runtimeKey, + tokenHash: credential.tokenHash, + timestamp: Date.now(), + }); + if (operation === 'issue' && current.status === 'active' + && current.token_hash === credential.tokenHash) { + return store.recordRuntimeAgentAuthority({ + organizationId, + runtimeKey: control.runtimeKey, + tokenHash: credential.tokenHash, + timestamp: Date.now(), + }); + } + return store.replaceRuntimeAgentAuthority({ + organizationId, + runtimeKey: control.runtimeKey, + tokenHash: credential.tokenHash, + expectedGeneration: current.generation, + expectedStatus: current.status, + timestamp: Date.now(), + }); + }); + generation = authority.generation; + changed = authority.changed || credential.changed; + } catch (error) { + if (credential?.tokenChanged) { + try { credentials.revoke(control.runtimeKey, credential.tokenHash); } catch {} + } + throw error; + } + } + process.stdout.write(`${JSON.stringify({ + ok: true, + status: operation === 'revoke' ? 'revoked' : operation === 'rotate' ? 'rotated' : 'issued', + changed, + generation, + })}\n`); + return 0; + } finally { + store.close(); + } +} + +try { process.exitCode = main(); } +catch (error) { + const code = typeof error?.code === 'string' ? error.code : 'runtime_boundary_violation'; + process.stdout.write(`${JSON.stringify({ ok: false, status: code })}\n`); + process.exitCode = 1; +} diff --git a/core/core/installations/examples/fictional-fragment.json b/core/core/installations/examples/fictional-fragment.json new file mode 100644 index 0000000..e540eb1 --- /dev/null +++ b/core/core/installations/examples/fictional-fragment.json @@ -0,0 +1,28 @@ +{ + "github": { + "summary": "Fictional example: clearer platform maintenance windows." + }, + "groups": [ + { + "id": "platform-tools", + "title": "Platform tools", + "icon": "calendar-clock" + } + ], + "changelog": [ + { + "kind": "improved", + "title": "Fictional maintenance window preview", + "description": "Platform owners can preview the next maintenance window before scheduling it.", + "group": "platform-tools", + "icon": "calendar-clock", + "details": "Fictional documentation fixture only: a preview shows the proposed window and affected platform services before it is saved.", + "audience": "platform", + "popup": { + "title": "Preview maintenance windows", + "description": "See the proposed window and affected services before saving." + } + } + ], + "afterUpdating": [] +} diff --git a/core/core/installations/examples/github-changelog.json b/core/core/installations/examples/github-changelog.json new file mode 100644 index 0000000..3a27bb8 --- /dev/null +++ b/core/core/installations/examples/github-changelog.json @@ -0,0 +1,128 @@ +{ + "github": { + "summary": "Smarter scheduling, clearer workforce planning, and more useful dashboard snapshots.", + "previousTag": "0.0.0" + }, + "groups": [ + { + "id": "scheduling", + "title": "Scheduling", + "icon": "calendar-clock" + }, + { + "id": "workforce", + "title": "Workforce", + "icon": "users" + }, + { + "id": "dashboard", + "title": "Dashboard", + "icon": "chart-column" + }, + { + "id": "maintenance", + "title": "Maintenance", + "icon": "refresh-cw" + } + ], + "changelog": [ + { + "kind": "added", + "title": "Capacity-aware shift templates", + "description": "Preview conflicts while building shifts.", + "group": "scheduling", + "icon": "calendar-clock", + "details": "Build shifts with capacity context and spot conflicts before publishing.", + "audience": "dsp", + "popup": { + "title": "Build shifts with capacity context", + "description": "Preview conflicts while creating shift templates." + }, + "github": { + "pullRequests": [{ "number": 101, "author": "example-author" }] + } + }, + { + "kind": "improved", + "title": "Safer recurring schedule edits", + "description": "Keep exception dates intact when editing recurring schedules.", + "group": "scheduling", + "icon": "calendar-clock", + "details": "Recurring schedule changes preserve dates that were already customized.", + "audience": "dsp", + "github": { + "pullRequests": [{ "number": 102, "author": "example-author" }] + } + }, + { + "kind": "added", + "title": "Skills and availability filters", + "description": "Find the right people faster when assigning work.", + "group": "workforce", + "icon": "users", + "details": "Filter assignments by the skills and availability needed for each shift.", + "audience": "dsp", + "popup": { + "title": "Find the right people faster", + "description": "Filter assignments by skills and availability." + }, + "github": { + "pullRequests": [{ "number": 103, "author": "example-author" }] + } + }, + { + "kind": "fixed", + "title": "Split-shift overtime totals", + "description": "Recalculate overtime totals after split shifts are edited.", + "group": "workforce", + "icon": "users", + "details": "Overtime totals now reflect edits made to each part of a split shift.", + "audience": "dsp", + "github": { + "pullRequests": [{ "number": 104, "author": "example-author" }] + } + }, + { + "kind": "improved", + "title": "Staffing gap snapshots", + "description": "Surface staffing gaps earlier in team snapshots.", + "group": "dashboard", + "icon": "chart-column", + "details": "Team snapshots bring emerging staffing gaps into view sooner.", + "audience": "dsp", + "popup": { + "title": "See staffing gaps earlier", + "description": "Team snapshots now surface emerging staffing gaps sooner." + }, + "github": { + "pullRequests": [{ "number": 105, "author": "example-author" }] + } + }, + { + "kind": "fixed", + "title": "Timezone-accurate dashboard exports", + "description": "Match exported dates to the selected timezone.", + "group": "dashboard", + "icon": "chart-column", + "details": "Dashboard exports now use the timezone selected for the view.", + "audience": "dsp", + "github": { + "pullRequests": [{ "number": 106, "author": "example-author" }] + } + }, + { + "kind": "changed", + "title": "Release verification diagnostics", + "description": "Refresh background-job diagnostics and test fixtures.", + "group": "maintenance", + "icon": "refresh-cw", + "details": "Background-job diagnostics and test fixtures now provide clearer release verification signals.", + "audience": "platform", + "github": { + "pullRequests": [{ "number": 107, "author": "example-author" }], + "maintenance": true + } + } + ], + "afterUpdating": [] +} diff --git a/core/core/installations/examples/helpers/oci-lifecycle-worker.js b/core/core/installations/examples/helpers/oci-lifecycle-worker.js new file mode 100644 index 0000000..804b85d --- /dev/null +++ b/core/core/installations/examples/helpers/oci-lifecycle-worker.js @@ -0,0 +1,178 @@ +'use strict'; +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const readline = require('node:readline'); +const { fork, spawnSync } = require('node:child_process'); +const { AccessStore } = require('../../../accounts/src/store'); +const { createAccessInstallationProvisioningAuthority, createAccessControlLiveAuthorityResolver, + createInstallationProvisioningReconciler } = require('../../../accounts/src/installation-provisioning'); +const { createAccessInstallationLifecycleAuthority } = require('../../../accounts/src/installation-lifecycle'); +const { createDurableInstallationProvisioner } = require('../../src/jobs'); +const { createProtectedOciHostClient } = require('../../src/oci-protected-client'); +const { createOciContainerAdapter } = require('../../src/oci-adapter'); +const { createOciRuntimeAgentCredentialPort } = require('../../src/oci-runtime-agent-credential'); +const { createOciInstallationLifecycle } = require('../../src/oci-lifecycle'); +const { createOciRuntimeLifecyclePort } = require('../../src/oci-runtime-lifecycle-port'); +const { CoreRuntimeAgentHub, createRuntimeAgentDispatchClient, runtimeAgentControlInvoke } = require('../../../agents/src'); +const { CoreRuntimeAgentControlServer } = require('../../../agents/src/control'); +const { INSTALLATION_ACTIVATION_RUNS, installationActivationEvidenceDigest } = require('../../../../shared/contracts/src'); +const config = JSON.parse(fs.readFileSync(process.argv[2], 'utf8')); +const runtimeKeys = ['runtime_lifecycle_alpha', 'runtime_lifecycle_beta']; +const credentials = createOciRuntimeAgentCredentialPort({ credentialRoot: path.join(config.controllerRoot, 'credentials') }); +function manifest(index, revision = 1, releaseId = 'dispatch_current_1') { + return { manifestVersion: 1, revision, organization: { id: `org_lifecycle_${index}`, stationCode: 'TEST', timezone: 'UTC' }, + runtime: { key: runtimeKeys[index], templateId: 'isolated_dsp_v1', releaseId } }; +} +function authority(value) { return { revision: value.revision, organization: value.organization, runtime: value.runtime }; } +const client = key => createRuntimeAgentDispatchClient({ runtimeKey: key, hub: { + invoke: async (runtimeKey, action, input) => { + try { return await runtimeAgentControlInvoke(config.controlSocket, runtimeKey, action, input); } + catch (error) { process.stderr.write(JSON.stringify({ controlFailure: error.code, action }) + '\n'); throw error; } + } } }); +async function hubMain() { + const authorities = Object.fromEntries(runtimeKeys.map(runtimeKey => [runtimeKey, + crypto.createHash('sha256').update(credentials.read(runtimeKey)).digest('hex')])); + const hub = new CoreRuntimeAgentHub({ socketPath: config.centralSocket, authorities }); + await hub.start(); + const control = new CoreRuntimeAgentControlServer({ socketPath: config.controlSocket, hub: { invoke: async (...args) => { + try { return await hub.invoke(...args); } catch (error) { process.stderr.write(JSON.stringify({ hubFailure: error.code, action: args[1], connected: hub.status().connected }) + '\n'); throw error; } + } } }); + await control.start(); + process.send({ ready: true }); + process.once('SIGTERM', async () => { await control.close(); await hub.close(); process.exit(0); }); +} +async function main() { + const input = readline.createInterface({ input: process.stdin }); + const pending = []; + input.on('line', line => pending.shift()?.(JSON.parse(line))); + const hostFixture = value => new Promise(resolve => { + pending.push(resolve); process.stdout.write(`${JSON.stringify(value)}\n`); + }); + const phase = value => process.stdout.write(`${JSON.stringify({ phase: value })}\n`); + for (const key of runtimeKeys) credentials.issue(key); + const hub = fork(__filename, [process.argv[2], '--hub'], { stdio: ['ignore', 'ignore', 'inherit', 'ipc'] }); + await new Promise((resolve, reject) => { hub.once('message', resolve); hub.once('error', reject); hub.once('exit', () => reject(new Error('hub exited'))); }); + const root = config.controllerRoot; + const store = new AccessStore({ databaseRoot: path.join(root, 'access'), database: path.join(root, 'access/access-control.sqlite3') }); + store.insertUser({ id: 'usr_fixture_owner', email: 'fixture@example.invalid', firstName: 'Synthetic', lastName: 'Fixture', + passwordHash: 'fixture-not-a-password', platformRole: 'owner', timestamp: Date.now() }); + let provisioner; + let lifecycleAuthority; + const host = createProtectedOciHostClient({ dispatchRequest: (request, dispatch) => { + try { return (Object.hasOwn(request.claim, 'generation') ? provisioner : lifecycleAuthority).dispatchHostRequest(request, dispatch); } + catch (error) { phase(`failed_host_${request.operation}_${error.code || error.message}_${error.hostStep || 'none'}`); throw error; } + } }); + const adapter = createOciContainerAdapter({ hostRegistry: host.hostRegistry, hostExecutor: host.hostExecutor, + releaseResolver: id => config.releases[id], credentialPort: credentials }); + provisioner = createDurableInstallationProvisioner({ stateRoot: path.join(root, 'provisioner'), + installationsRoot: path.join(root, 'unused-installations'), ociAdapter: adapter, leaseMs: 600_000, + liveAuthorityResolver: createAccessControlLiveAuthorityResolver({ store }) }); + const reconciler = createInstallationProvisioningReconciler({ store, provisioner, runtimeAgentCredentials: credentials }); + let sequence = 0; + const plans = []; + try { + for (let index = 0; index < 2; index += 1) { + const selected = manifest(index); + store.createOrganization({ id: selected.organization.id, name: `Synthetic ${index}`, abbreviation: `S${index}`, + timezone: 'UTC', status: 'active', createdBy: null, timestamp: Date.now() }); + store.insertStation(selected.organization.id, 'TEST', true, Date.now()); + store.createInstallation(selected.organization.id, selected.runtime.key, 'pending', Date.now(), 'dispatch_current_1', 'oci_container_v1'); + const provisioning = createAccessInstallationProvisioningAuthority({ store, organizationId: selected.organization.id, + authorityScope: 'fixture_authority', actorUserId: 'usr_fixture_owner' }); + const requested = provisioning.request({ operation: 'provision', idempotencyKey: `fixture:provision:${index}`, expectedRevision: 1 }); + const job = reconciler.dispatch(requested.id); + phase(`provision_${index}`); + const result = provisioner.runNext(`worker_fixture_${index}`); + assert.equal(result.status, 'succeeded', JSON.stringify(result)); + assert.equal(reconciler.reconcile(requested.id).status, 'completed'); + assert.equal(reconciler.reconcile(requested.id).status, 'completed'); + // Synthetic first-publication data is seeded by a fixed fixture executable + // in the test image, using the real provider publication store. + const seeded = await hostFixture({ seed: index }); + assert.equal(seeded.ok, true, JSON.stringify(seeded)); + const c = client(selected.runtime.key); + assert.equal((await c.workforce.day({ date: '2026-09-05', limit: 1, offset: 0 })).status, 'found'); + assert.equal((await c.sync.start('paycom-main-workforce')).status, 'started'); + const runs = INSTALLATION_ACTIVATION_RUNS.map((run, i) => ({ id: `run_fixture_${i}`, taskId: run.taskId, plan: run.plan, method: run.method })); + const pub = (value, runId, originRunId) => ({ id: value.publicationId, runId, originRunId, + contentSha256: value.contentSha256, batchBound: true }); + const body = { schemaVersion: 1, manifestRevision: 1, jobId: `job_activation_${index}`, runtimeKey: selected.runtime.key, + definitionDigest: 'a'.repeat(64), requestDigest: 'b'.repeat(64), previewDigest: 'c'.repeat(64), + batchId: `batch_fixture_${index}`, preparationRunId: 'run_periods_fixture', target: '2026-09-05', runs, + publications: { payPeriods: { id: seeded.payPeriods.publicationId, runId: 'run_periods_fixture', originRunId: 'run_periods_fixture', + contentSha256: seeded.payPeriods.contentSha256, batchBound: false }, roster: pub(seeded.roster, runs[0].id, 'run_fixture_roster'), + timecards: pub(seeded.timecards, runs[1].id, 'run_fixture_timecards'), resourceLinks: pub(seeded.links, runs[3].id, 'run_fixture_links') }, capturedAt: new Date().toISOString() }; + const evidence = { ...body, evidenceDigest: installationActivationEvidenceDigest(body) }; + const ctl = store.installationControl(selected.organization.id); + store.db.prepare("UPDATE installations SET status='ready',current_job_id=? WHERE organization_id=?").run(body.jobId, selected.organization.id); + store.db.prepare(`INSERT INTO installation_activation_jobs( + id,organization_id,operation,status,installation_state,installation_revision,manifest_revision, + runtime_key,authority_scope,idempotency_key,worker_id,fence,lease_expires_at,provider,profile_id, + provider_tested_at,evidence_json,evidence_digest,failure_code,created_at,started_at,finished_at,updated_at) + VALUES(?,?,'resume','succeeded','ready',?,1,?,'fixture_authority',?,'worker_fixture',1,NULL,'paycom','paycom-main',?,?,?,?,?,?,?,?)`) + .run(body.jobId, selected.organization.id, ctl.revision, selected.runtime.key, `fixture:activation:${index}`, + Date.now(), JSON.stringify(evidence), evidence.evidenceDigest, null, Date.now(), Date.now(), Date.now(), Date.now()); + } + const changed = await hostFixture({ seed: 0, changed: true }); + assert.equal(changed.ok, true); + assert.equal((await client(runtimeKeys[0]).workforce.day({ date: '2026-09-19', limit: 1, offset: 0 })).status, 'found'); + const betaBefore = await hostFixture({ inspectBeta: true }); + const run = async (operation, additions = {}, failPublication = false) => { + const organizationId = manifest(0).organization.id; + lifecycleAuthority = createAccessInstallationLifecycleAuthority({ store, organizationId, authorityScope: 'fixture_authority', + releaseCatalog: Object.keys(config.releases), destructionEnabled: true, leaseMs: 600_000 }); + const requested = lifecycleAuthority.request({ operation, expectedRevision: store.installationControl(organizationId).revision, + idempotencyKey: `fixture:lifecycle:${++sequence}`, ...additions }); + phase(operation + (failPublication ? '_partial_failure' : '')); + const lifecycle = createOciInstallationLifecycle({ authority: lifecycleAuthority, adapter, hostExecutor: host.hostExecutor, + backupManagerFactory: (plan, claim) => host.createBackupManager(plan, claim), runtimeFactory: plan => { + const port = createOciRuntimeLifecyclePort({ client: client(plan.runtimeKey) }); + const checkedPort = Object.fromEntries(Object.entries(port).map(([name, method]) => [name, async (...args) => { + try { return await method(...args); } catch (error) { phase(`failed_runtime_${name}_${error.code || error.message}`); throw error; } + }])); + return failPublication ? { ...checkedPort, verifyPublication: async () => { throw new Error('synthetic_failure_after_start'); } } : checkedPort; + } }); + const result = await lifecycle.run(requested.id, `worker_lifecycle_${sequence}`); + assert.equal(result.status, failPublication ? 'failed' : 'succeeded', JSON.stringify(result)); + assert.deepEqual(await hostFixture({ inspectBeta: true }), betaBefore); + return { result, backups: lifecycleAuthority.backups() }; + }; + const catalogFile = path.join(root, 'oci-releases.json'); + fs.writeFileSync(catalogFile, JSON.stringify({ schemaVersion: 1, releases: config.releases }), { mode: 0o600 }); + phase('backup_cli'); + const cli = spawnSync('/usr/bin/node', ['--no-warnings', path.resolve(__dirname, "../../bin/dispatch-installation-lifecycle"), + 'backup', manifest(0).organization.id], { encoding: 'utf8', timeout: 600_000, + env: { PATH: '/usr/bin:/bin', DISPATCH_ACCESS_CONTROL_DATABASE_ROOT: path.join(root, 'access'), + DISPATCH_INSTALLATIONS_ROOT: path.join(root, 'unused-installations'), DISPATCH_SYSTEMD_UNIT_ROOT: path.join(root, 'unused-units'), + DISPATCH_RUNTIME_AGENT_HUB_SOCKET: config.centralSocket, DISPATCH_RUNTIME_AGENT_CONTROL_SOCKET: config.controlSocket, + DISPATCH_OCI_RELEASE_CATALOG_FILE: catalogFile, DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT: path.join(root, 'credentials') } }); + assert.equal(cli.status, 0, cli.stdout + cli.stderr); + assert.equal(JSON.parse(cli.stdout).status, 'succeeded'); + assert.deepEqual(await hostFixture({ inspectBeta: true }), betaBefore); + const backup = { backups: createAccessInstallationLifecycleAuthority({ store, organizationId: manifest(0).organization.id, + authorityScope: 'fixture_authority', releaseCatalog: Object.keys(config.releases) }).backups() }; + await run('suspend'); + assert.equal((await client(runtimeKeys[0]).system.status()).ok, false); + assert.equal((await hostFixture({ mutateSentinel: true })).ok, true); + await run('restore', { backupId: backup.backups.find(value => value.purpose === 'manual').id }); + assert.equal((await hostFixture({ verifySentinel: true })).ok, true); + await run('resume'); + await run('upgrade', { releaseId: 'dispatch_fixture_2' }); + await run('upgrade', { releaseId: 'dispatch_fixture_3' }, true); + assert.equal(store.installationControl(manifest(0).organization.id).releaseId, 'dispatch_fixture_2'); + assert.equal((await client(runtimeKeys[0]).sync.status('paycom-main-workforce')).data.desiredState, 'running'); + await run('decommission'); + assert.equal((await hostFixture({ retained: true })).ok, true); + await run('destroy'); + assert.equal((await hostFixture({ destroyed: true })).ok, true); + phase('lifecycle_verified'); + } finally { + provisioner.close(); store.close(); hub.kill('SIGTERM'); + await new Promise(resolve => hub.once('exit', resolve)); input.close(); + } +} +(process.argv[3] === '--hub' ? hubMain() : main()).catch(error => { + process.stderr.write(`${error.stack}\n`); process.exitCode = 1; +}); diff --git a/core/core/installations/examples/installation-jobs.js b/core/core/installations/examples/installation-jobs.js new file mode 100644 index 0000000..e1ee9aa --- /dev/null +++ b/core/core/installations/examples/installation-jobs.js @@ -0,0 +1,187 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { + PRIVATE_DIRECTORY_MODE, + createInstallationLayoutManager, +} = require('../src'); +const { + INSTALLATION_JOB_SCHEMA_VERSION, + INSTALLATION_JOB_STAGES, + createInstallationJobStore, +} = require('../src/job-store'); + +const manage = { + scope: 'operator_fixture', + permission: 'platform.installations.manage', + operatorEnabled: true, +}; +const fixtureRegistration = { + fixture: true, + installationState: 'pending', + retainedData: false, +}; + +function manifest(id) { + return { + manifestVersion: 1, + revision: 1, + organization: { id: `org_${id}`, stationCode: 'TST1', timezone: 'America/Los_Angeles' }, + runtime: { key: `fixture_${id}`, templateId: 'isolated_dsp_v1', releaseId: 'dispatch_fixture_1' }, + }; +} + +function authority(value) { + return { + revision: value.revision, + organization: { ...value.organization }, + runtime: { ...value.runtime }, + }; +} + +function provision(key, revision) { + return { operation: 'provision', idempotencyKey: key, expectedRevision: revision }; +} + +function cancel(key, revision) { + return { operation: 'cancel', idempotencyKey: key, expectedRevision: revision }; +} + +function retry(key, revision) { + return { operation: 'retry', idempotencyKey: key, expectedRevision: revision }; +} + +function code(expected) { + return error => error?.code === expected; +} + +const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-jobs-exercise-')); +fs.chmodSync(fixtureRoot, PRIVATE_DIRECTORY_MODE); +const stateRoot = path.join(fixtureRoot, 'control'); +const installationsRoot = path.join(fixtureRoot, 'installations'); +fs.mkdirSync(stateRoot, { mode: PRIVATE_DIRECTORY_MODE }); +fs.mkdirSync(installationsRoot, { mode: PRIVATE_DIRECTORY_MODE }); +const layout = createInstallationLayoutManager({ installationsRoot }); +const alpha = manifest('alpha'); +const bravo = manifest('bravo'); +let store = null; + +try { + store = createInstallationJobStore({ stateRoot }); + store.registerFixture(alpha, authority(alpha), fixtureRegistration, 1_000); + store.registerFixture(bravo, authority(bravo), fixtureRegistration, 1_001); + + const alphaRequest = provision('fixture:provision:alpha', 1); + const alphaJob = store.request(alpha, authority(alpha), alphaRequest, manage, 'job_alpha', 1_010); + const replayed = store.request(alpha, authority(alpha), alphaRequest, manage, 'job_unused', 1_011); + assert.equal(replayed.id, alphaJob.id); + assert.equal(replayed.replayed, true); + store.close(); + store = null; + + const crashed = spawnSync(process.execPath, [ + '--no-warnings', + path.join(__dirname, "../tests/helpers/job-crash-worker.js"), + JSON.stringify({ stateRoot, installationsRoot }), + ], { encoding: 'utf8' }); + assert.equal(crashed.status, 86, crashed.stderr); + const interruptedClaim = JSON.parse(crashed.stdout); + + store = createInstallationJobStore({ stateRoot }); + const resumedClaim = store.claimNext('worker_resumed', 1_120, 100); + assert.throws(() => store.work(interruptedClaim, 1_121), code('installation_operation_in_progress')); + const resumedStage = store.work(resumedClaim, 1_122); + assert.equal(resumedStage.stage, INSTALLATION_JOB_STAGES[1]); + store.completeStage( + resumedClaim, + resumedStage.stage, + layout.inspect(resumedStage.manifest, resumedStage.authority), + 1_123, + ); + assert.equal(store.finishSucceeded(resumedClaim, 1_124).status, 'succeeded'); + + store.request( + bravo, + authority(bravo), + provision('fixture:provision:bravo-cancel', 1), + manage, + 'job_bravo_cancel', + 1_130, + ); + const cancelledClaim = store.claimNext('worker_cancelled', 1_131, 100); + store.request( + bravo, + authority(bravo), + cancel('fixture:cancel:bravo', 2), + manage, + 'job_cancel_request', + 1_132, + ); + assert.equal(store.work(cancelledClaim, 1_133).cancelRequested, true); + assert.equal(store.finishCancelled(cancelledClaim, 1_134).status, 'cancelled'); + + store.request( + bravo, + authority(bravo), + provision('fixture:provision:bravo-failure', 3), + manage, + 'job_bravo_failure', + 1_140, + ); + const failedClaim = store.claimNext('worker_failed', 1_141, 100); + const failed = store.finishFailed(failedClaim, 'runtime_layout_failed', 1_142); + assert.equal(failed.failure.code, 'runtime_layout_failed'); + const retryJob = store.request( + bravo, + authority(bravo), + retry('fixture:retry:bravo', 5), + manage, + 'job_bravo_retry', + 1_150, + ); + assert.equal(retryJob.operation, 'retry'); + const retryClaim = store.claimNext('worker_retry', 1_151, 100); + for (;;) { + const work = store.work(retryClaim, 1_152); + if (work.stage === null) break; + const receipt = work.stage === INSTALLATION_JOB_STAGES[0] + ? layout.materialize( + work.manifest, + work.authority, + mutation => store.mutateClaim(retryClaim, 1_152, mutation), + ) + : layout.inspect(work.manifest, work.authority); + store.completeStage(retryClaim, work.stage, receipt, 1_153); + } + assert.equal(store.finishSucceeded(retryClaim, 1_154).status, 'succeeded'); + + const alphaRoot = layout.derive(alpha, authority(alpha)).installationRoot; + const bravoRoot = layout.derive(bravo, authority(bravo)).installationRoot; + assert.notEqual(alphaRoot, bravoRoot); + assert.equal(path.relative(alphaRoot, bravoRoot).startsWith('..'), true); + assert.equal(layout.inspect(alpha, authority(alpha)).status, 'verified'); + assert.equal(layout.inspect(bravo, authority(bravo)).status, 'verified'); + const health = store.health(); + assert.deepEqual(health.jobs, { queued: 0, running: 0, succeeded: 2, failed: 1, cancelled: 1 }); + + process.stdout.write(`${JSON.stringify({ + ok: true, + status: 'verified', + schemaVersion: INSTALLATION_JOB_SCHEMA_VERSION, + fixtures: 2, + oneJobReplay: true, + exclusiveClaim: true, + checkpointResume: true, + staleFenceRejected: true, + cancellation: 'cooperative', + retry: 'verified', + isolated: true, + })}\n`); +} finally { + try { store?.close(); } catch {} + fs.rmSync(fixtureRoot, { recursive: true, force: true }); +} diff --git a/core/core/installations/examples/installation-layout.js b/core/core/installations/examples/installation-layout.js new file mode 100644 index 0000000..b311b2a --- /dev/null +++ b/core/core/installations/examples/installation-layout.js @@ -0,0 +1,84 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { + PRIVATE_DIRECTORY_MODE, + RELATIVE_DIRECTORIES, + createInstallationLayoutManager, +} = require('../src'); + +function manifest(id) { + return { + manifestVersion: 1, + revision: 1, + organization: { id: `org_${id}`, stationCode: 'TST1', timezone: 'America/Los_Angeles' }, + runtime: { key: `fixture_${id}`, templateId: 'isolated_dsp_v1', releaseId: 'dispatch_fixture_1' }, + }; +} + +function authority(value) { + return { + revision: value.revision, + organization: { ...value.organization }, + runtime: { ...value.runtime }, + }; +} + +const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-layout-exercise-')); +fs.chmodSync(fixtureRoot, PRIVATE_DIRECTORY_MODE); +const installationsRoot = path.join(fixtureRoot, 'installations'); +fs.mkdirSync(installationsRoot, { mode: PRIVATE_DIRECTORY_MODE }); +const cleanupAuthority = { fixture: true, installationState: 'failed', retainedData: false }; + +try { + const manager = createInstallationLayoutManager({ installationsRoot }); + const first = manifest('alpha'); + const second = manifest('bravo'); + const firstLayout = manager.derive(first, authority(first)); + const secondLayout = manager.derive(second, authority(second)); + assert.notEqual(firstLayout.installationRoot, secondLayout.installationRoot); + assert.equal(path.dirname(firstLayout.installationRoot), installationsRoot); + assert.equal(path.dirname(secondLayout.installationRoot), installationsRoot); + + const createdFirst = manager.materialize(first, authority(first)); + const createdSecond = manager.materialize(second, authority(second)); + const replayed = manager.materialize(first, authority(first)); + assert.deepEqual(createdFirst, { layoutVersion: 1, status: 'verified', directoryCount: RELATIVE_DIRECTORIES.length, changed: true }); + assert.equal(createdSecond.changed, true); + assert.equal(replayed.changed, false); + assert.equal(manager.inspect(first, authority(first)).status, 'verified'); + assert.equal(JSON.stringify(createdFirst).includes(fixtureRoot), false); + const runtimePaths = manager.runtimePaths(first, authority(first)); + const environment = manager.runtimeEnvironment(first, authority(first)); + assert.equal(runtimePaths.auth.databaseRoot, firstLayout.directories.authDataRoot); + assert.equal(runtimePaths.collection.databaseRoot, firstLayout.directories.collectionDataRoot); + assert.equal(Object.hasOwn(runtimePaths, 'accessControl'), false); + assert.equal(Object.hasOwn(environment, 'DISPATCH_ACCESS_CONTROL_DATABASE_ROOT'), false); + assert.equal(Object.hasOwn(environment, 'DISPATCH_LOCAL_ROOT'), false); + + for (const directory of Object.values(firstLayout.directories)) { + assert.equal(fs.lstatSync(directory).mode & 0o7777, PRIVATE_DIRECTORY_MODE); + } + + assert.equal(manager.removeEmpty(first, authority(first), cleanupAuthority).changed, true); + assert.equal(manager.removeEmpty(first, authority(first), cleanupAuthority).changed, false); + assert.equal(manager.inspect(second, authority(second)).status, 'verified'); + assert.equal(manager.removeEmpty(second, authority(second), cleanupAuthority).changed, true); + + process.stdout.write(`${JSON.stringify({ + ok: true, + status: 'verified', + layoutVersion: 1, + fixtures: 2, + directoriesPerFixture: RELATIVE_DIRECTORIES.length, + isolated: true, + idempotent: true, + componentPaths: 'verified', + cleanup: 'empty_only', + })}\n`); +} finally { + fs.rmSync(fixtureRoot, { recursive: true, force: true }); +} diff --git a/core/core/installations/examples/installation-services.js b/core/core/installations/examples/installation-services.js new file mode 100644 index 0000000..d0cf3a3 --- /dev/null +++ b/core/core/installations/examples/installation-services.js @@ -0,0 +1,580 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { spawn, spawnSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { + PRIVATE_DIRECTORY_MODE, + INSTALLATION_SERVICE_PLAN_VERSION, + INSTALLATION_AGENT_SERVICE_COUNT, + createRuntimeAgentCredentialManager, + createDurableInstallationProvisioner, + createInstallationLayoutManager, + createInstallationServiceManager, + createSystemdUserSupervisor, +} = require('../src'); +const { createManagedInstallationLifecycle } = require('../../../compatibility/provisioner/src/lifecycle.js'); +const { AccessStore } = require('../../accounts/src/store'); +const { + createAccessInstallationLifecycleAuthority, +} = require('../../accounts/src/installation-lifecycle'); +const { createRuntimeGatewayDispatchClient } = require('dispatch-dsp/runtime/gateway/src/index.js'); + +function manifest(id) { + return { + manifestVersion: 1, + revision: 1, + organization: { id: `org_${id}`, stationCode: 'TST1', timezone: 'America/Los_Angeles' }, + runtime: { + key: `fixture_${id}`, + templateId: 'isolated_dsp_v1', + releaseId: 'dispatch_fixture_1', + }, + }; +} + +function authority(value) { + return { + revision: value.revision, + organization: { ...value.organization }, + runtime: { ...value.runtime }, + }; +} + +function inactiveState(plan) { + return plan.units.map(unit => ({ + id: unit.id, + name: unit.name, + enabled: false, + active: false, + enableMode: 'none', + })); +} + +function rollback(serviceManager, supervisor, plan) { + const state = serviceManager.rollbackState(plan); + if (!state) return; + supervisor.stop(plan, operation => operation()); + supervisor.resetFailed(plan, operation => operation()); + supervisor.disable(plan, operation => operation()); + serviceManager.restoreFiles(plan); + supervisor.reload(plan, operation => operation()); + supervisor.restoreState(plan, state, operation => operation()); + serviceManager.finishRollback(plan); +} + +function cleanupCommittedFixture(supervisor, plan) { + supervisor.stop(plan, operation => operation()); + supervisor.resetFailed(plan, operation => operation()); + supervisor.disable(plan, operation => operation()); + for (const unit of [...plan.units].reverse()) { + if (!fs.existsSync(unit.installed)) continue; + const info = fs.lstatSync(unit.installed); + assert.equal(info.isFile(), true); + assert.equal(info.isSymbolicLink(), false); + assert.equal(info.uid, process.geteuid()); + assert.equal(info.nlink, 1); + assert.equal(info.mode & 0o7777, 0o600); + assert.equal(fs.realpathSync(unit.installed), unit.installed); + assert.equal(fs.readFileSync(unit.installed, 'utf8'), unit.content); + fs.unlinkSync(unit.installed); + } + supervisor.reload(plan, operation => operation()); + supervisor.resetFailed(plan, operation => operation()); + const stopped = supervisor.snapshot(plan); + assert.equal(stopped.every(value => !value.enabled && !value.active), true); +} + +function executeControl(plan, args) { + const unit = plan.units.find(value => value.id === 'collection_manager'); + assert.ok(unit); + const result = spawnSync(unit.healthCommand, args, { + cwd: unit.workingDirectory, + env: unit.environment, + encoding: 'utf8', + timeout: 15_000, + maxBuffer: 64 * 1024, + windowsHide: true, + }); + let value; + try { value = JSON.parse(result.stdout); } + catch { throw new Error('fixture control returned invalid output'); } + if (result.status !== 0 || result.signal !== null || value.ok !== true) { + throw new Error(`fixture control failed: ${typeof value.status === 'string' ? value.status : 'unknown'}`); + } + return value; +} + +async function verifyGateway(plan) { + const unit = plan.units.find(value => value.id === 'runtime_gateway'); + assert.ok(unit); + const client = createRuntimeGatewayDispatchClient({ + socketPath: unit.socketPath, + runtimeKey: plan.runtimeKey, + }); + const health = await client.health(); + assert.equal(health.ok, true); + assert.equal(health.status, 'ready'); + const system = await client.system.status(); + assert.equal(system.ok, true); + return system.status; +} + +async function startSyntheticHub(socketPath, authorities) { + const child = spawn(process.execPath, [ + path.join(__dirname, "../../agents/examples/synthetic-core-hub.js"), + ], { + env: { + PATH: process.env.PATH, + NODE_NO_WARNINGS: '1', + DISPATCH_RUNTIME_AGENT_HUB_SOCKET: socketPath, + DISPATCH_RUNTIME_AGENT_AUTHORITIES: JSON.stringify(authorities), + }, + stdio: ['pipe', 'pipe', 'pipe'], + }); + let buffer = ''; + let stderr = ''; + const queued = []; + const waiting = []; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stderr.on('data', chunk => { stderr = `${stderr}${chunk}`.slice(-4096); }); + child.stdout.on('data', chunk => { + buffer += chunk; + for (;;) { + const newline = buffer.indexOf('\n'); + if (newline < 0) break; + const line = buffer.slice(0, newline); + buffer = buffer.slice(newline + 1); + const waiter = waiting.shift(); + if (waiter) waiter.resolve(line); else queued.push(line); + } + }); + child.once('exit', () => { + while (waiting.length) waiting.shift().reject(new Error(stderr || 'runtime agent hub exited')); + }); + const next = (timeoutMs = 15_000) => new Promise((resolve, reject) => { + if (queued.length) return resolve(queued.shift()); + let timer; + const pending = { + resolve: value => { clearTimeout(timer); resolve(value); }, + reject: error => { clearTimeout(timer); reject(error); }, + }; + timer = setTimeout(() => { + const index = waiting.indexOf(pending); + if (index >= 0) waiting.splice(index, 1); + reject(new Error('runtime agent hub timed out')); + }, timeoutMs); + waiting.push(pending); + }); + const ready = JSON.parse(await next()); + assert.equal(ready.status, 'ready'); + return { + async command(value) { + child.stdin.write(`${JSON.stringify(value)}\n`); + return JSON.parse(await next()); + }, + async close() { + if (child.exitCode !== null) return; + child.kill('SIGTERM'); + await new Promise(resolve => child.once('exit', resolve)); + }, + }; +} + +function createFixtureLifecycleActivation(plan, supervisor) { + const unit = plan.units.find(value => value.id === 'runtime_gateway'); + assert.ok(unit); + const client = createRuntimeGatewayDispatchClient({ + socketPath: unit.socketPath, + runtimeKey: plan.runtimeKey, + }); + const current = async () => { + const result = await client.sync.status('fixture-main-sync'); + assert.equal(result.status, 'found'); + return result.data; + }; + return { + inspectSchedule: async () => ({ + syncWasRunning: (await current()).desiredState === 'running', + }), + quiesceSchedule: async syncWasRunning => { + if ((await current()).desiredState === 'running') { + assert.equal(executeControl(plan, ['stop-sync', 'fixture-main-sync']).status, 'stopped'); + } + const selected = await current(); + assert.equal(selected.desiredState, 'stopped'); + assert.equal(selected.activity, 'idle'); + assert.equal(selected.queuedRunCount, 0); + assert.equal(selected.activeRun, null); + return { syncWasRunning, changed: syncWasRunning }; + }, + restoreSchedule: async syncWasRunning => { + const before = await current(); + if (syncWasRunning && before.desiredState === 'stopped') { + assert.equal(executeControl(plan, ['start-sync', 'fixture-main-sync']).status, 'started'); + } else if (!syncWasRunning && before.desiredState === 'running') { + assert.equal(executeControl(plan, ['stop-sync', 'fixture-main-sync']).status, 'stopped'); + } + assert.equal((await current()).desiredState, syncWasRunning ? 'running' : 'stopped'); + return { + syncWasRunning, + changed: before.desiredState !== (syncWasRunning ? 'running' : 'stopped'), + }; + }, + verifyInfrastructure: async () => supervisor.health(plan), + verifyPublication: async () => { throw new Error('fixture_publication_not_available'); }, + }; +} + +async function executeFixtureWorker(plan) { + const collector = path.resolve(__dirname, "../../collection-manager/tests/fixture-collector.js"); + const specPath = path.join(plan.candidateRoot, 'fixture-collection-spec.json'); + const syncSchema = { + type: 'object', + properties: { + behavior: { type: 'string', enum: ['no_change'] }, + label: { type: 'string', maxLength: 64 }, + }, + required: ['behavior'], + additionalProperties: false, + }; + fs.writeFileSync(specPath, `${JSON.stringify({ + schemaVersion: 1, + collectors: [{ + id: 'fixture', + version: '1.0.0', + description: 'Credential-free service fixture', + command: collector, + sourceSchema: { + type: 'object', + properties: { tenant: { type: 'string', maxLength: 64 } }, + required: ['tenant'], + additionalProperties: false, + }, + methods: { + 'fixture.sync': { + description: 'Credential-free sync fixture', + inputSchema: syncSchema, + timeoutSeconds: 5, + maxAttempts: 1, + backoffSeconds: [], + concurrencyKeys: ['collector:{collector}'], + }, + }, + }], + sources: [{ + id: 'fixture-main', collector: 'fixture', authProfile: null, + config: { tenant: 'fixture' }, enabled: true, + }], + plans: [{ + id: 'fixture-sync-plan', source: 'fixture-main', method: 'fixture.sync', + schedule: { type: 'manual' }, input: { behavior: 'no_change', label: 'fixture' }, + dependsOn: [], enabled: true, + }], + syncs: [{ + id: 'fixture-main-sync', plan: 'fixture-sync-plan', intervalSeconds: 60, + jitterSeconds: 0, overlap: 'coalesce', settingsSchema: syncSchema, + settings: { behavior: 'no_change', label: 'fixture' }, desiredState: 'stopped', + }], + })}\n`, { mode: 0o600 }); + assert.equal(executeControl(plan, ['apply', specPath]).status, 'applied'); + const waitForRun = runId => { + const deadline = Date.now() + 15_000; + for (;;) { + const current = executeControl(plan, ['run-status', runId]); + if (current.status === 'succeeded') { + assert.equal(current.data.status, 'succeeded'); + return; + } + assert.ok(['queued', 'running'].includes(current.status)); + assert.ok(Date.now() < deadline); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 50); + } + }; + const started = executeControl(plan, ['start-sync', 'fixture-main-sync']); + assert.equal(started.status, 'started'); + waitForRun(started.data.run.id); + + const gatewayUnit = plan.units.find(unit => unit.id === 'runtime_gateway'); + const gateway = createRuntimeGatewayDispatchClient({ + socketPath: gatewayUnit.socketPath, + runtimeKey: plan.runtimeKey, + }); + assert.equal((await gateway.sync.status('fixture-main-sync')).status, 'found'); + const queued = await gateway.sync.runNow('fixture-main-sync', { + idempotencyKey: `gateway:${crypto.randomBytes(12).toString('hex')}`, + }); + assert.equal(queued.status, 'queued'); + waitForRun(queued.data.run.id); + assert.equal((await gateway.sync.status('fixture-main-sync')).data.activity, 'idle'); + return 2; +} + +async function main() { +const liveSystemd = process.env.DISPATCH_RUN_SYSTEMD_FIXTURE === '1'; +const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'dm4-')); +fs.chmodSync(fixtureRoot, PRIVATE_DIRECTORY_MODE); +const isolatedRoot = path.join(fixtureRoot, 'isolated fixtures'); +const installationsRoot = path.join(isolatedRoot, 'installations'); +const offlineUnitRoot = path.join(isolatedRoot, 'units'); +const controlRoot = path.join(isolatedRoot, 'control'); +const accessRoot = path.join(isolatedRoot, 'access'); +const centralRuntimeRoot = path.join(isolatedRoot, 'central-run'); +fs.mkdirSync(isolatedRoot, { mode: PRIVATE_DIRECTORY_MODE }); +fs.mkdirSync(installationsRoot, { mode: PRIVATE_DIRECTORY_MODE }); +fs.mkdirSync(offlineUnitRoot, { mode: PRIVATE_DIRECTORY_MODE }); +fs.mkdirSync(controlRoot, { mode: PRIVATE_DIRECTORY_MODE }); +fs.mkdirSync(accessRoot, { mode: PRIVATE_DIRECTORY_MODE }); +fs.mkdirSync(centralRuntimeRoot, { mode: PRIVATE_DIRECTORY_MODE }); +const suffix = crypto.randomBytes(6).toString('hex'); +const alpha = manifest(`a_${suffix}`); +const bravo = manifest(`b_${suffix}`); +const layoutManager = createInstallationLayoutManager({ installationsRoot }); +let serviceManager = null; +let supervisor = null; +let alphaPlan = null; +let bravoPlan = null; +let provisioner = null; +let accessStore = null; +let alphaCommitted = false; +let workerExecutions = 0; +let backupRestoreExercised = false; +let runtimeAgentHubController = null; + +try { + const credentials = createRuntimeAgentCredentialManager({ installationsRoot }); + const alphaCredential = credentials.issue(alpha.runtime.key); + const bravoCredential = credentials.issue(bravo.runtime.key); + layoutManager.materialize(alpha, authority(alpha)); + layoutManager.materialize(bravo, authority(bravo)); + const runtimeAgentHubSocket = path.join(centralRuntimeRoot, 'runtime-agent-hub.sock'); + runtimeAgentHubController = await startSyntheticHub(runtimeAgentHubSocket, { + [alpha.runtime.key]: alphaCredential.tokenHash, + [bravo.runtime.key]: bravoCredential.tokenHash, + }); + let unitRoot = offlineUnitRoot; + if (liveSystemd) { + if (typeof process.getuid !== 'function') throw new Error('systemd fixture requires a Unix uid'); + const systemdParent = `/run/user/${process.getuid()}/systemd`; + const parent = fs.lstatSync(systemdParent); + assert.equal(parent.isDirectory(), true); + assert.equal(parent.isSymbolicLink(), false); + assert.equal(parent.uid, process.geteuid()); + unitRoot = path.join(systemdParent, 'user'); + if (!fs.existsSync(unitRoot)) fs.mkdirSync(unitRoot, { mode: PRIVATE_DIRECTORY_MODE }); + } + serviceManager = createInstallationServiceManager({ unitRoot, runtimeAgentHubSocket }); + alphaPlan = serviceManager.plan(alpha, authority(alpha), layoutManager.derive(alpha, authority(alpha))); + bravoPlan = serviceManager.plan(bravo, authority(bravo), layoutManager.derive(bravo, authority(bravo))); + if (liveSystemd) { + serviceManager.render(bravoPlan); + serviceManager.validate(bravoPlan); + } else { + serviceManager.render(alphaPlan); + serviceManager.render(bravoPlan); + serviceManager.validate(alphaPlan); + serviceManager.validate(bravoPlan); + } + assert.equal(alphaPlan.units.some(unit => bravoPlan.units.some(other => other.name === unit.name)), false); + + if (!liveSystemd) { + serviceManager.install(alphaPlan, inactiveState(alphaPlan)); + serviceManager.restoreFiles(alphaPlan); + serviceManager.finishRollback(alphaPlan); + } else { + supervisor = createSystemdUserSupervisor({ runtimeOnly: true }); + provisioner = createDurableInstallationProvisioner({ + stateRoot: controlRoot, + installationsRoot, + unitRoot, + supervisor, + systemdRuntimeOnly: true, + idFactory: () => `job_fixture_service_${suffix}`, + runtimeAgentHubSocket, + }); + const fixtureRegistration = { + fixture: true, installationState: 'pending', retainedData: false, + }; + const manage = { + scope: 'operator_fixture', + permission: 'platform.installations.manage', + operatorEnabled: true, + }; + provisioner.registerFixture(alpha, authority(alpha), fixtureRegistration); + provisioner.request(alpha, authority(alpha), { + operation: 'provision', + idempotencyKey: `systemd_fixture_${suffix}`, + expectedRevision: 1, + }, manage); + const completed = provisioner.runNext(`worker_fixture_${suffix}`); + if (completed.status !== 'succeeded') { + const progress = provisioner.progress( + alpha, + authority(alpha), + { scope: 'operator_fixture', permission: 'platform.installations.read' }, + completed.id, + ); + throw new Error(`durable service fixture failed: ${completed.failure?.code || 'unknown'}:${progress.completedStages}`); + } + assert.equal(completed.installationState, 'provisioning'); + alphaCommitted = true; + supervisor.health(alphaPlan); + + serviceManager.install(bravoPlan, supervisor.snapshot(bravoPlan)); + supervisor.reload(bravoPlan, operation => operation()); + supervisor.enable(bravoPlan, operation => operation()); + supervisor.start(bravoPlan, operation => operation()); + supervisor.health(bravoPlan); + const gatewayStatuses = await Promise.all([verifyGateway(alphaPlan), verifyGateway(bravoPlan)]); + assert.equal(gatewayStatuses.every(status => ['ready', 'degraded'].includes(status)), true); + const crossedGateway = createRuntimeGatewayDispatchClient({ + socketPath: bravoPlan.units.find(unit => unit.id === 'runtime_gateway').socketPath, + runtimeKey: alphaPlan.runtimeKey, + }); + assert.equal((await crossedGateway.system.status()).status, 'runtime_identity_mismatch'); + const agentStatuses = []; + for (const plan of [alphaPlan, bravoPlan]) { + agentStatuses.push(await runtimeAgentHubController.command({ + action: 'system.status', runtimeKey: plan.runtimeKey, + })); + } + assert.equal(agentStatuses.every(result => result.ok), true); + workerExecutions += await executeFixtureWorker(alphaPlan); + + accessStore = new AccessStore({ + databaseRoot: accessRoot, + database: path.join(accessRoot, 'access-control.sqlite3'), + }); + accessStore.transaction(() => { + accessStore.createOrganization({ + id: alpha.organization.id, + name: 'Lifecycle service fixture', + abbreviation: 'LSF', + timezone: alpha.organization.timezone, + status: 'active', + createdBy: null, + timestamp: Date.now(), + }); + accessStore.insertStation(alpha.organization.id, alpha.organization.stationCode, true, Date.now()); + accessStore.createInstallation( + alpha.organization.id, alpha.runtime.key, 'ready', Date.now(), alpha.runtime.releaseId, + ); + }); + let lifecycleIds = 0; + const lifecycleAuthority = createAccessInstallationLifecycleAuthority({ + store: accessStore, + organizationId: alpha.organization.id, + authorityScope: 'systemd_fixture_lifecycle', + jobFactory: () => `life_fixture_${suffix}_${++lifecycleIds}`, + backupFactory: () => `backup_fixture_${suffix}_${++lifecycleIds}`, + }); + const lifecycle = createManagedInstallationLifecycle({ + authority: lifecycleAuthority, + installationsRoot, + unitRoot, + supervisor, + projectRoot: path.resolve(__dirname, "../../.."), + projectReleaseId: alpha.runtime.releaseId, + activationRuntimeFactory: () => createFixtureLifecycleActivation(alphaPlan, supervisor), + runtimeAgentHubSocket, + }); + const alphaLayout = layoutManager.derive(alpha, authority(alpha)); + const marker = path.join(alphaLayout.directories.providerDataRoot, 'lifecycle-fixture.txt'); + fs.writeFileSync(marker, 'before-restore\n', { mode: 0o600 }); + let control = accessStore.installationControl(alpha.organization.id); + const backupJob = lifecycleAuthority.request({ + operation: 'backup', + idempotencyKey: `systemd:lifecycle:backup:${suffix}`, + expectedRevision: control.revision, + }); + const backupResult = await lifecycle.run(backupJob.id, `life_worker_backup_${suffix}`); + assert.equal(backupResult.status, 'succeeded', JSON.stringify({ + result: backupResult, + job: lifecycleAuthority.inspect(backupJob.id), + })); + supervisor.health(alphaPlan); + const sourceBackup = lifecycleAuthority.backups()[0]; + assert.ok(sourceBackup); + + control = accessStore.installationControl(alpha.organization.id); + const suspendJob = lifecycleAuthority.request({ + operation: 'suspend', + idempotencyKey: `systemd:lifecycle:suspend:${suffix}`, + expectedRevision: control.revision, + }); + assert.equal((await lifecycle.run(suspendJob.id, `life_worker_suspend_${suffix}`)).status, 'succeeded'); + assert.equal(supervisor.snapshot(alphaPlan).every(value => !value.active), true); + fs.writeFileSync(marker, 'after-backup\n', { mode: 0o600 }); + + control = accessStore.installationControl(alpha.organization.id); + const restoreJob = lifecycleAuthority.request({ + operation: 'restore', + idempotencyKey: `systemd:lifecycle:restore:${suffix}`, + expectedRevision: control.revision, + backupId: sourceBackup.id, + }); + assert.equal((await lifecycle.run(restoreJob.id, `life_worker_restore_${suffix}`)).status, 'succeeded'); + assert.equal(fs.readFileSync(marker, 'utf8'), 'before-restore\n'); + assert.equal(supervisor.snapshot(alphaPlan).every(value => !value.active), true); + backupRestoreExercised = true; + supervisor.start(alphaPlan, operation => operation()); + supervisor.health(alphaPlan); + + supervisor.restartEvidence(alphaPlan, 'auth_broker', operation => operation()); + supervisor.restartEvidence(alphaPlan, 'runtime_agent', operation => operation()); + supervisor.restartEvidence(bravoPlan, 'collection_manager', operation => operation()); + workerExecutions += await executeFixtureWorker(bravoPlan); + supervisor.boundedRestartEvidence(bravoPlan, 'collection_manager', operation => operation()); + + rollback(serviceManager, supervisor, bravoPlan); + supervisor.health(alphaPlan); + serviceManager.finalizeSettled(alphaPlan); + cleanupCommittedFixture(supervisor, alphaPlan); + alphaCommitted = false; + assert.equal(fs.existsSync(alphaPlan.journal), false); + assert.equal(fs.existsSync(bravoPlan.journal), false); + } + + process.stdout.write(`${JSON.stringify({ + ok: true, + status: 'verified', + servicePlanVersion: INSTALLATION_SERVICE_PLAN_VERSION, + fixtures: 2, + servicesPerFixture: liveSystemd ? INSTALLATION_AGENT_SERVICE_COUNT : alphaPlan.units.length, + isolated: true, + systemdValidated: true, + lifecycle: liveSystemd ? 'exercised' : 'offline', + restartRecovery: liveSystemd, + restartBounded: liveSystemd, + workerExecutions, + durablePipeline: liveSystemd, + gatewayRouting: liveSystemd, + crossRouteRejected: liveSystemd, + rollback: liveSystemd ? 'verified' : 'file_transaction_verified', + backupRestore: backupRestoreExercised ? 'verified' : 'offline', + })}\n`); +} finally { + try { accessStore?.close(); } catch {} + try { provisioner?.close(); } catch {} + try { await runtimeAgentHubController?.close(); } catch {} + if (supervisor && serviceManager) { + for (const plan of [bravoPlan, alphaPlan]) { + if (!plan) continue; + try { rollback(serviceManager, supervisor, plan); } catch {} + } + if (alphaCommitted && alphaPlan) { + try { cleanupCommittedFixture(supervisor, alphaPlan); } catch {} + } + } + fs.rmSync(fixtureRoot, { recursive: true, force: true }); +} +} + +main().catch(() => { + process.stderr.write('installation service fixture failed\n'); + process.exitCode = 1; +}); diff --git a/core/core/installations/examples/oci-lifecycle-host-fixture.js b/core/core/installations/examples/oci-lifecycle-host-fixture.js new file mode 100644 index 0000000..59f6cb7 --- /dev/null +++ b/core/core/installations/examples/oci-lifecycle-host-fixture.js @@ -0,0 +1,261 @@ +'use strict'; +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const readline = require('node:readline'); +const { spawn, spawnSync } = require('node:child_process'); +const { DatabaseSync } = require('node:sqlite'); +const { hostAccountName, opaqueRuntimeSuffix } = require('../../runtime-host-identity'); +const { createOciDeploymentPlan } = require('../src/oci-deployment'); +const REPO = path.resolve(__dirname, "../../.."); +const KEYS = ['runtime_lifecycle_alpha', 'runtime_lifecycle_beta']; +const CONTROL = '/opt/dispatch-control'; +const RELEASES = '/opt/dispatch-oci-fixture-releases'; +const AUTHORITY = 'dispatch-lifecycle-authority'; +const CALLER = 'dispatch-lifecycle-caller'; +const HELPER = `${CONTROL}/current/host-helper-artifact/core/installations/bin/dispatch-oci-host-helper`; +const ISSUER = `${CONTROL}/current/host-helper-artifact/core/installations/bin/dispatch-oci-host-issuer`; +const ENV = { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' }; +function run(file, args, options = {}) { + const value = spawnSync(file, args, { encoding: 'utf8', timeout: 120_000, maxBuffer: 256 * 1024, env: ENV, ...options }); + if (!options.allowFailure) assert.equal(value.status, 0, `${path.basename(file)} ${args[0]}: ${value.stderr}`); + return value; +} +function removeTree(root) { + function writable(directory) { + fs.chmodSync(directory, 0o700); + for (const name of fs.readdirSync(directory)) { + const target = path.join(directory, name); + if (fs.lstatSync(target).isDirectory()) writable(target); + } + } + writable(root); fs.rmSync(root, { recursive: true }); +} +function seal(root) { + for (const name of fs.readdirSync(root)) { + const target = path.join(root, name); + if (fs.lstatSync(target).isDirectory()) seal(target); + } + fs.chmodSync(root, 0o555); +} +function hash(file) { return run('/usr/bin/sha256sum', [file], { timeout: 120_000 }).stdout.split(' ')[0]; } +async function lockedParent() { + const lock = '/run/dispatch-rootless-host-fixture.lock'; + run('/usr/bin/sudo', ['-n', '/usr/bin/mkdir', '--mode=0700', lock]); + try { return await parent(); } + finally { run('/usr/bin/sudo', ['-n', '/usr/bin/rmdir', lock]); } +} +async function parent() { + assert.equal(process.env.DISPATCH_RUN_OCI_LIFECYCLE_FIXTURE, '1', 'opt-in fixture required'); + assert.equal(fs.existsSync(CONTROL), false, 'existing control installation is preserved'); + const root = fs.mkdtempSync('/var/tmp/dispatch-full-lifecycle-'); + fs.chmodSync(root, 0o755); + const images = []; + try { + for (const [name, script] of [['helper', 'create-host-helper-artifact.js'], ['bridge', 'create-bridge-artifact.js']]) { + run('/usr/bin/node', [path.join(REPO, 'core/installations/src', script), path.join(root, name)]); + } + const build = path.join(root, 'build'); fs.mkdirSync(build, { mode: 0o755 }); + fs.copyFileSync(path.join(REPO, 'plugins/paycom/backend/tests/oci-lifecycle-seed.js'), path.join(build, 'seed.js')); + fs.copyFileSync(path.join(REPO, 'runtime/collection-manager/tests/fixture-collector.js'), path.join(build, 'collector')); + fs.chmodSync(path.join(build, 'collector'), 0o755); + fs.copyFileSync(path.join(REPO, 'plugins/paycom/backend/tests/helpers.js'), path.join(build, 'helpers.js')); + const releases = {}; + for (const [index, id] of ['dispatch_current_1', 'dispatch_fixture_2'].entries()) { + const tag = `ghcr.io/example-organization/dispatch-runtime:fixture-${process.pid}-${index}`; + fs.writeFileSync(path.join(build, 'Containerfile'), `FROM localhost/dispatch-runtime:dev\nUSER 0:0\nCOPY seed.js /opt/dispatch/fixture-seed.js\nCOPY helpers.js /opt/dispatch/plugins/paycom/backend/tests/helpers.js\nCOPY --chown=10001:10001 --chmod=0555 collector /opt/dispatch/fixture-collector\nLABEL io.dispatch.synthetic-fixture="${index}"\nUSER 10001:10001\nHEALTHCHECK --interval=30s --timeout=30s --start-period=30s --retries=3 CMD ["/usr/local/bin/node", "--no-warnings", "/opt/dispatch/runtime/supervisor/src/health.js"]\n`); + run('/usr/bin/podman', ['build', '--format=docker', '--pull=never', '--network=none', '-t', tag, build], { timeout: 300_000 }); + images.push(tag); + const inspected = JSON.parse(run('/usr/bin/podman', ['image', 'inspect', tag]).stdout)[0]; + assert.deepEqual(inspected.Healthcheck.Test, ['CMD', '/usr/local/bin/node', '--no-warnings', '/opt/dispatch/runtime/supervisor/src/health.js']); + const archive = path.join(root, `${id}.tar`); + run('/usr/bin/podman', ['save', '--format', 'oci-archive', '--output', archive, tag], { timeout: 300_000 }); + const archiveIndex = JSON.parse(run('/usr/bin/tar', ['-xOf', archive, 'index.json']).stdout); + const exportedDigest = archiveIndex.manifests[0].digest; + const exportedManifest = JSON.parse(run('/usr/bin/tar', ['-xOf', archive, `blobs/sha256/${exportedDigest.slice(7)}`]).stdout); + const embedded = run('/usr/bin/podman', ['run', '--rm', '--pull=never', '--network=none', '--read-only', + '--entrypoint=/usr/bin/sha256sum', tag, '/opt/dispatch/runtime-release-manifest.json']).stdout.split(' ')[0]; + releases[id] = { version: 2, backend: 'oci_container_v1', releaseId: id, channel: 'production', + image: `ghcr.io/example-organization/dispatch-runtime@${exportedDigest}`, imageDigest: exportedDigest, + imageId: exportedManifest.config.digest.slice(7), sourceCommit: inspected.Labels['org.opencontainers.image.revision'], platform: 'linux/amd64', + runtimeAgentProtocol: 1, runtimeGatewayProtocol: 1, embeddedManifestSha256: embedded, + imageArchiveSha256: hash(archive), bridgeManifestSha256: hash(path.join(root, 'bridge/manifest.json')) }; + } + releases.dispatch_fixture_3 = { ...releases.dispatch_fixture_2, releaseId: 'dispatch_fixture_3' }; + fs.writeFileSync(path.join(root, 'releases.json'), JSON.stringify(releases)); + const child = spawn('/usr/bin/sudo', ['-n', '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin', + '/usr/bin/node', '--no-warnings', __filename, '--root', root], { stdio: ['ignore', 'inherit', 'inherit'] }); + assert.equal(await new Promise(resolve => child.once('exit', resolve)), 0); + } finally { + for (const tag of images) run('/usr/bin/podman', ['image', 'rm', tag], { allowFailure: true }); + removeTree(root); + } +} +async function rootMain(source) { + assert.equal(process.geteuid(), 0); + process.umask(0o077); + assert.match(source, /^\/var\/tmp\/dispatch-full-lifecycle-[a-zA-Z0-9]+$/); + for (const file of [CONTROL, RELEASES, '/etc/dispatch/oci-host.json']) assert.equal(fs.existsSync(file), false); + for (const account of [AUTHORITY, CALLER, ...KEYS.map(hostAccountName)]) { + assert.equal(run('/usr/bin/getent', ['passwd', account], { allowFailure: true }).status, 2); + assert.equal(run('/usr/bin/getent', ['group', account], { allowFailure: true }).status, 2); + } + const references = ['dispatch-auth-broker.service', 'dispatch-collection-manager.service']; + const reference = () => references.map(name => run('/usr/bin/systemctl', ['show', name, '--property=MainPID,ActiveState']).stdout); + const referenceBefore = reference(); + const createdDirs = []; + const accounts = []; + const policies = []; + let configCreated = false; + const privateRoot = fs.mkdtempSync('/run/dispatch-full-lifecycle-'); fs.chmodSync(privateRoot, 0o755); + const releases = JSON.parse(fs.readFileSync(path.join(source, 'releases.json'))); + const makeDir = (target, mode, owner = 0) => { + fs.mkdirSync(target, { mode }); fs.chmodSync(target, mode); fs.chownSync(target, owner, owner); return target; + }; + const ensureDir = (target, mode) => { + if (!fs.existsSync(target)) { makeDir(target, mode); createdDirs.push(target); } + const stat = fs.lstatSync(target); assert.equal(stat.uid, 0); assert.equal(stat.gid, 0); + assert.equal(stat.mode & 0o7777, mode); assert.equal(fs.realpathSync(target), target); + }; + let controllerRoot; + function plan(index, releaseId = 'dispatch_current_1', revision = 1) { + const db = new DatabaseSync(path.join(privateRoot, 'state/oci-host.sqlite3'), { readOnly: true }); + let row; + try { row = db.prepare('SELECT * FROM allocations WHERE runtime_key=?').get(KEYS[index]); } finally { db.close(); } + if (!row) throw new Error('fixture allocation missing'); + const manifest = { manifestVersion: 1, revision, organization: { id: `org_lifecycle_${index}`, stationCode: 'TEST', timezone: 'UTC' }, + runtime: { key: KEYS[index], templateId: 'isolated_dsp_v1', releaseId } }; + return createOciDeploymentPlan(manifest, { revision, organization: manifest.organization, runtime: manifest.runtime }, + releases[releaseId], { name: row.account_name, uid: row.uid, gid: row.gid, subuidStart: row.subuid_start, + subgidStart: row.subgid_start, subidCount: row.subid_count }, + { version: 1, backend: 'oci_container_v1', channel: 'production', organizationId: manifest.organization.id, + runtimeKey: KEYS[index], manifestRevision: revision, releaseId }); + } + const asTenant = (selected, executable, args, options = {}) => run('/usr/sbin/runuser', ['--user', selected.account.name, + '--', '/usr/bin/env', '-i', `HOME=${selected.host.accountHome}`, `XDG_DATA_HOME=${selected.host.engineDataRoot}`, + `XDG_CONFIG_HOME=${selected.host.engineConfigRoot}`, `XDG_RUNTIME_DIR=/run/user/${selected.account.uid}`, + 'PATH=/usr/bin:/bin', executable, ...args], options); + function fixtureAction(message) { + if (Number.isInteger(message.seed)) { + const selected = plan(message.seed); + const result = asTenant(selected, '/usr/bin/podman', ['exec', selected.identity.containerName, + '/usr/local/bin/node', '--no-warnings', '/opt/dispatch/fixture-seed.js', ...(message.changed ? ['--changed'] : [])]); + asTenant(selected, '/usr/bin/tee', [path.join(selected.host.installationRoot, 'data/fixture-sentinel')], { input: 'before\n' }); + return { ok: true, ...JSON.parse(result.stdout) }; + } + if (message.inspectBeta) { + const selected = plan(1); + return { pid: run('/usr/bin/systemctl', ['show', selected.identity.unitName, '--property=MainPID', '--value']).stdout.trim(), + sentinel: asTenant(selected, '/usr/bin/cat', [path.join(selected.host.installationRoot, 'data/fixture-sentinel')]).stdout }; + } + if (message.destroyed) { + assert.equal(fs.existsSync(`/var/lib/dispatch/tenants/${opaqueRuntimeSuffix(KEYS[0])}`), false); + assert.equal(run('/usr/bin/getent', ['passwd', hostAccountName(KEYS[0])], { allowFailure: true }).status, 2); + return { ok: true }; + } + const selected = plan(0); + const sentinel = path.join(selected.host.installationRoot, 'data/fixture-sentinel'); + if (message.mutateSentinel) { asTenant(selected, '/usr/bin/tee', [sentinel], { input: 'after\n' }); return { ok: true }; } + if (message.verifySentinel) { assert.equal(asTenant(selected, '/usr/bin/cat', [sentinel]).stdout, 'before\n'); return { ok: true }; } + if (message.retained) { assert.equal(fs.existsSync(selected.host.installationRoot), true); return { ok: true }; } + throw new Error('invalid fixture action'); + } + try { + for (const name of [AUTHORITY, CALLER]) { + run('/usr/sbin/useradd', ['--system', '--user-group', '--no-create-home', '--shell', '/usr/sbin/nologin', name]); accounts.push(name); + } + const uid = name => Number(run('/usr/bin/id', ['-u', name]).stdout.trim()); + const authorityUid = uid(AUTHORITY), callerUid = uid(CALLER); + controllerRoot = makeDir(path.join(privateRoot, 'controller'), 0o700, authorityUid); + for (const name of ['credentials', 'provisioner', 'unused-installations']) makeDir(path.join(controllerRoot, name), 0o700, authorityUid); + makeDir(path.join(privateRoot, 'authority'), 0o700); makeDir(path.join(privateRoot, 'state'), 0o700); + ensureDir('/var/lib/dispatch', 0o755); ensureDir('/var/lib/dispatch/tenants', 0o755); ensureDir('/run/dispatch-runtime-agents', 0o711); + ensureDir('/etc/dispatch', 0o755); + makeDir(CONTROL, 0o755); makeDir(`${CONTROL}/releases`, 0o755); makeDir(`${CONTROL}/releases/synthetic-oci-control`, 0o755); + const artifact = `${CONTROL}/releases/synthetic-oci-control/host-helper-artifact`; + fs.cpSync(path.join(source, 'helper'), artifact, { recursive: true }); seal(artifact); seal(`${CONTROL}/releases/synthetic-oci-control`); + fs.symlinkSync(`${CONTROL}/releases/synthetic-oci-control`, `${CONTROL}/current`); + makeDir(RELEASES, 0o755); + for (const id of Object.keys(releases)) { + const root = makeDir(path.join(RELEASES, id), 0o755); + fs.cpSync(path.join(source, 'bridge'), path.join(root, 'bridge-artifact'), { recursive: true }); seal(path.join(root, 'bridge-artifact')); + fs.copyFileSync(path.join(source, `${id === 'dispatch_fixture_3' ? 'dispatch_fixture_2' : id}.tar`), path.join(root, 'runtime-image.tar')); + fs.chownSync(path.join(root, 'runtime-image.tar'), 0, 0); + fs.chmodSync(path.join(root, 'runtime-image.tar'), 0o444); seal(root); + } + const config = { stateRoot: path.join(privateRoot, 'state'), authorityRoot: path.join(privateRoot, 'authority'), + unitRoot: '/etc/systemd/system', releaseRoot: RELEASES, centralSocket: path.join(controllerRoot, 'runtime-agent-hub.sock'), + centralUid: authorityUid, controllerUid: 0, authorityUid, helperCallerUid: callerUid, helperCallerGid: callerUid, + controlReleaseId: 'synthetic-oci-control', helperManifestSha256: hash(path.join(artifact, 'manifest.json')) }; + fs.writeFileSync('/etc/dispatch/oci-host.json', JSON.stringify(config), { flag: 'wx', mode: 0o600 }); configCreated = true; + for (const [name, executable] of [[AUTHORITY, ISSUER], [CALLER, HELPER]]) { + const policy = `/etc/sudoers.d/${name}`; + fs.writeFileSync(policy, `Defaults:${name} env_reset,!setenv,secure_path="/usr/bin:/bin"\n` + + `Defaults:${name} env_delete += "NODE_OPTIONS NODE_PATH LD_PRELOAD LD_LIBRARY_PATH"\n` + + `${name} ALL=(root) NOPASSWD: NOSETENV: ${executable} ""\n`, { mode: 0o440, flag: 'wx' }); + fs.chmodSync(policy, 0o440); + policies.push(policy); run('/usr/sbin/visudo', ['-cf', policy]); + } + const workerConfig = path.join(controllerRoot, 'fixture.json'); + fs.writeFileSync(workerConfig, JSON.stringify({ controllerRoot, releases, centralSocket: config.centralSocket, + controlSocket: path.join(controllerRoot, 'runtime-agent-control.sock') }), { mode: 0o600 }); + fs.chownSync(workerConfig, authorityUid, authorityUid); + const worker = spawn('/usr/sbin/runuser', ['--user', AUTHORITY, '--', '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin', + '/usr/bin/node', '--no-warnings', path.join(__dirname, "./helpers/oci-lifecycle-worker.js"), workerConfig], { stdio: ['pipe', 'pipe', 'inherit'] }); + const output = readline.createInterface({ input: worker.stdout }); + let actionError; + output.on('line', line => { + try { + const value = JSON.parse(line); + if (value.phase) { + process.stdout.write(`${JSON.stringify(value)}\n`); + + } + else worker.stdin.write(`${JSON.stringify(fixtureAction(value))}\n`); + } catch (error) { actionError = error; worker.stdin.write(`${JSON.stringify({ ok: false, error: error.message })}\n`); } + }); + const status = await new Promise(resolve => worker.once('exit', resolve)); + if (actionError) throw actionError; + assert.equal(status, 0, 'lifecycle authority worker failed'); + } finally { + for (const key of KEYS) { + const name = hostAccountName(key), suffix = opaqueRuntimeSuffix(key); + for (const unit of [`dispatch-dsp-${suffix}.service`, `dispatch-runtime-agent-bridge-${suffix}.service`]) { + run('/usr/bin/systemctl', ['disable', '--now', unit], { allowFailure: true }); + fs.rmSync(`/etc/systemd/system/${unit}`, { force: true }); + run('/usr/bin/systemctl', ['reset-failed', unit], { allowFailure: true }); + } + if (run('/usr/bin/getent', ['passwd', name], { allowFailure: true }).status === 0) { + const index = KEYS.indexOf(key); const selected = plan(index); + asTenant(selected, '/usr/bin/podman', ['system', 'reset', '--force'], { allowFailure: true }); + run('/usr/bin/loginctl', ['disable-linger', name], { allowFailure: true }); + run('/usr/bin/systemctl', ['stop', `user@${selected.account.uid}.service`, `user-runtime-dir@${selected.account.uid}.service`], { allowFailure: true }); + run('/usr/sbin/usermod', ['--del-subuids', `${selected.account.subuidStart}-${selected.account.subuidStart + 65535}`, + '--del-subgids', `${selected.account.subgidStart}-${selected.account.subgidStart + 65535}`, name], { allowFailure: true }); + run('/usr/sbin/userdel', [name]); + if (run('/usr/bin/getent', ['group', name], { allowFailure: true }).status === 0) run('/usr/sbin/groupdel', [name]); + fs.rmSync(selected.host.tenantRoot, { recursive: true, force: true }); + fs.rmSync(selected.host.bridgeRoot, { recursive: true, force: true }); + } + assert.equal(run('/usr/bin/getent', ['passwd', name], { allowFailure: true }).status, 2); + for (const file of ['/etc/subuid', '/etc/subgid']) assert.equal(fs.readFileSync(file, 'utf8').includes(`${name}:`), false); + } + run('/usr/bin/systemctl', ['daemon-reload']); + for (const policy of policies) fs.unlinkSync(policy); + if (configCreated) fs.unlinkSync('/etc/dispatch/oci-host.json'); + for (const name of accounts.reverse()) { + run('/usr/sbin/userdel', [name]); + if (run('/usr/bin/getent', ['group', name], { allowFailure: true }).status === 0) run('/usr/sbin/groupdel', [name]); + } + for (const root of [CONTROL, RELEASES]) if (fs.existsSync(root)) removeTree(root); + fs.rmSync(privateRoot, { recursive: true, force: true }); + for (const directory of createdDirs.reverse()) fs.rmdirSync(directory); + assert.deepEqual(reference(), referenceBefore); + } + process.stdout.write(`${JSON.stringify({ status: 'oci_lifecycle_fixture_verified', tenants: 2, artifactsRemaining: 0 })}\n`); +} +(process.argv[2] === '--root' ? rootMain(process.argv[3]) : process.env.DISPATCH_RUN_OCI_LIFECYCLE_FIXTURE === '1' + ? lockedParent() : Promise.resolve()).catch(error => { + process.stderr.write(`${error.stack}\n`); process.exitCode = 1; +}); diff --git a/core/core/installations/examples/provider-activation.js b/core/core/installations/examples/provider-activation.js new file mode 100644 index 0000000..3f7bed5 --- /dev/null +++ b/core/core/installations/examples/provider-activation.js @@ -0,0 +1,252 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { + AccessStore, + AccessControlService, + createAccessControlLiveAuthorityResolver, + createAccessInstallationActivationAuthority, + createAccessInstallationProvisioningAuthority, + createInstallationProvisioningReconciler, +} = require('../../accounts/src'); +const { createDurableInstallationProvisioner } = require('../src/jobs'); +const { runManagedPaycomActivation } = require('../../../compatibility/provisioner/src/activation.js'); + +function createRoot() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-activation-fixture-')); + fs.chmodSync(root, 0o700); + return root; +} + +function artifactRuntime(installationRoot, testedAt, { failPublication = false } = {}) { + const configFile = path.join(installationRoot, 'config', 'activation.json'); + const dataRoot = path.join(installationRoot, 'data', 'providers', 'paycom'); + const stagingRoot = path.join(installationRoot, 'staging', 'providers', 'paycom'); + const active = path.join(dataRoot, 'first-publication.json'); + let definitionDigest = null; + let requestDigest = null; + return Object.freeze({ + async verifyInfrastructure(manifest) { + const info = fs.lstatSync(installationRoot); + assert.equal(info.isDirectory(), true); + assert.equal(info.mode & 0o7777, 0o700); + assert.equal(path.basename(installationRoot), manifest.runtime.key); + return { + runtimeKey: manifest.runtime.key, + runtime_layout: true, + service_supervision: true, + auth_broker: true, + collection_manager: true, + runtime_gateway: true, + }; + }, + async configure(definition) { + definitionDigest = definition.digest; + const selected = JSON.stringify({ digest: definition.digest, collectors: 1, sources: 1, plans: 15, syncs: 1 }); + const candidate = `${configFile}.candidate`; + fs.writeFileSync(candidate, selected, { mode: 0o600, flag: 'wx' }); + fs.renameSync(candidate, configFile); + assert.equal(fs.readFileSync(configFile, 'utf8'), selected); + return JSON.parse(selected); + }, + async testProvider(profileId) { + return { profileId, provider: 'paycom', status: 'authenticated', testedAt }; + }, + async publishFirst(request, operation) { + requestDigest = crypto.createHash('sha256').update(JSON.stringify(request)).digest('hex'); + fs.mkdirSync(dataRoot, { mode: 0o700 }); + fs.mkdirSync(stagingRoot, { mode: 0o700 }); + const batchId = `batch_${path.basename(installationRoot).replace(/^runtime_/, '')}`; + const candidate = path.join(stagingRoot, `${batchId}.json`); + const publication = JSON.stringify({ batchId, target: '2026-09-05', idempotencyKey: operation.idempotencyKey }); + fs.writeFileSync(candidate, publication, { mode: 0o600, flag: 'wx' }); + if (failPublication) { + fs.unlinkSync(candidate); + return { + batchId, preparationRunId: 'run_periods', status: 'failed', runCount: 5, + succeededRuns: 4, failedRuns: 1, cancelledRuns: 0, + }; + } + fs.renameSync(candidate, active); + assert.equal(fs.readFileSync(active, 'utf8'), publication); + return { + batchId, preparationRunId: 'run_periods', status: 'succeeded', runCount: 5, + succeededRuns: 5, failedRuns: 0, cancelledRuns: 0, + }; + }, + async verifyPublication(batchId, preparationRunId) { + const publication = JSON.parse(fs.readFileSync(active, 'utf8')); + return { + definitionDigest, + requestDigest, + previewDigest: 'a'.repeat(64), + batchId, + preparationRunId, + target: publication.target, + runs: [ + { id: 'run_roster', taskId: 'roster', plan: 'paycom-period-roster', method: 'roster.period' }, + { id: 'run_timecards', taskId: 'timecards', plan: 'paycom-period-timecards-from-roster', method: 'timecards.from-published-roster' }, + { id: 'run_timecards_audit', taskId: 'timecards-audit', plan: 'paycom-period-timecards-audit', method: 'timecards.audit' }, + { id: 'run_links', taskId: 'links', plan: 'paycom-period-resource-links', method: 'resource-links.period' }, + { id: 'run_links_audit', taskId: 'links-audit', plan: 'paycom-period-resource-links-audit', method: 'resource-links.audit' }, + ], + publications: { + payPeriods: { id: 'pub_periods', runId: 'run_periods', originRunId: 'run_periods', contentSha256: '1'.repeat(64), batchBound: false }, + roster: { id: 'pub_roster', runId: 'run_roster', originRunId: 'run_roster', contentSha256: '2'.repeat(64), batchBound: true }, + timecards: { id: 'pub_timecards', runId: 'run_timecards', originRunId: 'run_timecards', contentSha256: '3'.repeat(64), batchBound: true }, + resourceLinks: { id: 'pub_links', runId: 'run_links', originRunId: 'run_links', contentSha256: '4'.repeat(64), batchBound: true }, + }, + capturedAt: testedAt, + }; + }, + }); +} + +async function main() { + const root = createRoot(); + const accessRoot = path.join(root, 'access'); + const stateRoot = path.join(root, 'control'); + const installationsRoot = path.join(root, 'installations'); + fs.mkdirSync(stateRoot, { mode: 0o700 }); + fs.mkdirSync(installationsRoot, { mode: 0o700 }); + const store = new AccessStore({ + databaseRoot: accessRoot, + database: path.join(accessRoot, 'access-control.sqlite3'), + }); + let now = Date.parse('2026-09-02T21:30:00.000Z'); + const clock = () => ++now; + const service = new AccessControlService(store, { clock: () => new Date(now) }); + let provisioner; + try { + service.ensureLocalOrganization({ + organization: { id: 'local-dsp', name: 'Reference DSP' }, + site: { id: 'reference-site', code: 'REF1' }, + timezone: 'America/Los_Angeles', + }); + const referenceBefore = JSON.stringify(store.installationControl('local-dsp')); + store.transaction(() => { + store.insertUser({ + id: 'usr_platform_fixture', email: 'platform@example.invalid', firstName: 'Platform', lastName: 'Fixture', + passwordHash: 'fixture-hash-not-a-secret', platformRole: 'owner', timestamp: clock(), + }); + for (const suffix of ['alpha', 'bravo']) { + const organizationId = `org_activation_${suffix}`; + store.createOrganization({ + id: organizationId, name: `Activation ${suffix}`, abbreviation: suffix.toUpperCase(), + timezone: suffix === 'alpha' ? 'America/Chicago' : 'America/New_York', status: 'setup_required', + createdBy: 'usr_platform_fixture', timestamp: clock(), + }); + store.insertStation(organizationId, suffix === 'alpha' ? 'TST1' : 'TST2', true, clock()); + store.createInstallation(organizationId, `runtime_activation_${suffix}`, 'pending', clock()); + const roles = service.ensureSystemRoles(organizationId, 'usr_platform_fixture', clock()); + const userId = `usr_activation_${suffix}`; + store.insertUser({ + id: userId, email: `${suffix}@example.invalid`, firstName: suffix, lastName: 'Fixture', + passwordHash: 'fixture-hash-not-a-secret', platformRole: null, timestamp: clock(), + }); + store.createMembership({ + id: `mem_activation_${suffix}`, organizationId, userId, roleId: roles.owner.id, + createdBy: 'usr_platform_fixture', timestamp: clock(), + }); + } + }); + let nextJob = 0; + provisioner = createDurableInstallationProvisioner({ + stateRoot, + installationsRoot, + clock, + idFactory: () => `job_activation_provision_${++nextJob}`, + liveAuthorityResolver: createAccessControlLiveAuthorityResolver({ store }), + }); + const reconciler = createInstallationProvisioningReconciler({ store, provisioner, clock }); + for (const suffix of ['alpha', 'bravo']) { + createAccessInstallationProvisioningAuthority({ + store, + organizationId: `org_activation_${suffix}`, + authorityScope: 'platform_installation', + actorUserId: 'usr_platform_fixture', + clock, + requestFactory: () => `prq_activation_${suffix}`, + }).request({ + operation: 'provision', idempotencyKey: `activation:provision:${suffix}`, expectedRevision: 1, + }); + } + assert.deepEqual(reconciler.runPending('worker_activation_provision', 20), { + processed: 2, completed: 2, failed: 0, pending: 0, + }); + + const successfulAuthority = createAccessInstallationActivationAuthority({ + store, + organizationId: 'org_activation_alpha', + authorityScope: 'platform_activation', + idempotencyKey: 'activation:artifact:alpha', + workerId: 'worker_activation_alpha', + clock, + jobFactory: () => 'job_activation_alpha', + }); + const successful = await runManagedPaycomActivation({ + authority: successfulAuthority, + runtime: artifactRuntime( + path.join(installationsRoot, 'runtime_activation_alpha'), + new Date(now).toISOString(), + ), + }); + assert.equal(successful.status, 'ready'); + const alphaPublication = fs.readFileSync( + path.join(installationsRoot, 'runtime_activation_alpha', 'data', 'providers', 'paycom', 'first-publication.json'), + 'utf8', + ); + + const failedAuthority = createAccessInstallationActivationAuthority({ + store, + organizationId: 'org_activation_bravo', + authorityScope: 'platform_activation', + idempotencyKey: 'activation:artifact:bravo', + workerId: 'worker_activation_bravo', + clock, + jobFactory: () => 'job_activation_bravo', + }); + const failed = await runManagedPaycomActivation({ + authority: failedAuthority, + runtime: artifactRuntime( + path.join(installationsRoot, 'runtime_activation_bravo'), + new Date(now).toISOString(), + { failPublication: true }, + ), + }); + assert.equal(failed.status, 'first_publication_failed'); + assert.equal(store.installationControl('org_activation_alpha').status, 'ready'); + assert.equal(store.installationControl('org_activation_bravo').status, 'failed'); + assert.equal(fs.existsSync( + path.join(installationsRoot, 'runtime_activation_bravo', 'data', 'providers', 'paycom', 'first-publication.json'), + ), false); + assert.equal(fs.readFileSync( + path.join(installationsRoot, 'runtime_activation_alpha', 'data', 'providers', 'paycom', 'first-publication.json'), + 'utf8', + ), alphaPublication); + assert.equal(JSON.stringify(store.installationControl('local-dsp')), referenceBefore); + process.stdout.write(`${JSON.stringify({ + ok: true, + status: 'verified', + runtimes: 2, + successfulActivations: 1, + failedActivations: 1, + atomicPublication: true, + failureIsolation: true, + referenceInstallationPreserved: true, + })}\n`); + } finally { + try { provisioner?.close(); } catch {} + try { store.close(); } catch {} + fs.rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(() => { + process.stdout.write('{"ok":false,"status":"activation_fixture_failed"}\n'); + process.exitCode = 1; +}); diff --git a/core/core/installations/integration/systemd/dispatch-platform-update.service.in b/core/core/installations/integration/systemd/dispatch-platform-update.service.in new file mode 100644 index 0000000..34c43df --- /dev/null +++ b/core/core/installations/integration/systemd/dispatch-platform-update.service.in @@ -0,0 +1,16 @@ +[Unit] +Description=Dispatch Core update supervisor +After=network-online.target + +[Service] +Type=oneshot +# Pin this path independently. A Core rollout must not replace/restart this unit. +ExecStart=/usr/bin/node --no-warnings @UPDATER_ROOT@/core/installations/bin/dispatch-platform-update +Environment=PATH=/usr/bin:/bin +Environment=NODE_NO_WARNINGS=1 +EnvironmentFile=@LOCAL_ROOT@/config/provisioning.env +TimeoutStartSec=15min +TimeoutStopSec=30s +KillMode=control-group +UMask=0077 +NoNewPrivileges=false diff --git a/core/core/installations/integration/systemd/dispatch-platform-update.timer b/core/core/installations/integration/systemd/dispatch-platform-update.timer new file mode 100644 index 0000000..24907e7 --- /dev/null +++ b/core/core/installations/integration/systemd/dispatch-platform-update.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Check for queued Dispatch Core updates + +[Timer] +OnBootSec=15s +OnUnitInactiveSec=60s +AccuracySec=1s +Unit=dispatch-platform-update.service + +[Install] +WantedBy=timers.target diff --git a/core/core/installations/package.json b/core/core/installations/package.json new file mode 100644 index 0000000..12433fd --- /dev/null +++ b/core/core/installations/package.json @@ -0,0 +1,27 @@ +{ + "name": "dispatch-provisioner", + "version": "0.6.0", + "private": true, + "description": "Server-owned isolated runtime provisioning, lifecycle, reconciliation, and provider activation for Dispatch", + "type": "commonjs", + "main": "src/index.js", + "bin": { + "dispatch-installation-reconcile": "bin/dispatch-installation-reconcile", + "dispatch-managed-activation": "bin/dispatch-managed-activation", + "dispatch-installation-lifecycle": "bin/dispatch-installation-lifecycle", + "dispatch-runtime-agent-authority": "bin/dispatch-runtime-agent-authority", + "dispatch-oci-host-helper": "bin/dispatch-oci-host-helper", + "dispatch-oci-tenant-backup-helper": "bin/dispatch-oci-tenant-backup-helper", + "dispatch-oci-host-issuer": "bin/dispatch-oci-host-issuer" + }, + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "./scripts/build", + "test": "./scripts/test", + "verify": "./scripts/verify", + "verify:systemd": "./scripts/verify-systemd-fixture", + "verify:oci-lifecycle": "./scripts/verify-oci-lifecycle-fixture" + } +} diff --git a/core/core/installations/release-formats.json b/core/core/installations/release-formats.json new file mode 100644 index 0000000..7fe2ee2 --- /dev/null +++ b/core/core/installations/release-formats.json @@ -0,0 +1,29 @@ +{ + "manifest": "dispatch-release.json", + "notes": "dispatch-release-notes.json", + "changelog": "CHANGELOG.md", + "ociRuntime": "runtime-image.tar", + "formats": { + "legacy": { + "schemaVersion": 1, + "ciSuffix": "", + "checksums": "SHA256SUMS", + "notesSidecar": true, + "assets": { + "core": "dispatch-core.json", + "bridge": "dispatch-bridge.json", + "runtime": "dispatch-runtime.tar.gz" + } + }, + "split": { + "schemaVersion": 2, + "ciSuffix": "-split", + "checksums": null, + "notesSidecar": false, + "assets": { + "app": "dispatch-app.tar.gz", + "dependencies": "dispatch-dependencies.tar.gz" + } + } + } +} diff --git a/core/core/installations/runtime-dependencies.json b/core/core/installations/runtime-dependencies.json new file mode 100644 index 0000000..a7c19f8 --- /dev/null +++ b/core/core/installations/runtime-dependencies.json @@ -0,0 +1,5 @@ +{ + "node": "22.23.2", + "chrome": "151.0.7922.138", + "chromeArchiveSha256": "f2c1d48c310a2fc79aad59e985103902f27b70fd186b3f663ee67c4c722f5566" +} diff --git a/core/core/installations/scripts/build b/core/core/installations/scripts/build new file mode 100755 index 0000000..d831d43 --- /dev/null +++ b/core/core/installations/scripts/build @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +node --no-warnings --check "$ROOT/../../shared/paths/runtime-paths.js" +shopt -s globstar nullglob +for file in "$ROOT"/{src,examples,tests}/**/*.js "$ROOT"/bin/*; do + node --no-warnings --check "$file" +done +for file in "$ROOT"/tooling/*; do + bash -n "$file" +done +node --no-warnings -e 'require(process.argv[1])' "$ROOT/src" +printf '%s\n' '{"ok":true,"status":"built"}' diff --git a/core/core/installations/scripts/test b/core/core/installations/scripts/test new file mode 100755 index 0000000..d5abda4 --- /dev/null +++ b/core/core/installations/scripts/test @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +export NODE_NO_WARNINGS=1 +python3 -I "$ROOT/tests/native-runtime-archive.test.py" +python3 -I "$ROOT/tests/release-package.test.py" +exec node --test "$ROOT"/tests/*.test.js diff --git a/core/core/installations/scripts/verify b/core/core/installations/scripts/verify new file mode 100755 index 0000000..eaeaa61 --- /dev/null +++ b/core/core/installations/scripts/verify @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +"$ROOT/tooling/build" +"$ROOT/tooling/test" +for file in "$ROOT"/examples/*.js; do + node --no-warnings "$file" +done diff --git a/core/core/installations/scripts/verify-live-dsps b/core/core/installations/scripts/verify-live-dsps new file mode 100755 index 0000000..d798105 --- /dev/null +++ b/core/core/installations/scripts/verify-live-dsps @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/../../.." && pwd)" +exec /usr/bin/node --no-warnings "$ROOT/core/installations/tests/live-dsps/runner.js" "$@" diff --git a/core/core/installations/scripts/verify-native-dsp-lab b/core/core/installations/scripts/verify-native-dsp-lab new file mode 100755 index 0000000..b7c8b4b --- /dev/null +++ b/core/core/installations/scripts/verify-native-dsp-lab @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/../../.." && pwd)" +exec python3 "$ROOT/core/installations/tests/native-lab/run.py" "$@" diff --git a/core/core/installations/scripts/verify-oci-lifecycle-fixture b/core/core/installations/scripts/verify-oci-lifecycle-fixture new file mode 100755 index 0000000..6883c73 --- /dev/null +++ b/core/core/installations/scripts/verify-oci-lifecycle-fixture @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +DISPATCH_RUN_OCI_LIFECYCLE_FIXTURE=1 node --no-warnings "$ROOT/examples/oci-lifecycle-host-fixture.js" diff --git a/core/core/installations/scripts/verify-systemd-fixture b/core/core/installations/scripts/verify-systemd-fixture new file mode 100755 index 0000000..d9b0eee --- /dev/null +++ b/core/core/installations/scripts/verify-systemd-fixture @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +RUNTIME_UNIT_ROOT="/run/user/$(id -u)/systemd/user" +BEFORE_AUTH="$(systemctl --user show dispatch-auth-broker.service -p MainPID --value)" +BEFORE_MANAGER="$(systemctl --user show dispatch-collection-manager.service -p MainPID --value)" +shopt -s nullglob globstar +before_journals=("$RUNTIME_UNIT_ROOT"/**/.dispatch-service-*.json) +DISPATCH_RUN_SYSTEMD_FIXTURE=1 node --no-warnings "$ROOT/examples/installation-services.js" +AFTER_AUTH="$(systemctl --user show dispatch-auth-broker.service -p MainPID --value)" +AFTER_MANAGER="$(systemctl --user show dispatch-collection-manager.service -p MainPID --value)" +test "$BEFORE_AUTH" = "$AFTER_AUTH" +test "$BEFORE_MANAGER" = "$AFTER_MANAGER" +systemctl --user is-active --quiet dispatch-auth-broker.service +systemctl --user is-active --quiet dispatch-collection-manager.service +after_journals=("$RUNTIME_UNIT_ROOT"/**/.dispatch-service-*.json) +test "${before_journals[*]-}" = "${after_journals[*]-}" +artifacts=("$RUNTIME_UNIT_ROOT"/**/dispatch-runtime-fixture_*) +test "${#artifacts[@]}" -eq 0 +if systemctl --user list-units --all --plain --no-legend 'dispatch-runtime-fixture_*' | read -r _; then + exit 1 +fi +printf '%s\n' '{"ok":true,"status":"systemd_fixture_verified","fixtures":2,"referenceServicesUnchanged":true,"artifactsRemaining":0}' diff --git a/core/core/installations/src/activation.js b/core/core/installations/src/activation.js new file mode 100644 index 0000000..eeac4da --- /dev/null +++ b/core/core/installations/src/activation.js @@ -0,0 +1,265 @@ +'use strict'; + +const { + INSTALLATION_IDENTIFIER_RE, + INSTALLATION_ACTIVATION_EVIDENCE_VERSION, + INSTALLATION_READINESS_GATES, + installationActivationEvidence, + installationActivationEvidenceDigest, + installationActivationResult, + installationFailure, + installationJob, + installationReadiness, + serverInstallationActivation, + serverInstallationManifest, +} = require('../../../shared/contracts/src'); +const { + PAYCOM_PROFILE_ID, + managedPaycomFirstPublicationRequest, +} = require('../../../shared/paycom-activation'); + +const ACTIVATION_INFRASTRUCTURE_GATES = Object.freeze([ + 'runtime_layout', + 'service_supervision', + 'auth_broker', + 'collection_manager', + 'runtime_gateway', +]); + +function fail(code) { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, fields, code = 'installation_not_ready') { + if (!plain(value) || Object.keys(value).sort().join(',') !== [...fields].sort().join(',')) fail(code); + return value; +} + +function timestamp(value, code = 'installation_not_ready') { + if (typeof value !== 'string' || Number.isNaN(Date.parse(value))) fail(code); + return value; +} + +function authorityContext(value) { + exact(value, ['manifest', 'manifestAuthority', 'installation', 'job', 'owner']); + const manifest = serverInstallationManifest(value.manifest, value.manifestAuthority); + exact(value.installation, ['state', 'revision', 'currentJobId']); + if (!['waiting_for_provider_auth', 'verifying', 'ready'].includes(value.installation.state) + || !Number.isSafeInteger(value.installation.revision) || value.installation.revision < 1 + || value.installation.currentJobId !== null + && (typeof value.installation.currentJobId !== 'string' + || !INSTALLATION_IDENTIFIER_RE.test(value.installation.currentJobId))) fail('installation_not_ready'); + exact(value.owner, ['active']); + if (value.owner.active !== true) fail('installation_not_ready'); + let job = null; + if (value.installation.state === 'verifying' || value.installation.state === 'ready') { + job = installationJob(value.job); + const expectedStatus = value.installation.state === 'ready' ? 'succeeded' : 'running'; + if (job.status !== expectedStatus || job.installationState !== value.installation.state + || job.id !== value.installation.currentJobId) fail('installation_not_ready'); + } else if (value.job !== null || value.installation.currentJobId !== null) fail('installation_not_ready'); + return Object.freeze({ + manifest, + manifestAuthority: value.manifestAuthority, + installation: Object.freeze({ ...value.installation }), + job, + owner: Object.freeze({ active: true }), + }); +} + +function infrastructureEvidence(value, runtimeKey) { + exact(value, ['runtimeKey', ...ACTIVATION_INFRASTRUCTURE_GATES], 'runtime_health_failed'); + if (value.runtimeKey !== runtimeKey) fail('runtime_identity_mismatch'); + for (const gate of ACTIVATION_INFRASTRUCTURE_GATES) { + if (value[gate] !== true) fail('runtime_health_failed'); + } + return value; +} + +function configurationEvidence(value, expectedDigest) { + exact(value, ['digest', 'collectors', 'sources', 'plans', 'syncs'], 'runtime_health_failed'); + if (value.digest !== expectedDigest + || value.collectors !== 1 || value.sources !== 1 || value.plans !== 15 || value.syncs !== 1) { + fail('runtime_health_failed'); + } + return value; +} + +function providerEvidence(value) { + exact(value, ['profileId', 'provider', 'status', 'testedAt'], 'provider_auth_required'); + if (value.profileId !== PAYCOM_PROFILE_ID || value.provider !== 'paycom' || value.status !== 'authenticated') { + fail('provider_auth_required'); + } + timestamp(value.testedAt, 'provider_auth_required'); + return Object.freeze({ ...value }); +} + +function publicationEvidence(value) { + exact(value, ['batchId', 'preparationRunId', 'status', 'runCount', 'succeededRuns', 'failedRuns', 'cancelledRuns'], + 'first_publication_failed'); + if (typeof value.batchId !== 'string' || !INSTALLATION_IDENTIFIER_RE.test(value.batchId) + || typeof value.preparationRunId !== 'string' || !INSTALLATION_IDENTIFIER_RE.test(value.preparationRunId) + || value.status !== 'succeeded' || !Number.isSafeInteger(value.runCount) || value.runCount < 1 + || value.succeededRuns !== value.runCount || value.failedRuns !== 0 || value.cancelledRuns !== 0) { + fail('first_publication_failed'); + } + return Object.freeze({ ...value }); +} + +function readiness(context) { + const gates = Object.fromEntries(INSTALLATION_READINESS_GATES.map(gate => [gate, 'passed'])); + return installationReadiness({ + manifestRevision: context.manifest.revision, + jobId: context.job.id, + runtimeKey: context.manifest.runtime.key, + gates, + }); +} + +function activationDependencies(value) { + exact(value, ['authority', 'runtime', 'clock'], 'runtime_boundary_violation'); + const authority = value.authority; + const runtime = value.runtime; + if (!authority || !['inspect', 'begin', 'heartbeat', 'commit', 'fail'] + .every(method => typeof authority[method] === 'function') + || !runtime || !['verifyInfrastructure', 'configure', 'testProvider', 'publishFirst', 'verifyPublication'] + .every(method => typeof runtime[method] === 'function') + || typeof value.clock !== 'function') fail('runtime_boundary_violation'); + return value; +} + +async function runManagedPaycomActivation(options) { + const selected = activationDependencies({ + authority: options?.authority, + runtime: options?.runtime, + clock: options?.clock === undefined ? Date.now : options.clock, + }); + let context = null; + let committed = false; + try { + context = authorityContext(await selected.authority.inspect()); + if (context.installation.state === 'ready') { + return installationActivationResult({ + ok: true, + status: 'ready', + state: 'ready', + revision: context.installation.revision, + manifestRevision: context.manifest.revision, + gates: INSTALLATION_READINESS_GATES.length, + }); + } + const definition = options.definitionFactory ? options.definitionFactory(context) : null; + infrastructureEvidence(await selected.runtime.verifyInfrastructure(context.manifest), context.manifest.runtime.key); + + if (context.installation.state === 'waiting_for_provider_auth') { + const authenticated = providerEvidence(await selected.runtime.testProvider(PAYCOM_PROFILE_ID)); + context = authorityContext(await selected.authority.begin(authenticated)); + } + const heartbeat = async () => { + const current = authorityContext(await selected.authority.heartbeat()); + if (current.installation.state !== 'verifying' || current.job.id !== context.job.id + || JSON.stringify(current.manifest) !== JSON.stringify(context.manifest)) { + fail('installation_operation_in_progress'); + } + context = current; + }; + await heartbeat(); + const configured = await selected.runtime.configure(definition); + const definitionDigest = definition ? definition.digest : configured?.digest; + if (!/^[a-f0-9]{64}$/.test(definitionDigest || '')) fail('installation_not_ready'); + configurationEvidence(configured, definitionDigest); + await heartbeat(); + infrastructureEvidence(await selected.runtime.verifyInfrastructure(context.manifest), context.manifest.runtime.key); + await heartbeat(); + const request = managedPaycomFirstPublicationRequest(); + const batch = publicationEvidence(await selected.runtime.publishFirst( + request, + { idempotencyKey: `activation:${context.job.id}`, heartbeat }, + )); + const audited = exact(await selected.runtime.verifyPublication(batch.batchId, batch.preparationRunId), [ + 'definitionDigest', 'requestDigest', 'previewDigest', 'batchId', 'preparationRunId', 'target', + 'runs', 'publications', 'capturedAt', + ], 'first_publication_failed'); + if (audited.definitionDigest !== definitionDigest) fail('first_publication_failed'); + const evidencePayload = { + schemaVersion: INSTALLATION_ACTIVATION_EVIDENCE_VERSION, + manifestRevision: context.manifest.revision, + jobId: context.job.id, + runtimeKey: context.manifest.runtime.key, + definitionDigest: audited.definitionDigest, + requestDigest: audited.requestDigest, + previewDigest: audited.previewDigest, + batchId: audited.batchId, + preparationRunId: audited.preparationRunId, + target: audited.target, + runs: audited.runs, + publications: audited.publications, + capturedAt: audited.capturedAt, + }; + const evidence = installationActivationEvidence({ + ...evidencePayload, + evidenceDigest: installationActivationEvidenceDigest(evidencePayload), + }); + await heartbeat(); + infrastructureEvidence(await selected.runtime.verifyInfrastructure(context.manifest), context.manifest.runtime.key); + await heartbeat(); + + const current = authorityContext(await selected.authority.inspect()); + if (current.installation.state !== 'verifying' || current.job.id !== context.job.id + || JSON.stringify(current.manifest) !== JSON.stringify(context.manifest)) fail('installation_not_ready'); + context = current; + const report = readiness(context); + const activation = Object.freeze({ + manifest: context.manifest, + job: context.job, + readiness: report, + evidence, + }); + const activationAuthority = Object.freeze({ + manifestAuthority: context.manifestAuthority, + jobId: context.job.id, + }); + serverInstallationActivation(activation, activationAuthority); + const result = await selected.authority.commit(activation, activationAuthority); + exact(result, ['state', 'revision'], 'installation_not_ready'); + if (result.state !== 'ready' || !Number.isSafeInteger(result.revision) || result.revision < 1) { + fail('installation_not_ready'); + } + committed = true; + return installationActivationResult({ + ok: true, + status: 'ready', + state: result.state, + revision: result.revision, + manifestRevision: context.manifest.revision, + gates: INSTALLATION_READINESS_GATES.length, + }); + } catch (error) { + const failure = installationFailure(error); + if (context?.installation.state === 'verifying' && context.job && !committed + && failure.code !== 'installation_operation_in_progress') { + try { await selected.authority.fail(context.job.id, failure); } catch {} + } + return installationActivationResult({ + ok: false, + status: failure.code, + failure, + }); + } +} + +module.exports = { + ACTIVATION_INFRASTRUCTURE_GATES, + authorityContext, + infrastructureEvidence, + configurationEvidence, + providerEvidence, + publicationEvidence, + readiness, + runManagedPaycomActivation, +}; diff --git a/core/core/installations/src/assemble-system-recovery.js b/core/core/installations/src/assemble-system-recovery.js new file mode 100644 index 0000000..80eade5 --- /dev/null +++ b/core/core/installations/src/assemble-system-recovery.js @@ -0,0 +1,201 @@ +'use strict'; +const fs = require('node:fs'), + path = require('node:path'), + crypto = require('node:crypto'); +const capsule = require('./recovery-capsule'), + { recoveryRoots } = require('./host-recovery-bundle'); +const hash = (bytes) => crypto.createHash('sha256').update(bytes).digest('hex'); +const fail = () => { + throw Error('system_recovery_invalid'); +}; +// Assemble only after every component has downloaded and authenticated. No +// installed path changes until the resulting combined capsule verifies. +function assembleSystemRecovery({ components, destination, systemSchedule = null }) { + if ( + !Array.isArray(components) || + components.filter((c) => c.kind === 'core').length !== 1 || + components.some((c) => !['core', 'dsp'].includes(c.kind)) + ) + fail(); + const manifests = components.map((c) => { + const directory = path.join(c.directory, 'recovery'), + raw = JSON.parse(fs.readFileSync(path.join(directory, 'recovery.json'))); + const m = capsule.verify(directory, c.digest, recoveryRoots(raw.metadata, raw.roots)); + if ( + m.metadata.kind !== c.kind || + (c.kind === 'core' && m.metadata.scope !== 'core') || + (c.kind === 'dsp' && m.metadata.organizationId !== c.organizationId) + ) + fail(); + return { ...c, directory, manifest: m }; + }); + const core = manifests.find((c) => c.kind === 'core'), + metadata = { + ...core.manifest.metadata, + scope: 'system', + accounts: [], + services: [], + installations: [], + hostAllocations: [], + }; + const entries = new Map(), + roots = new Set(); + fs.mkdirSync(destination, { mode: 0o700 }); + fs.mkdirSync(path.join(destination, 'files'), { mode: 0o700 }); + for (const part of manifests) { + if (part.manifest.metadata.localRoot !== metadata.localRoot) fail(); + for (const [field, key] of [ + ['accounts', 'uid'], + ['services', 'file'], + ['installations', 'organization_id'], + ]) + for (const value of part.manifest.metadata[field]) { + const existing = metadata[field].find((r) => r[key] === value[key]); + if (existing && JSON.stringify(existing) !== JSON.stringify(value)) fail(); + if (!existing) metadata[field].push(value); + } + for (const allocation of part.manifest.metadata.hostAllocations || []) { + if ( + part.kind !== 'dsp' || + !part.manifest.metadata.installations.some((i) => i.runtime_key === allocation.runtime_key) + ) + fail(); + metadata.hostAllocations.push(allocation); + } + for (const root of part.manifest.roots) roots.add(root); + for (const item of part.manifest.entries) { + const prior = entries.get(item.path), + { payload, artifact, ...shape } = item; + if (prior) { + const { payload: _, ...priorShape } = prior; + if (JSON.stringify(priorShape) !== JSON.stringify(shape)) fail(); + continue; + } + const entry = { ...shape }; + if (item.type === 'file') { + entry.payload = `files/${String(entries.size).padStart(8, '0')}`; + capsule.streamFile( + path.join(part.directory, payload), + path.join(destination, entry.payload), + ); + } + entries.set(entry.path, entry); + } + } + const databaseEntry = entries.get( + path.join(metadata.localRoot, 'data/access-control/access-control.sqlite3'), + ); + if (!databaseEntry || databaseEntry.type !== 'file') fail(); + const database = path.join(destination, databaseEntry.payload), + { DatabaseSync } = require('node:sqlite'), + db = new DatabaseSync(database); + try { + require('../../accounts/src/core-backup').verifyCoreDatabase(db); + db.exec('PRAGMA foreign_keys=ON; BEGIN IMMEDIATE'); + const insert = (table, row) => { + if (!row || typeof row !== 'object') fail(); + const columns = db + .prepare(`PRAGMA table_info(${table})`) + .all() + .map((c) => c.name); + if (Object.keys(row).some((k) => !columns.includes(k))) fail(); + const value = { ...row }; + for (const key of ['created_by', 'actor_user_id']) + if (value[key] && !db.prepare('SELECT 1 FROM users WHERE id=?').get(value[key])) + value[key] = null; + const keys = Object.keys(value); + db.prepare( + `INSERT INTO ${table} (${keys.join(',')}) VALUES (${keys.map(() => '?').join(',')})`, + ).run(...Object.values(value)); + }; + for (const part of manifests.filter((c) => c.kind === 'dsp')) { + const info = JSON.parse(fs.readFileSync(path.join(part.directory, '../dsp.json'))), + value = info.metadata, + id = part.organizationId; + if ( + info.kind !== 'dsp' || + value.organizationId !== id || + value.organization?.id !== id || + value.recovery?.installation?.organization_id !== id || + value.manifest?.organization?.id !== id + ) + fail(); + const runtime = part.manifest.metadata.installations.find((i) => i.organization_id === id); + if (!runtime || runtime.runtime_key !== value.recovery.installation.runtime_key) fail(); + for (const user of value.users) { + if (user.platform_role !== null) fail(); + insert('users', { ...user, auth_version: user.auth_version + 1 }); + } + insert('organizations', value.organization); + for (const table of ['stations', 'roles', 'memberships']) + for (const row of value[table]) { + if (row.organization_id !== id) fail(); + insert(table, row); + } + for (const row of value.permissions) { + if (!value.roles.some((r) => r.id === row.role_id)) fail(); + insert('role_permissions', row); + } + if (value.profile) insert('organization_profiles', value.profile); + insert('installations', { + ...value.recovery.installation, + status: value.organization.status === 'suspended' ? 'suspended' : 'ready', + current_job_id: null, + setup_worker_id: null, + setup_lease_expires_at: null, + }); + // Keep completed readiness and suspension evidence: later removal, + // restoration and upgrades must still prove publication continuity. + const provisioning = value.recovery.provisioning; + if (provisioning) { + if (provisioning.organization_id !== id || provisioning.runtime_key !== runtime.runtime_key || provisioning.status !== 'completed') fail(); + insert('installation_provisioning_requests', provisioning); + } + const activation = value.recovery.activation; + if (activation) { + if (activation.organization_id !== id || activation.runtime_key !== runtime.runtime_key || activation.status !== 'succeeded') fail(); + insert('installation_activation_jobs', activation); + } + for (const job of value.recovery.lifecycle || []) { + if (job.organization_id !== id || job.runtime_key !== runtime.runtime_key || job.status !== 'succeeded' || !['resume', 'upgrade', 'suspend'].includes(job.operation)) fail(); + insert('installation_lifecycle_jobs', job); + } + if (value.recovery.authority) { + if (value.recovery.authority.organization_id !== id) fail(); + insert('runtime_agent_authorities', value.recovery.authority); + } + if (value.schedule) { + if (value.schedule.scope !== id) fail(); + insert('backup_scope_settings', value.schedule); + } + } + if (systemSchedule) + db.prepare("INSERT INTO backup_scope_settings VALUES('system',1,?,?)").run( + JSON.stringify(systemSchedule), + Date.now(), + ); + db.exec('COMMIT; PRAGMA wal_checkpoint(TRUNCATE)'); + if ( + db.prepare('PRAGMA foreign_key_check').all().length || + db.prepare('PRAGMA quick_check').get().quick_check !== 'ok' + ) + fail(); + } finally { + db.close(); + } + const bytes = fs.readFileSync(database); + databaseEntry.sha256 = hash(bytes); + databaseEntry.size = bytes.length; + const manifest = { + schemaVersion: 1, + metadata, + roots: [...roots], + entries: [...entries.values()], + }, + encoded = JSON.stringify(manifest) + '\n'; + fs.writeFileSync(path.join(destination, 'recovery.json'), encoded, { mode: 0o600 }); + const digest = hash(encoded); + capsule.verify(destination, digest, recoveryRoots(metadata, manifest.roots)); + return { directory: destination, digest }; +} +module.exports = { assembleSystemRecovery }; diff --git a/core/core/installations/src/backup-archive-status.js b/core/core/installations/src/backup-archive-status.js new file mode 100644 index 0000000..3ac5415 --- /dev/null +++ b/core/core/installations/src/backup-archive-status.js @@ -0,0 +1,25 @@ +'use strict'; +const path = require('node:path'); +const { publicRootJson, RECEIPTS } = require('./offsite-policy'); +function backupArchiveStatus() { + try { + const catalog = publicRootJson(path.join(RECEIPTS, 'catalog.json'), true, 0, 4 * 1024 * 1024); + const status = publicRootJson(path.join(RECEIPTS, 'status.json'), true); + return { + status: + status?.status === 'verified' && Date.now() - status.checkedAt < 300000 + ? 'connected' + : status + ? 'attention' + : 'unavailable', + checkedAt: status?.checkedAt ? new Date(status.checkedAt).toISOString() : null, + backups: catalog?.backups || {}, + sets: catalog?.sets || {}, + deletions: catalog?.deletions || {}, + usage: catalog?.usage || {status:'unavailable', checkedAt:null}, + }; + } catch { + return { status: 'attention', backups: {} }; + } +} +module.exports = { backupArchiveStatus }; diff --git a/core/core/installations/src/backup-archives.js b/core/core/installations/src/backup-archives.js new file mode 100644 index 0000000..ea63431 --- /dev/null +++ b/core/core/installations/src/backup-archives.js @@ -0,0 +1,515 @@ +'use strict'; +const fs = require('node:fs'), + path = require('node:path'), + crypto = require('node:crypto'); +const { DatabaseSync } = require('node:sqlite'); +const { tree, verifySnapshot, createRestic, WORK } = require('./offsite-backup'); +const { atomic, privateJson } = require('./release-delivery-files'); +const { RECEIPTS, receiptKey } = require('./offsite-policy'); +const { + HOST_TENANT_ROOT, + opaqueRuntimeSuffix, + hostAccountName, +} = require('../../runtime-host-identity'); +const { createR2BackupStorage } = require('./r2-backup-storage'); +const hash = (value) => crypto.createHash('sha256').update(value).digest('hex'); +const validId = (id) => /^(backup|breq)_[a-f0-9]{32}$/.test(id); +function fail() { + throw Object.assign(Error('backup_archive_unavailable'), { code: 'backup_archive_unavailable' }); +} +function checkedDirectory(p, uid) { + const s = fs.lstatSync(p); + if (!s.isDirectory() || s.uid !== uid || s.mode & 0o077 || fs.realpathSync(p) !== p) fail(); + return s; +} +function pathsFor(config, row) { + if (!validId(row.id)) fail(); + if (row.kind === 'core') + return { + source: path.join(config.localRoot, 'backups/scheduled-core', row.id), + uid: config.coreUid, + }; + const m = JSON.parse(row.metadata_json), + runtimeKey = m.manifest?.runtime?.key; + if (!/^[a-z][a-z0-9_-]{2,95}$/.test(runtimeKey) || m.organizationId !== row.organization_id) + fail(); + const installation = path.join( + HOST_TENANT_ROOT, + opaqueRuntimeSuffix(runtimeKey), + 'runtime', + runtimeKey, + ); + const stat = fs.lstatSync(installation); + if (stat.uid === 0 || stat.uid === config.coreUid) fail(); + checkedDirectory(installation, stat.uid); + checkedDirectory(path.join(installation, 'backups'), stat.uid); + return { source: path.join(installation, 'backups', row.id), uid: stat.uid }; +} +function createBackupArchives( + config, + { + runFactory = createRestic, + storage = createR2BackupStorage(config), + clock = Date.now, + workRoot = WORK, + receiptRoot = RECEIPTS, + ownerUid = 0, + pathResolver = pathsFor, + parallelExport = ownerUid === 0 && runFactory === createRestic ? require('./parallel-backup-exports').parallelBackupExports : null, + recoveryCapture = ownerUid === 0 ? require('./host-recovery-bundle').captureHostRecovery : null, + } = {}, +) { + const root = path.join(workRoot, 'archives'); + fs.mkdirSync(root, { recursive: true, mode: 0o700 }); + checkedDirectory(root, ownerUid); + const rootFile = (id) => path.join(root, `${id}.json`); + function environment(id, days) { + if (!validId(id) || ![null, 7, 30, 90, 365].includes(days)) fail(); + return { + ...config.environment, + RESTIC_REPOSITORY: `s3:https://${config.accountId}.r2.cloudflarestorage.com/${config.bucket}/archives/${days === null ? 'all' : days}/${id}`, + }; + } + function runFor(id, days) { + const call = runFactory(environment(id, days)); + return (args, cwd) => call(['--no-lock', ...args], cwd); + } + function record(id) { + return fs.existsSync(rootFile(id)) ? privateJson(rootFile(id), ownerUid) : null; + } + async function exportRecord(row) { + const existing = record(row.id), + metadataDigest = hash(row.metadata_json); + if (existing) { + if (existing.metadataDigest !== metadataDigest) fail(); + return existing; + } + const { source, uid } = pathResolver(config, row); + if (!fs.existsSync(path.join(source, 'manifest.json'))) return null; + const timing = {}, startedAt = clock(); + let stageStarted = startedAt; + const mark = stage => { const now = clock(); timing[stage] = { startedAt: stageStarted, finishedAt: now, durationMs: Math.max(0, now - stageStarted) }; stageStarted = now; }; + const checked = verifySnapshot(source, uid), + work = fs.mkdtempSync(path.join(workRoot, 'archive-transfer-')); + try { + const bundle = path.join(work, 'bundle'); + fs.mkdirSync(bundle, { mode: 0o700 }); + const copied = tree(source, uid, path.join(bundle, 'snapshot')); + if (copied.digest !== checked.tree.digest) fail(); + atomic(path.join(bundle, 'dsp.json'), { + schemaVersion: 1, + id: row.id, + kind: row.kind, + metadata: JSON.parse(row.metadata_json), + retentionDays: row.retention_days, + }); + mark('snapshot_copy'); + const recovery = recoveryCapture ? recoveryCapture({ config, destination: path.join(bundle, 'recovery'), + kind: row.kind, organizationId: row.organization_id, snapshotSource: source, shareReleases: true }) : null; + if (recovery) atomic(path.join(bundle, 'recovery-proof.json'), recovery); + mark('recovery_capture'); + const expected = tree(bundle, ownerUid), + run = runFor(row.id, row.retention_days); + // Repositories are unique per snapshot: no cross-backup deduplication or + // shared data deletion. A root flock serializes writers; --no-lock avoids + // mutable restic locks inside the R2 retention-locked archive prefix. + try { + run(['cat', 'config']); + } catch { + run(['init', '--repository-version', '2']); + } + mark('repository_prepare'); + const lines = run(['backup', '--host', 'dispatch', '--tag', row.id, '--', 'bundle'], work); + const snapshotId = lines.find((l) => l?.message_type === 'summary')?.snapshot_id; + if (!/^[a-f0-9]{64}$/.test(snapshotId)) fail(); + // Local integrity was checked above. A successful encrypted upload is the + // completion boundary; restore and full readback are explicit operations. + mark('encrypted_upload'); + const recoveryArtifacts = recovery ? [...new Map(JSON.parse(fs.readFileSync(path.join(bundle, 'recovery/recovery.json'))).entries + .filter(entry => entry.artifact).map(entry => [entry.artifact.digest, entry.artifact])).values()] : []; + const verifiedAt = clock(), + manifest = JSON.parse(fs.readFileSync(path.join(source, 'manifest.json'))); + const receipt = { + schemaVersion: 1, + id: row.id, + organizationId: row.organization_id, + kind: row.kind, + status: 'verified', + verification: 'upload', + timings: timing, recoveryArtifacts, + metadataDigest, + digest: checked.digest, + bundleDigest: expected.digest, + snapshotId, + size: expected.size, + retentionDays: row.retention_days, + verifiedAt, + expiresAt: row.retention_days === null ? null : verifiedAt + row.retention_days * 86400000, + format: manifest.version, + trigger: row.category || manifest.purpose || 'scheduled', + deletedAt: null, + ...(recovery ? { recoveryDigest: recovery.sha256, organizationIds: recovery.organizationIds, + ...(recovery.organizationInventoryVersion === 1 ? { organizationInventoryVersion: 1 } : {}) } : {}), + }; + atomic(rootFile(row.id), receipt); + atomic( + path.join(receiptRoot, receiptKey(source) + '.json'), + { schemaVersion: 1, status: 'verified', digest: checked.digest, snapshotId, verifiedAt, + ...(recovery ? { recoveryDigest: recovery.sha256 } : {}) }, + 0o644, + ); + fs.chmodSync(path.join(receiptRoot, receiptKey(source) + '.json'), 0o644); + return receipt; + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } + } + function hydrate(row) { + const rec = record(row.id); + if (!rec || rec.deletedAt || rec.metadataDigest !== hash(row.metadata_json)) fail(); + const { source, uid } = pathResolver(config, row); + if (fs.existsSync(source)) { + if (verifySnapshot(source, uid).digest !== rec.digest) fail(); + return; + } + const work = fs.mkdtempSync(path.join(workRoot, 'archive-restore-')); + let staging = null; + try { + runFor(row.id, rec.retentionDays)(['restore', rec.snapshotId, '--target', work, '--verify']); + const bundle = path.join(work, 'bundle'); + if (tree(bundle, ownerUid).digest !== rec.bundleDigest) fail(); + if (verifySnapshot(path.join(bundle, 'snapshot'), ownerUid).digest !== rec.digest) fail(); + const stagingRoot = '/var/lib/dispatch-restore-staging'; + if (!fs.existsSync(stagingRoot)) { + fs.mkdirSync(stagingRoot, { mode: 0o711 }); + fs.chmodSync(stagingRoot, 0o711); + } + const parent = fs.lstatSync(stagingRoot); + if ( + parent.uid !== 0 || + !parent.isDirectory() || + (parent.mode & 0o7777) !== 0o711 || + fs.realpathSync(stagingRoot) !== stagingRoot + ) + fail(); + staging = fs.mkdtempSync(path.join(stagingRoot, 'import-')); + tree(path.join(bundle, 'snapshot'), ownerUid, path.join(staging, 'snapshot')); + const chown = (directory) => { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const p = path.join(directory, entry.name); + if (entry.isDirectory()) chown(p); + else { + if (!entry.isFile()) fail(); + fs.chownSync(p, uid, uid); + } + } + fs.chownSync(directory, uid, uid); + }; + chown(staging); + if (row.kind === 'core') { + const imported = require('node:child_process').spawnSync('/usr/sbin/runuser', ['--user', String(config.coreUser || require('./host-recovery-bundle').account(config.coreUid).name), '--', '/usr/bin/node', '--no-warnings', path.resolve(__dirname, "../bin/dispatch-core-backup-import")], { + input: JSON.stringify({id:row.id,localRoot:config.localRoot,source:path.join(staging,'snapshot'),digest:rec.digest})+'\n', encoding:'utf8',timeout:240000,maxBuffer:4096,env:{PATH:'/usr/bin:/bin',LANG:'C.UTF-8'} + }); + if(imported.status!==0 || imported.stdout.trim()!=='{"ok":true}') fail(); + return; + } + const helper = path.resolve(__dirname, "../bin/dispatch-backup-import"), + helperStat = fs.lstatSync(helper); + if ( + helperStat.uid !== 0 || + !helperStat.isFile() || + helperStat.nlink !== 1 || + helperStat.mode & 0o022 || + fs.realpathSync(helper) !== helper + ) + fail(); + const runtimeKey = JSON.parse(row.metadata_json).manifest.runtime.key; + const imported = require('node:child_process').spawnSync( + '/usr/sbin/runuser', + ['--user', hostAccountName(runtimeKey), '--', '/usr/bin/node', '--no-warnings', helper], + { + input: + JSON.stringify({ + id: row.id, + runtimeKey, + digest: rec.digest, + source: path.join(staging, 'snapshot'), + }) + '\n', + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8' }, + encoding: 'utf8', + timeout: 240000, + maxBuffer: 4096, + }, + ); + if (imported.status !== 0 || imported.stdout.trim() !== '{"ok":true}') fail(); + if (verifySnapshot(source, uid).digest !== rec.digest) fail(); + } finally { + if (staging) fs.rmSync(staging, { recursive: true, force: true }); + fs.rmSync(work, { recursive: true, force: true }); + } + } + function removeLocal(row, receipt) { + let selected; + try { + selected = pathResolver(config, row); + } catch (error) { + if (error.code === 'ENOENT') return; + throw error; + } + if (!fs.existsSync(selected.source)) return; + const helper = path.resolve(__dirname, "../bin/dispatch-backup-expire"); + const stat = fs.lstatSync(helper); + if ( + stat.uid !== 0 || + !stat.isFile() || + stat.nlink !== 1 || + stat.mode & 0o022 || + fs.realpathSync(helper) !== helper + ) + fail(); + const { spawnSync } = require('node:child_process'); + const lookup = spawnSync('/usr/bin/getent', ['passwd', String(selected.uid)], { + encoding: 'utf8', + timeout: 10000, + maxBuffer: 4096, + }); + const account = lookup.stdout?.trim().split(':'); + if ( + lookup.status !== 0 || + !account || + Number(account[2]) !== selected.uid || + selected.uid === 0 + ) + fail(); + const result = spawnSync( + '/usr/sbin/runuser', + ['--user', account[0], '--', '/usr/bin/node', '--no-warnings', helper], + { + input: JSON.stringify({ source: selected.source, digest: receipt.digest }) + '\n', + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8' }, + encoding: 'utf8', + timeout: 240000, + maxBuffer: 4096, + }, + ); + if (result.status !== 0 || result.stdout.trim() !== '{"ok":true}') fail(); + } + + async function scan() { + await require('./recover-archive-catalog').recoverArchiveCatalog({ config, storage, runFor, workRoot, + ownerUid, record, save: (id, value) => atomic(rootFile(id), value), clock }); + await storage.ensureLocks(); + const dbFile = path.join(config.localRoot, 'data/access-control/access-control.sqlite3'); + const stat = fs.lstatSync(dbFile); + if ( + stat.uid !== config.coreUid || + !stat.isFile() || + stat.nlink !== 1 || + stat.mode & 0o077 || + fs.realpathSync(dbFile) !== dbFile + ) + fail(); + const db = new DatabaseSync(dbFile, { readOnly: true }); + db.exec('PRAGMA trusted_schema=OFF; PRAGMA busy_timeout=3000;'); + let rows, requests, jobs, deletions = [], installationBackups = [], allRecords = [], removed = [], heldRuntimes = [], archiveDeletions = [], backupSets = []; + try { + if (!db.prepare("SELECT 1 FROM sqlite_schema WHERE name='platform_backup_records'").get()) + return { verified: 0, failed: 0, managedIds: [] }; + if (db.prepare("SELECT 1 FROM sqlite_schema WHERE name='installation_lifecycle_jobs'").get()) { + deletions = db.prepare("SELECT j.* FROM installation_lifecycle_jobs j JOIN installations i ON i.current_job_id=j.id WHERE j.operation='destroy' AND j.authority_scope IN ('platform_removal','platform_lifecycle')").all(); + installationBackups = db.prepare('SELECT id,organization_id FROM installation_backups').all(); + allRecords = db.prepare('SELECT * FROM platform_backup_records').all(); + } + if (db.prepare("SELECT 1 FROM sqlite_schema WHERE name='dsp_removals'").get()) { + removed = db.prepare('SELECT organization_id FROM dsp_removals').all().map(r => r.organization_id); + heldRuntimes = db.prepare('SELECT runtime_key FROM installations WHERE organization_id IN (SELECT organization_id FROM dsp_removals)').all().map(r => r.runtime_key); + } + if(db.prepare("SELECT 1 FROM sqlite_schema WHERE name='backup_sets'").get()) backupSets=db.prepare('SELECT * FROM backup_sets').all(); + if(db.prepare("SELECT 1 FROM sqlite_schema WHERE name='backup_deletions'").get()) archiveDeletions=db.prepare("SELECT * FROM backup_deletions WHERE status='queued'").all(); + rows = db + .prepare( + db.prepare("SELECT 1 FROM sqlite_schema WHERE name='backup_categories'").get() + ? "SELECT r.*,c.category FROM platform_backup_records r LEFT JOIN backup_categories c ON c.backup_id=r.id WHERE r.deleted_at IS NULL ORDER BY r.created_at DESC" + : 'SELECT * FROM platform_backup_records WHERE deleted_at IS NULL ORDER BY created_at DESC' , + ) + .all(); + requests = db + .prepare("SELECT * FROM platform_backup_requests WHERE status IN ('queued','running')") + .all(); + jobs = db + .prepare( + "SELECT backup_id,safety_backup_id,stage_receipts_json FROM installation_lifecycle_jobs WHERE status IN ('queued','running')", + ) + .all(); + } finally { + db.close(); + } + const deletion = await require('./dsp-backup-deletion').purgeDspBackups({ config, jobs: deletions, + backups: installationBackups, records: allRecords, sets:backupSets, storage, run: runFactory(config.environment), + workRoot, receiptRoot, ownerUid, clock }); + let verified = 0, + failed = deletion.failed; + const catalog = { schemaVersion: 1, backups: {}, deletions: {} }; + const pinned = new Set( + requests.filter((r) => r.kind === 'restore' || JSON.parse(r.input_json).action === 'restore').map((r) => JSON.parse(r.input_json).backupId), + ); + for(const r of requests) if(r.kind==='core') pinned.add(r.id); + const pinDb = new DatabaseSync(dbFile, { readOnly: true }); + try { + if (pinDb.prepare("SELECT 1 FROM sqlite_schema WHERE name='platform_rollout_backups'").get()) { + for (const set of pinDb.prepare(`SELECT s.members_json FROM backup_sets s JOIN platform_rollout_backups b ON b.set_id=s.id + JOIN platform_rollouts r ON r.id=b.rollout_id WHERE r.status!='completed'`).all()) { + for (const member of JSON.parse(set.members_json)) { + if (member.backupId) pinned.add(member.backupId); + const request = pinDb.prepare('SELECT id,kind,job_id FROM platform_backup_requests WHERE id=?').get(member.requestId); + if (request?.kind === 'core') pinned.add(request.id); + else if (request?.job_id) { + const job = pinDb.prepare('SELECT backup_id FROM installation_lifecycle_jobs WHERE id=?').get(request.job_id); + if (job?.backup_id) pinned.add(job.backup_id); + } + } + } + } + } finally { pinDb.close(); } + for (const job of jobs) { + if (job.backup_id) pinned.add(job.backup_id); + if (job.safety_backup_id) pinned.add(job.safety_backup_id); + const request = JSON.parse(job.stage_receipts_json).__request; + if (request) { + const id = JSON.parse(request).backupId; + if (id) pinned.add(id); + } + } + // Start independent exports together before assembling the catalog. Remote + // deletion and retention changes remain serialized under the parent lock. + const parallelResults = parallelExport ? await parallelExport(rows.filter(row => + !record(row.id) && !removed.includes(row.organization_id) + && !deletion.deletingOrganizations.has(row.organization_id) + && !(row.kind === 'core' && deletion.deletingOrganizations.size) + && !archiveDeletions.some(d => d.backup_id === row.id)), { discover: () => pendingExports(dbFile, record) }) : new Map(); + const visible = new Set(rows.slice(0, 1000).map((r) => r.id)); + for (const id of pinned) visible.add(id); + for (const row of rows) { + if (row.kind === 'core' && deletion.deletingOrganizations.size) continue; + if (deletion.deletingOrganizations.has(row.organization_id)) continue; + try { + const held = removed.includes(row.organization_id); + let rec = record(row.id); + if (rec && rec.metadataDigest !== hash(row.metadata_json)) fail(); + if (archiveDeletions.some(d=>d.backup_id===row.id) && !pinned.has(row.id)) { + await storage.withDeletionAccess(['all',7,30,90,365].map(t=>`archives/${t}/`),async()=>{ + for(const retentionDays of [null,7,30,90,365]) await storage.removePermanent({id:row.id,retentionDays}); + }); + rec={...rec,id:row.id,organizationId:row.organization_id,kind:row.kind,metadataDigest:hash(row.metadata_json),deletedAt:clock(),status:'destroyed'}; + if(fs.existsSync(pathResolver(config,row).source)) { rec.digest ||= verifySnapshot(pathResolver(config,row).source,pathResolver(config,row).uid).digest; removeLocal(row,rec); } + rec.localPruned=true; + atomic(rootFile(row.id),rec); + } + if ( + rec && + !rec.deletedAt && + rec.expiresAt !== null && + clock() >= rec.expiresAt && + !pinned.has(row.id) && !held + ) { + await storage.removeExpired(rec, clock()); + rec = { ...rec, deletedAt: clock(), status: 'expired' }; + atomic(rootFile(row.id), rec); + } + if (rec?.deletedAt && !rec.localPruned && !pinned.has(row.id) && !held) { + removeLocal(row, rec); + rec = { ...rec, localPruned: true }; + atomic(rootFile(row.id), rec); + } + if (!rec && !held) { + if (parallelResults.has(row.id)) { + if (!parallelResults.get(row.id).ok) throw Error('backup_upload_failed'); + rec = record(row.id); + } else rec = await exportRecord(row); + } + if (!rec) continue; + if ( + requests.some((r) => (r.kind === 'restore' || JSON.parse(r.input_json).action === 'restore') && JSON.parse(r.input_json).backupId === row.id) + ) + hydrate(row); + if (rec.status === 'verified' && rec.recoveryDigest && !pinned.has(row.id) && !held) { + // Cloudflare is the history store. Keep a local snapshot only while + // an active lifecycle operation needs it for compensation or restore. + removeLocal(row, rec); + } + let localReady = false; + try { + const p = pathResolver(config, row); + localReady = fs.existsSync(path.join(p.source, 'manifest.json')); + } catch {} + if (visible.has(row.id)) + catalog.backups[row.id] = { + status: rec.status, + metadataDigest: rec.metadataDigest, + size: rec.size, + expiresAt: held ? null : rec.expiresAt, + retained: held, + format: rec.format, + trigger: rec.trigger, + verification: rec.verification || 'restore', + localReady, + verifiedAt: rec.verifiedAt, + }; + if (!rec.deletedAt) verified++; + } catch { + failed++; + if (visible.has(row.id)) catalog.backups[row.id] = { status: 'failed', metadataDigest: hash(row.metadata_json), checkedAt: clock(), failureCode: 'backup_upload_failed' }; + for(const d of archiveDeletions.filter(d=>d.backup_id===row.id)) catalog.deletions[d.id]={status:'failed',failureCode:'backup_deletion_failed'}; + } + } + const setDb=new DatabaseSync(dbFile,{readOnly:true}); + try { await require('./legacy-pre-update-backups').retireLegacyPreUpdateBackups({ db: setDb, config, record, run: runFactory(config.environment), storage, receiptRoot, ownerUid }); } + catch { failed++; } + try { if(setDb.prepare("SELECT 1 FROM sqlite_schema WHERE name='backup_sets'").get()) catalog.sets=await require('./system-backup-manifests').syncSystemManifests({config,db:setDb,runFactory,workRoot,record,storage,clock,excludedOrganizations:deletion.deletingOrganizations}); } + catch { failed++; } finally {setDb.close();} + catalog.usage = await require('./backup-storage-usage').measureStorageUsage({ + storage, config, workRoot, ownerUid, clock, records:allRecords.length ? allRecords : rows, sets:backupSets, + version:[rows.map(r => r.id), catalog.backups, catalog.sets, archiveDeletions], + }); + atomic(path.join(receiptRoot, 'catalog.json'), catalog, 0o644); + fs.chmodSync(path.join(receiptRoot, 'catalog.json'), 0o644); + return { verified, failed, managedIds: rows.map((r) => r.id), deletedRuntimeKeys: deletion.deletedRuntimeKeys, heldRuntimes }; + } + function check() { + let checked = 0; + const artifacts = new Set(); + for (const name of fs.readdirSync(root)) { + if (!/^(backup|breq)_[a-f0-9]{32}\.json$/.test(name)) continue; + const rec = record(name.slice(0, -5)); + if (rec.deletedAt) continue; + runFor(rec.id, rec.retentionDays)(['check', '--read-data']); + for (const artifact of rec.recoveryArtifacts || []) { + if (!/^[a-f0-9]{64}$/.test(artifact.digest)) fail(); + if (artifacts.has(artifact.digest)) continue; + require('./recovery-artifacts').resticReader(config)(`recovery-artifacts/${artifact.digest}`, ['--no-lock', 'check', '--read-data']); + artifacts.add(artifact.digest); + } + checked++; + } + return checked; + } + return { scan, exportRecord, hydrate, check }; +} +module.exports = { createBackupArchives, pathsFor }; + +// Re-read removal/deletion intent for every admission. The exporter keeps its +// global lock while this read-only discovery admits independently ready work. +function pendingExports(dbFile, record) { + const db = new DatabaseSync(dbFile, {readOnly:true}); + try { + db.exec('PRAGMA trusted_schema=OFF; PRAGMA busy_timeout=3000; BEGIN'); + const has = table => Boolean(db.prepare("SELECT 1 FROM sqlite_schema WHERE name=?").get(table)); + const removed = new Set(has('dsp_removals') ? db.prepare('SELECT organization_id FROM dsp_removals').all().map(r => r.organization_id) : []); + const deleting = new Set(has('installation_lifecycle_jobs') ? db.prepare("SELECT organization_id FROM installation_lifecycle_jobs WHERE operation='destroy' AND status IN ('queued','running')").all().map(r => r.organization_id) : []); + const archives = new Set(has('backup_deletions') ? db.prepare("SELECT backup_id FROM backup_deletions WHERE status='queued'").all().map(r => r.backup_id) : []); + return db.prepare('SELECT * FROM platform_backup_records WHERE deleted_at IS NULL ORDER BY created_at DESC').all() + .filter(row => !record(row.id) && !removed.has(row.organization_id) && !deleting.has(row.organization_id) + && !(row.kind === 'core' && (removed.size || deleting.size)) && !archives.has(row.id)); + } finally { db.close(); } +} +module.exports.pendingExports = pendingExports; diff --git a/core/core/installations/src/backup-export-worker.js b/core/core/installations/src/backup-export-worker.js new file mode 100644 index 0000000..f7d38d6 --- /dev/null +++ b/core/core/installations/src/backup-export-worker.js @@ -0,0 +1,19 @@ +'use strict'; +const { workerData, parentPort } = require('node:worker_threads'); +const { DatabaseSync } = require('node:sqlite'); +const path = require('node:path'); +async function main() { + if (process.geteuid() !== 0 || !/^(backup|breq)_[a-f0-9]{32}$/.test(workerData?.backupId)) throw Error(); + const config = require('./offsite-backup').loadConfig(); + const db = new DatabaseSync(path.join(config.localRoot, 'data/access-control/access-control.sqlite3'), { readOnly: true }); + let row; + try { + row = db.prepare(`SELECT r.*,c.category FROM platform_backup_records r LEFT JOIN backup_categories c ON c.backup_id=r.id + WHERE r.id=? AND r.deleted_at IS NULL`).get(workerData.backupId); + } finally { db.close(); } + if (!row || !require('./backup-archives').pendingExports(path.join(config.localRoot, 'data/access-control/access-control.sqlite3'), () => null) + .some(candidate => candidate.id === row.id)) throw Error(); + await require('./backup-archives').createBackupArchives(config).exportRecord(row); + parentPort.postMessage({ ok: true }); +} +main().catch(() => { process.exitCode = 1; }); diff --git a/core/core/installations/src/backup-readiness.js b/core/core/installations/src/backup-readiness.js new file mode 100644 index 0000000..52a37a1 --- /dev/null +++ b/core/core/installations/src/backup-readiness.js @@ -0,0 +1,95 @@ +'use strict'; +// Advisory metadata inspection while services are live. Snapshot/restore proofs +// remain mandatory under the ordinary stopped-service backup boundary. +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { HOST_TENANT_ROOT, opaqueRuntimeSuffix } = require('../../runtime-host-identity'); +const { publicRootJson, RECEIPTS } = require('./offsite-policy'); +const { atomic } = require('./release-delivery-files'); +const FILE = path.join(RECEIPTS, 'readiness.json'); +const localIdentity = root => crypto.createHash('sha256').update(root).digest('hex'); + +function inspectInstallation(root, uid, { maxEntries = 100000, now = Date.now, deadline = now() + 15000 } = {}) { + const issues = new Set(); + let entries = 0, bytes = 0; + const device = fs.lstatSync(root).dev; + function visit(file, scope) { + if (++entries > maxEntries || now() > deadline) { issues.add('inspection_limit'); return; } + let info; + try { info = fs.lstatSync(file); } catch { issues.add('tree_changed_or_missing'); return; } + if (info.isSymbolicLink()) { issues.add(scope === 'state' ? 'state_symlink' : 'backup_symlink'); return; } + if (info.uid !== uid || info.dev !== device) { issues.add('backup_owner_or_device'); return; } + if (info.isDirectory()) { + if (fs.realpathSync(file) !== file) { issues.add('backup_symlink'); return; } + if ((info.mode & 0o7777) !== 0o700) issues.add(/\/state\/auth-broker\/browser-sessions\/[a-f0-9]{32}\/chrome\/\.cache(?:\/fontconfig)?$/.test(file) + ? 'browser_cache_directory_permissions' : 'backup_directory_permissions'); + const directory = fs.opendirSync(file); + try { + let entry; + while ((entry = directory.readSync())) { + if (entries >= maxEntries || now() > deadline) { issues.add('inspection_limit'); break; } + visit(path.join(file, entry.name), scope); + } + } finally { directory.closeSync(); } + } else if (info.isFile()) { + if (info.mode & 0o077) issues.add('backup_file_permissions'); + if (info.nlink !== 1) issues.add('backup_hardlink'); + bytes += info.size; + if (info.size > 2 * 1024 ** 3 || bytes > 8 * 1024 ** 3) issues.add('backup_size_limit'); + } else issues.add('backup_nonregular_file'); + } + const rootInfo = fs.lstatSync(root); + if (!rootInfo.isDirectory() || rootInfo.uid !== uid || (rootInfo.mode & 0o7777) !== 0o700 || fs.realpathSync(root) !== root) { + return { status: 'attention', issues: ['unsafe_installation_root'], entries: 0, bytes: 0 }; + } + for (const scope of ['data', 'state', 'config', 'secrets/auth-broker']) visit(path.join(root, scope), scope); + const backupRoot = path.join(root, 'backups'); + const backup = fs.lstatSync(backupRoot); + if (!backup.isDirectory() || backup.uid !== uid || (backup.mode & 0o7777) !== 0o700 || fs.realpathSync(backupRoot) !== backupRoot) issues.add('unsafe_backup_directory'); + else { + const space = fs.statfsSync(backupRoot); + if (space.bavail * space.bsize < bytes * 3 + 64 * 1024 ** 2) issues.add('backup_insufficient_space'); + } + return { status: issues.size ? 'attention' : 'ready', issues: [...issues].sort(), entries, bytes }; +} + +function inspectHost(config, { base = HOST_TENANT_ROOT, inspect = inspectInstallation, now = Date.now } = {}) { + const members = [], deadline = now() + 45000; + if (fs.existsSync(base)) { + const stat = fs.lstatSync(base); + if (!stat.isDirectory() || stat.uid !== 0 || stat.mode & 0o022 || fs.realpathSync(base) !== base) throw Error('unsafe_tenant_root'); + for (const suffix of fs.readdirSync(base)) { + if (!/^[a-f0-9]{20}$/.test(suffix)) continue; + const parent = path.join(base, suffix, 'runtime'); + if (fs.realpathSync(parent) !== parent) throw Error('unsafe_tenant_root'); + for (const runtime of fs.readdirSync(parent)) { + if (opaqueRuntimeSuffix(runtime) !== suffix || members.length >= 100 || now() > deadline) throw Error('backup_readiness_limit'); + const root = path.join(parent, runtime), uid = fs.lstatSync(root).uid; + if (uid === 0 || uid === config.coreUid) throw Error('unsafe_tenant_root'); + try { members.push({ runtimeKey: runtime, ...inspect(root, uid, { deadline, now }) }); } + catch { members.push({ runtimeKey: runtime, status: 'attention', issues: ['inspection_failed'] }); } + } + } + } + return { schemaVersion: 1, localRootHash: localIdentity(config.localRoot), checkedAt: now(), + status: members.some(m => m.status !== 'ready') ? 'attention' : 'ready', members }; +} + +function readReadiness(localRoot, { file = FILE, uid = 0, now = Date.now } = {}) { + const result = publicRootJson(file, true, uid, 65536); + if (!result) return { status: 'unavailable' }; + if (result.schemaVersion !== 1 || result.localRootHash !== localIdentity(localRoot) + || !Number.isSafeInteger(result.checkedAt) || result.checkedAt > now() + 5000 + || !['ready', 'attention'].includes(result.status) || !Array.isArray(result.members)) throw Error('backup_readiness_invalid'); + return now() - result.checkedAt > 120000 ? { status: 'stale', checkedAt: result.checkedAt } : result; +} + +function install(executable, localRoot) { + const unit = '[Unit]\nDescription=Inspect Dispatch backup readiness\n\n[Service]\nType=oneshot\nUMask=0077\n' + + `ExecStart=/usr/bin/node --no-warnings ${executable}\nTimeoutStartSec=60s\n`; + atomic('/etc/systemd/system/dispatch-backup-readiness.service', unit, 0o644); + atomic('/etc/systemd/system/dispatch-backup-readiness.timer', '[Unit]\nDescription=Refresh Dispatch backup readiness\n\n[Timer]\nOnBootSec=30s\nOnUnitInactiveSec=60s\n\n[Install]\nWantedBy=timers.target\n', 0o644); + atomic('/etc/systemd/system/dispatch-backup-readiness.path', `[Unit]\nDescription=Check backups before release publication\n\n[Path]\nPathChanged=${localRoot}/run/release-preflight\nUnit=dispatch-backup-readiness.service\n\n[Install]\nWantedBy=multi-user.target\n`, 0o644); +} +module.exports = { FILE, localIdentity, inspectInstallation, inspectHost, readReadiness, install }; diff --git a/core/core/installations/src/backup-scratch.js b/core/core/installations/src/backup-scratch.js new file mode 100644 index 0000000..c8c8ee1 --- /dev/null +++ b/core/core/installations/src/backup-scratch.js @@ -0,0 +1,32 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +// Call only while holding the offsite worker lock, before starting any transfer. +// Interrupted exports/restores can contain full tenant history after a reboot. +function cleanupBackupScratch(root, ownerUid = 0) { + const directory = fs.lstatSync(root); + if (!directory.isDirectory() || directory.uid !== ownerUid || directory.mode & 0o077 + || fs.realpathSync(root) !== root) throw Error('unsafe_backup_scratch'); + for (const name of fs.readdirSync(root)) { + if (!/^(archive-transfer|archive-restore|rediscover|transfer|canary)-[A-Za-z0-9]{6}$/.test(name)) continue; + const file = path.join(root, name), info = fs.lstatSync(file); + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== ownerUid + || info.mode & 0o077 || fs.realpathSync(file) !== file) throw Error('unsafe_backup_scratch'); + fs.rmSync(file, { recursive: true }); + } +} +function cleanupRestoreStaging(root = '/var/lib/dispatch-restore-staging', ownerUid = 0) { + if (!fs.existsSync(root)) return; + const parent = fs.lstatSync(root); + if (!parent.isDirectory() || parent.uid !== ownerUid || (parent.mode & 0o7777) !== 0o711 + || fs.realpathSync(root) !== root) throw Error('unsafe_backup_scratch'); + for (const name of fs.readdirSync(root)) { + if (!/^import-[A-Za-z0-9]{6}$/.test(name)) continue; + const file = path.join(root, name), info = fs.lstatSync(file); + // Import directories are handed to the selected DSP UID. The parent stays + // root-owned and non-writable, so DSPs cannot create or replace these names. + if (!info.isDirectory() || info.isSymbolicLink() || info.mode & 0o077 + || fs.realpathSync(file) !== file) throw Error('unsafe_backup_scratch'); + fs.rmSync(file, { recursive: true }); + } +} +module.exports = { cleanupBackupScratch, cleanupRestoreStaging }; diff --git a/core/core/installations/src/backup-storage-usage.js b/core/core/installations/src/backup-storage-usage.js new file mode 100644 index 0000000..9ff88b8 --- /dev/null +++ b/core/core/installations/src/backup-storage-usage.js @@ -0,0 +1,131 @@ +"use strict"; +const fs = require("node:fs"), + path = require("node:path"), + crypto = require("node:crypto"); +const { atomic } = require("./release-delivery-files"); +const { publicRootJson } = require("./offsite-policy"); +const empty = () => ({ bytes: 0, backupCount: 0 }); +function summarize(inventory, records, sets) { + const core = empty(), + other = empty(), + dsps = new Map(); + const byId = new Map(records.map((row) => [row.id, row])); + const add = (a, b) => { + if (!Number.isSafeInteger(b) || b < 0 || !Number.isSafeInteger(a + b)) + throw Error("backup_usage_invalid"); + return a + b; + }; + let bytes = 0; + for (const [id, size] of Object.entries(inventory.archives)) { + const row = byId.get(id); + let scope = other; + if (row?.kind === "core") scope = core; + else if (row?.organization_id) { + if (!dsps.has(row.organization_id)) + dsps.set(row.organization_id, { + organizationId: row.organization_id, + ...empty(), + }); + scope = dsps.get(row.organization_id); + } + scope.bytes = add(scope.bytes, size); + scope.backupCount++; + bytes = add(bytes, size); + } + let manifestBytes = 0; + for (const size of Object.values(inventory.sets)) + manifestBytes = add(manifestBytes, size); + bytes = add(add(add(bytes, manifestBytes), inventory.legacyBytes), inventory.artifactBytes || 0); + return { + bytes, + backupCount: Object.keys(inventory.archives).length, + core, + dsps: [...dsps.values()], + other, + manifestBytes, + legacyBytes: inventory.legacyBytes, + artifactBytes: inventory.artifactBytes || 0, + sets: sets.map((set) => { + const ids = new Set( + JSON.parse(set.members_json) + .map((m) => m.backupId) + .filter(Boolean), + ); + let componentBytes = 0, + backupCount = 0; + for (const id of ids) + if (Object.hasOwn(inventory.archives, id)) { + componentBytes = add(componentBytes, inventory.archives[id]); + backupCount++; + } + const manifestBytes = inventory.sets[set.id] || 0; + return { + id: set.id, + bytes: add(componentBytes, manifestBytes), + backupCount, + manifestBytes, + }; + }), + }; +} +async function measureStorageUsage({ + storage, + config, + workRoot, + ownerUid, + records, + sets, + version, + clock = Date.now, +}) { + const file = path.join(workRoot, "storage-usage.json"); + const storageKey = crypto + .createHash("sha256") + .update(JSON.stringify([config.accountId, config.bucket, config.prefix])) + .digest("hex"); + const signature = crypto + .createHash("sha256") + .update(JSON.stringify([storageKey, version])) + .digest("hex"); + let cached; + try { + if (fs.existsSync(file)) + cached = publicRootJson(file, false, ownerUid, 16 * 1024 * 1024); + } catch {} + if ( + cached?.storageKey !== storageKey || + !Number.isSafeInteger(cached?.checkedAt) + ) + cached = null; + try { + if ( + !cached || + cached.signature !== signature || + clock() - cached.checkedAt >= 300000 || + clock() < cached.checkedAt + ) { + const inventory = await storage.usage(); + // Validate the complete inventory before publishing or caching any totals. + summarize(inventory, records, sets); + cached = { storageKey, signature, checkedAt: clock(), inventory }; + atomic(file, cached); + } + return { + ...summarize(cached.inventory, records, sets), + status: "ready", + checkedAt: cached.checkedAt, + }; + } catch { + // Usage reporting cannot block a backup or turn a failed measurement into 0 B. + if (cached) + try { + return { + ...summarize(cached.inventory, records, sets), + status: "stale", + checkedAt: cached.checkedAt, + }; + } catch {} + return { status: "unavailable", checkedAt: null }; + } +} +module.exports = { summarize, measureStorageUsage }; diff --git a/core/core/installations/src/backups.js b/core/core/installations/src/backups.js new file mode 100644 index 0000000..7924992 --- /dev/null +++ b/core/core/installations/src/backups.js @@ -0,0 +1,502 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { INSTALLATION_IDENTIFIER_RE } = require('../../../shared/contracts/src/installation'); + +const BACKUP_FORMAT_VERSION = 1; +const BACKUP_ROOTS = Object.freeze(['data', 'state']); +const MAX_BACKUP_FILES = 100_000; +const MAX_BACKUP_BYTES = 8 * 1024 * 1024 * 1024; +const MAX_BACKUP_FILE_BYTES = 2 * 1024 * 1024 * 1024; + +function fail(code = 'backup_failed') { throw Object.assign(new Error(code), { code }); } +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} +function exact(value, allowed, required, code = 'runtime_boundary_violation') { + if (!plain(value)) fail(code); + const keys = Object.keys(value); + if (keys.some(key => !allowed.includes(key)) || required.some(key => !Object.hasOwn(value, key))) fail(code); +} +function identifier(value) { + if (typeof value !== 'string' || !INSTALLATION_IDENTIFIER_RE.test(value)) fail('runtime_boundary_violation'); + return value; +} +function lstatMaybe(target) { + try { return fs.lstatSync(target); } + catch (error) { if (error?.code === 'ENOENT') return null; throw error; } +} +function canonicalDirectory(target, expectedDevice = null, code = 'backup_failed') { + const info = lstatMaybe(target); + if (!info || !info.isDirectory() || info.isSymbolicLink() || info.uid !== process.geteuid() + || (info.mode & 0o7777) !== 0o700 || expectedDevice !== null && info.dev !== expectedDevice) fail(code); + let canonical; + try { canonical = fs.realpathSync(target); } catch { fail(code); } + if (canonical !== target) fail(code); + return info; +} +function safeFile(target, expectedDevice, code = 'backup_failed') { + const info = lstatMaybe(target); + if (!info || !info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() + || info.nlink !== 1 || info.dev !== expectedDevice || (info.mode & 0o077) !== 0 + || info.size > MAX_BACKUP_FILE_BYTES) fail(code); + return info; +} +function digestBuffer(value) { return crypto.createHash('sha256').update(value).digest('hex'); } +function readFileDigest(target, device, code = 'backup_failed') { + const before = safeFile(target, device, code); + let handle; + try { + handle = fs.openSync(target, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)); + const opened = fs.fstatSync(handle); + if (!opened.isFile() || opened.uid !== process.geteuid() || opened.nlink !== 1 + || opened.dev !== device || opened.ino !== before.ino || (opened.mode & 0o077) !== 0 + || opened.size > MAX_BACKUP_FILE_BYTES) fail(code); + const hash = crypto.createHash('sha256'); + const buffer = Buffer.allocUnsafe(1024 * 1024); + let size = 0; + while (true) { + const read = fs.readSync(handle, buffer, 0, buffer.length, null); + if (read === 0) break; + hash.update(buffer.subarray(0, read)); + size += read; + } + const after = fs.fstatSync(handle); + if (after.dev !== opened.dev || after.ino !== opened.ino || after.size !== opened.size + || after.mtimeMs !== opened.mtimeMs || size !== opened.size) fail(code); + return { size, sha256: hash.digest('hex') }; + } finally { if (handle !== undefined) fs.closeSync(handle); } +} +function syncDirectory(directory) { + let handle; + try { handle = fs.openSync(directory, fs.constants.O_RDONLY); fs.fsyncSync(handle); } + finally { if (handle !== undefined) fs.closeSync(handle); } +} +function writePrivate(target, content, parentDevice) { + const parent = path.dirname(target); + canonicalDirectory(parent, parentDevice); + let handle; + try { + handle = fs.openSync(target, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY + | (fs.constants.O_NOFOLLOW || 0), 0o600); + fs.writeFileSync(handle, content); + fs.fsyncSync(handle); + } finally { if (handle !== undefined) fs.closeSync(handle); } + safeFile(target, parentDevice); + syncDirectory(parent); +} +function makePrivate(target, parentDevice) { + const parent = path.dirname(target); + canonicalDirectory(parent, parentDevice); + fs.mkdirSync(target, { mode: 0o700 }); + canonicalDirectory(target, parentDevice); + syncDirectory(parent); +} +function scanTree(root, label, device) { + canonicalDirectory(root, device); + const entries = []; + let fileCount = 0; + let totalBytes = 0; + function visit(directory, relative) { + canonicalDirectory(directory, device); + const names = fs.readdirSync(directory).sort(); + for (const name of names) { + if (!name || name === '.' || name === '..' || name.includes('/') || name.includes('\0')) fail(); + const target = path.join(directory, name); + const childRelative = relative ? `${relative}/${name}` : name; + const info = lstatMaybe(target); + if (!info || info.isSymbolicLink() || info.uid !== process.geteuid() || info.dev !== device) fail(); + if (info.isDirectory()) { + if ((info.mode & 0o7777) !== 0o700) fail(); + entries.push({ path: `${label}/${childRelative}`, type: 'directory' }); + visit(target, childRelative); + } else if (info.isFile()) { + const digested = readFileDigest(target, device); + fileCount += 1; + totalBytes += digested.size; + if (fileCount > MAX_BACKUP_FILES || totalBytes > MAX_BACKUP_BYTES) fail(); + entries.push({ + path: `${label}/${childRelative}`, + type: 'file', + size: digested.size, + sha256: digested.sha256, + }); + } else fail(); + } + } + visit(root, ''); + return { entries, fileCount, totalBytes }; +} +function scanRoots(roots, labels = BACKUP_ROOTS) { + const device = canonicalDirectory(roots[0]).dev; + const result = { entries: [], fileCount: 0, totalBytes: 0 }; + for (const [label, root] of labels.map(label => [label, rootsByLabel(roots, labels)[label]])) { + const scanned = scanTree(root, label, device); + result.entries.push(...scanned.entries); + result.fileCount += scanned.fileCount; + result.totalBytes += scanned.totalBytes; + } + result.entries.sort((left, right) => left.path.localeCompare(right.path)); + result.treeDigest = digestBuffer(JSON.stringify(result.entries)); + return result; +} +function rootsByLabel(roots, labels = BACKUP_ROOTS) { + return Object.freeze(Object.fromEntries(labels.map((label, index) => [label, roots[index]]))); +} +function copyTree(source, destination, device) { + makePrivate(destination, device); + for (const name of fs.readdirSync(source).sort()) { + const from = path.join(source, name); + const to = path.join(destination, name); + const info = lstatMaybe(from); + if (!info || info.isSymbolicLink() || info.uid !== process.geteuid() || info.dev !== device) fail(); + if (info.isDirectory()) { + canonicalDirectory(from, device); + copyTree(from, to, device); + } else if (info.isFile()) { + const parent = path.dirname(to); + canonicalDirectory(parent, device); + const before = safeFile(from, device); + let input; + let output; + try { + input = fs.openSync(from, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)); + const opened = fs.fstatSync(input); + if (opened.dev !== device || opened.ino !== before.ino || opened.uid !== process.geteuid() + || opened.nlink !== 1 || !opened.isFile() || (opened.mode & 0o077) !== 0) fail(); + output = fs.openSync(to, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY + | (fs.constants.O_NOFOLLOW || 0), 0o600); + const buffer = Buffer.allocUnsafe(1024 * 1024); + let copied = 0; + while (true) { + const read = fs.readSync(input, buffer, 0, buffer.length, null); + if (read === 0) break; + let written = 0; + while (written < read) written += fs.writeSync(output, buffer, written, read - written); + copied += read; + } + const after = fs.fstatSync(input); + if (after.dev !== opened.dev || after.ino !== opened.ino || after.size !== opened.size + || after.mtimeMs !== opened.mtimeMs || copied !== opened.size) fail(); + fs.fsyncSync(output); + } finally { + if (output !== undefined) fs.closeSync(output); + if (input !== undefined) fs.closeSync(input); + } + safeFile(to, device); + syncDirectory(parent); + } else fail(); + } +} +function removeTree(target, device, code = 'backup_failed') { + const info = lstatMaybe(target); + if (!info) return; + if (info.isSymbolicLink() || info.uid !== process.geteuid() || info.dev !== device) fail(code); + if (info.isDirectory()) { + canonicalDirectory(target, device, code); + for (const name of fs.readdirSync(target)) removeTree(path.join(target, name), device, code); + syncDirectory(target); + fs.rmdirSync(target); + syncDirectory(path.dirname(target)); + } else if (info.isFile()) { + safeFile(target, device, code); + fs.unlinkSync(target); + syncDirectory(path.dirname(target)); + } else fail(code); +} +function validateSpec(spec, { source = false } = {}) { + exact(spec, source + ? ['id', 'purpose', 'manifestRevision', 'releaseId', 'status', 'treeDigest', 'fileCount', 'totalBytes'] + : ['id', 'purpose', 'manifestRevision', 'releaseId', 'status'], + source + ? ['id', 'purpose', 'manifestRevision', 'releaseId', 'status', 'treeDigest', 'fileCount', 'totalBytes'] + : ['id', 'purpose', 'manifestRevision', 'releaseId', 'status']); + identifier(spec.id); + if (!['manual', 'upgrade', 'restore_safety', 'decommission'].includes(spec.purpose) + || !Number.isSafeInteger(spec.manifestRevision) || spec.manifestRevision < 1 + || typeof spec.releaseId !== 'string' || !/^[a-z][a-z0-9_.-]{2,95}$/.test(spec.releaseId) + || !['reserved', 'available'].includes(spec.status)) fail('runtime_boundary_violation'); + if (source && (spec.status !== 'available' || !/^[a-f0-9]{64}$/.test(spec.treeDigest) + || !Number.isSafeInteger(spec.fileCount) || spec.fileCount < 0 + || !Number.isSafeInteger(spec.totalBytes) || spec.totalBytes < 0)) fail('restore_failed'); + return spec; +} + +function createInstallationBackupManager(options) { + exact(options, ['layout', 'full'], ['layout']); + const labels = options.full ? ['data', 'state', 'config', 'auth-secrets'] : BACKUP_ROOTS; + const version = options.full ? 2 : BACKUP_FORMAT_VERSION; + const layout = options.layout; + if (!plain(layout) || !plain(layout.directories)) fail('runtime_boundary_violation'); + const installationRoot = layout.installationRoot; + const roots = labels.map(label => Object.freeze({ + data: layout.directories.dataRoot, + state: layout.directories.stateRoot, + config: layout.directories.configRoot, + 'auth-secrets': layout.directories.authSecretsRoot, + })[label]); + const backupsRoot = layout.directories.backupsRoot; + if (![installationRoot, backupsRoot, ...roots].every(value => typeof value === 'string' && path.isAbsolute(value))) { + fail('runtime_boundary_violation'); + } + + function snapshot(specValue, mutate) { + const spec = validateSpec(specValue); + if (typeof mutate !== 'function') fail('runtime_boundary_violation'); + const installation = canonicalDirectory(installationRoot); + canonicalDirectory(backupsRoot, installation.dev); + for (const root of roots) canonicalDirectory(root, installation.dev); + const target = path.join(backupsRoot, spec.id); + const temporary = path.join(backupsRoot, `.creating-${spec.id}`); + if (path.dirname(target) !== backupsRoot || path.dirname(temporary) !== backupsRoot) fail('runtime_boundary_violation'); + if (lstatMaybe(target)) return inspect(spec); + const required = scanRoots(roots, labels).totalBytes; + const space = fs.statfsSync(backupsRoot); + // Leave room for the snapshot and a full restore with the previous tree retained. + if (space.bavail * space.bsize < required * 3 + 64 * 1024 * 1024) fail('backup_failed'); + return mutate(() => { + if (lstatMaybe(temporary)) removeTree(temporary, installation.dev); + makePrivate(temporary, installation.dev); + const payload = path.join(temporary, 'payload'); + makePrivate(payload, installation.dev); + for (const [index, label] of labels.entries()) { + copyTree(roots[index], path.join(payload, label), installation.dev); + } + const scanned = scanRoots(labels.map(label => path.join(payload, label)), labels); + const metadata = { + version, + backupId: spec.id, + purpose: spec.purpose, + manifestRevision: spec.manifestRevision, + releaseId: spec.releaseId, + fileCount: scanned.fileCount, + totalBytes: scanned.totalBytes, + treeDigest: scanned.treeDigest, + entries: scanned.entries, + }; + writePrivate(path.join(temporary, 'manifest.json'), `${JSON.stringify(metadata)}\n`, installation.dev); + syncDirectory(temporary); + fs.renameSync(temporary, target); + syncDirectory(backupsRoot); + return inspect({ ...spec, status: 'available', treeDigest: scanned.treeDigest, + fileCount: scanned.fileCount, totalBytes: scanned.totalBytes }); + }); + } + + function legacy(spec) { + if (!options.full || spec.status !== 'available') return null; + const file = path.join(backupsRoot, identifier(spec.id), 'manifest.json'); + const device = canonicalDirectory(backupsRoot).dev; + if (safeFile(file, device).size > 32 * 1024 * 1024) fail(); + const manifest = JSON.parse(fs.readFileSync(file, 'utf8')); + return manifest.version === 1 ? createInstallationBackupManager({ layout }) : null; + } + + function inspect(specValue) { + const prior = legacy(specValue); if (prior) return prior.inspect(specValue); + const spec = validateSpec(specValue, { source: specValue.status === 'available' }); + const installation = canonicalDirectory(installationRoot); + canonicalDirectory(backupsRoot, installation.dev); + const target = path.join(backupsRoot, spec.id); + canonicalDirectory(target, installation.dev); + const manifestFile = path.join(target, 'manifest.json'); + const info = safeFile(manifestFile, installation.dev); + if (info.size < 2 || info.size > 32 * 1024 * 1024) fail(); + const raw = fs.readFileSync(manifestFile, 'utf8'); + if (!raw.endsWith('\n') || raw.includes('\0')) fail(); + let manifest; + try { manifest = JSON.parse(raw.slice(0, -1)); } catch { fail(); } + exact(manifest, [ + 'version', 'backupId', 'purpose', 'manifestRevision', 'releaseId', 'fileCount', + 'totalBytes', 'treeDigest', 'entries', + ], [ + 'version', 'backupId', 'purpose', 'manifestRevision', 'releaseId', 'fileCount', + 'totalBytes', 'treeDigest', 'entries', + ]); + const payloadRoots = labels.map(label => path.join(target, 'payload', label)); + const scanned = scanRoots(payloadRoots, labels); + if (manifest.version !== version || manifest.backupId !== spec.id + || manifest.purpose !== spec.purpose || manifest.manifestRevision !== spec.manifestRevision + || manifest.releaseId !== spec.releaseId || manifest.fileCount !== scanned.fileCount + || manifest.totalBytes !== scanned.totalBytes || manifest.treeDigest !== scanned.treeDigest + || JSON.stringify(manifest.entries) !== JSON.stringify(scanned.entries) + || spec.status === 'available' && (spec.treeDigest !== scanned.treeDigest + || spec.fileCount !== scanned.fileCount || spec.totalBytes !== scanned.totalBytes)) fail(); + return Object.freeze({ + status: 'snapshot', changed: false, fileCount: scanned.fileCount, + totalBytes: scanned.totalBytes, treeDigest: scanned.treeDigest, + }); + } + + function restore(sourceValue, operationIdValue, mutate) { + const prior = legacy(sourceValue); if (prior) return prior.restore(sourceValue, operationIdValue, mutate); + const source = validateSpec(sourceValue, { source: true }); + const operationId = identifier(operationIdValue); + if (typeof mutate !== 'function') fail('runtime_boundary_violation'); + inspect(source); + const installation = canonicalDirectory(installationRoot); + const sourcePayload = path.join(backupsRoot, source.id, 'payload'); + const work = path.join(backupsRoot, `.restore-${operationId}`); + const byLabel = rootsByLabel(roots, labels); + return mutate(() => { + const journalFile = path.join(work, 'journal.json'); + const committedFile = path.join(work, 'committed.json'); + + function readRecord(target, requiredKeys) { + const info = safeFile(target, installation.dev, 'lifecycle_compensation_failed'); + if (info.size < 2 || info.size > 4 * 1024) fail('lifecycle_compensation_failed'); + const raw = fs.readFileSync(target, 'utf8'); + if (!raw.endsWith('\n') || raw.includes('\0')) fail('lifecycle_compensation_failed'); + let value; + try { value = JSON.parse(raw.slice(0, -1)); } catch { fail('lifecycle_compensation_failed'); } + exact(value, requiredKeys, requiredKeys, 'lifecycle_compensation_failed'); + if (value.version !== 1 || value.operationId !== operationId + || value.treeDigest !== source.treeDigest) fail('lifecycle_compensation_failed'); + return value; + } + + function verifiedReceipt() { + const restored = scanRoots(roots, labels); + if (restored.treeDigest !== source.treeDigest || restored.fileCount !== source.fileCount + || restored.totalBytes !== source.totalBytes) fail('restore_failed'); + return Object.freeze({ + status: 'restored', changed: true, fileCount: restored.fileCount, + totalBytes: restored.totalBytes, treeDigest: restored.treeDigest, + }); + } + + function finishCommitted() { + readRecord(journalFile, ['version', 'operationId', 'treeDigest']); + readRecord(committedFile, ['version', 'operationId', 'treeDigest']); + let receipt; + try { receipt = verifiedReceipt(); } catch { fail('lifecycle_compensation_failed'); } + for (const label of labels) { + removeTree(path.join(work, `previous-${label}`), installation.dev, 'lifecycle_compensation_failed'); + removeTree(path.join(work, `candidate-${label}`), installation.dev, 'lifecycle_compensation_failed'); + } + removeTree(work, installation.dev, 'lifecycle_compensation_failed'); + syncDirectory(backupsRoot); + return receipt; + } + + function recoverWork() { + if (!lstatMaybe(work)) return null; + canonicalDirectory(work, installation.dev, 'restore_failed'); + const hasPrevious = BACKUP_ROOTS.some(label => lstatMaybe(path.join(work, `previous-${label}`))); + if (!lstatMaybe(journalFile)) { + if (hasPrevious) fail('lifecycle_compensation_failed'); + removeTree(work, installation.dev, 'restore_failed'); + syncDirectory(backupsRoot); + return null; + } + readRecord(journalFile, ['version', 'operationId', 'treeDigest']); + if (lstatMaybe(committedFile)) return finishCommitted(); + for (const label of [...labels].reverse()) { + const previous = path.join(work, `previous-${label}`); + if (!lstatMaybe(previous)) { + if (!lstatMaybe(byLabel[label])) fail('lifecycle_compensation_failed'); + continue; + } + if (lstatMaybe(byLabel[label])) removeTree(byLabel[label], installation.dev, 'restore_failed'); + fs.renameSync(previous, byLabel[label]); + syncDirectory(path.dirname(byLabel[label])); + syncDirectory(work); + syncDirectory(installationRoot); + } + removeTree(work, installation.dev, 'restore_failed'); + syncDirectory(installationRoot); + syncDirectory(backupsRoot); + return null; + } + try { + const recovered = recoverWork(); + if (recovered) return recovered; + makePrivate(work, installation.dev); + for (const label of labels) { + copyTree(path.join(sourcePayload, label), path.join(work, `candidate-${label}`), installation.dev); + } + const record = { version: 1, operationId, treeDigest: source.treeDigest }; + writePrivate(journalFile, `${JSON.stringify(record)}\n`, installation.dev); + syncDirectory(work); + syncDirectory(backupsRoot); + for (const label of labels) { + fs.renameSync(byLabel[label], path.join(work, `previous-${label}`)); + syncDirectory(path.dirname(byLabel[label])); + syncDirectory(work); + fs.renameSync(path.join(work, `candidate-${label}`), byLabel[label]); + syncDirectory(work); + syncDirectory(path.dirname(byLabel[label])); + } + verifiedReceipt(); + writePrivate(committedFile, `${JSON.stringify(record)}\n`, installation.dev); + return finishCommitted(); + } catch (error) { + let rollbackError = null; + let recovered = null; + try { recovered = recoverWork(); } catch (selected) { rollbackError = selected; } + if (recovered) return recovered; + if (error?.code === 'installation_operation_in_progress') throw error; + if (rollbackError?.code === 'installation_operation_in_progress') throw rollbackError; + if (rollbackError) fail('lifecycle_compensation_failed'); + fail('restore_failed'); + } + }); + } + + function inspectRestored(sourceValue) { + const prior = legacy(sourceValue); if (prior) return prior.inspectRestored(sourceValue); + const source = validateSpec(sourceValue, { source: true }); + inspect(source); + const installation = canonicalDirectory(installationRoot); + for (const root of roots) canonicalDirectory(root, installation.dev, 'restore_failed'); + const restored = scanRoots(roots, labels); + if (restored.treeDigest !== source.treeDigest || restored.fileCount !== source.fileCount + || restored.totalBytes !== source.totalBytes) fail('restore_failed'); + return Object.freeze({ + status: 'verified', changed: false, fileCount: restored.fileCount, + totalBytes: restored.totalBytes, treeDigest: restored.treeDigest, + }); + } + + function destroy(approval, mutate) { + exact(approval, ['installationState', 'retainedData', 'destructionApproved'], + ['installationState', 'retainedData', 'destructionApproved']); + if (approval.installationState !== 'decommissioned' || approval.retainedData !== true + || approval.destructionApproved !== true || typeof mutate !== 'function') { + fail('runtime_boundary_violation'); + } + if (!lstatMaybe(installationRoot)) { + canonicalDirectory(path.dirname(installationRoot)); + return mutate(() => { + syncDirectory(path.dirname(installationRoot)); + if (lstatMaybe(installationRoot)) fail('destruction_failed'); + return Object.freeze({ status: 'destroyed', changed: false }); + }); + } + const installation = canonicalDirectory(installationRoot); + const parent = path.dirname(installationRoot); + canonicalDirectory(parent, installation.dev, 'destruction_failed'); + return mutate(() => { + removeTree(installationRoot, installation.dev, 'destruction_failed'); + syncDirectory(parent); + if (lstatMaybe(installationRoot)) fail('destruction_failed'); + return Object.freeze({ status: 'destroyed', changed: true }); + }); + } + + function verifyDestroyed() { + if (lstatMaybe(installationRoot)) fail('destruction_failed'); + return Object.freeze({ status: 'absent', changed: false }); + } + + return Object.freeze({ snapshot, inspect, restore, inspectRestored, destroy, verifyDestroyed }); +} + +module.exports = { + BACKUP_FORMAT_VERSION, + BACKUP_ROOTS, + createInstallationBackupManager, +}; diff --git a/core/core/installations/src/core-artifact-layout.js b/core/core/installations/src/core-artifact-layout.js new file mode 100644 index 0000000..6df1be3 --- /dev/null +++ b/core/core/installations/src/core-artifact-layout.js @@ -0,0 +1,36 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const sha = value => crypto.createHash('sha256').update(value).digest('hex'); + +// Render host-local deployment files around an already verified portable code tree. +function finishCoreArtifact(root, config) { + const write = (relative, contents, mode = 0o444) => { + const file = path.join(root, relative); fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o755 }); + fs.writeFileSync(file, contents, { flag: 'wx', mode }); + }; + write('deployment.json', JSON.stringify(config) + '\n'); + for (const action of ['apply', 'verify', 'switch-host', 'prepare-backup']) write(action, + `#!/usr/bin/node --no-warnings\n'use strict';\nrequire([__dirname,'code','core','installations','src','core-systemd-deployment'].join('/')).main('${action}', __dirname).catch(() => { process.stderr.write('core_stage_failed\\n'); process.exitCode=1; });\n`, 0o555); + const source = `/opt/dispatch-platform/releases/${config.releaseId}/core-artifact/code`; + const environment = `Environment=PATH=/usr/bin:/bin\nEnvironment=NODE_NO_WARNINGS=1\nEnvironment=DISPATCH_LOCAL_ROOT=${config.localRoot}\nEnvironmentFile=${config.localRoot}/config/provisioning.env\n`; + write('units/dispatch-platform-update.service', `[Unit]\nDescription=Dispatch Core update supervisor\nAfter=network-online.target\n\n[Service]\nType=oneshot\n${environment}ExecStart=/usr/bin/node --no-warnings ${source}/core/installations/bin/dispatch-platform-update\nTimeoutStartSec=90min\nTimeoutStopSec=30s\nKillMode=control-group\nUMask=0077\nNoNewPrivileges=false\n`); + write('units/dispatch-dashboard.service', `[Unit]\nDescription=Dispatch Platform\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=simple\nWorkingDirectory=${source}/dashboard\n${environment}EnvironmentFile=-${config.localRoot}/config/dashboard.env\nExecStart=${source}/bin/dispatch-dashboard --installation-operator --installation-backend native_service_v1 --port ${config.port} --secure-cookies --public-origin ${config.publicOrigin}\nRestart=always\nRestartSec=3\nKillMode=control-group\nTimeoutStopSec=15s\nUMask=0077\nNoNewPrivileges=true\n\n[Install]\nWantedBy=default.target\n`); + write('units/dispatch-installation-reconcile.service', `[Unit]\nDescription=Dispatch isolated DSP reconciliation\nAfter=dispatch-dashboard.service\nWants=dispatch-dashboard.service\n\n[Service]\nType=oneshot\nWorkingDirectory=${source}\n${environment}ExecStart=/usr/bin/node --no-warnings ${source}/core/installations/bin/dispatch-installation-reconcile --limit=20\nTimeoutStartSec=3h\nTimeoutStopSec=30s\nKillMode=control-group\nUMask=0077\nNoNewPrivileges=false\n`); + for (const unit of ['dispatch-platform-update', 'dispatch-installation-reconcile']) write(`units/${unit}.timer`, + `[Unit]\nDescription=Recover missed Dispatch worker wakeups\n\n[Timer]\nOnBootSec=15s\nOnUnitInactiveSec=60s\nAccuracySec=1s\nUnit=${unit}.service\n\n[Install]\nWantedBy=timers.target\n`); + const files = []; + const visit = directory => { + for (const name of fs.readdirSync(directory).sort()) { + const file = path.join(directory, name); const stat = fs.lstatSync(file); + if (stat.isDirectory()) { visit(file); fs.chmodSync(file, 0o555); } + else files.push({ path: path.relative(root, file), mode: (stat.mode & 0o777).toString(8), sha256: sha(fs.readFileSync(file)) }); + } + }; + visit(root); + const manifest = JSON.stringify({ schemaVersion: 1, releaseId: config.releaseId, sourceCommit: config.sourceCommit, files }) + '\n'; + write('manifest.json', manifest); fs.chmodSync(root, 0o555); + return sha(manifest); +} +module.exports = { finishCoreArtifact }; diff --git a/core/core/installations/src/core-backup-files.js b/core/core/installations/src/core-backup-files.js new file mode 100644 index 0000000..90d3cb7 --- /dev/null +++ b/core/core/installations/src/core-backup-files.js @@ -0,0 +1,83 @@ +'use strict'; +const fs = require('node:fs'), + path = require('node:path'); +// Fixed Core-owned files. In particular, runtime registration tokens and the +// provisioner database belong to DSP recovery and cannot enter a Core archive. +const FILES = [ + 'config/dashboard.env', + 'config/provisioning.env', + 'config/platform-releases.json', + 'config/oci-releases.json', + 'config/cloudflared/config.yml', + 'secrets/email/cloudflare-api-token', + 'secrets/turnstile/secret-key', +]; +function filesAt(root) { + const directory = path.join(root, 'secrets/cloudflared'); + if (!fs.existsSync(directory)) return FILES; + if (fs.realpathSync(directory) !== directory || !fs.lstatSync(directory).isDirectory()) + throw Error('core_backup_invalid'); + const names = fs.readdirSync(directory); + if (names.some((name) => !/^[-a-zA-Z0-9_]{1,100}\.json$/.test(name))) + throw Error('core_backup_invalid'); + return [...FILES, ...names.map((name) => 'secrets/cloudflared/' + name)]; +} +function isCoreFile(name) { + return FILES.includes(name) || /^secrets\/cloudflared\/[-a-zA-Z0-9_]{1,100}\.json$/.test(name); +} +function recoveryFileRoots(localRoot, snapshotSource) { + const source = path.join(snapshotSource, 'core-files'); + return filesAt(source).filter(name => fs.existsSync(path.join(source, name))) + .map(name => ({source: path.join(source, name), target: path.join(localRoot, name)})); +} +function capture(localRoot, destination) { + for (const name of filesAt(localRoot)) { + const file = path.join(localRoot, name); + if (!fs.existsSync(file)) continue; + const stat = fs.lstatSync(file); + if ( + !stat.isFile() || + stat.nlink !== 1 || + fs.realpathSync(file) !== file || + stat.uid !== process.geteuid() + ) + throw Error('core_backup_invalid'); + const target = path.join(destination, name); + fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 }); + fs.copyFileSync(file, target, fs.constants.COPYFILE_EXCL); + fs.chmodSync(target, 0o600); + } +} +function restore(localRoot, source) { + const writes = []; + // Restore absence as well as content, including credentials created after + // this snapshot. The same operation also makes safety rollback exact. + for (const name of new Set([...filesAt(source), ...filesAt(localRoot)])) { + const file = path.join(source, name); + const bytes = fs.existsSync(file) ? fs.readFileSync(file) : null, + target = path.join(localRoot, name); + const parent = path.dirname(target); + let existing = parent; + while (!fs.existsSync(existing)) existing = path.dirname(existing); + if (fs.realpathSync(existing) !== existing) throw Error('core_backup_invalid'); + if (bytes !== null) fs.mkdirSync(parent, { recursive: true, mode: 0o700 }); + if (!fs.existsSync(parent)) continue; + if (fs.realpathSync(parent) !== parent) throw Error('core_backup_invalid'); + const stat = fs.lstatSync(target, {throwIfNoEntry: false}); + if (stat) { + if ( + !stat.isFile() || + stat.nlink !== 1 || + stat.uid !== process.geteuid() || + fs.realpathSync(target) !== target + ) + throw Error('core_backup_invalid'); + } + writes.push({ target, bytes }); + } + for (const { target, bytes } of writes) { + if (bytes === null) fs.rmSync(target, {force: true}); + else require('./release-delivery-files').atomic(target, bytes.toString('utf8'), 0o600); + } +} +module.exports = { capture, restore, FILES, isCoreFile, recoveryFileRoots }; diff --git a/core/core/installations/src/core-database-probe.js b/core/core/installations/src/core-database-probe.js new file mode 100644 index 0000000..50d6eae --- /dev/null +++ b/core/core/installations/src/core-database-probe.js @@ -0,0 +1,11 @@ +'use strict'; +function verifyCoreDatabase(db) { + if (db.prepare('PRAGMA quick_check').get().quick_check !== 'ok' || db.prepare('PRAGMA foreign_key_check').all().length) throw Error('core_database_unhealthy'); + for (const table of ['users', 'organizations', 'sessions', 'platform_rollouts', 'runtime_agent_authorities']) db.prepare(`SELECT count(*) FROM ${table}`).get(); + db.exec('SAVEPOINT dispatch_recovery_probe'); + try { + db.exec('CREATE TABLE dispatch_recovery_probe(value TEXT NOT NULL) STRICT; INSERT INTO dispatch_recovery_probe VALUES (\'verified\')'); + if (db.prepare('SELECT value FROM dispatch_recovery_probe').get().value !== 'verified') throw Error(); + } finally { db.exec('ROLLBACK TO dispatch_recovery_probe; RELEASE dispatch_recovery_probe'); } +} +module.exports = { verifyCoreDatabase }; diff --git a/core/core/installations/src/core-recovery-host.js b/core/core/installations/src/core-recovery-host.js new file mode 100644 index 0000000..5403c74 --- /dev/null +++ b/core/core/installations/src/core-recovery-host.js @@ -0,0 +1,260 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const http = require('node:http'); +const { spawnSync } = require('node:child_process'); +const { DatabaseSync, backup } = require('node:sqlite'); +const { atomic, privateJson, hashFileSync } = require('./release-delivery-files'); +const { verifyCoreArtifact } = require('./platform-core-update'); +const { verifyCoreDatabase } = require('./core-database-probe'); +const UNITS = ['dispatch-dashboard.service', 'dispatch-installation-reconcile.service']; +const TIMER = 'dispatch-installation-reconcile.timer'; +const fail = code => { throw Object.assign(new Error(code), { code }); }; +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +function serviceCommand(args) { + const result = spawnSync('/usr/bin/systemctl', ['--user', ...args], { encoding: 'utf8', timeout: 60_000, maxBuffer: 16384 }); + if (result.status !== 0 || result.error) fail('core_service_failed'); + return result.stdout.trim(); +} +function privateFile(file) { + const stat = fs.lstatSync(file); + if (!stat.isFile() || stat.isSymbolicLink() || stat.uid !== process.geteuid() || stat.nlink !== 1 + || (stat.mode & 0o077) || fs.realpathSync(file) !== file) fail('core_storage_invalid'); + return stat; +} +function identity(config) { return { releaseId: config.releaseId, version: config.version, sourceCommit: config.sourceCommit }; } +function rootArtifact(id) { + if (!/^[a-z][a-z0-9_.-]{2,95}$/.test(id)) fail('core_identity_invalid'); + const root = `/opt/dispatch-platform/releases/${id}/core-artifact`; + const manifestSha256 = hashFileSync(path.join(root, 'manifest.json')); + const config = JSON.parse(fs.readFileSync(path.join(root, 'deployment.json'))); + if (config.releaseId !== id) fail('core_identity_invalid'); + verifyCoreArtifact(id, { ...config, core: { artifactPath: root, manifestSha256 } }); + return { root, config }; +} +function switchHost(root, checkOnly = false) { + const args = checkOnly ? ['-n', '-l', '--', path.join(root, 'switch-host')] : ['-n', path.join(root, 'switch-host')]; + const result = spawnSync('/usr/bin/sudo', args, { encoding: 'utf8', timeout: 30_000, maxBuffer: 4096, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' } }); + if (result.status !== 0 || result.error) fail('core_host_switch_failed'); +} +function readHealth(config, nonce) { + return new Promise((resolve, reject) => { + const request = http.get({ hostname: '127.0.0.1', port: config.port, path: '/api/platform/core-health', + headers: { Host: new URL(config.publicOrigin).host, 'CF-Visitor': '{"scheme":"https"}', + ...(nonce ? { 'X-Dispatch-Recovery-Probe': nonce } : {}) }, timeout: 5000 }, response => { + let body = ''; + response.on('data', chunk => { body += chunk; if (body.length > 4096) request.destroy(new Error()); }); + response.on('end', () => { try { const value = JSON.parse(body); if (response.statusCode !== 200 || !value.ok) throw Error(); resolve(value.data); } catch (error) { reject(error); } }); + }); + request.on('timeout', () => request.destroy(new Error())); request.on('error', reject); + }); +} +async function assertHealth(config, nonce) { + const value = await readHealth(config, nonce); + if (Object.entries(identity(config)).some(([k, v]) => value[k] !== v) || (nonce && value.recoveryProbe !== 'passed')) fail('core_health_failed'); +} +async function waitHealth(config, nonce) { + for (let i = 0; i < 30; i++) { try { await assertHealth(config, nonce); return; } catch { await sleep(1000); } } + fail('core_health_failed'); +} +async function checkedBackup(source, target) { + privateFile(source); + const db = new DatabaseSync(source, { readOnly: true }); + try { if (db.prepare('PRAGMA quick_check').get().quick_check !== 'ok') fail('core_backup_invalid'); await backup(db, target); } + finally { db.close(); } + fs.chmodSync(target, 0o600); + const fd = fs.openSync(target, 'r'); try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); } +} +async function snapshotDatabase(database, directory) { + const snapshot = path.join(directory, 'access-control-before.sqlite3'); + // A failed partial snapshot is never registered in the durable journal. + await checkedBackup(database, snapshot); + const db = new DatabaseSync(snapshot, { readOnly: true }); + try { + if (db.prepare('PRAGMA quick_check').get().quick_check !== 'ok' || db.prepare('PRAGMA foreign_key_check').all().length) fail('core_backup_invalid'); + } finally { db.close(); } + const receipt = { sha256: hashFileSync(snapshot), size: privateFile(snapshot).size }; + atomic(path.join(directory, 'manifest.json'), { version: 1, kind: 'core', ...receipt }); + return receipt; +} +async function restoreDatabase(database, directory, receipt) { + const snapshot = path.join(directory, 'access-control-before.sqlite3'); + if (privateFile(snapshot).size !== receipt.size || hashFileSync(snapshot) !== receipt.sha256) fail('core_backup_invalid'); + privateFile(database); + // SQLite's backup transaction restores in place, including WAL handling, while + // the idle independent supervisor may still hold a connection to this database. + await checkedBackup(snapshot, database); + const db = new DatabaseSync(database); + try { verifyCoreDatabase(db); } finally { db.close(); } +} +function createHostRecovery(config, artifactRoot, ports = {}) { + const command = ports.command || serviceCommand; + const artifact = ports.artifact || rootArtifact; + const switchRelease = ports.switchHost || switchHost; + const pause = ports.sleep || sleep; + const offsitePolicy = ports.offsitePolicy || require('./offsite-policy'); + const health = ports.health || { read: readHealth, assert: assertHealth, wait: waitHealth }; + const database = path.join(config.localRoot, 'data/access-control/access-control.sqlite3'); + const gate = path.join(config.localRoot, 'config/core-maintenance.json'); + function sharedBackup(directory) { + const db = new DatabaseSync(database, { readOnly: true }); + try { + if (!db.prepare("SELECT 1 FROM sqlite_schema WHERE name='platform_rollout_backups'").get()) return false; + const progress = require('../../accounts/src/rollout-backups').rolloutBackupProgress(db, path.basename(path.dirname(directory))); + if (!progress) return false; + if (progress.status !== 'completed') fail('core_backup_invalid'); + return true; + } finally { db.close(); } + } + async function preflight() { + const runtimes = require('./release-catalog').loadPrivateOciReleaseCatalog(path.join(config.localRoot, 'config/oci-releases.json')); + if (runtimes[config.releaseId]?.backend === 'native_service_v1') { + // The first native update is launched by the previous dashboard, which + // cannot enforce the new rollout guard. Check again in candidate code + // before preparing backup services or stopping the running Core. + privateFile(database); + const current = new DatabaseSync(database, { readOnly: true }); + try { + if (current.prepare("SELECT 1 FROM installations WHERE backend<>'native_service_v1' AND status<>'decommissioned'").get()) { + fail('native_migration_required'); + } + } finally { current.close(); } + } + const prepareBackup = path.join(artifactRoot, 'prepare-backup'); + if (fs.existsSync(prepareBackup)) { + artifact(config.releaseId); + const result = spawnSync('/usr/bin/sudo', ['-n', prepareBackup], { encoding: 'utf8', timeout: 30000, + maxBuffer: 4096, env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8' } }); + if (result.status !== 0 || result.error) fail('offsite_backup_unavailable'); + const deadline = Date.now() + 300000; + while (true) { + try { offsitePolicy.assertOffsiteReady(); break; } + catch { if (Date.now() >= deadline) fail('offsite_backup_unavailable'); await pause(1000); } + } + } + offsitePolicy.assertOffsiteReady(); + if (privateJson(gate, process.geteuid(), true)) fail('core_maintenance_conflict'); + const candidate = artifact(config.releaseId); + const value = await health.read(config); + const prior = artifact(value.releaseId); + await health.assert(prior.config); + if (prior.config.releaseId === config.releaseId) fail('core_prior_release_required'); + for (const key of ['localRoot', 'unitRoot', 'port', 'publicOrigin']) if (prior.config[key] !== config[key]) fail('core_host_config_changed'); + if (candidate.root !== artifactRoot) fail('core_identity_invalid'); + switchRelease(prior.root, true); switchRelease(artifactRoot, true); + const files = {}; + for (const name of UNITS) { + const file = path.join(config.unitRoot, name); const stat = privateFile(file); + if (stat.size > 16384) fail('core_unit_invalid'); + files[name] = fs.readFileSync(file, 'utf8'); + // Restoration invokes only the existing immutable release's fixed entrypoint. + if (!files[name].includes(prior.root + '/code')) fail('core_unit_invalid'); + } + privateFile(database); + const space = fs.statfsSync(config.localRoot); + let size = fs.statSync(database).size; + if (fs.existsSync(database + '-wal')) size += privateFile(database + '-wal').size; + if (space.bavail * space.bsize < size * 4 + 128 * 1024 * 1024) fail('core_backup_space_unavailable'); + // Old and new releases must explicitly retain the same wire protocol. The + // candidate's database migrations are trialled on a copy before service stops. + const candidateSchemaModule = require(path.join(artifactRoot, 'code/core/accounts/src/schema')); + const candidateSchema = candidateSchemaModule.SCHEMA_VERSION; + const previousSchema = require(path.join(prior.root, 'code/core/accounts/src/schema')).SCHEMA_VERSION; + if (candidateSchema !== previousSchema && !candidateSchemaModule.REVIEWED_UPGRADE_SCHEMAS?.includes(previousSchema)) fail('core_schema_transition_requires_review'); + for (const field of ['runtimeAgentProtocol', 'runtimeGatewayProtocol']) { + if (!runtimes[config.releaseId] || !runtimes[prior.config.releaseId] + || runtimes[config.releaseId][field] !== runtimes[prior.config.releaseId][field]) fail('core_protocol_incompatible'); + } + const trialRoot = fs.mkdtempSync(path.join(config.localRoot, 'core-preflight-')); + fs.chmodSync(trialRoot, 0o700); + try { + const trial = path.join(trialRoot, 'access-control.sqlite3'); + await checkedBackup(database, trial); + const { AccessStore } = require(path.join(artifactRoot, 'code/core/accounts/src/store')); + const store = new AccessStore({ databaseRoot: trialRoot, database: trial }); + try { verifyCoreDatabase(store.db); } finally { store.close(); } + } finally { fs.rmSync(trialRoot, { recursive: true, force: true }); } + return { ...identity(prior.config), units: files, timerWasActive: command(['show', TIMER, '--property=ActiveState', '--value']) === 'active' }; + } + return { + preflight, + armRecovery(journal) { + const service = `[Unit]\nDescription=Recover an interrupted Dispatch Core update\n\n[Service]\nType=oneshot\nUMask=0077\nExecStart=/usr/bin/node --no-warnings ${artifactRoot}/code/core/installations/bin/dispatch-core-recover watch ${JSON.stringify(config.localRoot)} ${journal.rolloutId}\nTimeoutStartSec=10min\nNoNewPrivileges=false\n`; + const timer = '[Unit]\nDescription=Check interrupted Dispatch Core updates\n\n[Timer]\nOnActiveSec=15s\nOnUnitInactiveSec=15s\nAccuracySec=1s\n\n[Install]\nWantedBy=timers.target\n'; + atomic(path.join(config.unitRoot, 'dispatch-core-recovery.service'), service); + atomic(path.join(config.unitRoot, 'dispatch-core-recovery.timer'), timer); + command(['daemon-reload']); command(['enable', '--now', 'dispatch-core-recovery.timer']); + }, + disarmRecovery() { command(['disable', '--now', 'dispatch-core-recovery.timer']); }, + enterMaintenance: journal => atomic(gate, { rolloutId: journal.rolloutId, releaseId: journal.releaseId, nonce: journal.nonce }), + async drain() { + command(['stop', TIMER]); const deadline = Date.now() + 480_000; + while (['active', 'activating', 'deactivating'].includes(command(['show', 'dispatch-installation-reconcile.service', '--property=ActiveState', '--value']))) { + if (Date.now() >= deadline) fail('core_worker_busy'); await pause(1000); + } + }, + stopCandidate() { command(['stop', 'dispatch-dashboard.service']); }, + snapshot: async directory => { + const receipt = await snapshotDatabase(database, directory); + if (sharedBackup(directory)) atomic(path.join(directory, 'manifest.json'), { version: 1, kind: 'core', ...receipt, localOnly: true }); + return receipt; + }, + verifyOffsite: (directory, receipt) => { + if (sharedBackup(directory)) return; + const releases = require('./release-catalog').loadPrivateOciReleaseCatalog(path.join(config.localRoot, 'config/oci-releases.json')); + const native = releases[config.releaseId]?.backend === 'native_service_v1'; + return offsitePolicy.waitForOffsiteBackup(directory, receipt.sha256, () => {}, { required: native, recoveryRequired: native }); + }, + restore: (directory, receipt) => restoreDatabase(database, directory, receipt), + installCandidate() { + switchRelease(artifactRoot); + for (const name of UNITS) { + const bytes = fs.readFileSync(path.join(artifactRoot, 'units', name)); + atomic(path.join(config.unitRoot, name), bytes.toString()); + atomic(path.join(config.localRoot, 'config/systemd/user', name), bytes.toString()); + } + command(['daemon-reload']); + }, + startCandidate() { command(['start', 'dispatch-dashboard.service']); }, + async verifyCandidate(journal) { + await health.wait(config, journal.nonce); + // Require sustained success while client traffic is still blocked. + for (let i = 0; i < 3; i++) { await pause(5000); await health.assert(config, journal.nonce); } + if (command(['is-active', 'dispatch-dashboard.service']) !== 'active') fail('core_health_failed'); + }, + verifyPromoted: () => health.wait(config), + updateSupervisor() { + const file = path.join(artifactRoot, 'units/dispatch-platform-update.service'); + if (!fs.existsSync(file)) return; + // daemon-reload changes the next invocation without stopping the updater + // that is verifying this release. Retention waits for that process to exit. + atomic(path.join(config.unitRoot, 'dispatch-platform-update.service'), fs.readFileSync(file, 'utf8')); + for (const name of ['dispatch-platform-update.timer', TIMER]) { + const source = path.join(artifactRoot, 'units', name); + if (fs.existsSync(source)) atomic(path.join(config.unitRoot, name), fs.readFileSync(source, 'utf8')); + } + command(['daemon-reload']); + }, + restoreServices(prior) { + const previous = artifact(prior.releaseId); + switchRelease(previous.root); + for (const name of UNITS) { + if (typeof prior.units[name] !== 'string' || !prior.units[name].includes(previous.root + '/code')) fail('core_unit_invalid'); + atomic(path.join(config.unitRoot, name), prior.units[name]); + atomic(path.join(config.localRoot, 'config/systemd/user', name), prior.units[name]); + } + command(['daemon-reload']); + }, + startPrior() { command(['start', 'dispatch-dashboard.service']); }, + verifyPrior: prior => health.wait({ ...config, ...identity(prior) }), + releaseMaintenance(journal) { + const value = privateJson(gate, process.geteuid(), true); + if (!value) return; + if (value.rolloutId !== journal.rolloutId || value.nonce !== journal.nonce) fail('core_maintenance_conflict'); + fs.unlinkSync(gate); const fd = fs.openSync(path.dirname(gate), 'r'); try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); } + }, + restoreScheduling(prior) { if (prior.timerWasActive) command(['start', TIMER]); }, + }; +} +module.exports = { createHostRecovery, snapshotDatabase, restoreDatabase, assertHealth, readHealth, rootArtifact }; diff --git a/core/core/installations/src/core-recovery.js b/core/core/installations/src/core-recovery.js new file mode 100644 index 0000000..a051df6 --- /dev/null +++ b/core/core/installations/src/core-recovery.js @@ -0,0 +1,108 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { atomic, privateJson } = require('./release-delivery-files'); +const failure = code => Object.assign(new Error(code), { code }); +const TERMINAL = new Set(['promoted', 'recovered']); + +// The journal is outside the database being restored. Persist intent before every +// destructive operation; after promotion or restored-service startup, never rewind data. +function createCoreRecovery({ localRoot, context, adapter }) { + if (!/^rollout_[a-f0-9]{32}$/.test(context.rolloutId) || !Number.isSafeInteger(context.attempt) || context.attempt < 1) throw failure('core_context_invalid'); + const root = path.join(localRoot, 'backups/platform-core', context.rolloutId); + fs.mkdirSync(root, { recursive: true, mode: 0o700 }); + const stat = fs.lstatSync(root); + if (!stat.isDirectory() || stat.uid !== process.geteuid() || (stat.mode & 0o777) !== 0o700 || fs.realpathSync(root) !== root) throw failure('core_recovery_storage_invalid'); + const file = path.join(root, 'recovery.json'); + let journal = privateJson(file, process.geteuid(), true); + if (journal && (journal.schemaVersion !== 1 || journal.releaseId !== context.releaseId || journal.rolloutId !== context.rolloutId + || !Number.isSafeInteger(journal.attempt) || journal.attempt < 1 || !/^[a-f0-9]{64}$/.test(journal.nonce) + || !journal.prior || typeof journal.prior !== 'object' + || (journal.snapshot && (!/^[a-f0-9]{64}$/.test(journal.snapshot.sha256) || !Number.isSafeInteger(journal.snapshot.size) || journal.snapshot.size < 1)) + || !['preparing', 'prepared', 'switching', 'verifying', 'promoted', 'recovering', 'restored', 'recovered'].includes(journal.phase))) throw failure('core_recovery_journal_invalid'); + const save = phase => { journal.phase = phase; atomic(file, journal); }; + const directory = () => path.join(root, `attempt-${journal.attempt}`); + async function recover() { + if (!journal) return; + if (journal.phase === 'promoted') throw failure('core_already_promoted'); + if (journal.phase === 'recovered') return; + // 'restored' is durable BEFORE the old service can accept even one new write. + if (journal.phase !== 'restored') { + save('recovering'); + await adapter.stopCandidate(); + if (journal.snapshot) await adapter.restore(directory(), journal.snapshot); + await adapter.restoreServices(journal.prior); + save('restored'); + } + await adapter.startPrior(journal.prior); + await adapter.verifyPrior(journal.prior); + await adapter.releaseMaintenance(journal); + await adapter.restoreScheduling(journal.prior); + save('recovered'); + } + async function guarded(fn) { + try { return await fn(); } + catch (error) { + // Simulated power loss is injected by tests by killing a child, never by a + // production flag. All ordinary failures attempt recovery immediately. + if (journal && !TERMINAL.has(journal.phase)) { + try { await recover(); } + catch { throw failure('core_recovery_required'); } + } + throw error; + } + } + async function apply() { + return guarded(async () => { + if (journal?.phase === 'promoted') { await adapter.verifyPromoted(); return; } + if (journal && journal.phase !== 'recovered') { + await recover(); throw failure('core_interrupted_update_recovered'); + } + if (journal && context.attempt <= journal.attempt) throw failure('core_update_recovered'); + const prior = await adapter.preflight(); // Nothing is stopped before this passes. + journal = { schemaVersion: 1, rolloutId: context.rolloutId, releaseId: context.releaseId, + attempt: context.attempt, phase: 'preparing', prior, snapshot: null, nonce: crypto.randomBytes(32).toString('hex') }; + save('preparing'); + fs.mkdirSync(directory(), { mode: 0o700 }); + if (adapter.armRecovery) await adapter.armRecovery(journal); + await adapter.enterMaintenance(journal); + await adapter.drain(); + await adapter.stopCandidate(); + journal.snapshot = await adapter.snapshot(directory()); + save('prepared'); + if (adapter.verifyOffsite) await adapter.verifyOffsite(directory(), journal.snapshot); + save('switching'); + await adapter.installCandidate(); + await adapter.startCandidate(); + save('verifying'); + }); + } + async function verify() { + return guarded(async () => { + if (!journal) throw failure('core_recovery_journal_missing'); + if (journal.phase === 'promoted') { + await adapter.verifyPromoted(); + if (adapter.updateSupervisor) await adapter.updateSupervisor(); + await adapter.releaseMaintenance(journal); + await adapter.restoreScheduling(journal.prior); + if (adapter.disarmRecovery) await adapter.disarmRecovery(); + return; + } + if (journal.phase !== 'verifying') { + await recover(); throw failure('core_interrupted_update_recovered'); + } + await adapter.verifyCandidate(journal); + // Commit before opening traffic. A later retry may finish opening traffic, + // but must not restore a pre-update database after clients have written. + save('promoted'); + if (adapter.updateSupervisor) await adapter.updateSupervisor(); + await adapter.releaseMaintenance(journal); + await adapter.restoreScheduling(journal.prior); + if (adapter.disarmRecovery) await adapter.disarmRecovery(); + }); + } + return { apply, verify, recover, view: () => journal }; +} +module.exports = { createCoreRecovery }; diff --git a/core/core/installations/src/core-systemd-deployment.js b/core/core/installations/src/core-systemd-deployment.js new file mode 100644 index 0000000..b56fbd2 --- /dev/null +++ b/core/core/installations/src/core-systemd-deployment.js @@ -0,0 +1,75 @@ +'use strict'; + +// Release-local entrypoints used by the independent platform updater. The bundle +// is verified before invocation; no paths or commands come from a browser. +const fs = require('node:fs'); +const path = require('node:path'); +const { verifyPreparedHostArtifact } = require('./oci-host-artifact'); +function atomic(file, contents, mode = 0o600) { + const tmp = `${file}.new-${process.pid}`; + fs.writeFileSync(tmp, contents, { flag: 'wx', mode }); + const fd = fs.openSync(tmp, 'r'); try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); } + fs.renameSync(tmp, file); + const dir = fs.openSync(path.dirname(file), 'r'); try { fs.fsyncSync(dir); } finally { fs.closeSync(dir); } +} +async function input() { + const chunks = []; let size = 0; + for await (const chunk of process.stdin) { size += chunk.length; if (size > 4096) throw new Error(); chunks.push(chunk); } + return JSON.parse(Buffer.concat(chunks).toString('utf8')); +} +async function main(action, artifactRoot) { + const config = JSON.parse(fs.readFileSync(path.join(artifactRoot, 'deployment.json'), 'utf8')); + if (artifactRoot !== `/opt/dispatch-platform/releases/${config.releaseId}/core-artifact`) throw new Error(); + if (action === 'prepare-backup') { + if (process.geteuid() !== 0 || !fs.existsSync('/etc/dispatch/offsite-backup.json')) throw new Error(); + if (require('./core-recovery-host').rootArtifact(config.releaseId).root !== artifactRoot) throw new Error(); + const unit = `dispatch-backup-enable-${config.releaseId}.service`; + atomic(`/etc/systemd/system/${unit}`, `[Unit]\nDescription=Prepare complete Dispatch recovery backups\nAfter=network-online.target\n\n[Service]\nType=oneshot\nUMask=0077\nExecStart=/usr/bin/node --no-warnings ${artifactRoot}/code/core/installations/bin/dispatch-offsite-backup enable\nTimeoutStartSec=1h\n`, 0o644); + for (const args of [['daemon-reload'], ['start', '--no-block', unit]]) { + if (require('node:child_process').spawnSync('/usr/bin/systemctl', args, { timeout: 30000 }).status !== 0) throw new Error(); + } + return; + } + if (action === 'switch-host') { + if (process.geteuid() !== 0) throw new Error(); + const helper = `/opt/dispatch-control/releases/${config.releaseId}/host-helper-artifact`; + verifyPreparedHostArtifact(`${helper}/core/installations/bin/dispatch-oci-host-issuer`, config.releaseId, config.helperManifestSha256, 'dispatch-oci-host-issuer'); + const hostFile = '/etc/dispatch/oci-host.json'; + const current = JSON.parse(fs.readFileSync(hostFile, 'utf8')); + current.controlReleaseId = config.releaseId; current.helperManifestSha256 = config.helperManifestSha256; + atomic(hostFile, JSON.stringify(current) + '\n'); + const link = `/opt/dispatch-control/current.new-${process.pid}`; + fs.symlinkSync(`/opt/dispatch-control/releases/${config.releaseId}`, link); + fs.renameSync(link, '/opt/dispatch-control/current'); + // The watcher remains a separate service, but shares this installed Core + // release's code. Reloading its definition does not interrupt a running job. + atomic('/etc/systemd/system/dispatch-release-watch.service', + `[Unit]\nDescription=Discover and prepare Dispatch releases\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nExecStart=/usr/bin/node --no-warnings ${artifactRoot}/code/core/installations/bin/dispatch-release-watch\nEnvironment=PATH=/usr/bin:/bin\nUMask=0022\nTimeoutStartSec=45min\nTimeoutStopSec=30s\nKillMode=control-group\nPrivateTmp=true\nProtectSystem=full\nReadWritePaths=/etc/sudoers.d /etc/apparmor.d\nProtectKernelTunables=true\nProtectControlGroups=true\n`, 0o644); + require('./release-ready-notify').install(config.localRoot); + if (require('node:child_process').spawnSync('/usr/bin/systemctl', ['daemon-reload'], { timeout: 30000 }).status !== 0) throw new Error(); + if (require('node:child_process').spawnSync('/usr/bin/systemctl', ['enable', '--now', 'dispatch-release-watch.path'], { timeout:30000 }).status !== 0) throw new Error(); + // Bootstrap the independently supervised backup worker from this verified + // release. It confirms encrypted uploads before making backups + // mandatory. A Core restart must not kill an in-progress backup export. + if (fs.existsSync('/etc/dispatch/offsite-backup.json')) { + const { spawnSync } = require('node:child_process'); + const enableUnit = `dispatch-backup-enable-${config.releaseId}.service`; + atomic(`/etc/systemd/system/${enableUnit}`, + `[Unit]\nDescription=Verify and enable Dispatch backup protection\nAfter=network-online.target\nWants=network-online.target\n\n[Service]\nType=oneshot\nUMask=0077\nExecStart=/usr/bin/node --no-warnings ${artifactRoot}/code/core/installations/bin/dispatch-offsite-backup enable\nTimeoutStartSec=1h\n`, 0o644); + for (const args of [['daemon-reload'], ['start', '--no-block', enableUnit]]) { + if (spawnSync('/usr/bin/systemctl', args, {timeout:30000}).status !== 0) throw new Error(); + } + } + return; + } + if (process.geteuid() === 0 || !['apply', 'verify'].includes(action)) throw new Error(); + const context = await input(); + if (context.protocolVersion !== 1 || context.action !== action || context.releaseId !== config.releaseId + || context.sourceCommit !== config.sourceCommit || context.version !== config.version + || !/^rollout_[a-f0-9]{32}$/.test(context.rolloutId)) throw new Error(); + const recovery = require('./core-recovery').createCoreRecovery({ localRoot: config.localRoot, context, + adapter: require('./core-recovery-host').createHostRecovery(config, artifactRoot) }); + await recovery[action](); + process.stdout.write(JSON.stringify({ ok: true, releaseId: config.releaseId, version: config.version, sourceCommit: config.sourceCommit }) + '\n'); +} +module.exports = { main }; diff --git a/core/core/installations/src/create-bridge-artifact.js b/core/core/installations/src/create-bridge-artifact.js new file mode 100644 index 0000000..4a2843b --- /dev/null +++ b/core/core/installations/src/create-bridge-artifact.js @@ -0,0 +1,37 @@ +'use strict'; + +const path = require('node:path'); +const { createImmutableArtifact } = require('./immutable-artifact'); + +const PROJECT_ROOT = path.resolve(__dirname, "../../.."); +const BRIDGE_ARTIFACT_FILES = Object.freeze([ + 'core/agent-bridge/src/bridge.js', + 'core/agent-bridge/src/forwarding.js', + 'core/agent-bridge/src/service-cli.js', + 'core/runtime-host-identity.js', + 'shared/contracts/src/input.js', + 'shared/contracts/src/paycom-setup.js', + 'shared/contracts/src/connections.js', + 'shared/contracts/src/installation.js', + 'shared/contracts/src/result.js', + 'shared/contracts/src/sync.js', + 'shared/contracts/src/workforce.js', + 'shared/agent/capacity.js', + 'shared/agent/framing.js', + 'shared/agent/protocol.js', + 'shared/gateway/protocol.js', + 'shared/gateway/strict-json.js', +]); + +function main(argv = process.argv.slice(2)) { + if (argv.length !== 1) throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); + const receipt = createImmutableArtifact({ + projectRoot: PROJECT_ROOT, target: argv[0], sourceFiles: BRIDGE_ARTIFACT_FILES, + }); + process.stdout.write(`${JSON.stringify({ ok: true, status: 'bridge_artifact_created', ...receipt })}\n`); +} + +if (require.main === module) { + try { main(); } catch { process.stderr.write('bridge artifact creation failed\n'); process.exitCode = 1; } +} +module.exports = { BRIDGE_ARTIFACT_FILES, main }; diff --git a/core/core/installations/src/create-host-helper-artifact.js b/core/core/installations/src/create-host-helper-artifact.js new file mode 100644 index 0000000..375a97d --- /dev/null +++ b/core/core/installations/src/create-host-helper-artifact.js @@ -0,0 +1,55 @@ +'use strict'; + +const path = require('node:path'); +const { createImmutableArtifact } = require('./immutable-artifact'); + +const PROJECT_ROOT = path.resolve(__dirname, "../../.."); +const HOST_HELPER_ARTIFACT_FILES = Object.freeze([ + 'core/installations/bin/dispatch-oci-host-helper', + 'core/installations/bin/dispatch-oci-host-issuer', + 'core/installations/bin/dispatch-oci-tenant-backup-helper', + 'core/installations/src/oci-host-helper.js', + 'core/installations/src/oci-helper-input.js', + 'core/installations/src/oci-host-authority.js', + 'core/installations/src/oci-host-issuer.js', + 'core/installations/src/oci-host-artifact.js', + 'core/installations/src/oci-host-account-registry.js', + 'core/installations/src/oci-host-executor.js', + 'core/installations/src/oci-deployment.js', + 'core/installations/src/native-deployment.js', + 'core/installations/src/native-runtime-artifact.js', + 'core/installations/src/native-runtime-archive.py', + 'core/installations/src/backups.js', + 'core/runtime-host-identity.js', + 'shared/paths/runtime-paths.js', + 'shared/contracts/src/input.js', + 'shared/contracts/src/paycom-setup.js', + 'shared/contracts/src/connections.js', + 'shared/contracts/src/installation.js', + 'shared/contracts/src/publication-baseline.js', + 'shared/contracts/src/result.js', + 'shared/contracts/src/sync.js', + 'shared/contracts/src/workforce.js', + 'shared/agent/framing.js', + 'shared/agent/protocol.js', + 'shared/gateway/protocol.js', + 'shared/gateway/strict-json.js', +]); +const EXECUTABLES = new Set([ + 'core/installations/bin/dispatch-oci-host-helper', + 'core/installations/bin/dispatch-oci-host-issuer', + 'core/installations/bin/dispatch-oci-tenant-backup-helper', +]); + +function main(argv = process.argv.slice(2)) { + if (argv.length !== 1) throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); + const receipt = createImmutableArtifact({ + projectRoot: PROJECT_ROOT, target: argv[0], sourceFiles: HOST_HELPER_ARTIFACT_FILES, executables: EXECUTABLES, includeModes: true, + }); + process.stdout.write(`${JSON.stringify({ ok: true, status: 'host_helper_artifact_created', ...receipt })}\n`); +} + +if (require.main === module) { + try { main(); } catch { process.stderr.write('host helper artifact creation failed\n'); process.exitCode = 1; } +} +module.exports = { HOST_HELPER_ARTIFACT_FILES, main }; diff --git a/core/core/installations/src/create-platform-core-artifact.js b/core/core/installations/src/create-platform-core-artifact.js new file mode 100644 index 0000000..f79a7ba --- /dev/null +++ b/core/core/installations/src/create-platform-core-artifact.js @@ -0,0 +1,59 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { spawnSync } = require('node:child_process'); +const { PROJECT_ROOT } = require('../../../shared/paths/runtime-paths'); +const sha = value => crypto.createHash('sha256').update(value).digest('hex'); +function git(args) { + const result = spawnSync('/usr/bin/git', args, { cwd: PROJECT_ROOT, encoding: 'utf8', maxBuffer: 1024 * 1024 }); + if (result.status !== 0) throw new Error(); return result.stdout; +} +function main(argv = process.argv.slice(2)) { + if (argv.length !== 2) throw new Error(); + const [configFile, target] = argv; + const config = JSON.parse(fs.readFileSync(configFile, 'utf8')); + if (Object.keys(config).sort().join(',') !== 'localRoot,port,publicOrigin,releaseId,unitRoot,version' + || !/^[a-z][a-z0-9_.-]{2,95}$/.test(config.releaseId) + || !/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?$/.test(config.version) + || !/^https:\/\/[a-zA-Z0-9.-]+(?::[0-9]+)?$/.test(config.publicOrigin) + || !Number.isInteger(config.port) || config.port < 1024 || config.port > 65535) throw new Error(); + for (const value of [config.localRoot, config.unitRoot, target]) { + if (typeof value !== 'string' || !/^\/[a-zA-Z0-9_./-]+$/.test(value) || path.resolve(value) !== value + || value === PROJECT_ROOT || value.startsWith(`${PROJECT_ROOT}/`)) throw new Error(); + } + if (git(['status', '--porcelain']).trim()) throw new Error('clean_checkout_required'); + config.sourceCommit = git(['rev-parse', 'HEAD']).trim(); + if (fs.existsSync(target)) throw new Error(); + fs.mkdirSync(target, { mode: 0o700 }); + require('./create-host-helper-artifact').main([path.join(target, 'host-helper-artifact')]); + config.helperManifestSha256 = sha(fs.readFileSync(path.join(target, 'host-helper-artifact/manifest.json'))); + const root = path.join(target, 'core-artifact'); fs.mkdirSync(root, { mode: 0o755 }); + const write = (relative, contents, mode = 0o444) => { + const file = path.join(root, relative); fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o755 }); + fs.writeFileSync(file, contents, { flag: 'wx', mode }); + }; + for (const entry of git(['ls-tree', '-r', '-z', 'HEAD']).split('\0').filter(Boolean)) { + const match = /^(\d+) blob ([a-f0-9]{40})\t(.+)$/.exec(entry); + if (!match) throw new Error('unsupported_git_entry'); + const [, mode, object, relative] = match; + if (!(/^(core|host|dashboard|shared|sdk|runtime|plugins)\//.test(relative) || ['bin/dispatch-api', 'bin/dispatch-dashboard', 'bin/dispatch-access-admin', 'bin/dispatch-plugin-collector'].includes(relative)) + || /\/(tests|examples|docs)\//.test(relative)) continue; + if (relative.startsWith('plugins/') && !/^plugins\/[^/]+\/(dashboard\/|dispatch-plugin\.json$)/.test(relative)) continue; + if (!['100644', '100755'].includes(mode)) throw new Error('unsupported_git_mode'); + const blob = spawnSync('/usr/bin/git', ['cat-file', 'blob', object], { cwd: PROJECT_ROOT, maxBuffer: 16 * 1024 * 1024 }); + if (blob.status !== 0 || blob.error) throw new Error('git_blob_unavailable'); + write(`code/${relative}`, blob.stdout, mode === '100755' ? 0o555 : 0o444); + } + for (const file of require('./release-frontend').buildFrontend(PROJECT_ROOT, config.sourceCommit, target)) { + write(file.path, Buffer.from(file.data, 'base64')); + } + const manifestSha256 = require('./core-artifact-layout').finishCoreArtifact(root, config); + process.stdout.write(JSON.stringify({ status: 'platform_core_artifact_created', sourceCommit: config.sourceCommit, + core: { artifactPath: `/opt/dispatch-platform/releases/${config.releaseId}/core-artifact`, manifestSha256 }, + helperManifestSha256: config.helperManifestSha256 }) + '\n'); +} +if (require.main === module) { + try { main(); } catch { process.stderr.write('platform_core_artifact_failed\n'); process.exitCode = 1; } +} +module.exports = { main }; diff --git a/core/core/installations/src/diagnostics-activation.js b/core/core/installations/src/diagnostics-activation.js new file mode 100644 index 0000000..ed03bee --- /dev/null +++ b/core/core/installations/src/diagnostics-activation.js @@ -0,0 +1,124 @@ +"use strict"; +// Synthetic provider evidence is restricted to explicitly created diagnostic DSPs. +const assert = require("node:assert/strict"); +const { + createAccessInstallationActivationAuthority, +} = require("../../accounts/src/installation-activation"); +const { + runManagedPaycomActivation, + ACTIVATION_INFRASTRUCTURE_GATES, +} = require("./activation"); +const { + INSTALLATION_ACTIVATION_RUNS, +} = require("../../../shared/contracts/src"); +async function activateSyntheticDsp({ + store, + organizationId, + workerId, + invoke, +}) { + if ( + !store.db + .prepare("SELECT 1 FROM diagnostic_dsps WHERE organization_id=?") + .get(organizationId) + ) + throw new Error("diagnostic_dsp_required"); + const authority = createAccessInstallationActivationAuthority({ + store, + organizationId, + authorityScope: "platform_diagnostics", + workerId, + idempotencyKey: "diagnostics:activation:" + organizationId, + releaseId: store.installationControl(organizationId).releaseId, + }); + const context = authority.inspect(), + key = context.manifest.runtime.key; + let audited; + function audit(seed) { + const runs = INSTALLATION_ACTIVATION_RUNS.map((run, i) => ({ + id: `run_fixture_${i}`, + taskId: run.taskId, + plan: run.plan, + method: run.method, + })); + const pub = (value, runId, originRunId) => ({ + id: value.publicationId, + runId, + originRunId, + contentSha256: value.contentSha256, + batchBound: true, + }); + return { + definitionDigest: "a".repeat(64), + requestDigest: "b".repeat(64), + previewDigest: "c".repeat(64), + batchId: "batch_lab_" + organizationId, + preparationRunId: "run_periods_fixture", + target: seed.target || "2026-09-05", + runs, + publications: { + payPeriods: { + id: seed.payPeriods.publicationId, + runId: "run_periods_fixture", + originRunId: "run_periods_fixture", + contentSha256: seed.payPeriods.contentSha256, + batchBound: false, + }, + roster: pub(seed.roster, runs[0].id, "run_fixture_roster"), + timecards: pub(seed.timecards, runs[1].id, "run_fixture_timecards"), + resourceLinks: pub(seed.links, runs[3].id, "run_fixture_links"), + }, + capturedAt: new Date().toISOString(), + }; + } + const result = await runManagedPaycomActivation({ + authority, + runtime: { + verifyInfrastructure: async () => { + const health = await invoke(key, "health", {}); + assert.equal(health.ok, true); + return { + runtimeKey: key, + ...Object.fromEntries( + ACTIVATION_INFRASTRUCTURE_GATES.map((k) => [k, true]), + ), + }; + }, + testProvider: async () => ({ + profileId: "paycom-main", + provider: "paycom", + status: "authenticated", + testedAt: new Date().toISOString(), + }), + configure: async () => { + const response = await invoke(key, "diagnostics.seed", { + requestId: organizationId, + }); + if (!response?.ok) + throw Object.assign(new Error("first_publication_failed"), { + code: "first_publication_failed", + }); + audited = audit(response.data); + return { + digest: audited.definitionDigest, + collectors: 1, + sources: 1, + plans: 15, + syncs: 1, + }; + }, + publishFirst: async () => ({ + batchId: audited.batchId, + preparationRunId: audited.preparationRunId, + status: "succeeded", + runCount: audited.runs.length, + succeededRuns: audited.runs.length, + failedRuns: 0, + cancelledRuns: 0, + }), + verifyPublication: async () => audited, + }, + }); + return result; +} +module.exports = { activateSyntheticDsp }; diff --git a/core/core/installations/src/diagnostics-worker.js b/core/core/installations/src/diagnostics-worker.js new file mode 100644 index 0000000..94a728e --- /dev/null +++ b/core/core/installations/src/diagnostics-worker.js @@ -0,0 +1,31 @@ +'use strict'; + +const { activateSyntheticDsp } = require('./diagnostics-activation'); + +function createDiagnosticsWorker({ store, invoke, activate = activateSyntheticDsp }) { + async function runPending(workerId, limit = 20) { + const rows = store.db.prepare(`SELECT d.*,i.status AS installation_status FROM diagnostic_dsps d + JOIN installations i ON i.organization_id=d.organization_id JOIN organizations o ON o.id=d.organization_id + WHERE d.status='pending' AND i.backend='native_service_v1' AND o.status!='suspended' + AND i.status IN ('waiting_for_provider_auth','verifying','ready','failed') ORDER BY d.created_at LIMIT ?`).all(limit); + let completed = 0, failed = 0; + for (const [index, row] of rows.entries()) { + let status = 'failed'; + try { + if (row.installation_status !== 'failed') { + const result = await activate({ store, organizationId: row.organization_id, workerId: `${workerId}_${index}`, invoke }); + if (result.ok) status = 'ready'; + else if (result.status === 'installation_operation_in_progress') continue; + } + } catch (error) { + if (error.code === 'installation_operation_in_progress') continue; + } + store.db.prepare("UPDATE diagnostic_dsps SET status=? WHERE organization_id=? AND status='pending'") + .run(status, row.organization_id); + if (status === 'ready') completed += 1; else failed += 1; + } + return { processed: rows.length, completed, failed }; + } + return { runPending }; +} +module.exports = { createDiagnosticsWorker }; diff --git a/core/core/installations/src/drain-worker.js b/core/core/installations/src/drain-worker.js new file mode 100644 index 0000000..89bfca6 --- /dev/null +++ b/core/core/installations/src/drain-worker.js @@ -0,0 +1,44 @@ +'use strict'; +// Drain bounded, productive passes. A wait on Core, uploads, a lease, or user +// input exits cleanly; the owning service/event or fallback timer wakes us. +async function drain(run, { maxPasses = 100, clock = Date.now, budgetMs = 2 * 60 * 60 * 1000 } = {}) { + const deadline = clock() + budgetMs; + let result; + for (let pass = 0; pass < maxPasses && clock() < deadline; pass++) { + result = await run(); + if (!result?.progressed || result.failed) break; + } + return result; +} +function progressKey(db) { + return JSON.stringify([ + db.prepare('SELECT id,status,updated_at FROM platform_rollouts ORDER BY created_at DESC LIMIT 1').all(), + db.prepare("SELECT rollout_id,organization_id,status,job_id FROM platform_rollout_members WHERE rollout_id=(SELECT id FROM platform_rollouts ORDER BY created_at DESC LIMIT 1) ORDER BY position").all(), + db.prepare("SELECT id,status,phase,job_id FROM platform_backup_requests WHERE status IN ('queued','running') ORDER BY id").all(), + db.prepare("SELECT id,status,next_stage FROM installation_lifecycle_jobs WHERE status IN ('queued','running') ORDER BY id").all(), + db.prepare("SELECT id,status FROM installation_onboarding_requests WHERE status IN ('queued','running') ORDER BY id").all(), + db.prepare("SELECT organization_id,status FROM diagnostic_dsps WHERE status <> 'ready' ORDER BY organization_id").all(), + ]); +} +function backupQueueKey(config) { + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(require('node:path').join(config.localRoot, 'data/access-control/access-control.sqlite3'), { readOnly: true }); + try { + return JSON.stringify(['platform_backup_records', 'installation_backups'].map(table => { + if (!db.prepare("SELECT 1 FROM sqlite_schema WHERE name=?").get(table)) return []; + return db.prepare(table === 'installation_backups' + ? "SELECT id,status,completed_at FROM installation_backups ORDER BY id" + : "SELECT id,deleted_at FROM platform_backup_records ORDER BY id").all(); + })); + } finally { db.close(); } +} +function workPending(config) { + const { DatabaseSync } = require('node:sqlite'); + const db = new DatabaseSync(require('node:path').join(config.localRoot, 'data/access-control/access-control.sqlite3'), { readOnly: true }); + try { + return Boolean(db.prepare("SELECT 1 FROM platform_rollouts WHERE status='running' LIMIT 1").get() + || db.prepare("SELECT 1 FROM platform_backup_requests WHERE status IN ('queued','running') LIMIT 1").get() + || db.prepare("SELECT 1 FROM installation_lifecycle_jobs WHERE status IN ('queued','running') LIMIT 1").get()); + } finally { db.close(); } +} +module.exports = { drain, progressKey, backupQueueKey, workPending }; diff --git a/core/core/installations/src/dsp-backup-deletion.js b/core/core/installations/src/dsp-backup-deletion.js new file mode 100644 index 0000000..e9e4f2b --- /dev/null +++ b/core/core/installations/src/dsp-backup-deletion.js @@ -0,0 +1,110 @@ +'use strict'; +// Runs only inside the root exporter's flock. Core sees an opaque completion +// receipt; storage credentials and restic object identifiers never enter its API. +const fs = require('node:fs'); +const path = require('node:path'); +const { atomic, privateJson } = require('./release-delivery-files'); +const { receiptKey, publicRootJson } = require('./offsite-policy'); +const { HOST_TENANT_ROOT, opaqueRuntimeSuffix } = require('../../runtime-host-identity'); +const fail = () => { throw Object.assign(Error('offsite_backup_unavailable'), { code: 'offsite_backup_unavailable' }); }; +const idPattern = /^(backup|breq)_[a-f0-9]{32}$/; + +function mayContainOrganization(proof, organizationId) { + // Older receipts only enumerated installed runtimes, omitting shared identity + // records. Only the complete inventory can prove a Core archive unrelated. + return !(proof?.organizationInventoryVersion === 1 && Array.isArray(proof.organizationIds) + && proof.organizationIds.every(id => typeof id === 'string' && /^org_[a-z0-9_]+$/.test(id)) + && !proof.organizationIds.includes(organizationId)); +} + +async function purgeDspBackups({ config, jobs, backups, records, sets = [], storage, run, workRoot, receiptRoot, + ownerUid = 0, clock = Date.now }) { + const deletedRuntimeKeys = [], deletingOrganizations = new Set(); + let failed = 0; + for (const job of jobs) { + deletingOrganizations.add(job.organization_id); + deletedRuntimeKeys.push(job.runtime_key); + try { + if (!/^life_[a-f0-9]{32}$/.test(job.id) || !/^[a-z][a-z0-9_-]{2,95}$/.test(job.runtime_key) + || !['platform_removal', 'platform_lifecycle'].includes(job.authority_scope) || JSON.parse(JSON.parse(job.stage_receipts_json).__request).operation !== 'destroy') fail(); + const proofFile = path.join(receiptRoot, `deleted-${job.id}.json`); + if (fs.existsSync(proofFile)) { + const proof = publicRootJson(proofFile, false, ownerUid); + if (proof.status !== 'destroyed' || proof.jobId !== job.id || proof.organizationId !== job.organization_id + || proof.runtimeKey !== job.runtime_key) fail(); + continue; + } + const selected = records.filter(row => { + if (row.organization_id === job.organization_id) return true; + if (row.kind !== 'core') return false; + if (!idPattern.test(row.id)) fail(); + const file = path.join(workRoot, 'archives', `${row.id}.json`); + const proof = fs.existsSync(file) ? privateJson(file, ownerUid) : null; + if (proof && (proof.id !== row.id || proof.kind !== 'core' || proof.organizationId !== null)) fail(); + if (!proof && JSON.parse(row.metadata_json || '{}').scope === 'core') { + const source = path.join(config.localRoot, 'backups/scheduled-core', row.id); + if (fs.existsSync(source)) { + require('./offsite-backup').verifySnapshot(source, config.coreUid); + const manifest = JSON.parse(fs.readFileSync(path.join(source, 'manifest.json'))); + // A newly created Core snapshot may not yet have an export receipt. + // Its validated isolated payload still proves it contains no DSP. + if (manifest.version === 3 && manifest.scope === 'core') return false; + } + } + return mayContainOrganization(proof, job.organization_id); + }); + if (selected.some(row => !['dsp', 'core'].includes(row.kind) || !idPattern.test(row.id))) fail(); + const ids = new Set([...selected.map(row => row.id), ...backups.filter(row => row.organization_id === job.organization_id).map(row => row.id)]); + if ([...ids].some(id => !idPattern.test(id))) fail(); + const installation = path.join(HOST_TENANT_ROOT, opaqueRuntimeSuffix(job.runtime_key), 'runtime', job.runtime_key); + const tags = new Set([...ids].map(id => receiptKey(path.join(installation, 'backups', id)))); + for (const row of selected.filter(row => row.kind === 'core')) tags.add(receiptKey(path.join(config.localRoot, 'backups/scheduled-core', row.id))); + for (const name of fs.readdirSync(receiptRoot)) { + if (!/^[a-f0-9]{64}\.json$/.test(name)) continue; + const proof = publicRootJson(path.join(receiptRoot, name), false, ownerUid, 1024 * 1024); + if (Array.isArray(proof.organizationIds) && mayContainOrganization(proof, job.organization_id)) tags.add(name.slice(0, -5)); + } + const snapshots = () => { + const result = run(['snapshots']).flat(); + if (result.some(item => !/^[a-f0-9]{64}$/.test(item.id) || !Array.isArray(item.tags))) fail(); + return result; + }; + const before = snapshots(); + const targets = before.filter(item => item.tags.some(tag => tags.has(tag))); + if (targets.some(item => item.tags.length !== 1 || item.hostname !== 'dispatch')) fail(); + const remaining = before.filter(item => !targets.includes(item)).map(item => item.id).sort(); + const tiers = [null, 7, 30, 90, 365]; + const prefixes = ids.size ? tiers.map(tier => `archives/${tier === null ? 'all' : tier}/`) : []; + // Prune is necessary on retry even when an earlier attempt already forgot + // the target snapshot IDs. Restic retains all still-referenced shared data. + if (ids.size || targets.length) prefixes.push(...['data/', 'index/', 'snapshots/'].map(part => `${config.prefix}/${part}`)); + await storage.withDeletionAccess(prefixes, async () => { + for (const set of sets.filter(set=>JSON.parse(set.members_json).some(member=>member.organizationId===job.organization_id))) { + if(!/^breq_[a-f0-9]{32}$/.test(set.id))fail(); + await storage.removeSet(set.id); + fs.rmSync(path.join(workRoot,`set-${set.id}.json`),{force:true}); + } + for (const id of ids) for (const retentionDays of tiers) await storage.removePermanent({ id, retentionDays }); + if (targets.length) run(['forget', ...targets.map(item => item.id)]); + if (ids.size || targets.length) run(['prune', '--max-unused', '0']); + if (JSON.stringify(snapshots().map(item => item.id).sort()) !== JSON.stringify(remaining)) fail(); + run(['check', '--read-data']); + }); + for (const row of selected) { + const file = path.join(workRoot, 'archives', `${row.id}.json`); + if (fs.existsSync(file)) { + const receipt = privateJson(file, ownerUid); + if (receipt.organizationId !== row.organization_id || receipt.id !== row.id) fail(); + atomic(file, { ...receipt, status: 'destroyed', deletedAt: clock() }); + } + } + for (const tag of tags) fs.rmSync(path.join(receiptRoot, `${tag}.json`), { force: true }); + // Publish only after every remote deletion and lock restoration succeeds. + atomic(proofFile, { schemaVersion: 1, status: 'destroyed', jobId: job.id, + organizationId: job.organization_id, runtimeKey: job.runtime_key, completedAt: clock() }, 0o644); + fs.chmodSync(proofFile, 0o644); + } catch { failed++; } + } + return { deletedRuntimeKeys, deletingOrganizations, failed }; +} +module.exports = { purgeDspBackups, mayContainOrganization }; diff --git a/core/core/installations/src/github-release-notes.js b/core/core/installations/src/github-release-notes.js new file mode 100644 index 0000000..53cd4ba --- /dev/null +++ b/core/core/installations/src/github-release-notes.js @@ -0,0 +1,78 @@ +'use strict'; +// GitHub-only authoring metadata never enters the installation manifest, rich +// Updates sidecar, or dashboard popup. +const REPO = 'https://github.com/example-organization/dispatch-platform'; +const fail = () => { throw Object.assign(new Error('release_notes_invalid'), { code: 'release_notes_invalid' }); }; +function fields(value, allowed) { + if (!value || typeof value !== 'object' || Array.isArray(value) + || Object.keys(value).some(key => !allowed.includes(key))) fail(); +} +function pullRequests(value) { + if (!Array.isArray(value) || value.length > 20) fail(); + const numbers = new Set(); + for (const reference of value) { + const number = typeof reference === 'number' ? reference : reference?.number; + if (!Number.isSafeInteger(number) || number < 1 || numbers.has(number)) fail(); + numbers.add(number); + // Integer references remain readable for older authoring inputs. New inputs + // pair each PR with its verified GitHub author login, including bot logins. + if (typeof reference !== 'number') { + fields(reference, ['number', 'author']); + if (typeof reference.author !== 'string' + || !/^[a-zA-Z0-9][a-zA-Z0-9-]{0,38}(?:\[bot\])?$/.test(reference.author)) fail(); + } + } +} +function githubAuthoring(input) { + if (!input || Array.isArray(input)) return { input }; + const hasSummary = Object.hasOwn(input, 'github'); + const hasEntries = input.changelog?.some(change => Object.hasOwn(change, 'github')); + if (!hasSummary && !hasEntries) return { input }; + if (!Array.isArray(input.changelog)) fail(); + const metadata = hasSummary ? input.github : {}; + fields(metadata, ['summary', 'previousTag']); + if (Object.hasOwn(metadata, 'summary') && (typeof metadata.summary !== 'string' + || !metadata.summary.trim() || metadata.summary.length > 600 || /[\x00-\x1f\x7f]/.test(metadata.summary))) fail(); + if (Object.hasOwn(metadata, 'previousTag') && (typeof metadata.previousTag !== 'string' + || !/^[A-Za-z0-9][A-Za-z0-9._/+\-]{0,159}$/.test(metadata.previousTag))) fail(); + const changes = input.changelog.map(change => { + const github = Object.hasOwn(change, 'github') ? change.github : {}; + fields(github, ['pullRequests', 'maintenance']); + if (Object.hasOwn(github, 'maintenance') && typeof github.maintenance !== 'boolean') fail(); + if (Object.hasOwn(github, 'pullRequests')) pullRequests(github.pullRequests); + return github; + }); + const { github, ...content } = input; + return { input: { ...content, changelog: input.changelog.map(({ github, ...change }) => change) }, + github: { ...metadata, changes } }; +} +// All authored fields are plain text. Only the renderer supplies Markdown and links. +const escape = value => value.replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>') + .replace(/[\\`*_{}\[\]()#+.!|~\-]/g, '\\$&'); +function markdown(version, changelog, notes, github = {}) { + const entries = (notes?.changelog || changelog).map((item, index) => ({ ...item, github: github.changes?.[index] || {} })); + const category = item => item.github.maintenance ? 'Maintenance' + : ({ added: 'New', changed: 'Improved', improved: 'Improved', fixed: 'Fixed', removed: 'Removed' })[item.kind]; + const row = item => { + const title = escape(item.title); + const description = item.description ? ` — ${escape(item.description)}` : ''; + const links = item.github.pullRequests?.length ? ` ${item.github.pullRequests.map(reference => { + const { number, author } = typeof reference === 'number' ? { number: reference } : reference; + const credit = author ? `by [@${escape(author)}](https://github.com/${encodeURIComponent(author)}) · ` : ''; + return `${credit}[PR #${number}](${REPO}/pull/${number})`; + }).join('; ')}` : ''; + const details = item.details ? `\n\n
\n Details\n\n ${escape(item.details).replaceAll('\n', '\n ')}\n\n
\n` : ''; + return `- **${title}**${description}${links}${details}`; + }; + // The release page already displays its title; its body starts with the summary. + const sections = []; + if (github.summary) sections.push(escape(github.summary)); + if (notes?.afterUpdating.length) sections.push(`## After updating\n\n${notes.afterUpdating.map(item => `- **${escape(item.title)}** — ${escape(item.description)}`).join('\n')}`); + for (const heading of ['New', 'Improved', 'Fixed', 'Removed', 'Maintenance']) { + const items = entries.filter(item => category(item) === heading); + if (items.length) sections.push(`## ${heading}\n\n${items.map(row).join('\n')}`); + } + if (github.previousTag) sections.push(`---\n\n[Full changelog](${REPO}/compare/${encodeURIComponent(github.previousTag)}...${encodeURIComponent(version)})`); + return `${sections.join('\n\n')}\n`; +} +module.exports = { githubAuthoring, markdown }; diff --git a/core/core/installations/src/host-recovery-bundle.js b/core/core/installations/src/host-recovery-bundle.js new file mode 100644 index 0000000..7beaa2d --- /dev/null +++ b/core/core/installations/src/host-recovery-bundle.js @@ -0,0 +1,447 @@ +'use strict'; +// Root-only recovery inventory. Account names, service names and root paths are +// derived from trusted host configuration, never from an API-supplied path. +const fs = require('node:fs'), path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { DatabaseSync } = require('node:sqlite'); +const capsule = require('./recovery-capsule'); +const { HOST_TENANT_ROOT, opaqueRuntimeSuffix, hostAccountName } = require('../../runtime-host-identity'); +const fail = () => { throw Error('host_recovery_unavailable'); }; +const SHARED_UNITS = ['dispatch-dashboard.service', 'dispatch-installation-reconcile.service', 'dispatch-installation-reconcile.timer', + 'dispatch-platform-update.service', 'dispatch-platform-update.timer', 'dispatch-cloudflared.service', 'dispatch-dashboard-tunnel.service', + 'dispatch-release-watch.service', 'dispatch-release-watch.timer', 'dispatch-release-watch.path', + 'dispatch-release-delivery.service', 'dispatch-release-delivery.timer', 'dispatch-offsite-backup.service', 'dispatch-offsite-backup.timer', 'dispatch-offsite-backup.path', 'dispatch-recovery-prewarm.service', 'dispatch-recovery-prewarm.timer']; +const PACKAGES = ['ca-certificates', 'python3', 'restic', 'patchelf', 'dbus-user-session', 'apparmor-utils', 'libnss3', + 'libatk-bridge2.0-0t64', 'libx11-xcb1', 'libxcomposite1', 'libxdamage1', 'libxrandr2', 'libgbm1', + 'libasound2t64', 'libcups2t64', 'libgtk-3-0t64', 'fonts-liberation']; +function command(binary, args, options = {}) { + const result = spawnSync(binary, args, { encoding: 'utf8', timeout: 60000, maxBuffer: 1024 * 1024, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8' }, ...options }); + if (result.status !== 0 || result.error) fail(); + return result.stdout.trim(); +} +function account(value) { + const parts = command('/usr/bin/getent', ['passwd', String(value)]).split(':'); + if (parts.length !== 7 || !/^[a-z_][a-z0-9_-]{0,63}$/.test(parts[0]) || !/^[1-9][0-9]*$/.test(parts[2]) + || !/^[1-9][0-9]*$/.test(parts[3]) || !path.isAbsolute(parts[5])) fail(); + return { name: parts[0], uid: Number(parts[2]), gid: Number(parts[3]), home: parts[5] }; +} +function systemctl(selected, args) { + if (selected.scope === 'system') return command('/usr/bin/systemctl', args); + return command('/usr/sbin/runuser', ['--user', selected.account.name, '--', '/usr/bin/env', + `XDG_RUNTIME_DIR=/run/user/${selected.account.uid}`, `DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/${selected.account.uid}/bus`, + '/usr/bin/systemctl', '--user', ...args]); +} +function supportedHost({ source = false } = {}) { + if (process.platform !== 'linux' || process.arch !== 'x64' || process.geteuid() !== 0) fail(); + const os = fs.readFileSync('/etc/os-release', 'utf8'); + if (!/^ID=ubuntu$/m.test(os) || !(source ? /^VERSION_ID="(?:24|26)\.04"$/m : /^VERSION_ID="24\.04"$/m).test(os)) fail(); +} +function captureHostRecovery({ config, destination, kind = 'core', organizationId = null, snapshotSource = null, shareReleases = false }) { + supportedHost({ source: true }); + const scopedCore = kind === 'core' && snapshotSource && JSON.parse(fs.readFileSync(path.join(snapshotSource, 'manifest.json'))).scope === 'core'; + const coreAccount = account(config.coreUid), userRoot = path.join(coreAccount.home, '.config/systemd/user'); + const database = path.join(config.localRoot, 'data/access-control/access-control.sqlite3'); + const db = new DatabaseSync(database, { readOnly: true }); + let installations; + try { + if (kind === 'core' && !scopedCore && db.prepare("SELECT 1 FROM installation_lifecycle_jobs WHERE status='running'").get()) fail(); + installations = scopedCore ? [] : db.prepare(`SELECT i.organization_id,i.runtime_key,i.release_id,i.backend,i.status,o.status AS organization_status + FROM installations i JOIN organizations o ON o.id=i.organization_id WHERE i.status NOT IN ('decommissioned','decommissioning') + ${kind === 'dsp' ? 'AND i.organization_id=?' : ''}`).all(...(kind === 'dsp' ? [organizationId] : [])); + if(kind==='dsp'&&snapshotSource) { + const job=db.prepare('SELECT j.* FROM installation_backups b JOIN installation_lifecycle_jobs j ON j.id=b.lifecycle_job_id WHERE b.id=? AND b.organization_id=?').get(path.basename(snapshotSource),organizationId); + if(job&&installations.length===1){const receipts=JSON.parse(job.stage_receipts_json);installations[0].status=job.starting_state;installations[0].sync_was_running=receipts.inspect_schedule?.syncWasRunning===true;} + } + } finally { db.close(); } + if (kind === 'dsp' && installations.length !== 1 || installations.some(i => i.backend !== 'native_service_v1')) fail(); + const accounts = [coreAccount], roots = [], services = [], releases = new Set(), hostAllocations=[]; + if(kind==='dsp') { + const host=require('./release-delivery-files').privateJson('/etc/dispatch/oci-host.json',0); + const registry=new DatabaseSync(path.join(host.stateRoot,'oci-host.sqlite3'),{readOnly:true}); + try {for(const item of installations){const allocation=registry.prepare('SELECT * FROM allocations WHERE runtime_key=?').get(item.runtime_key);if(!allocation)fail();hostAllocations.push(allocation);}}finally{registry.close();} + } + // The fenced host issuer runs as a separate unprivileged account. Restoring + // its sudo rule and numeric configuration without that account leaves new + // DSP provisioning broken even when existing DSP health checks pass. + if (kind === 'core' && fs.existsSync('/etc/dispatch/oci-host.json')) { + const host = require('./release-delivery-files').privateJson('/etc/dispatch/oci-host.json', 0); + for (const uid of [host.authorityUid, host.helperCallerUid]) { + if (!Number.isSafeInteger(uid) || uid < 1) fail(); + const selected = account(uid); + if (uid === host.helperCallerUid && selected.gid !== host.helperCallerGid) fail(); + if (!accounts.some(existing => existing.uid === uid)) accounts.push(selected); + } + } + function add(source, extra = {}) { + if (fs.existsSync(source)) roots.push({ source, target: source, ...extra }); + } + function unit(name, scope) { + const source = path.join(scope === 'system' ? '/etc/systemd/system' : userRoot, name); + if (!fs.existsSync(source)) return; + const real = fs.realpathSync(source); + if (!fs.statSync(real).isFile() || real !== source && !real.startsWith(path.join(config.localRoot, 'config/systemd') + '/')) fail(); + const text = fs.readFileSync(source, 'utf8'); + for (const match of text.matchAll(/\/opt\/(dispatch-(?:platform|runtime|control|updater|release-delivery))\/releases\/([a-z0-9][a-z0-9_.-]{2,95})\//g)) { + releases.add(`/opt/${match[1]}/releases/${match[2]}`); + } + const service = { name, scope, account: coreAccount, file: source }; + const state = systemctl(service, ['show', name, '--property=ActiveState', '--value']); + service.active = ['active', 'activating'].includes(state); + service.enabled = systemctl(service, ['show', name, '--property=UnitFileState', '--value']) === 'enabled'; + services.push(service); add(source); + } + if (kind === 'core') { + for (const name of SHARED_UNITS) { unit(name, 'user'); unit(name, 'system'); } + if (scopedCore) { + // Enumerate platform-owned roots positively: newly added DSP stores do + // not become Core backup payload merely by sharing the local directory. + const data=path.join(config.localRoot,'data'); + add(data,{exclude:fs.readdirSync(data).filter(name=>name!=='access-control'),overrides:{'access-control/access-control.sqlite3':path.join(snapshotSource,'access-control-before.sqlite3')}}); + roots.push(...require('./core-backup-files').recoveryFileRoots(config.localRoot, snapshotSource)); + add(path.join(config.localRoot,'config/systemd')); + } else for (const child of ['data','state','config','secrets']) add(path.join(config.localRoot,child), + child==='config'?{exclude:['core-maintenance.json']}:child==='data'&&snapshotSource?{overrides:{'access-control/access-control.sqlite3':path.join(snapshotSource,'access-control-before.sqlite3')}}:{}); + add('/etc/dispatch'); if (!scopedCore) add('/var/lib/dispatch-host'); add('/opt/dispatch-control/current'); + // Preserve discovery/verification metadata for earlier offsite archives. + // Their encrypted payloads remain remote; transfer scratch space is excluded. + if (!scopedCore) { add('/var/lib/dispatch-backup/archives'); add('/var/lib/dispatch-backup-receipts'); } + if (fs.existsSync('/opt/dispatch-control/current')) releases.add(fs.realpathSync('/opt/dispatch-control/current')); + for (const binary of ['cloudflared']) { + const source = [`/usr/bin/${binary}`, `/usr/local/bin/${binary}`, path.join(coreAccount.home, '.local/bin', binary)].find(file => fs.existsSync(file)); + if (source) add(fs.realpathSync(source), { target: `/usr/local/lib/dispatch-recovery/${binary}` }); + } + add('/etc/apparmor.d/dispatch-native-chrome'); + } + for (const item of installations) { + if (item.status === 'pending' && spawnSync('/usr/bin/getent', ['passwd', hostAccountName(item.runtime_key)]).status === 2) continue; + const runtimeAccount = account(hostAccountName(item.runtime_key)); accounts.push(runtimeAccount); + if (runtimeAccount.uid < 20000 || runtimeAccount.uid > 59999) fail(); + const suffix = opaqueRuntimeSuffix(item.runtime_key), root = path.join(HOST_TENANT_ROOT, suffix); + if (kind === 'dsp') add(path.join(config.localRoot, 'secrets/oci-runtime-agents', `${item.runtime_key}.token`)); + add(root, { excludeContents: [`runtime/${item.runtime_key}/run`, `runtime/${item.runtime_key}/backups`, + `runtime/${item.runtime_key}/staging`], exclude: ['engine-data', 'engine-config'], + ...(kind === 'dsp' && snapshotSource ? { overrides: Object.fromEntries([ + ['data', 'data'], ['state', 'state'], ['config', 'config'], ['secrets/auth-broker', 'auth-secrets'], + ].filter(([, child]) => fs.existsSync(path.join(snapshotSource, 'payload', child))) + .map(([target, child]) => [`runtime/${item.runtime_key}/${target}`, path.join(snapshotSource, 'payload', child)])) } : {}) }); + releases.add(`/opt/dispatch-runtime/releases/${item.release_id}`); + unit(`dispatch-runtime-agent-bridge-${suffix}.service`, 'system'); unit(`dispatch-dsp-${suffix}.service`, 'system'); + } + for (const root of [...releases]) { + const match = /^\/opt\/dispatch-platform\/releases\/([a-z0-9][a-z0-9_.-]{2,95})$/.exec(root); + if (match) for (const base of ['dispatch-runtime', 'dispatch-control']) { + const peer = `/opt/${base}/releases/${match[1]}`; + if (fs.existsSync(peer)) releases.add(peer); + } + } + // Finish shared immutable recovery payloads before freezing any legacy writers. + const artifacts = shareReleases ? [...releases].map(root => require('./recovery-artifacts').prepareRelease(config, root)) : []; + if (!shareReleases) for (const root of releases) add(root); + if (kind === 'core') for (const name of fs.readdirSync('/etc/sudoers.d')) { + if (!/^dispatch[-_a-z0-9.]*$/.test(name)) continue; + const source = path.join('/etc/sudoers.d', name), text = fs.readFileSync(source, 'utf8'); + const mentioned = [...text.matchAll(/\/opt\/(dispatch-[a-z]+)\/releases\/([a-z][a-z0-9_.-]{2,95})\//g)] + .map(m => `/opt/${m[1]}/releases/${m[2]}`); + if (mentioned.every(root => releases.has(root))) add(source); + } + let organizationIds = installations.map(i => i.organization_id); + const stopped = []; + let nodeStage; + try { + // Scoped exports read sealed lifecycle/Core snapshots. Stopping their owners + // here strands fenced jobs until lease expiry (and dashboard Requires= + // dependencies also stop reconciliation). Legacy whole-host capture alone + // still freezes writers, after the active-job checks above. + const writers = captureWriters({ services, snapshotSource, scopedCore, kind }); + for (const service of writers.sort((a, b) => Number(b.name.endsWith('.timer')) - Number(a.name.endsWith('.timer')))) { + stopped.push(service); systemctl(service, ['stop', service.name]); + } + if (kind === 'core' && !scopedCore) { + const frozen = new DatabaseSync(database, { readOnly: true }); + try { + if (frozen.prepare("SELECT 1 FROM installation_lifecycle_jobs WHERE status IN ('queued','running')").get() + || frozen.prepare("SELECT 1 FROM installation_provisioning_requests WHERE status IN ('pending','dispatched')").get()) fail(); + const current = frozen.prepare(`SELECT i.organization_id,i.runtime_key,i.release_id,i.backend,i.status,o.status AS organization_status + FROM installations i JOIN organizations o ON o.id=i.organization_id WHERE i.status NOT IN ('decommissioned','decommissioning')`).all(); + if (JSON.stringify(current) !== JSON.stringify(installations)) fail(); + organizationIds = frozen.prepare('SELECT id FROM organizations').all().map(row => row.id); + if (snapshotSource) { + const saved = new DatabaseSync(path.join(snapshotSource, 'access-control-before.sqlite3'), { readOnly: true }); + try { organizationIds.push(...saved.prepare('SELECT id FROM organizations').all().map(row => row.id)); } + finally { saved.close(); } + } + organizationIds = [...new Set(organizationIds)].sort(); + } finally { frozen.close(); } + } + if (kind === 'core') { + nodeStage = fs.mkdtempSync('/var/tmp/dispatch-node-capture-'); + const portable = path.join(nodeStage, 'runtime'); + require('./portable-node').bundleNode('/usr/bin/node', portable); + add(portable, { target: '/usr/local/lib/dispatch-node' }); + } + const metadata = { kind, ...(scopedCore ? {scope:'core'} : {}), organizationId, platform: 'ubuntu-24.04-amd64', localRoot: config.localRoot, + accounts, hostAllocations, services: servicesForRecovery(kind,services,installations), installations, packages: PACKAGES, + legacyEngine: [...releases].some(root => fs.existsSync(path.join(root, 'runtime-image.tar'))), createdAt: Date.now() }; + const captured = capsule.capture(destination, roots, metadata, new Set([0, ...accounts.map(a => a.uid)])); + const proof = require('./recovery-artifacts').append(destination, captured, artifacts); + return { ...proof, organizationIds, ...(kind === 'core' ? { organizationInventoryVersion: 1 } : {}) }; + } finally { + if (nodeStage) fs.rmSync(nodeStage, { recursive: true, force: true }); + let failure; + for (const service of stopped.reverse()) { + try { systemctl(service, ['start', service.name]); } catch (error) { failure = error; } + } + if (failure) throw failure; + } +} +function captureWriters({ services, snapshotSource, scopedCore, kind }) { + if (snapshotSource && (scopedCore || kind === 'dsp')) return []; + return services.filter(s => s.active && (s.name.endsWith('.timer') || s.name === 'dispatch-dashboard.service' + || s.name === 'dispatch-installation-reconcile.service' || /^dispatch-dsp-/.test(s.name)) + && !['dispatch-offsite-backup.timer', 'dispatch-platform-update.timer'].includes(s.name)); +} +function servicesForRecovery(kind,services,installations) { + // A DSP archive can be exported while its fenced backup job has the service + // stopped. Record the state to resume, without changing the capture cleanup. + if(kind!=='dsp')return services; + return services.map(service=>{ + const item=installations.find(i=>[`dispatch-dsp-${opaqueRuntimeSuffix(i.runtime_key)}.service`,`dispatch-runtime-agent-bridge-${opaqueRuntimeSuffix(i.runtime_key)}.service`].includes(service.name)); + if(!item)return service; + const running=item.status==='ready'&&item.organization_status==='active'; + return {...service,active:running,enabled:running}; + }); +} +function recoveryRoots(metadata, roots) { + if (!metadata || metadata.platform !== 'ubuntu-24.04-amd64' || !['core', 'dsp'].includes(metadata.kind) + || !Array.isArray(metadata.accounts) || !metadata.accounts.length || !Array.isArray(metadata.services) + || !Array.isArray(metadata.installations) || !Array.isArray(roots)) fail(); + const core = metadata.accounts[0]; + for (const a of metadata.accounts) { + if (!/^[a-z_][a-z0-9_-]{0,63}$/.test(a.name) || !Number.isSafeInteger(a.uid) || a.uid < 1 + || !Number.isSafeInteger(a.gid) || a.gid < 1 || !path.isAbsolute(a.home) + || path.resolve(a.home) !== a.home || /[\0\r\n]/.test(a.home)) fail(); + } + if (!core.home.startsWith('/home/') || core.home.split('/').length !== 3 + || !metadata.localRoot.startsWith(core.home + '/') || path.resolve(metadata.localRoot) !== metadata.localRoot) fail(); + const allowed = new Set(metadata.kind === 'core' ? ['/etc/dispatch', '/var/lib/dispatch-host', '/opt/dispatch-control/current', + '/var/lib/dispatch-backup/archives', '/var/lib/dispatch-backup-receipts', + '/etc/apparmor.d/dispatch-native-chrome', '/usr/local/lib/dispatch-node', '/usr/local/lib/dispatch-recovery/cloudflared', + ...['data', 'state', 'config', 'config/systemd', 'secrets','secrets/email', 'secrets/turnstile','secrets/cloudflared'].map(child => path.join(metadata.localRoot, child))] : []); + if (metadata.kind === 'core') + for (const root of roots) + if (require('./core-backup-files').isCoreFile(path.relative(metadata.localRoot, root))) allowed.add(root); + for (const item of metadata.installations) { + if (!/^[a-z][a-z0-9_-]{2,95}$/.test(item.organization_id) || item.backend !== 'native_service_v1') fail(); + const suffix = opaqueRuntimeSuffix(item.runtime_key), selected = metadata.accounts.find(a => a.name === hostAccountName(item.runtime_key)); + if (!selected && item.status === 'pending') continue; + if (!selected || selected.uid < 20000 || selected.uid > 59999 || selected.gid !== selected.uid + || selected.home !== `${HOST_TENANT_ROOT}/${suffix}/home`) fail(); + allowed.add(`${HOST_TENANT_ROOT}/${suffix}`); + allowed.add(path.join(metadata.localRoot, 'secrets/oci-runtime-agents', `${item.runtime_key}.token`)); + } + for (const service of metadata.services) { + if (!['user', 'system'].includes(service.scope) || typeof service.active !== 'boolean' || typeof service.enabled !== 'boolean' + || service.scope === 'user' && metadata.kind !== 'core' + || !SHARED_UNITS.includes(service.name) && !metadata.installations.some(item => + [`dispatch-dsp-${opaqueRuntimeSuffix(item.runtime_key)}.service`, + `dispatch-runtime-agent-bridge-${opaqueRuntimeSuffix(item.runtime_key)}.service`].includes(service.name))) fail(); + const expected = path.join(service.scope === 'system' ? '/etc/systemd/system' : path.join(core.home, '.config/systemd/user'), service.name); + if (service.file !== expected || JSON.stringify(service.account) !== JSON.stringify(core)) fail(); + allowed.add(expected); + } + for (const root of roots) { + if (/^\/opt\/dispatch-(platform|runtime|control|updater|release-delivery)\/releases\/[a-z0-9][a-z0-9_.-]{2,95}$/.test(root) + || metadata.kind === 'core' && /^\/etc\/sudoers\.d\/dispatch[-_a-z0-9.]*$/.test(root)) allowed.add(root); + if (!allowed.has(root)) fail(); + } + return allowed; +} +async function restoreHostRecovery({ directory, digest, installPackages = true }) { + supportedHost(); + const bytes = fs.readFileSync(path.join(directory, 'recovery.json')); + if (bytes.length > 128 * 1024 ** 2 || require('node:crypto').createHash('sha256').update(bytes).digest('hex') !== digest) fail(); + const raw = JSON.parse(bytes), allowed = recoveryRoots(raw.metadata, raw.roots); + const manifest = capsule.verify(directory, digest, allowed), metadata = manifest.metadata; + if (metadata.kind !== 'core') fail(); // Individual DSP restores use the fenced lifecycle API. + // The downloaded capsule is already on disk. Reserve space for staging, + // cross-filesystem promotion and installed prerequisites before changing accounts. + const payloadBytes = manifest.entries.reduce((sum, entry) => sum + (entry.type === 'file' ? entry.size : 0), 0); + const space = fs.statfsSync('/var/tmp'); + if (!Number.isSafeInteger(payloadBytes) || space.bavail * space.bsize < payloadBytes * 2 + 1024 ** 3) throw Error('recovery_space_unavailable'); + for (const root of manifest.roots) { + try { fs.lstatSync(root); fail(); } catch (error) { if (error.code !== 'ENOENT') throw error; } + } + for (const selected of metadata.accounts) { + const existing = spawnSync('/usr/bin/getent', ['passwd', selected.name], { encoding: 'utf8' }); + if (existing.status === 0) { + if (JSON.stringify(account(selected.name)) !== JSON.stringify(selected)) fail(); + continue; + } + if (spawnSync('/usr/bin/getent', ['passwd', String(selected.uid)]).status === 0 + || spawnSync('/usr/bin/getent', ['group', String(selected.gid)]).status === 0) fail(); + } + if (installPackages) { + command('/usr/bin/apt-get', ['update'], { timeout: 600000 }); + command('/usr/bin/apt-get', ['install', '--yes', '--no-install-recommends', ...PACKAGES, + ...(metadata.legacyEngine === true ? ['podman', 'uidmap', 'passt'] : [])], + { timeout: 1200000, env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', DEBIAN_FRONTEND: 'noninteractive' } }); + } + for (const selected of metadata.accounts) { + if (spawnSync('/usr/bin/getent', ['passwd', selected.name]).status === 0) continue; + command('/usr/sbin/groupadd', ['--gid', String(selected.gid), selected.name]); + // Service accounts must not acquire unrelated subordinate-ID ranges. + command('/usr/sbin/useradd', ['--system', '--uid', String(selected.uid), '--gid', String(selected.gid), '--no-create-home', + '--home-dir', selected.home, '--shell', '/usr/sbin/nologin', selected.name]); + } + const core = metadata.accounts[0]; + fs.mkdirSync(core.home, { recursive: true, mode: 0o700 }); + fs.chownSync(core.home, core.uid, core.gid); + capsule.installFresh(directory, digest, allowed); + for (const [directory, mode] of [[HOST_TENANT_ROOT, 0o755], ['/run/dispatch-runtime-agents', 0o711]]) { + fs.mkdirSync(directory, {recursive:true, mode}); + fs.chownSync(directory, 0, 0); fs.chmodSync(directory, mode); + } + for (const directory of [metadata.localRoot, path.join(core.home, '.config'), path.join(core.home, '.config/systemd'), path.join(core.home, '.config/systemd/user')]) { + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); fs.chownSync(directory, core.uid, core.gid); fs.chmodSync(directory, 0o700); + } + for (const child of ['run', 'tmp', 'logs', 'staging', 'backups', 'installations']) { + const directory = path.join(metadata.localRoot, child); + fs.mkdirSync(directory, { mode: 0o700 }); fs.chownSync(directory, core.uid, core.gid); + } + // File-scoped capsules create ancestors without application ownership. + // Recreate Core's writable private directories before starting its workers. + for (const child of ['config', 'config/cloudflared', 'secrets', 'secrets/email', 'secrets/turnstile', 'secrets/cloudflared', 'secrets/oci-runtime-agents', 'data', 'data/access-control', 'state']) { + const directory = path.join(metadata.localRoot, child); + fs.mkdirSync(directory, {recursive:true, mode:0o700}); + fs.chownSync(directory, core.uid, core.gid); fs.chmodSync(directory, 0o700); + } + for (const name of ['cloudflared']) { + const source = `/usr/local/lib/dispatch-recovery/${name}`; + if (!fs.existsSync(source)) continue; + const temp = `/usr/bin/.dispatch-restore-${name}`; + fs.copyFileSync(source, temp, fs.constants.COPYFILE_EXCL); fs.chmodSync(temp, 0o755); fs.renameSync(temp, `/usr/bin/${name}`); + fs.unlinkSync(source); + } + if (fs.existsSync('/usr/local/lib/dispatch-recovery')) fs.rmdirSync('/usr/local/lib/dispatch-recovery'); + if (!fs.existsSync('/usr/local/lib/dispatch-node/node')) fail(); + const builtinRoot = '/usr/local/lib/dispatch-node/host-files/usr/share/nodejs'; + if (fs.existsSync(builtinRoot)) fs.cpSync(builtinRoot, '/usr/share/nodejs', { recursive: true, force: true }); + fs.symlinkSync('/usr/local/lib/dispatch-node/node', '/usr/bin/.dispatch-restore-node'); + fs.renameSync('/usr/bin/.dispatch-restore-node', '/usr/bin/node'); + for (const service of metadata.services) { + const file = fs.realpathSync(service.file); + const text = fs.readFileSync(file, 'utf8'); + const updated = text.replace(/^ExecStart=(?:\/usr\/local\/bin|\/home\/[a-z_][a-z0-9_-]*\/\.local\/bin)\/(node|cloudflared)(?=\s)/gm, 'ExecStart=/usr/bin/$1'); + if (updated !== text) fs.writeFileSync(file, updated); + } + for (const item of metadata.installations) { + const selected = metadata.accounts.find(a => a.name === hostAccountName(item.runtime_key)); + if (!selected) continue; + const runtime = path.join(HOST_TENANT_ROOT, opaqueRuntimeSuffix(item.runtime_key), 'runtime', item.runtime_key); + for (const child of Object.values(require('../../../shared/paths/runtime-paths').MANAGED_INSTALLATION_DIRECTORY_FIELDS)) { + const directory = path.join(runtime, child); + let parent = directory; + while (!fs.existsSync(parent)) parent = path.dirname(parent); + if (fs.realpathSync(parent) !== parent) fail(); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + if (fs.realpathSync(directory) !== directory || !fs.lstatSync(directory).isDirectory()) fail(); + fs.chownSync(directory, selected.uid, selected.gid); fs.chmodSync(directory, 0o700); + } + const bridge = `/run/dispatch-runtime-agents/${opaqueRuntimeSuffix(item.runtime_key)}`; + fs.mkdirSync(bridge, { recursive: true, mode: 0o711 }); fs.chmodSync(bridge, 0o711); + } + // Restoring returns to the recorded application version. An interrupted + // rollout is paused instead of replaying an update against restored data. + const db = new DatabaseSync(path.join(metadata.localRoot, 'data/access-control/access-control.sqlite3')); + try { + db.exec("PRAGMA trusted_schema=OFF; PRAGMA foreign_keys=ON; BEGIN IMMEDIATE"); + db.prepare("UPDATE platform_rollouts SET status='paused' WHERE status!='completed'").run(); + db.prepare("UPDATE platform_rollout_core SET status='failed',failure_code='restored_from_backup' WHERE status!='succeeded'").run(); + db.prepare("UPDATE platform_backup_requests SET status='failed',phase='failed',failure_code='restored_from_backup' WHERE status IN ('queued','running')").run(); + db.exec('COMMIT; PRAGMA wal_checkpoint(TRUNCATE)'); + } finally { db.close(); } + if(metadata.scope==='core'||metadata.scope==='system') { + const host=require('./release-delivery-files').privateJson('/etc/dispatch/oci-host.json',0); + for(const directory of [host.stateRoot,host.authorityRoot]) { + if(!directory.startsWith('/var/lib/dispatch-host/')||path.resolve(directory)!==directory)fail(); + fs.mkdirSync(directory,{recursive:true,mode:0o700});fs.chmodSync(directory,0o700); + } + const registryModule='/opt/dispatch-control/current/host-helper-artifact/core/installations/src/oci-host-account-registry'; + const registry=require(registryModule).createOciHostAccountRegistry({stateRoot:host.stateRoot,identityAvailable:()=>false});registry.close(); + const ledger=new DatabaseSync(path.join(host.stateRoot,'oci-host.sqlite3')); + try { + ledger.exec('BEGIN IMMEDIATE'); + for(const allocation of metadata.hostAllocations||[]) { + const item=metadata.installations.find(i=>i.runtime_key===allocation.runtime_key),owner=metadata.accounts.find(a=>a.name===allocation.account_name); + if(!item||!owner||allocation.account_name!==hostAccountName(item.runtime_key)||allocation.uid!==owner.uid||allocation.gid!==owner.gid||allocation.status!=='active')fail(); + ledger.prepare('INSERT INTO allocations VALUES(?,?,?,?,?,?,?,?,?,?)').run(allocation.runtime_key,allocation.account_name,allocation.uid,allocation.gid,allocation.subuid_start,allocation.subgid_start,allocation.subid_count,allocation.status,allocation.created_at,allocation.updated_at); + } + ledger.exec('COMMIT; PRAGMA wal_checkpoint(TRUNCATE)'); + }finally{ledger.close();} + } + if (fs.existsSync('/etc/apparmor.d/dispatch-native-chrome')) command('/usr/sbin/apparmor_parser', ['-r', '/etc/apparmor.d/dispatch-native-chrome']); + fs.mkdirSync('/var/lib/dispatch-backup', { recursive: true, mode: 0o700 }); + fs.chmodSync('/var/lib/dispatch-backup', 0o700); + fs.chownSync('/var/lib/dispatch-backup', 0, 0); + require('./release-delivery-files').atomic('/var/lib/dispatch-backup/rediscover.json', { schemaVersion: 1 }); + command('/usr/bin/systemctl', ['daemon-reload']); + command('/usr/bin/loginctl', ['enable-linger', core.name]); + command('/usr/bin/systemctl', ['start', `user@${core.uid}.service`]); + systemctl({ scope: 'user', account: core }, ['daemon-reload']); + const suspended = service => metadata.installations.some(item => + (item.organization_status === 'suspended' || item.status === 'suspended') + && [`dispatch-dsp-${opaqueRuntimeSuffix(item.runtime_key)}.service`, + `dispatch-runtime-agent-bridge-${opaqueRuntimeSuffix(item.runtime_key)}.service`].includes(service.name)); + for (const service of metadata.services) { + if (suspended(service)) systemctl(service, ['disable', service.name]); + else if (service.enabled) systemctl(service, ['enable', service.name]); + } + const active = metadata.services.filter(service => !suspended(service) && (service.active + || service.enabled && service.name.endsWith('.timer') || service.name === 'dispatch-dashboard.service')); + for (const service of active.sort((a, b) => Number(a.name.endsWith('.timer')) - Number(b.name.endsWith('.timer')))) { + if (/^dispatch-dsp-/.test(service.name) && metadata.installations.some(item => item.organization_status === 'suspended' + && service.name === `dispatch-dsp-${opaqueRuntimeSuffix(item.runtime_key)}.service`)) continue; + systemctl(service, ['start', '--no-block', service.name]); + } + const dashboard = metadata.services.find(service => service.name === 'dispatch-dashboard.service'); + if (!dashboard) fail(); + const match = /\/opt\/dispatch-platform\/releases\/([a-z][a-z0-9_.-]{2,95})\/core-artifact\/code\//.exec(fs.readFileSync(dashboard.file, 'utf8')); + if (!match) fail(); + const deployment = JSON.parse(fs.readFileSync(`/opt/dispatch-platform/releases/${match[1]}/core-artifact/deployment.json`)); + if (deployment.localRoot !== metadata.localRoot || deployment.releaseId !== match[1]) fail(); + const controlSetting = /^DISPATCH_RUNTIME_AGENT_CONTROL_SOCKET=(.+)$/m.exec(fs.readFileSync(path.join(metadata.localRoot, 'config/provisioning.env'), 'utf8')); + const controlSocket = controlSetting?.[1].replace(/^['"]|['"]$/g, '') || path.join(metadata.localRoot, 'run/runtime-agent-control.sock'); + const controlModule = `/opt/dispatch-platform/releases/${match[1]}/core-artifact/code/core/agents/src/control`; + const restoreSchedule = `const call=(op,args)=>require(process.argv[1]).runtimeAgentControlInvoke(process.argv[2],process.argv[3],op,args,{timeoutMs:5000});(async()=>{const id='paycom-main-workforce',want=process.argv[4]==='true'?'running':'stopped';let r=await call('sync.status',{id});if(!r.ok)throw Error();if(r.data.desiredState!==want){r=await call(want==='running'?'sync.start':'sync.stop',{id});if(!r.ok)throw Error();}r=await call('sync.status',{id});if(!r.ok||r.data.desiredState!==want)throw Error();})().catch(()=>process.exitCode=1);`; + const probeRuntime = `require(process.argv[1]).runtimeAgentControlInvoke(process.argv[2],process.argv[3],'health',{}, {timeoutMs:5000}).then(r=>{if(!r.ok)process.exitCode=1}).catch(()=>process.exitCode=1);`; + let healthy = false; + const deadline = Date.now() + 120000; + while (Date.now() < deadline) { + try { + const body = await new Promise((resolve, reject) => { + const request = require('node:http').get({ hostname: '127.0.0.1', port: deployment.port, path: '/api/platform/core-health', + headers: { Host: new URL(deployment.publicOrigin).host, 'CF-Visitor': '{"scheme":"https"}' }, timeout: 2000 }, response => { + let text = ''; response.on('data', chunk => { text += chunk; if (text.length > 4096) request.destroy(); }); + response.on('end', () => { try { if (response.statusCode !== 200) throw Error(); resolve(JSON.parse(text)); } catch (error) { reject(error); } }); + }); + request.on('error', reject); request.on('timeout', () => request.destroy(Error('restore_health_timeout'))); + }); + if (body.ok && body.data.releaseId === deployment.releaseId && body.data.sourceCommit === deployment.sourceCommit + && active.filter(service => /^dispatch-(dsp|runtime-agent-bridge)-/.test(service.name)) + .every(service => systemctl(service, ['is-active', service.name]) === 'active')) { + for (const item of metadata.installations) { + if (!active.some(service => service.name === `dispatch-dsp-${opaqueRuntimeSuffix(item.runtime_key)}.service`)) continue; + command('/usr/sbin/runuser', ['--user', core.name, '--', '/usr/bin/node', '--no-warnings', '-e', probeRuntime, + controlModule, controlSocket, item.runtime_key], { timeout: 10000 }); + if(typeof item.sync_was_running==='boolean')command('/usr/sbin/runuser',['--user',core.name,'--','/usr/bin/node','--no-warnings','-e',restoreSchedule,controlModule,controlSocket,item.runtime_key,String(item.sync_was_running)],{timeout:20000}); + } + healthy = true; break; + } + } catch {} + await new Promise(resolve => setTimeout(resolve, 1000)); + } + if (!healthy) throw Error('restore_service_health_failed'); + return { status: 'restored', dsps: metadata.installations.length, coreVerified: true }; +} +module.exports = { captureWriters, servicesForRecovery, captureHostRecovery, restoreHostRecovery, recoveryRoots, supportedHost, account, command, systemctl, PACKAGES }; diff --git a/core/core/installations/src/immutable-artifact.js b/core/core/installations/src/immutable-artifact.js new file mode 100644 index 0000000..144eb49 --- /dev/null +++ b/core/core/installations/src/immutable-artifact.js @@ -0,0 +1,58 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +function fail() { throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); } +function sha256(value) { return crypto.createHash('sha256').update(value).digest('hex'); } + +function createImmutableArtifact({ projectRoot, target, sourceFiles, executables = new Set(), includeModes = false }) { + if (typeof target !== 'string' || !path.isAbsolute(target) || path.resolve(target) !== target) fail(); + try { fs.lstatSync(target); fail(); } catch (error) { if (error?.code !== 'ENOENT') throw error; } + fs.mkdirSync(target, { mode: 0o755 }); + const files = []; + try { + for (const relative of sourceFiles) { + const source = path.join(projectRoot, relative); + const info = fs.lstatSync(source); + if (path.relative(projectRoot, source) !== relative || !info.isFile() || info.isSymbolicLink() + || info.nlink !== 1 || info.uid !== process.geteuid() || (info.mode & 0o022) !== 0 + || fs.realpathSync(source) !== source) fail(); + const content = fs.readFileSync(source); + const destination = path.join(target, relative); + fs.mkdirSync(path.dirname(destination), { recursive: true, mode: 0o755 }); + const mode = executables.has(relative) ? 0o555 : 0o444; + fs.writeFileSync(destination, content, { mode, flag: 'wx' }); + files.push({ path: relative, ...(includeModes ? { mode: mode.toString(8) } : {}), sha256: sha256(content) }); + } + files.sort((left, right) => left.path.localeCompare(right.path)); + const manifest = `${JSON.stringify({ version: 1, files })}\n`; + fs.writeFileSync(path.join(target, 'manifest.json'), manifest, { mode: 0o444, flag: 'wx' }); + const directories = []; + function collect(directory) { + directories.push(directory); + for (const name of fs.readdirSync(directory)) { + const child = path.join(directory, name); + if (fs.lstatSync(child).isDirectory()) collect(child); + } + } + collect(target); + for (const directory of directories.sort((a, b) => b.length - a.length)) fs.chmodSync(directory, 0o555); + return { files: files.length, manifestSha256: sha256(manifest) }; + } catch (error) { + try { + const writable = directory => { + fs.chmodSync(directory, 0o700); + for (const name of fs.readdirSync(directory)) { + const child = path.join(directory, name); + if (fs.lstatSync(child).isDirectory()) writable(child); + } + }; + writable(target); + fs.rmSync(target, { recursive: true, force: true }); + } catch {} + throw error; + } +} + +module.exports = { createImmutableArtifact }; diff --git a/core/core/installations/src/index.js b/core/core/installations/src/index.js new file mode 100644 index 0000000..f1c7e57 --- /dev/null +++ b/core/core/installations/src/index.js @@ -0,0 +1,23 @@ +'use strict'; + +module.exports = Object.freeze({ + ...require('./layout'), + ...require('./activation'), + ...require('./services'), + ...require('./systemd-user'), + ...require('./jobs'), + ...require('./backups'), + ...require('./lifecycle'), + ...require('./lifecycle-reconcile'), + ...require('./release-catalog'), + ...require('./runtime-agent-credential'), + ...require('./oci-deployment'), + ...require('./oci-host-account-registry'), + ...require('./oci-host-executor'), + ...require('./oci-host-helper'), + ...require('./oci-host-helper-client'), + ...require('./oci-runtime-agent-credential'), + ...require('./oci-adapter'), + ...require('./oci-lifecycle'), + ...require('./oci-runtime-lifecycle-port'), +}); diff --git a/core/core/installations/src/install-command.js b/core/core/installations/src/install-command.js new file mode 100644 index 0000000..3b39bcb --- /dev/null +++ b/core/core/installations/src/install-command.js @@ -0,0 +1,75 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { configuration, CONFIG, TOKEN, STATE } = require('./release-delivery-config'); +const { privateJson } = require('./release-delivery-files'); +const { identity } = require('./release-delivery-contract'); +const usage = 'dispatch-install check|setup|prepare|update|status [--config FILE] [--core FILE] [--version VERSION --commit SHA]'; +function parse(argv) { + const [action, ...args] = argv; + if (!['check', 'setup', 'prepare', 'update', 'status'].includes(action)) throw Error(usage); + const input = { action }; + for (let i = 0; i < args.length; i += 2) { + const key = { '--config': 'config', '--core': 'core', '--version': 'version', '--commit': 'sourceCommit' }[args[i]]; + if (!key || input[key] !== undefined || !args[i + 1] || args[i + 1].startsWith('--')) throw Error(usage); + input[key] = args[i + 1]; + } + for (const key of ['config', 'core']) if (input[key] && (!path.isAbsolute(input[key]) || path.resolve(input[key]) !== input[key])) throw Error(usage); + if (input.version || input.sourceCommit || ['prepare', 'update'].includes(action)) identity(input.version, input.sourceCommit); + if (action === 'setup' && (!input.config || !input.core)) throw Error(usage); + if (input.core && action !== 'setup' || input.config && !['setup', 'check'].includes(action)) throw Error(usage); + return input; +} +function preflight(config, { exists = fs.existsSync, stat = fs.lstatSync, run = spawnSync, platform = process.platform, arch = process.arch } = {}) { + const missing = []; + if (platform !== 'linux' || arch !== 'x64') missing.push('linux_amd64_required'); + for (const file of ['/usr/bin/node', '/usr/bin/python3', '/usr/bin/systemctl', '/usr/bin/setpriv', '/usr/bin/flock', '/usr/sbin/visudo']) if (!exists(file)) missing.push(`missing:${file}`); + const user = run('/usr/bin/getent', ['passwd', String(config.uid)], { encoding: 'utf8', timeout: 5000 }); + if (user.status !== 0 || Number(user.stdout.split(':')[3]) !== config.gid) missing.push('configured_service_account_required'); + for (const directory of [config.localRoot, config.unitRoot]) { + if (!exists(directory)) missing.push(`missing:${directory}`); + else { const value = stat(directory); if (!value.isDirectory() || value.isSymbolicLink() || value.uid !== config.uid || value.mode & 0o022) missing.push(`unsafe:${directory}`); } + } + for (const filename of ['oci-releases.json', 'platform-releases.json', 'provisioning.env']) if (!exists(path.join(config.localRoot, 'config', filename))) missing.push(`missing:config/${filename}`); + + if (!exists(TOKEN)) missing.push('release_delivery_setup_required'); + return { ok: missing.length === 0, status: missing.length ? 'prerequisites_missing' : 'ready', missing }; +} +function command(executable, args, options = {}) { + const result = spawnSync(executable, args, { encoding: 'utf8', timeout: 45 * 60_000, maxBuffer: 1024 * 1024, ...options }); + if (result.error || result.status !== 0) throw Error('installation_command_failed'); + return result.stdout; +} +function asService(config, args) { + return command('/usr/bin/setpriv', [`--reuid=${config.uid}`, `--regid=${config.gid}`, '--clear-groups', process.execPath, '--no-warnings', + path.resolve(__dirname, "../../../bin/dispatch-access-admin"), ...args], { env: { PATH: '/usr/bin:/bin', DISPATCH_LOCAL_ROOT: config.localRoot } }); +} +function status(config, input) { + const progress = privateJson(path.join(STATE, 'preparation-progress.json'), 0, true); + const result = { preparation: !input.version || progress?.version === input.version ? progress : null }; + if (input.version) result.rollout = JSON.parse(asService(config, ['rollout-status', '--local-root', config.localRoot, '--version', input.version, '--commit', input.sourceCommit])); + return result; +} +async function main(argv) { + if (argv.length === 1 && argv[0] === '--help') { process.stdout.write(usage + '\n'); return; } + const input = parse(argv); + if (process.geteuid() !== 0) throw Error('installation_requires_root'); + const config = configuration(privateJson(input.config || CONFIG, 0)); + if (input.action === 'check') { + const result = preflight(config); process.stdout.write(JSON.stringify(result) + '\n'); process.exitCode = result.ok ? 0 : 1; return; + } + if (input.action === 'setup') { + command(process.execPath, [path.resolve(__dirname, "../bin/dispatch-release-delivery-install"), input.config, input.core], { stdio: 'inherit' }); return; + } + if (input.action === 'status') { process.stdout.write(JSON.stringify(status(config, input)) + '\n'); return; } + const check = preflight(config); + if (input.action === 'update' && !fs.existsSync('/etc/dispatch/oci-host.json')) { check.ok = false; check.status = 'prerequisites_missing'; check.missing.push('host_control_configuration_required'); } + if (!check.ok) { process.stdout.write(JSON.stringify(check) + '\n'); process.exitCode = 1; return; } + const prepared = JSON.parse(command(process.execPath, [path.resolve(__dirname, "../bin/dispatch-release-watch"), '--version', input.version, '--commit', input.sourceCommit])); + if (!['release_ready', 'idle'].includes(prepared.status)) { process.stdout.write(JSON.stringify(prepared) + '\n'); process.exitCode = 1; return; } + if (input.action === 'update') { + process.stdout.write(asService(config, ['rollout-start', '--local-root', config.localRoot, '--version', input.version, '--commit', input.sourceCommit])); + } else process.stdout.write(JSON.stringify(prepared) + '\n'); +} +module.exports = { main, parse, preflight, status }; diff --git a/core/core/installations/src/install-layout.js b/core/core/installations/src/install-layout.js new file mode 100644 index 0000000..d358794 --- /dev/null +++ b/core/core/installations/src/install-layout.js @@ -0,0 +1,47 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { atomic, privateJson } = require('./release-delivery-files'); +// Host accounts and host-control authority are explicit prerequisites. This +// initializes only missing service-owned files, never existing data or secrets. +function initializeLayout(config) { + const account = spawnSync('/usr/bin/getent', ['passwd', String(config.uid)], { encoding: 'utf8', timeout: 5000 }); + if (account.status !== 0 || Number(account.stdout.split(':')[3]) !== config.gid) throw Error('configured_service_account_required'); + function directory(selected, create = true) { + if (!fs.existsSync(selected)) { + directory(path.dirname(selected), create); + if (create) { fs.mkdirSync(selected, { mode: 0o700 }); fs.chownSync(selected, config.uid, config.gid); } + } else { + const info = fs.lstatSync(selected); + if (!info.isDirectory() || info.isSymbolicLink() || ![0, config.uid].includes(info.uid) || info.mode & 0o022 || fs.realpathSync(selected) !== selected) throw Error('unsafe_installation_directory'); + } + } + const directories = [config.localRoot, config.unitRoot, ...['config', 'data', 'state/provisioner', 'secrets/oci-runtime-agents', 'run', 'installations', 'backups', 'logs'].map(name => path.join(config.localRoot, name))]; + for (const selected of directories) { + directory(selected, false); + if (fs.existsSync(selected) && fs.statSync(selected).uid !== config.uid) throw Error('installation_directory_owner_mismatch'); + } + const files = { 'oci-releases.json': { schemaVersion: 1, releases: {} }, 'platform-releases.json': { schemaVersion: 1, releases: {} } }; + for (const name of Object.keys(files)) { + const file = path.join(config.localRoot, 'config', name); + if (fs.existsSync(file)) privateJson(file, config.uid); + } + for (const selected of directories) directory(selected); + const env = { + DISPATCH_LOCAL_ROOT: config.localRoot, + DISPATCH_ACCESS_CONTROL_DATABASE_ROOT: path.join(config.localRoot, 'data/access-control'), + DISPATCH_PROVISIONER_STATE_ROOT: path.join(config.localRoot, 'state/provisioner'), + DISPATCH_INSTALLATIONS_ROOT: path.join(config.localRoot, 'installations'), + DISPATCH_SYSTEMD_UNIT_ROOT: config.unitRoot, + DISPATCH_RUNTIME_AGENT_HUB_SOCKET: path.join(config.localRoot, 'run/runtime-agent-hub.sock'), + DISPATCH_RUNTIME_AGENT_CONTROL_SOCKET: path.join(config.localRoot, 'run/runtime-agent-control.sock'), + DISPATCH_OCI_RELEASE_CATALOG_FILE: path.join(config.localRoot, 'config/oci-releases.json'), + DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT: path.join(config.localRoot, 'secrets/oci-runtime-agents'), + }; + files['provisioning.env'] = Object.entries(env).map(([key, value]) => `${key}=${value}`).join('\n') + '\n'; + for (const [name, value] of Object.entries(files)) { + const file = path.join(config.localRoot, 'config', name); + if (!fs.existsSync(file)) { atomic(file, value); fs.chownSync(file, config.uid, config.gid); } + } +} +module.exports = { initializeLayout }; diff --git a/core/core/installations/src/job-schema.js b/core/core/installations/src/job-schema.js new file mode 100644 index 0000000..b93029e --- /dev/null +++ b/core/core/installations/src/job-schema.js @@ -0,0 +1,177 @@ +'use strict'; +const { DatabaseSync } = require('node:sqlite'); +const { INSTALLATION_STATES, INSTALLATION_JOB_STATES } = require('../../../shared/contracts/src'); +const INSTALLATION_JOB_SCHEMA_VERSION = 3; +function fail(code) { throw Object.assign(new Error(code), { code }); } +const EXPECTED_TABLE_COLUMNS = Object.freeze({ + installations: Object.freeze([ + 'organization_id', 'runtime_key', 'status', 'revision', 'generation', 'manifest_json', + 'current_job_id', 'fixture', 'created_at', 'updated_at', + ]), + jobs: Object.freeze([ + 'id', 'organization_id', 'operation', 'status', 'installation_state', 'installation_revision', + 'starting_state', 'generation', 'pipeline_id', 'pipeline_version', 'stages_json', 'next_stage', + 'manifest_json', 'fence', 'attempt', 'max_attempts', 'worker_id', 'lease_expires_at', + 'cancel_requested', 'failure_code', 'created_at', 'started_at', 'finished_at', 'updated_at', + ]), + operation_requests: Object.freeze([ + 'organization_id', 'authority_scope', 'idempotency_key', 'request_json', 'result_job_id', 'created_at', + ]), + job_checkpoints: Object.freeze(['job_id', 'stage_index', 'stage', 'receipt_json', 'completed_at']), + job_attempts: Object.freeze([ + 'job_id', 'attempt', 'fence', 'worker_id', 'status', 'failure_code', 'started_at', 'finished_at', + ]), + job_compensations: Object.freeze([ + 'job_id', 'intent', 'failure_code', 'status', 'attempt', 'max_attempts', + 'created_at', 'finished_at', 'updated_at', + ]), + live_job_authorizations: Object.freeze([ + 'job_id', 'organization_id', 'runtime_key', 'authorized_at', + ]), +}); + +function initializeSchema(db, schemaVersion = INSTALLATION_JOB_SCHEMA_VERSION) { + if (![1, 2, INSTALLATION_JOB_SCHEMA_VERSION].includes(schemaVersion)) fail('runtime_boundary_violation'); + const compensationSchema = schemaVersion >= 2 ? ` + CREATE TABLE job_compensations ( + job_id TEXT PRIMARY KEY REFERENCES jobs(id), + intent TEXT NOT NULL CHECK(intent IN ('failed','cancelled')), + failure_code TEXT, + status TEXT NOT NULL CHECK(status IN ('queued','running','succeeded','failed')), + attempt INTEGER NOT NULL CHECK(attempt>=1), + max_attempts INTEGER NOT NULL CHECK(max_attempts>=1 AND max_attempts<=8), + created_at INTEGER NOT NULL, + finished_at INTEGER, + updated_at INTEGER NOT NULL, + CHECK((intent='failed')=(failure_code IS NOT NULL)), + CHECK(attempt<=max_attempts), + CHECK((status IN ('succeeded','failed'))=(finished_at IS NOT NULL)) + ) STRICT;` : ''; + const liveAuthorizationSchema = schemaVersion >= 3 ? ` + CREATE TABLE live_job_authorizations ( + job_id TEXT PRIMARY KEY REFERENCES jobs(id), + organization_id TEXT NOT NULL REFERENCES installations(organization_id), + runtime_key TEXT NOT NULL, + authorized_at INTEGER NOT NULL + ) STRICT;` : ''; + db.exec(`BEGIN IMMEDIATE; + CREATE TABLE installations ( + organization_id TEXT PRIMARY KEY, + runtime_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL CHECK(status IN (${INSTALLATION_STATES.map(state => `'${state}'`).join(',')})), + revision INTEGER NOT NULL CHECK(revision>=1), + generation INTEGER NOT NULL CHECK(generation>=0), + manifest_json TEXT NOT NULL, + current_job_id TEXT, + fixture INTEGER NOT NULL CHECK(fixture IN (0,1)), + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ) STRICT; + CREATE TABLE jobs ( + id TEXT PRIMARY KEY, + organization_id TEXT NOT NULL REFERENCES installations(organization_id), + operation TEXT NOT NULL CHECK(operation IN ('provision','retry')), + status TEXT NOT NULL CHECK(status IN (${INSTALLATION_JOB_STATES.map(state => `'${state}'`).join(',')})), + installation_state TEXT NOT NULL CHECK(installation_state IN (${INSTALLATION_STATES.map(state => `'${state}'`).join(',')})), + installation_revision INTEGER NOT NULL CHECK(installation_revision>=1), + starting_state TEXT NOT NULL CHECK(starting_state IN (${INSTALLATION_STATES.map(state => `'${state}'`).join(',')})), + generation INTEGER NOT NULL CHECK(generation>=1), + pipeline_id TEXT NOT NULL, + pipeline_version INTEGER NOT NULL CHECK(pipeline_version>=1), + stages_json TEXT NOT NULL, + next_stage INTEGER NOT NULL CHECK(next_stage>=0), + manifest_json TEXT NOT NULL, + fence INTEGER NOT NULL CHECK(fence>=0), + attempt INTEGER NOT NULL CHECK(attempt>=0), + max_attempts INTEGER NOT NULL CHECK(max_attempts>=1 AND max_attempts<=32), + worker_id TEXT, + lease_expires_at INTEGER, + cancel_requested INTEGER NOT NULL CHECK(cancel_requested IN (0,1)), + failure_code TEXT, + created_at INTEGER NOT NULL, + started_at INTEGER, + finished_at INTEGER, + updated_at INTEGER NOT NULL, + CHECK((status='failed')=(failure_code IS NOT NULL)), + CHECK((worker_id IS NULL)=(lease_expires_at IS NULL)) + ) STRICT; + CREATE UNIQUE INDEX one_active_installation_job + ON jobs(organization_id) WHERE status IN ('queued','running'); + CREATE TABLE operation_requests ( + organization_id TEXT NOT NULL REFERENCES installations(organization_id), + authority_scope TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + request_json TEXT NOT NULL, + result_job_id TEXT NOT NULL REFERENCES jobs(id), + created_at INTEGER NOT NULL, + PRIMARY KEY(organization_id,authority_scope,idempotency_key) + ) STRICT; + CREATE TABLE job_checkpoints ( + job_id TEXT NOT NULL REFERENCES jobs(id), + stage_index INTEGER NOT NULL CHECK(stage_index>=0), + stage TEXT NOT NULL, + receipt_json TEXT NOT NULL, + completed_at INTEGER NOT NULL, + PRIMARY KEY(job_id,stage_index), + UNIQUE(job_id,stage) + ) STRICT; + CREATE TABLE job_attempts ( + job_id TEXT NOT NULL REFERENCES jobs(id), + attempt INTEGER NOT NULL CHECK(attempt>=1), + fence INTEGER NOT NULL CHECK(fence>=1), + worker_id TEXT NOT NULL, + status TEXT NOT NULL CHECK(status IN ('running','interrupted','succeeded','failed','cancelled')), + failure_code TEXT, + started_at INTEGER NOT NULL, + finished_at INTEGER, + PRIMARY KEY(job_id,attempt) + ) STRICT; + ${compensationSchema} + ${liveAuthorizationSchema} + PRAGMA user_version=${schemaVersion}; + COMMIT; + `); +} + +function normalizedSchema(db) { + return db.prepare(`SELECT type,name,tbl_name,sql FROM sqlite_schema + WHERE sql IS NOT NULL AND name NOT LIKE 'sqlite_%' ORDER BY type,name`).all() + .map(row => ({ + type: row.type, + name: row.name, + table: row.tbl_name, + sql: row.sql.replace(/\s+/g, ' ').trim(), + })); +} + +function validateSchema(db, schemaVersion = INSTALLATION_JOB_SCHEMA_VERSION) { + const version = db.prepare('PRAGMA user_version').get().user_version; + if (version !== schemaVersion) fail('runtime_boundary_violation'); + const integrity = db.prepare('PRAGMA quick_check(1)').all(); + if (integrity.length !== 1 || integrity[0].quick_check !== 'ok') fail('runtime_boundary_violation'); + const expectedColumns = Object.freeze(Object.fromEntries(Object.entries(EXPECTED_TABLE_COLUMNS) + .filter(([table]) => schemaVersion >= 3 || table !== 'live_job_authorizations') + .filter(([table]) => schemaVersion >= 2 || table !== 'job_compensations'))); + const expectedTables = Object.keys(expectedColumns).sort(); + const actualTables = db.prepare(`SELECT name FROM sqlite_schema + WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name`).all().map(row => row.name); + if (JSON.stringify(actualTables) !== JSON.stringify(expectedTables)) fail('runtime_boundary_violation'); + for (const [table, expected] of Object.entries(expectedColumns)) { + const columns = db.prepare(`PRAGMA table_info(${table})`).all().map(row => row.name); + if (JSON.stringify(columns) !== JSON.stringify(expected)) fail('runtime_boundary_violation'); + } + let reference = null; + try { + reference = new DatabaseSync(':memory:'); + initializeSchema(reference, schemaVersion); + if (JSON.stringify(normalizedSchema(db)) !== JSON.stringify(normalizedSchema(reference))) { + fail('runtime_boundary_violation'); + } + } finally { + try { reference?.close(); } catch {} + } + if (db.prepare('PRAGMA foreign_key_check').all().length !== 0) fail('runtime_boundary_violation'); +} + + +module.exports = { INSTALLATION_JOB_SCHEMA_VERSION, initializeSchema, validateSchema }; diff --git a/core/core/installations/src/job-store.js b/core/core/installations/src/job-store.js new file mode 100644 index 0000000..f3544bb --- /dev/null +++ b/core/core/installations/src/job-store.js @@ -0,0 +1,1432 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const { PROJECT_ROOT } = require('../../../shared/paths/runtime-paths'); +const { + INSTALLATION_IDENTIFIER_RE, + INSTALLATION_JOB_STATES, + serverInstallationManifest, + installationOperation, + assertInstallationOperationAllowed, + installationTransition, + installationRetryTransition, + installationJob, + installationFailure, +} = require('../../../shared/contracts/src'); +const { + INSTALLATION_SERVICE_PLAN_VERSION, + INSTALLATION_AGENT_SERVICE_COUNT, +} = require('./services'); + +const { INSTALLATION_JOB_SCHEMA_VERSION, initializeSchema, validateSchema } = require('./job-schema'); +const INSTALLATION_JOB_PIPELINE_ID = 'installation_layout_v1'; +const INSTALLATION_JOB_PIPELINE_VERSION = 1; +const INSTALLATION_JOB_STAGES = Object.freeze(['runtime_layout_materialize', 'runtime_layout_verify']); +const INSTALLATION_SERVICE_PIPELINE_ID = 'installation_services_v1'; +const INSTALLATION_SERVICE_PIPELINE_VERSION = 1; +const INSTALLATION_SERVICE_STAGES = Object.freeze([ + 'runtime_layout_materialize', + 'runtime_layout_verify', + 'runtime_service_render', + 'runtime_service_validate', + 'runtime_service_install', + 'runtime_service_start', + 'runtime_service_verify', +]); +const INSTALLATION_OCI_PIPELINE_ID = 'installation_oci_container_v1'; +const INSTALLATION_NATIVE_PIPELINE_ID = 'installation_native_service_v1'; +const INSTALLATION_OCI_PIPELINE_VERSION = 1; +const INSTALLATION_OCI_STAGES = Object.freeze([ + 'runtime_oci_host_account', + 'runtime_oci_image_reconcile', + 'runtime_oci_bridge_reconcile', + 'runtime_oci_container_reconcile', + 'runtime_oci_verify', +]); +const INSTALLATION_PIPELINES = Object.freeze({ + [INSTALLATION_NATIVE_PIPELINE_ID]: Object.freeze({ + id: INSTALLATION_NATIVE_PIPELINE_ID, version: INSTALLATION_OCI_PIPELINE_VERSION, stages: INSTALLATION_OCI_STAGES, + }), + [INSTALLATION_JOB_PIPELINE_ID]: Object.freeze({ + id: INSTALLATION_JOB_PIPELINE_ID, + version: INSTALLATION_JOB_PIPELINE_VERSION, + stages: INSTALLATION_JOB_STAGES, + }), + [INSTALLATION_SERVICE_PIPELINE_ID]: Object.freeze({ + id: INSTALLATION_SERVICE_PIPELINE_ID, + version: INSTALLATION_SERVICE_PIPELINE_VERSION, + stages: INSTALLATION_SERVICE_STAGES, + }), + [INSTALLATION_OCI_PIPELINE_ID]: Object.freeze({ + id: INSTALLATION_OCI_PIPELINE_ID, + version: INSTALLATION_OCI_PIPELINE_VERSION, + stages: INSTALLATION_OCI_STAGES, + }), +}); +const INSTALLATION_BACKEND_PIPELINES = Object.freeze({ + systemd_user: INSTALLATION_SERVICE_PIPELINE_ID, + oci_container_v1: INSTALLATION_OCI_PIPELINE_ID, + native_service_v1: INSTALLATION_NATIVE_PIPELINE_ID, +}); +const PROVISIONER_DATABASE_NAME = 'provisioner.sqlite3'; +const PRIVATE_DIRECTORY_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; +const MAX_DATABASE_BYTES = 64 * 1024 * 1024; +const MAX_STAGE_RECEIPT_BYTES = 2048; +const INSTALLATION_JOB_MAX_ATTEMPTS = 8; +const INSTALLATION_ROLLBACK_MAX_ATTEMPTS = 3; +const MIN_LEASE_MS = 100; +const MAX_LEASE_MS = 10 * 60 * 1000; + +function fail(code) { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, allowed, required, code = 'runtime_boundary_violation') { + if (!plain(value)) fail(code); + const keys = Object.keys(value); + if (keys.some(key => !allowed.includes(key)) || required.some(key => !Object.hasOwn(value, key))) fail(code); + return value; +} + +function absolute(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) fail('runtime_boundary_violation'); + return value; +} + +function contains(root, candidate) { + const relative = path.relative(root, candidate); + return relative === '' || (relative !== '..' && !relative.startsWith(`..${path.sep}`) + && !path.isAbsolute(relative)); +} + +function lstatMaybe(target) { + try { return fs.lstatSync(target); } + catch (error) { + if (error?.code === 'ENOENT') return null; + fail('runtime_boundary_violation'); + } +} + +function canonicalDirectory(target, mode, code = 'runtime_boundary_violation') { + const selected = absolute(target); + const info = lstatMaybe(selected); + if (!info || !info.isDirectory() || info.isSymbolicLink() || info.uid !== process.geteuid() + || (info.mode & 0o7777) !== mode) fail(code); + let canonical; + try { canonical = fs.realpathSync(selected); } catch { fail(code); } + if (canonical !== selected) fail(code); + return Object.freeze({ dev: info.dev, ino: info.ino }); +} + +function sameIdentity(left, right) { + return Boolean(left && right && left.dev === right.dev && left.ino === right.ino); +} + +function safeRegularFile(target, expectedDevice, { allowEmpty = false } = {}) { + const selected = absolute(target); + const info = lstatMaybe(selected); + if (!info || !info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() + || info.nlink !== 1 || info.dev !== expectedDevice || (info.mode & 0o7777) !== PRIVATE_FILE_MODE + || (!allowEmpty && info.size < 1) || info.size > MAX_DATABASE_BYTES) { + fail('runtime_boundary_violation'); + } + let canonical; + try { canonical = fs.realpathSync(selected); } catch { fail('runtime_boundary_violation'); } + if (canonical !== selected) fail('runtime_boundary_violation'); + return Object.freeze({ dev: info.dev, ino: info.ino }); +} + +function canonicalProjectRoot(value) { + const selected = absolute(value); + const info = lstatMaybe(selected); + if (!info || !info.isDirectory() || info.isSymbolicLink()) fail('runtime_boundary_violation'); + let canonical; + try { canonical = fs.realpathSync(selected); } catch { fail('runtime_boundary_violation'); } + if (canonical !== selected) fail('runtime_boundary_violation'); + return selected; +} + +function timestamp(value) { + if (!Number.isSafeInteger(value) || value < 0) fail('installation_operation_failed'); + return value; +} + +function leaseDuration(value) { + if (!Number.isSafeInteger(value) || value < MIN_LEASE_MS || value > MAX_LEASE_MS) { + fail('runtime_boundary_violation'); + } + return value; +} + +function identifier(value, code = 'runtime_boundary_violation') { + if (typeof value !== 'string' || !INSTALLATION_IDENTIFIER_RE.test(value)) fail(code); + return value; +} + +function parseJson(value, code = 'runtime_boundary_violation') { + try { + const parsed = JSON.parse(value); + if (!plain(parsed) && !Array.isArray(parsed)) fail(code); + return parsed; + } catch (error) { + if (error?.code === code) throw error; + fail(code); + } +} + +function manifestAuthority(manifest) { + return Object.freeze({ + revision: manifest.revision, + organization: Object.freeze({ ...manifest.organization }), + runtime: Object.freeze({ ...manifest.runtime }), + }); +} + +function mutationAuthority(value) { + exact(value, ['scope', 'permission', 'operatorEnabled'], ['scope', 'permission', 'operatorEnabled']); + identifier(value.scope); + if (value.permission !== 'platform.installations.manage' || value.operatorEnabled !== true) { + fail('installation_operation_not_allowed'); + } + return Object.freeze({ scope: value.scope }); +} + +function fixtureRegistrationAuthority(value) { + exact( + value, + ['fixture', 'installationState', 'retainedData'], + ['fixture', 'installationState', 'retainedData'], + ); + if (value.fixture !== true || value.installationState !== 'pending' || value.retainedData !== false) { + fail('runtime_boundary_violation'); + } +} + +function liveRegistrationAuthority(value) { + exact( + value, + ['source', 'installationState', 'organizationStatus', 'retainedData'], + ['source', 'installationState', 'organizationStatus', 'retainedData'], + ); + if (value.source !== 'access_control' || value.installationState !== 'pending' + || !['pending_owner', 'setup_required', 'active'].includes(value.organizationStatus) + || value.retainedData !== false) { + fail('runtime_boundary_violation'); + } +} + +function liveJobAuthorization(value, jobId) { + exact( + value, + ['source', 'installationState', 'currentJobId', 'organizationStatus'], + ['source', 'installationState', 'currentJobId', 'organizationStatus'], + ); + if (value.source !== 'access_control' || value.installationState !== 'provisioning' + || value.currentJobId !== jobId + || !['pending_owner', 'setup_required', 'active'].includes(value.organizationStatus)) { + fail('runtime_boundary_violation'); + } +} + +function readAuthority(value) { + exact(value, ['scope', 'permission'], ['scope', 'permission']); + identifier(value.scope); + if (value.permission !== 'platform.installations.read') fail('installation_operation_not_allowed'); + return Object.freeze({ scope: value.scope }); +} + +function claimToken(value) { + exact(value, ['jobId', 'workerId', 'fence', 'generation'], + ['jobId', 'workerId', 'fence', 'generation'], 'installation_operation_in_progress'); + identifier(value.jobId, 'installation_operation_in_progress'); + identifier(value.workerId, 'installation_operation_in_progress'); + if (!Number.isSafeInteger(value.fence) || value.fence < 1 + || !Number.isSafeInteger(value.generation) || value.generation < 1) { + fail('installation_operation_in_progress'); + } + return value; +} + +function publicJob(row, replayed = false) { + if (!row) fail('installation_operation_not_found'); + return installationJob({ + id: row.id, + operation: row.operation, + status: row.status, + installationState: row.installation_state, + revision: row.installation_revision, + replayed, + failure: row.failure_code === null ? null : installationFailure(row.failure_code), + }); +} + +function sanitizedLayoutReceipt(value) { + exact(value, ['layoutVersion', 'status', 'directoryCount', 'changed'], + ['layoutVersion', 'status', 'directoryCount', 'changed'], 'runtime_layout_failed'); + if (value.layoutVersion !== 1 || value.status !== 'verified' + || !Number.isSafeInteger(value.directoryCount) || value.directoryCount < 1 || value.directoryCount > 64 + || typeof value.changed !== 'boolean') fail('runtime_layout_failed'); + const selected = Object.freeze({ + layoutVersion: value.layoutVersion, + status: value.status, + directoryCount: value.directoryCount, + changed: value.changed, + }); + if (Buffer.byteLength(JSON.stringify(selected), 'utf8') > MAX_STAGE_RECEIPT_BYTES) fail('runtime_layout_failed'); + return selected; +} + +function pipelineDefinition(id, version) { + const selected = INSTALLATION_PIPELINES[id]; + if (!selected || selected.version !== version) fail('runtime_boundary_violation'); + return selected; +} + +function pipelineIdForBackend(backend) { + const selected = INSTALLATION_BACKEND_PIPELINES[backend]; + if (!selected) fail('runtime_boundary_violation'); + return selected; +} + +function validatedStageSnapshot(value, selectedPipeline) { + if (!Array.isArray(value) || value.length !== selectedPipeline.stages.length + || value.some((stage, index) => stage !== selectedPipeline.stages[index])) { + fail('runtime_boundary_violation'); + } + return Object.freeze([...value]); +} + +function sanitizedServiceReceipt(stage, value) { + const status = Object.freeze({ + runtime_service_render: 'rendered', + runtime_service_validate: 'validated', + runtime_service_install: 'installed', + runtime_service_start: 'started', + runtime_service_verify: 'healthy', + })[stage]; + exact(value, ['servicePlanVersion', 'status', 'serviceCount', 'changed'], + ['servicePlanVersion', 'status', 'serviceCount', 'changed'], 'service_installation_failed'); + if (value.servicePlanVersion !== INSTALLATION_SERVICE_PLAN_VERSION || value.status !== status + || value.serviceCount !== INSTALLATION_AGENT_SERVICE_COUNT + || typeof value.changed !== 'boolean') fail('service_installation_failed'); + const selected = Object.freeze({ + servicePlanVersion: value.servicePlanVersion, + status: value.status, + serviceCount: value.serviceCount, + changed: value.changed, + }); + if (Buffer.byteLength(JSON.stringify(selected), 'utf8') > MAX_STAGE_RECEIPT_BYTES) { + fail('service_installation_failed'); + } + return selected; +} + +function sanitizedOciReceipt(stage, value) { + const status = Object.freeze({ + runtime_oci_host_account: 'host_account_ready', + runtime_oci_image_reconcile: 'image_ready', + runtime_oci_bridge_reconcile: 'bridge_ready', + runtime_oci_container_reconcile: 'container_ready', + runtime_oci_verify: 'healthy', + })[stage]; + exact(value, ['ociDeploymentPlanVersion', 'status', 'changed'], + ['ociDeploymentPlanVersion', 'status', 'changed'], 'service_installation_failed'); + if (value.ociDeploymentPlanVersion !== 1 || value.status !== status + || typeof value.changed !== 'boolean') fail('service_installation_failed'); + return Object.freeze({ + ociDeploymentPlanVersion: value.ociDeploymentPlanVersion, + status: value.status, + changed: value.changed, + }); +} + +function sanitizedStageReceipt(stage, value) { + if (INSTALLATION_JOB_STAGES.includes(stage)) return sanitizedLayoutReceipt(value); + if (INSTALLATION_SERVICE_STAGES.includes(stage)) return sanitizedServiceReceipt(stage, value); + if (INSTALLATION_OCI_STAGES.includes(stage)) return sanitizedOciReceipt(stage, value); + fail('runtime_boundary_violation'); +} + +function validateStoredCheckpoints(db, row, stages = null) { + const selectedPipeline = pipelineDefinition(row.pipeline_id, row.pipeline_version); + const selectedStages = stages || validatedStageSnapshot(parseJson(row.stages_json), selectedPipeline); + if (!Number.isInteger(row.next_stage) || row.next_stage < 0 || row.next_stage > selectedStages.length) { + fail('runtime_boundary_violation'); + } + const checkpoints = db.prepare(`SELECT stage_index,stage,receipt_json FROM job_checkpoints + WHERE job_id=? ORDER BY stage_index`).all(row.id); + if (checkpoints.length !== row.next_stage) fail('runtime_boundary_violation'); + for (let index = 0; index < checkpoints.length; index += 1) { + const checkpoint = checkpoints[index]; + if (checkpoint.stage_index !== index || checkpoint.stage !== selectedStages[index]) { + fail('runtime_boundary_violation'); + } + const receipt = sanitizedStageReceipt(checkpoint.stage, parseJson(checkpoint.receipt_json)); + if (checkpoint.receipt_json !== JSON.stringify(receipt)) fail('runtime_boundary_violation'); + } + return selectedStages; +} + +function validateAllStoredCheckpoints(db) { + for (const row of db.prepare(`SELECT id,pipeline_id,pipeline_version,stages_json,next_stage + FROM jobs ORDER BY id`).all()) validateStoredCheckpoints(db, row); +} + +function createInstallationJobStore(options) { + exact(options, ['stateRoot', 'projectRoot', 'clock', 'pipelineId'], ['stateRoot']); + const stateRoot = absolute(options.stateRoot); + if (stateRoot === path.parse(stateRoot).root) fail('runtime_boundary_violation'); + const projectRoot = canonicalProjectRoot(options.projectRoot === undefined ? PROJECT_ROOT : options.projectRoot); + const clock = options.clock === undefined ? null : options.clock; + if (clock !== null && typeof clock !== 'function') fail('runtime_boundary_violation'); + const configuredPipelineId = options.pipelineId === undefined + ? INSTALLATION_JOB_PIPELINE_ID : options.pipelineId; + const configuredPipeline = pipelineDefinition( + configuredPipelineId, + INSTALLATION_PIPELINES[configuredPipelineId]?.version, + ); + if (contains(projectRoot, stateRoot) || contains(stateRoot, projectRoot)) fail('runtime_boundary_violation'); + const rootIdentity = canonicalDirectory(stateRoot, PRIVATE_DIRECTORY_MODE); + const database = path.join(stateRoot, PROVISIONER_DATABASE_NAME); + const allowedEntries = new Set([ + PROVISIONER_DATABASE_NAME, + `${PROVISIONER_DATABASE_NAME}-wal`, + `${PROVISIONER_DATABASE_NAME}-shm`, + ]); + let db = null; + let databaseIdentity = null; + let closed = false; + + function assertRoot() { + const current = canonicalDirectory(stateRoot, PRIVATE_DIRECTORY_MODE); + if (!sameIdentity(rootIdentity, current)) fail('runtime_boundary_violation'); + } + + function rootEntries() { + assertRoot(); + let entries; + try { entries = fs.readdirSync(stateRoot); } catch { fail('runtime_boundary_violation'); } + if (entries.some(entry => !allowedEntries.has(entry))) fail('runtime_boundary_violation'); + return entries; + } + + function assertStorage({ allowEmpty = false } = {}) { + if (closed || !db) fail('installation_operation_failed'); + rootEntries(); + const currentDatabase = safeRegularFile(database, rootIdentity.dev, { allowEmpty }); + if (databaseIdentity && !sameIdentity(databaseIdentity, currentDatabase)) fail('runtime_boundary_violation'); + for (const suffix of ['-wal', '-shm']) { + const candidate = `${database}${suffix}`; + if (lstatMaybe(candidate)) safeRegularFile(candidate, rootIdentity.dev, { allowEmpty: true }); + } + } + + const initialEntries = rootEntries(); + if (lstatMaybe(database)) { + safeRegularFile(database, rootIdentity.dev, { allowEmpty: true }); + } else { + if (initialEntries.length !== 0) fail('runtime_boundary_violation'); + let handle; + try { + handle = fs.openSync( + database, + fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_RDWR | (fs.constants.O_NOFOLLOW || 0), + PRIVATE_FILE_MODE, + ); + } catch { fail('runtime_boundary_violation'); } + try { fs.closeSync(handle); } catch { fail('runtime_boundary_violation'); } + safeRegularFile(database, rootIdentity.dev, { allowEmpty: true }); + } + + try { + db = new DatabaseSync(database); + db.exec('PRAGMA busy_timeout=3000; PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON; PRAGMA trusted_schema=OFF;'); + let version = db.prepare('PRAGMA user_version').get().user_version; + if (!Number.isSafeInteger(version) || ![0, 1, 2, INSTALLATION_JOB_SCHEMA_VERSION].includes(version)) { + fail('runtime_boundary_violation'); + } + if (version === 0) { + const existingTables = db.prepare(`SELECT COUNT(*) AS count FROM sqlite_schema + WHERE type='table' AND name NOT LIKE 'sqlite_%'`).get().count; + if (existingTables !== 0) fail('runtime_boundary_violation'); + initializeSchema(db); + version = INSTALLATION_JOB_SCHEMA_VERSION; + } + if (version === 1) { + validateSchema(db, 1); + db.exec(`BEGIN IMMEDIATE; + CREATE TABLE job_compensations ( + job_id TEXT PRIMARY KEY REFERENCES jobs(id), + intent TEXT NOT NULL CHECK(intent IN ('failed','cancelled')), + failure_code TEXT, + status TEXT NOT NULL CHECK(status IN ('queued','running','succeeded','failed')), + attempt INTEGER NOT NULL CHECK(attempt>=1), + max_attempts INTEGER NOT NULL CHECK(max_attempts>=1 AND max_attempts<=8), + created_at INTEGER NOT NULL, + finished_at INTEGER, + updated_at INTEGER NOT NULL, + CHECK((intent='failed')=(failure_code IS NOT NULL)), + CHECK(attempt<=max_attempts), + CHECK((status IN ('succeeded','failed'))=(finished_at IS NOT NULL)) + ) STRICT; + PRAGMA user_version=2; + COMMIT; + `); + version = 2; + } + if (version === 2) { + validateSchema(db, 2); + db.exec(`BEGIN IMMEDIATE; + CREATE TABLE live_job_authorizations ( + job_id TEXT PRIMARY KEY REFERENCES jobs(id), + organization_id TEXT NOT NULL REFERENCES installations(organization_id), + runtime_key TEXT NOT NULL, + authorized_at INTEGER NOT NULL + ) STRICT; + PRAGMA user_version=${INSTALLATION_JOB_SCHEMA_VERSION}; + COMMIT; + `); + version = INSTALLATION_JOB_SCHEMA_VERSION; + } + validateSchema(db); + validateAllStoredCheckpoints(db); + databaseIdentity = safeRegularFile(database, rootIdentity.dev); + assertStorage(); + } catch (error) { + try { db?.close(); } catch {} + db = null; + closed = true; + const failure = installationFailure(error); + fail(failure.code); + } + + function transaction(callback) { + assertStorage(); + db.exec('BEGIN IMMEDIATE'); + try { + const result = callback(); + db.exec('COMMIT'); + assertStorage(); + return result; + } catch (error) { + try { db.exec('ROLLBACK'); } catch {} + throw error; + } + } + + function transactionTime(fallback) { + return timestamp(clock === null ? fallback : clock()); + } + + function installationRow(manifest) { + const row = db.prepare('SELECT * FROM installations WHERE organization_id=?').get(manifest.organization.id); + if (!row) fail('installation_not_found'); + if (row.runtime_key !== manifest.runtime.key || row.manifest_json !== JSON.stringify(manifest)) { + fail('runtime_identity_mismatch'); + } + return row; + } + + function jobRow(jobId) { + return db.prepare('SELECT * FROM jobs WHERE id=?').get(jobId) || null; + } + + function jobAuthorized(row, installation) { + if (installation.fixture === 1) return true; + if (installation.fixture !== 0) return false; + const authorization = db.prepare('SELECT * FROM live_job_authorizations WHERE job_id=?').get(row.id); + return Boolean(authorization && authorization.organization_id === installation.organization_id + && authorization.runtime_key === installation.runtime_key); + } + + function compensationRow(jobId) { + return db.prepare('SELECT * FROM job_compensations WHERE job_id=?').get(jobId) || null; + } + + function serviceCompensationRequired(row) { + const service = row.pipeline_id === INSTALLATION_SERVICE_PIPELINE_ID + && row.pipeline_version === INSTALLATION_SERVICE_PIPELINE_VERSION + && row.next_stage >= INSTALLATION_SERVICE_STAGES.indexOf('runtime_service_render'); + const oci = [INSTALLATION_OCI_PIPELINE_ID, INSTALLATION_NATIVE_PIPELINE_ID].includes(row.pipeline_id) + && row.pipeline_version === INSTALLATION_OCI_PIPELINE_VERSION; + return service || oci; + } + + function retrySource(installation) { + if (!installation.current_job_id) fail('installation_operation_not_found'); + const currentJob = jobRow(installation.current_job_id); + if (!currentJob) fail('installation_operation_not_found'); + if (currentJob.status === 'failed') return currentJob; + if (currentJob.status !== 'cancelled' || currentJob.operation !== 'retry' + || currentJob.starting_state !== 'failed') { + fail('installation_operation_not_allowed'); + } + const prior = db.prepare(`SELECT * FROM jobs WHERE organization_id=? AND status='failed' + AND generation { + const now = transactionTime(at); + const existing = db.prepare('SELECT * FROM installations WHERE organization_id=?').get(manifest.organization.id); + if (existing) { + if (existing.runtime_key !== manifest.runtime.key || existing.manifest_json !== manifestJson + || existing.fixture !== 1) fail('runtime_identity_mismatch'); + return Object.freeze({ state: existing.status, revision: existing.revision, generation: existing.generation }); + } + try { + db.prepare(`INSERT INTO installations( + organization_id,runtime_key,status,revision,generation,manifest_json,current_job_id,fixture,created_at,updated_at + ) VALUES(?,?,'pending',1,0,?,NULL,1,?,?)`) + .run(manifest.organization.id, manifest.runtime.key, manifestJson, now, now); + } catch (error) { + if (String(error?.message).includes('UNIQUE constraint failed')) fail('runtime_identity_mismatch'); + throw error; + } + const row = installationRow(manifest); + return Object.freeze({ state: row.status, revision: row.revision, generation: row.generation }); + }); + } + + function registerLive(manifestValue, authorityValue, registrationAuthorityValue, at) { + const manifest = serverInstallationManifest(manifestValue, authorityValue); + liveRegistrationAuthority(registrationAuthorityValue); + if (manifest.runtime.key === 'local' || manifest.runtime.key.startsWith('fixture_')) { + fail('runtime_boundary_violation'); + } + const manifestJson = JSON.stringify(manifest); + return transaction(() => { + const now = transactionTime(at); + const existing = db.prepare('SELECT * FROM installations WHERE organization_id=?').get(manifest.organization.id); + if (existing) { + if (existing.runtime_key !== manifest.runtime.key || existing.manifest_json !== manifestJson + || existing.fixture !== 0) fail('runtime_identity_mismatch'); + return Object.freeze({ state: existing.status, revision: existing.revision, generation: existing.generation }); + } + try { + db.prepare(`INSERT INTO installations( + organization_id,runtime_key,status,revision,generation,manifest_json,current_job_id,fixture,created_at,updated_at + ) VALUES(?,?,'pending',1,0,?,NULL,0,?,?)`) + .run(manifest.organization.id, manifest.runtime.key, manifestJson, now, now); + } catch (error) { + if (String(error?.message).includes('UNIQUE constraint failed')) fail('runtime_identity_mismatch'); + throw error; + } + const row = installationRow(manifest); + return Object.freeze({ state: row.status, revision: row.revision, generation: row.generation }); + }); + } + + function insertRequest(manifest, authority, operation, resultJobId, now) { + db.prepare(`INSERT INTO operation_requests( + organization_id,authority_scope,idempotency_key,request_json,result_job_id,created_at + ) VALUES(?,?,?,?,?,?)`).run( + manifest.organization.id, + authority.scope, + operation.idempotencyKey, + JSON.stringify(operation), + resultJobId, + now, + ); + } + + function createJob(manifest, operation, jobId, installation, destination, now, selectedPipeline) { + identifier(jobId, 'installation_operation_failed'); + const nextRevision = installation.revision + 1; + const nextGeneration = installation.generation + 1; + db.prepare(`INSERT INTO jobs( + id,organization_id,operation,status,installation_state,installation_revision,starting_state,generation, + pipeline_id,pipeline_version,stages_json,next_stage,manifest_json,fence,attempt,max_attempts,worker_id, + lease_expires_at,cancel_requested,failure_code,created_at,started_at,finished_at,updated_at + ) VALUES(?,? ,?,'queued',?,?,?, ?,?,?,?,0,?,0,0,?,NULL,NULL,0,NULL,?,NULL,NULL,?)`).run( + jobId, + manifest.organization.id, + operation.operation, + destination, + nextRevision, + installation.status, + nextGeneration, + selectedPipeline.id, + selectedPipeline.version, + JSON.stringify(selectedPipeline.stages), + JSON.stringify(manifest), + INSTALLATION_JOB_MAX_ATTEMPTS, + now, + now, + ); + db.prepare(`UPDATE installations SET status=?,revision=?,generation=?,current_job_id=?,updated_at=? + WHERE organization_id=? AND revision=? AND generation=?`).run( + destination, + nextRevision, + nextGeneration, + jobId, + now, + manifest.organization.id, + installation.revision, + installation.generation, + ); + const changed = db.prepare('SELECT changes() AS count').get().count; + if (changed !== 1) fail('installation_revision_conflict'); + return jobRow(jobId); + } + + function request( + manifestValue, authorityValue, operationValue, requestAuthorityValue, jobId, at, + pipelineIdValue = configuredPipeline.id, + ) { + const manifest = serverInstallationManifest(manifestValue, authorityValue); + const operation = installationOperation(operationValue); + const authority = mutationAuthority(requestAuthorityValue); + const selectedPipeline = INSTALLATION_PIPELINES[pipelineIdValue]; + if (!selectedPipeline) fail('runtime_boundary_violation'); + if (!['provision', 'retry', 'cancel'].includes(operation.operation)) { + fail('installation_operation_not_allowed'); + } + + return transaction(() => { + const now = transactionTime(at); + let installation = installationRow(manifest); + const prior = db.prepare(`SELECT request_json,result_job_id FROM operation_requests + WHERE organization_id=? AND authority_scope=? AND idempotency_key=?`).get( + manifest.organization.id, + authority.scope, + operation.idempotencyKey, + ); + if (prior) { + if (prior.request_json !== JSON.stringify(operation)) fail('idempotency_conflict'); + return publicJob(jobRow(prior.result_job_id), true); + } + if (operation.expectedRevision !== installation.revision) fail('installation_revision_conflict'); + + const active = db.prepare("SELECT * FROM jobs WHERE organization_id=? AND status IN ('queued','running')") + .get(manifest.organization.id) || null; + if (operation.operation === 'cancel') { + assertInstallationOperationAllowed(installation.status, 'cancel', { + activeJob: active ? publicJob(active) : null, + }); + if (active.status === 'queued') { + installationTransition(installation.status, active.starting_state); + const nextRevision = installation.revision + 1; + const jobChanged = db.prepare(`UPDATE jobs SET status='cancelled',installation_state=?,installation_revision=?, + finished_at=?,updated_at=? WHERE id=? AND status='queued' AND generation=?`).run( + active.starting_state, nextRevision, now, now, active.id, active.generation, + ).changes; + if (jobChanged !== 1) fail('installation_operation_in_progress'); + const installationChanged = db.prepare(`UPDATE installations SET status=?,revision=?,updated_at=? + WHERE organization_id=? AND revision=? AND generation=? AND current_job_id=?`).run( + active.starting_state, + nextRevision, + now, + manifest.organization.id, + installation.revision, + active.generation, + active.id, + ).changes; + if (installationChanged !== 1) fail('installation_revision_conflict'); + } else { + const changed = db.prepare(`UPDATE jobs SET cancel_requested=1,updated_at=? + WHERE id=? AND status='running' AND generation=?`).run( + now, active.id, active.generation, + ).changes; + if (changed !== 1) fail('installation_operation_in_progress'); + } + insertRequest(manifest, authority, operation, active.id, now); + return publicJob(jobRow(active.id)); + } + + if (active) fail('installation_operation_in_progress'); + let destination; + if (operation.operation === 'provision') { + assertInstallationOperationAllowed(installation.status, 'provision'); + destination = installationTransition(installation.status, 'provisioning').to; + } else { + assertInstallationOperationAllowed(installation.status, 'retry'); + const failed = retrySource(installation); + if (failed.pipeline_id !== selectedPipeline.id + || failed.pipeline_version !== selectedPipeline.version) fail('runtime_identity_mismatch'); + destination = installationRetryTransition(publicJob(failed)).to; + } + const created = createJob(manifest, operation, jobId, installation, destination, now, selectedPipeline); + insertRequest(manifest, authority, operation, created.id, now); + installation = installationRow(manifest); + if (installation.current_job_id !== created.id || installation.status !== destination) { + fail('installation_operation_failed'); + } + return publicJob(created); + }); + } + + function authorizeLive(manifestValue, authorityValue, requestAuthorityValue, jobIdValue, + authorizationValue, at) { + const manifest = serverInstallationManifest(manifestValue, authorityValue); + const requestAuthority = mutationAuthority(requestAuthorityValue); + const jobId = identifier(jobIdValue); + liveJobAuthorization(authorizationValue, jobId); + return transaction(() => { + const now = transactionTime(at); + const installation = installationRow(manifest); + if (installation.fixture !== 0 || installation.status !== 'provisioning' + || installation.current_job_id !== jobId) fail('installation_operation_not_allowed'); + const row = jobRow(jobId); + if (!row || row.organization_id !== installation.organization_id || row.status !== 'queued' + || row.installation_state !== 'provisioning' || row.generation !== installation.generation) { + fail('installation_operation_in_progress'); + } + const operationRequest = db.prepare(`SELECT authority_scope FROM operation_requests + WHERE organization_id=? AND result_job_id=?`).get(installation.organization_id, jobId); + if (!operationRequest || operationRequest.authority_scope !== requestAuthority.scope) { + fail('installation_operation_not_allowed'); + } + const prior = db.prepare('SELECT * FROM live_job_authorizations WHERE job_id=?').get(jobId); + if (prior) { + if (prior.organization_id !== installation.organization_id + || prior.runtime_key !== installation.runtime_key) fail('runtime_identity_mismatch'); + return publicJob(row, true); + } + db.prepare(`INSERT INTO live_job_authorizations(job_id,organization_id,runtime_key,authorized_at) + VALUES(?,?,?,?)`).run(jobId, installation.organization_id, installation.runtime_key, now); + const readBack = db.prepare('SELECT * FROM live_job_authorizations WHERE job_id=?').get(jobId); + if (!readBack || readBack.organization_id !== installation.organization_id + || readBack.runtime_key !== installation.runtime_key) fail('runtime_boundary_violation'); + return publicJob(row); + }); + } + + function validatedClaim(value, at) { + const claim = claimToken(value); + const now = timestamp(at); + const row = jobRow(claim.jobId); + if (!row || row.status !== 'running' || row.worker_id !== claim.workerId + || row.fence !== claim.fence || row.generation !== claim.generation + || row.lease_expires_at <= now) fail('installation_operation_in_progress'); + const installation = db.prepare('SELECT * FROM installations WHERE organization_id=?').get(row.organization_id); + if (!installation || installation.current_job_id !== row.id + || installation.generation !== row.generation || installation.status !== row.installation_state + || !jobAuthorized(row, installation)) fail('installation_operation_in_progress'); + return { row, installation }; + } + + function beginCompensation(claimValue, intentValue, error, at) { + if (!['failed', 'cancelled'].includes(intentValue)) fail('runtime_boundary_violation'); + const selectedFailure = intentValue === 'failed' ? installationFailure(error) : null; + return transaction(() => { + const now = transactionTime(at); + const { row } = validatedClaim(claimValue, now); + if (!serviceCompensationRequired(row) || compensationRow(row.id)) fail('runtime_boundary_violation'); + if (intentValue === 'cancelled' && !row.cancel_requested) fail('installation_operation_not_allowed'); + updateAttempt(row, intentValue === 'cancelled' ? 'cancelled' : 'failed', selectedFailure?.code || null, now); + db.prepare(`INSERT INTO job_compensations( + job_id,intent,failure_code,status,attempt,max_attempts,created_at,finished_at,updated_at + ) VALUES(?,?,?,'running',1,?,?,NULL,?)`).run( + row.id, + intentValue, + selectedFailure?.code || null, + INSTALLATION_ROLLBACK_MAX_ATTEMPTS, + now, + now, + ); + return true; + }); + } + + function claimNext(workerIdValue, at, leaseMsValue) { + const workerId = identifier(workerIdValue); + const leaseMs = leaseDuration(leaseMsValue); + return transaction(() => { + const now = transactionTime(at); + const selected = db.prepare(`SELECT j.* FROM jobs j JOIN installations i ON i.organization_id=j.organization_id + WHERE (j.status='queued' OR (j.status='running' AND j.lease_expires_at<=?) + OR (j.status='running' AND j.worker_id IS NULL AND EXISTS( + SELECT 1 FROM job_compensations c WHERE c.job_id=j.id AND c.status='queued' + ))) + AND i.current_job_id=j.id AND i.generation=j.generation AND i.status=j.installation_state + AND (i.fixture=1 OR (i.fixture=0 AND EXISTS( + SELECT 1 FROM live_job_authorizations a WHERE a.job_id=j.id + AND a.organization_id=i.organization_id AND a.runtime_key=i.runtime_key + ))) + ORDER BY CASE j.status WHEN 'running' THEN 0 ELSE 1 END,j.created_at,j.id LIMIT 1`).get(now); + if (!selected) return null; + if (selected.max_attempts !== INSTALLATION_JOB_MAX_ATTEMPTS) fail('runtime_boundary_violation'); + const compensation = compensationRow(selected.id); + if (compensation) { + if (!['queued', 'running'].includes(compensation.status) + || compensation.max_attempts !== INSTALLATION_ROLLBACK_MAX_ATTEMPTS) { + fail('runtime_boundary_violation'); + } + if (compensation.attempt >= compensation.max_attempts) { + const installation = db.prepare('SELECT * FROM installations WHERE organization_id=?') + .get(selected.organization_id); + installationTransition(installation.status, 'failed'); + const nextRevision = installation.revision + 1; + const compensationChanged = db.prepare(`UPDATE job_compensations SET status='failed', + intent='failed',failure_code='service_installation_failed',finished_at=?,updated_at=? + WHERE job_id=? AND status IN ('queued','running') AND attempt=?`).run( + now, now, selected.id, compensation.attempt, + ).changes; + if (compensationChanged !== 1) fail('installation_operation_in_progress'); + const jobChanged = db.prepare(`UPDATE jobs SET status='failed',installation_state='failed', + installation_revision=?,worker_id=NULL,lease_expires_at=NULL,cancel_requested=0, + failure_code='service_installation_failed',finished_at=?,updated_at=? + WHERE id=? AND status='running' AND generation=? AND fence=?`).run( + nextRevision, now, now, selected.id, selected.generation, selected.fence, + ).changes; + if (jobChanged !== 1) fail('installation_operation_in_progress'); + const installationChanged = db.prepare(`UPDATE installations SET status='failed',revision=?,updated_at=? + WHERE organization_id=? AND revision=? AND generation=? AND current_job_id=?`).run( + nextRevision, now, selected.organization_id, installation.revision, selected.generation, selected.id, + ).changes; + if (installationChanged !== 1) fail('installation_revision_conflict'); + return Object.freeze({ terminalJob: publicJob(jobRow(selected.id)) }); + } + const nextFence = selected.fence + 1; + const jobChanged = db.prepare(`UPDATE jobs SET fence=?,worker_id=?,lease_expires_at=?,updated_at=? + WHERE id=? AND status='running' AND generation=? + AND (worker_id IS NULL OR lease_expires_at<=?)`).run( + nextFence, workerId, now + leaseMs, now, selected.id, selected.generation, now, + ).changes; + if (jobChanged !== 1) fail('installation_operation_in_progress'); + const compensationChanged = db.prepare(`UPDATE job_compensations SET status='running', + attempt=attempt+1,updated_at=? WHERE job_id=? AND status IN ('queued','running') AND attempt=?`).run( + now, selected.id, compensation.attempt, + ).changes; + if (compensationChanged !== 1) fail('installation_operation_in_progress'); + const claimed = jobRow(selected.id); + return Object.freeze({ + jobId: claimed.id, + workerId, + fence: claimed.fence, + generation: claimed.generation, + }); + } + if (selected.status === 'running' && serviceCompensationRequired(selected) + && (selected.cancel_requested === 1 || selected.attempt >= selected.max_attempts)) { + const intent = selected.cancel_requested === 1 ? 'cancelled' : 'failed'; + const failureCode = intent === 'failed' ? 'installation_operation_failed' : null; + const attemptChanged = db.prepare(`UPDATE job_attempts SET status=?,failure_code=?,finished_at=? + WHERE job_id=? AND attempt=? AND fence=? AND status='running'`).run( + intent === 'cancelled' ? 'cancelled' : 'interrupted', + failureCode, + now, + selected.id, + selected.attempt, + selected.fence, + ).changes; + if (attemptChanged !== 1) fail('installation_operation_in_progress'); + const nextFence = selected.fence + 1; + const jobChanged = db.prepare(`UPDATE jobs SET fence=?,worker_id=?,lease_expires_at=?,updated_at=? + WHERE id=? AND status='running' AND generation=? AND fence=? AND lease_expires_at<=?`).run( + nextFence, workerId, now + leaseMs, now, + selected.id, selected.generation, selected.fence, now, + ).changes; + if (jobChanged !== 1) fail('installation_operation_in_progress'); + db.prepare(`INSERT INTO job_compensations( + job_id,intent,failure_code,status,attempt,max_attempts,created_at,finished_at,updated_at + ) VALUES(?,?,?,'running',1,?,?,NULL,?)`).run( + selected.id, + intent, + failureCode, + INSTALLATION_ROLLBACK_MAX_ATTEMPTS, + now, + now, + ); + const claimed = jobRow(selected.id); + return Object.freeze({ + jobId: claimed.id, + workerId, + fence: claimed.fence, + generation: claimed.generation, + }); + } + if (selected.status === 'running' && selected.cancel_requested === 1) { + const installation = db.prepare('SELECT * FROM installations WHERE organization_id=?') + .get(selected.organization_id); + installationTransition(installation.status, selected.starting_state); + const nextRevision = installation.revision + 1; + const attemptChanged = db.prepare(`UPDATE job_attempts SET status='cancelled',failure_code=NULL, + finished_at=? WHERE job_id=? AND attempt=? AND fence=? AND status='running'`).run( + now, selected.id, selected.attempt, selected.fence, + ).changes; + if (attemptChanged !== 1) fail('installation_operation_in_progress'); + const jobChanged = db.prepare(`UPDATE jobs SET status='cancelled',installation_state=?, + installation_revision=?,worker_id=NULL,lease_expires_at=NULL,cancel_requested=0, + failure_code=NULL,finished_at=?,updated_at=? WHERE id=? AND status='running' + AND generation=? AND fence=? AND attempt=? AND cancel_requested=1 AND lease_expires_at<=?`).run( + selected.starting_state, + nextRevision, + now, + now, + selected.id, + selected.generation, + selected.fence, + selected.attempt, + now, + ).changes; + if (jobChanged !== 1) fail('installation_operation_in_progress'); + const installationChanged = db.prepare(`UPDATE installations SET status=?,revision=?,updated_at=? + WHERE organization_id=? AND revision=? AND generation=? AND current_job_id=?`).run( + selected.starting_state, + nextRevision, + now, + selected.organization_id, + installation.revision, + selected.generation, + selected.id, + ).changes; + if (installationChanged !== 1) fail('installation_revision_conflict'); + return Object.freeze({ terminalJob: publicJob(jobRow(selected.id)) }); + } + if (selected.status === 'running' && selected.attempt >= selected.max_attempts) { + const installation = db.prepare('SELECT * FROM installations WHERE organization_id=?') + .get(selected.organization_id); + installationTransition(installation.status, 'failed'); + const nextRevision = installation.revision + 1; + const interrupted = db.prepare(`UPDATE job_attempts SET status='interrupted',failure_code='installation_operation_failed', + finished_at=? WHERE job_id=? AND attempt=? AND fence=? AND status='running'`).run( + now, selected.id, selected.attempt, selected.fence, + ).changes; + if (interrupted !== 1) fail('installation_operation_in_progress'); + const jobChanged = db.prepare(`UPDATE jobs SET status='failed',installation_state='failed',installation_revision=?, + worker_id=NULL,lease_expires_at=NULL,cancel_requested=0,failure_code='installation_operation_failed', + finished_at=?,updated_at=? WHERE id=? AND status='running' AND generation=? AND fence=? + AND attempt=? AND lease_expires_at<=?`).run( + nextRevision, now, now, selected.id, selected.generation, selected.fence, selected.attempt, now, + ).changes; + if (jobChanged !== 1) fail('installation_operation_in_progress'); + const changed = db.prepare(`UPDATE installations SET status='failed',revision=?,updated_at=? + WHERE organization_id=? AND revision=? AND generation=? AND current_job_id=?`).run( + nextRevision, + now, + selected.organization_id, + installation.revision, + selected.generation, + selected.id, + ).changes; + if (changed !== 1) fail('installation_revision_conflict'); + return Object.freeze({ terminalJob: publicJob(jobRow(selected.id)) }); + } + if (selected.status === 'running') { + const interrupted = db.prepare(`UPDATE job_attempts SET status='interrupted',failure_code='installation_operation_failed',finished_at=? + WHERE job_id=? AND attempt=? AND fence=? AND status='running'`).run( + now, selected.id, selected.attempt, selected.fence, + ).changes; + if (interrupted !== 1) fail('installation_operation_in_progress'); + } + const nextFence = selected.fence + 1; + const nextAttempt = selected.attempt + 1; + const changed = db.prepare(`UPDATE jobs SET status='running',fence=?,attempt=?,worker_id=?,lease_expires_at=?, + started_at=COALESCE(started_at,?),updated_at=? WHERE id=? AND ( + status='queued' OR (status='running' AND lease_expires_at<=?) + ) AND generation=?`).run( + nextFence, nextAttempt, workerId, now + leaseMs, now, now, selected.id, now, selected.generation, + ).changes; + if (changed !== 1) fail('installation_operation_in_progress'); + db.prepare(`INSERT INTO job_attempts( + job_id,attempt,fence,worker_id,status,failure_code,started_at,finished_at + ) VALUES(?,?,?,?,'running',NULL,?,NULL)`).run( + selected.id, nextAttempt, nextFence, workerId, now, + ); + const claimed = jobRow(selected.id); + return Object.freeze({ + jobId: claimed.id, + workerId, + fence: claimed.fence, + generation: claimed.generation, + }); + }); + } + + function renew(claimValue, at, leaseMsValue) { + const leaseMs = leaseDuration(leaseMsValue); + return transaction(() => { + const now = transactionTime(at); + const { row } = validatedClaim(claimValue, now); + const changed = db.prepare(`UPDATE jobs SET lease_expires_at=?,updated_at=? + WHERE id=? AND worker_id=? AND fence=? AND generation=? AND status='running' + AND lease_expires_at>?`).run( + now + leaseMs, now, row.id, row.worker_id, row.fence, row.generation, now, + ).changes; + if (changed !== 1) fail('installation_operation_in_progress'); + return true; + }); + } + + function mutateClaim(claimValue, at, mutation) { + if (typeof mutation !== 'function') fail('runtime_boundary_violation'); + return transaction(() => { + const startedAt = transactionTime(at); + const { row } = validatedClaim(claimValue, startedAt); + validateStoredCheckpoints(db, row); + const result = mutation(); + const { row: current } = validatedClaim(claimValue, transactionTime(at)); + validateStoredCheckpoints(db, current); + return result; + }); + } + + function work(claimValue, at) { + assertStorage(); + const { row, installation } = validatedClaim(claimValue, at); + if (row.max_attempts !== INSTALLATION_JOB_MAX_ATTEMPTS) fail('runtime_boundary_violation'); + const selectedPipeline = pipelineDefinition(row.pipeline_id, row.pipeline_version); + const stages = validateStoredCheckpoints( + db, + row, + validatedStageSnapshot(parseJson(row.stages_json), selectedPipeline), + ); + const manifest = serverInstallationManifest(parseJson(row.manifest_json), manifestAuthority(parseJson(row.manifest_json))); + if (manifest.organization.id !== row.organization_id) fail('runtime_identity_mismatch'); + const compensation = compensationRow(row.id); + if (compensation && compensation.status !== 'running') fail('runtime_boundary_violation'); + return Object.freeze({ + operation: row.operation, + pipelineId: selectedPipeline.id, + stage: compensation ? null : stages[row.next_stage] || null, + completedStages: row.next_stage, + totalStages: stages.length, + cancelRequested: Boolean(row.cancel_requested), + compensating: Boolean(compensation), + compensationIntent: compensation + ? (row.cancel_requested ? 'cancelled' : compensation.intent) : null, + compensationFailure: compensation?.failure_code || null, + fixture: installation.fixture === 1, + revision: row.installation_revision, + manifest, + leaseExpiresAt: row.lease_expires_at, + authority: manifestAuthority(manifest), + }); + } + + function completeStage(claimValue, stage, receiptValue, at) { + return transaction(() => { + const now = transactionTime(at); + const { row } = validatedClaim(claimValue, now); + if (compensationRow(row.id)) fail('installation_operation_not_allowed'); + const selectedPipeline = pipelineDefinition(row.pipeline_id, row.pipeline_version); + const stages = validateStoredCheckpoints( + db, + row, + validatedStageSnapshot(parseJson(row.stages_json), selectedPipeline), + ); + if (!stages.includes(stage)) fail('runtime_boundary_violation'); + const receipt = sanitizedStageReceipt(stage, receiptValue); + const receiptJson = JSON.stringify(receipt); + const stageIndex = stages.indexOf(stage); + if (row.cancel_requested) fail('installation_operation_not_allowed'); + if (stageIndex < row.next_stage) { + const completed = db.prepare('SELECT receipt_json FROM job_checkpoints WHERE job_id=? AND stage_index=?') + .get(row.id, stageIndex); + if (!completed || completed.receipt_json !== receiptJson) fail('runtime_boundary_violation'); + return false; + } + if (stageIndex !== row.next_stage) fail('runtime_boundary_violation'); + db.prepare(`INSERT INTO job_checkpoints(job_id,stage_index,stage,receipt_json,completed_at) + VALUES(?,?,?,?,?)`).run(row.id, stageIndex, stage, receiptJson, now); + const changed = db.prepare(`UPDATE jobs SET next_stage=next_stage+1,updated_at=? + WHERE id=? AND next_stage=? AND worker_id=? AND fence=? AND generation=? AND status='running' + AND lease_expires_at>?`).run( + now, row.id, stageIndex, row.worker_id, row.fence, row.generation, now, + ).changes; + if (changed !== 1) fail('installation_operation_in_progress'); + return true; + }); + } + + function updateAttempt(row, status, failureCode, now) { + const changed = db.prepare(`UPDATE job_attempts SET status=?,failure_code=?,finished_at=? + WHERE job_id=? AND attempt=? AND fence=? AND status='running'`).run( + status, failureCode, now, row.id, row.attempt, row.fence, + ).changes; + if (changed !== 1) fail('installation_operation_in_progress'); + } + + function finishSucceeded(claimValue, at) { + return transaction(() => { + const now = transactionTime(at); + const { row } = validatedClaim(claimValue, now); + if (compensationRow(row.id)) fail('installation_operation_not_allowed'); + const selectedPipeline = pipelineDefinition(row.pipeline_id, row.pipeline_version); + const stages = validateStoredCheckpoints( + db, + row, + validatedStageSnapshot(parseJson(row.stages_json), selectedPipeline), + ); + if (row.cancel_requested || row.next_stage !== stages.length) fail('installation_operation_not_allowed'); + updateAttempt(row, 'succeeded', null, now); + const changed = db.prepare(`UPDATE jobs SET status='succeeded',worker_id=NULL,lease_expires_at=NULL, + cancel_requested=0,finished_at=?,updated_at=? WHERE id=? AND worker_id=? AND fence=? + AND generation=? AND status='running' AND lease_expires_at>?`).run( + now, now, row.id, row.worker_id, row.fence, row.generation, now, + ).changes; + if (changed !== 1) fail('installation_operation_in_progress'); + return publicJob(jobRow(row.id)); + }); + } + + + function finishFailed(claimValue, error, at) { + const failure = installationFailure(error); + return transaction(() => { + const now = transactionTime(at); + const { row, installation } = validatedClaim(claimValue, now); + if (compensationRow(row.id)) fail('installation_operation_not_allowed'); + if (row.cancel_requested) fail('installation_operation_not_allowed'); + installationTransition(installation.status, 'failed'); + const nextRevision = installation.revision + 1; + updateAttempt(row, 'failed', failure.code, now); + const jobChanged = db.prepare(`UPDATE jobs SET status='failed',installation_state='failed',installation_revision=?, + worker_id=NULL,lease_expires_at=NULL,cancel_requested=0,failure_code=?,finished_at=?,updated_at=? + WHERE id=? AND worker_id=? AND fence=? AND generation=? AND status='running' + AND lease_expires_at>?`).run( + nextRevision, failure.code, now, now, row.id, row.worker_id, row.fence, row.generation, now, + ).changes; + if (jobChanged !== 1) fail('installation_operation_in_progress'); + db.prepare(`UPDATE installations SET status='failed',revision=?,updated_at=? + WHERE organization_id=? AND revision=? AND generation=? AND current_job_id=?`).run( + nextRevision, now, row.organization_id, installation.revision, row.generation, row.id, + ); + if (db.prepare('SELECT changes() AS count').get().count !== 1) fail('installation_revision_conflict'); + return publicJob(jobRow(row.id)); + }); + } + + + function finishCompensation(claimValue, at) { + return transaction(() => { + const now = transactionTime(at); + const { row, installation } = validatedClaim(claimValue, now); + const compensation = compensationRow(row.id); + if (!compensation || compensation.status !== 'running') fail('installation_operation_not_allowed'); + const intent = row.cancel_requested ? 'cancelled' : compensation.intent; + const destination = intent === 'cancelled' ? row.starting_state : 'failed'; + installationTransition(installation.status, destination); + const nextRevision = installation.revision + 1; + const compensationChanged = db.prepare(`UPDATE job_compensations SET status='succeeded', + finished_at=?,updated_at=? WHERE job_id=? AND status='running' AND attempt=?`).run( + now, now, row.id, compensation.attempt, + ).changes; + if (compensationChanged !== 1) fail('installation_operation_in_progress'); + const jobChanged = db.prepare(`UPDATE jobs SET status=?,installation_state=?,installation_revision=?, + worker_id=NULL,lease_expires_at=NULL,cancel_requested=0,failure_code=?,finished_at=?,updated_at=? + WHERE id=? AND worker_id=? AND fence=? AND generation=? AND status='running' AND lease_expires_at>?`).run( + intent === 'cancelled' ? 'cancelled' : 'failed', + destination, + nextRevision, + intent === 'cancelled' ? null : compensation.failure_code, + now, + now, + row.id, + row.worker_id, + row.fence, + row.generation, + now, + ).changes; + if (jobChanged !== 1) fail('installation_operation_in_progress'); + const installationChanged = db.prepare(`UPDATE installations SET status=?,revision=?,updated_at=? + WHERE organization_id=? AND revision=? AND generation=? AND current_job_id=?`).run( + destination, nextRevision, now, row.organization_id, installation.revision, row.generation, row.id, + ).changes; + if (installationChanged !== 1) fail('installation_revision_conflict'); + return publicJob(jobRow(row.id)); + }); + } + + function failCompensation(claimValue, at) { + return transaction(() => { + const now = transactionTime(at); + const { row, installation } = validatedClaim(claimValue, now); + const compensation = compensationRow(row.id); + if (!compensation || compensation.status !== 'running') fail('installation_operation_not_allowed'); + if (compensation.attempt < compensation.max_attempts) { + const compensationChanged = db.prepare(`UPDATE job_compensations SET status='queued',updated_at=? + WHERE job_id=? AND status='running' AND attempt=?`).run( + now, row.id, compensation.attempt, + ).changes; + if (compensationChanged !== 1) fail('installation_operation_in_progress'); + const jobChanged = db.prepare(`UPDATE jobs SET worker_id=NULL,lease_expires_at=NULL,updated_at=? + WHERE id=? AND worker_id=? AND fence=? AND generation=? AND status='running' + AND lease_expires_at>?`).run( + now, row.id, row.worker_id, row.fence, row.generation, now, + ).changes; + if (jobChanged !== 1) fail('installation_operation_in_progress'); + return publicJob(jobRow(row.id)); + } + installationTransition(installation.status, 'failed'); + const nextRevision = installation.revision + 1; + const compensationChanged = db.prepare(`UPDATE job_compensations SET status='failed',intent='failed', + failure_code='service_installation_failed',finished_at=?,updated_at=? + WHERE job_id=? AND status='running' AND attempt=?`).run( + now, now, row.id, compensation.attempt, + ).changes; + if (compensationChanged !== 1) fail('installation_operation_in_progress'); + const jobChanged = db.prepare(`UPDATE jobs SET status='failed',installation_state='failed', + installation_revision=?,worker_id=NULL,lease_expires_at=NULL,cancel_requested=0, + failure_code='service_installation_failed',finished_at=?,updated_at=? + WHERE id=? AND worker_id=? AND fence=? AND generation=? AND status='running' AND lease_expires_at>?`).run( + nextRevision, now, now, row.id, row.worker_id, row.fence, row.generation, now, + ).changes; + if (jobChanged !== 1) fail('installation_operation_in_progress'); + const installationChanged = db.prepare(`UPDATE installations SET status='failed',revision=?,updated_at=? + WHERE organization_id=? AND revision=? AND generation=? AND current_job_id=?`).run( + nextRevision, now, row.organization_id, installation.revision, row.generation, row.id, + ).changes; + if (installationChanged !== 1) fail('installation_revision_conflict'); + return publicJob(jobRow(row.id)); + }); + } + + function finishCancelled(claimValue, at) { + return transaction(() => { + const now = transactionTime(at); + const { row, installation } = validatedClaim(claimValue, now); + if (compensationRow(row.id)) fail('installation_operation_not_allowed'); + if (!row.cancel_requested) fail('installation_operation_not_allowed'); + installationTransition(installation.status, row.starting_state); + const nextRevision = installation.revision + 1; + updateAttempt(row, 'cancelled', null, now); + const jobChanged = db.prepare(`UPDATE jobs SET status='cancelled',installation_state=?,installation_revision=?, + worker_id=NULL,lease_expires_at=NULL,cancel_requested=0,finished_at=?,updated_at=? WHERE id=? + AND worker_id=? AND fence=? AND generation=? AND status='running' AND lease_expires_at>?`).run( + row.starting_state, nextRevision, now, now, row.id, + row.worker_id, row.fence, row.generation, now, + ).changes; + if (jobChanged !== 1) fail('installation_operation_in_progress'); + db.prepare(`UPDATE installations SET status=?,revision=?,updated_at=? + WHERE organization_id=? AND revision=? AND generation=? AND current_job_id=?`).run( + row.starting_state, nextRevision, now, row.organization_id, installation.revision, row.generation, row.id, + ); + if (db.prepare('SELECT changes() AS count').get().count !== 1) fail('installation_revision_conflict'); + return publicJob(jobRow(row.id)); + }); + } + + function current(manifestValue, authorityValue, requestAuthorityValue) { + const manifest = serverInstallationManifest(manifestValue, authorityValue); + readAuthority(requestAuthorityValue); + assertStorage(); + const installation = installationRow(manifest); + if (!installation.current_job_id) return null; + return publicJob(jobRow(installation.current_job_id)); + } + + function installation(manifestValue, authorityValue, requestAuthorityValue) { + const manifest = serverInstallationManifest(manifestValue, authorityValue); + readAuthority(requestAuthorityValue); + assertStorage(); + const row = installationRow(manifest); + return Object.freeze({ state: row.status, revision: row.revision, generation: row.generation }); + } + + function progress(manifestValue, authorityValue, requestAuthorityValue, jobIdValue) { + const manifest = serverInstallationManifest(manifestValue, authorityValue); + readAuthority(requestAuthorityValue); + const jobId = identifier(jobIdValue); + assertStorage(); + const installation = installationRow(manifest); + const row = jobRow(jobId); + if (!row || row.organization_id !== installation.organization_id) { + fail('installation_operation_not_found'); + } + const attempts = db.prepare('SELECT COUNT(*) AS count FROM job_attempts WHERE job_id=?').get(jobId).count; + const selectedPipeline = pipelineDefinition(row.pipeline_id, row.pipeline_version); + const stages = validatedStageSnapshot(parseJson(row.stages_json), selectedPipeline); + return Object.freeze({ + completedStages: row.next_stage, + totalStages: stages.length, + attempts, + }); + } + + function health() { + assertStorage(); + const integrityOk = db.prepare('PRAGMA quick_check').get().quick_check === 'ok'; + const jobs = Object.fromEntries(INSTALLATION_JOB_STATES.map(status => [ + status, + db.prepare('SELECT COUNT(*) AS count FROM jobs WHERE status=?').get(status).count, + ])); + return Object.freeze({ + ok: integrityOk, + status: integrityOk ? 'ready' : 'failed', + schemaVersion: INSTALLATION_JOB_SCHEMA_VERSION, + databaseIntegrity: integrityOk ? 'ok' : 'failed', + installations: db.prepare('SELECT COUNT(*) AS count FROM installations').get().count, + jobs: Object.freeze(jobs), + }); + } + + function close() { + if (closed) return; + try { db.exec('PRAGMA wal_checkpoint(TRUNCATE)'); } catch {} + try { db.close(); } finally { + db = null; + closed = true; + } + if (!lstatMaybe(stateRoot)) return; + assertRoot(); + const currentDatabase = safeRegularFile(database, rootIdentity.dev); + if (!sameIdentity(databaseIdentity, currentDatabase)) fail('runtime_boundary_violation'); + } + + return Object.freeze({ + registerFixture, + registerLive, + request, + authorizeLive, + claimNext, + renew, + mutateClaim, + work, + beginCompensation, + finishCompensation, + failCompensation, + completeStage, + finishSucceeded, + finishFailed, + finishCancelled, + current, + installation, + progress, + health, + close, + }); +} + +module.exports = { + INSTALLATION_JOB_SCHEMA_VERSION, + INSTALLATION_JOB_PIPELINE_ID, + INSTALLATION_JOB_PIPELINE_VERSION, + INSTALLATION_JOB_STAGES, + INSTALLATION_SERVICE_PIPELINE_ID, + INSTALLATION_SERVICE_PIPELINE_VERSION, + INSTALLATION_SERVICE_STAGES, + INSTALLATION_OCI_PIPELINE_ID, + INSTALLATION_NATIVE_PIPELINE_ID, + INSTALLATION_OCI_PIPELINE_VERSION, + INSTALLATION_OCI_STAGES, + INSTALLATION_BACKEND_PIPELINES, + pipelineIdForBackend, + INSTALLATION_JOB_MAX_ATTEMPTS, + INSTALLATION_ROLLBACK_MAX_ATTEMPTS, + PROVISIONER_DATABASE_NAME, + PRIVATE_FILE_MODE, + MIN_LEASE_MS, + MAX_LEASE_MS, + createInstallationJobStore, +}; diff --git a/core/core/installations/src/jobs.js b/core/core/installations/src/jobs.js new file mode 100644 index 0000000..d75124f --- /dev/null +++ b/core/core/installations/src/jobs.js @@ -0,0 +1,537 @@ +'use strict'; + +const crypto = require('node:crypto'); +const path = require('node:path'); +const { installationFailure, serverInstallationManifest } = require('../../../shared/contracts/src'); +const { createInstallationLayoutManager } = require('./layout'); +const { createInstallationServiceManager } = require('./services'); +const { createSystemdUserSupervisor, DEFAULT_WAIT_TIMEOUT_MS } = require('./systemd-user'); +const { + INSTALLATION_JOB_STAGES, + INSTALLATION_JOB_PIPELINE_ID, + INSTALLATION_SERVICE_PIPELINE_ID, + INSTALLATION_SERVICE_STAGES, + INSTALLATION_OCI_PIPELINE_ID, + INSTALLATION_NATIVE_PIPELINE_ID, + INSTALLATION_OCI_STAGES, + pipelineIdForBackend, + MIN_LEASE_MS, + MAX_LEASE_MS, + createInstallationJobStore, +} = require('./job-store'); + +const DEFAULT_JOB_LEASE_MS = 30_000; +const SERVICE_HEALTH_LEASE_MARGIN_MS = 5_000; + +function serviceHealthLeaseMs(plan, configuredLeaseMs) { + return Math.min(MAX_LEASE_MS, Math.max( + configuredLeaseMs, + DEFAULT_WAIT_TIMEOUT_MS * plan.units.length + SERVICE_HEALTH_LEASE_MARGIN_MS, + )); +} + +function fail(code) { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, allowed, required, code = 'runtime_boundary_violation') { + if (!plain(value)) fail(code); + const keys = Object.keys(value); + if (keys.some(key => !allowed.includes(key)) || required.some(key => !Object.hasOwn(value, key))) fail(code); +} + +function absolute(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) fail('runtime_boundary_violation'); + return value; +} + +function overlaps(left, right) { + const relative = path.relative(left, right); + return relative === '' || (relative !== '..' && !relative.startsWith(`..${path.sep}`) + && !path.isAbsolute(relative)); +} + +function selectedTime(clock) { + const value = clock(); + if (!Number.isSafeInteger(value) || value < 0) fail('installation_operation_failed'); + return value; +} + +function defaultJobId() { + return `job_${crypto.randomUUID().replaceAll('-', '')}`; +} + +function sanitizedCall(callback) { + try { return callback(); } + catch (error) { + const failure = installationFailure(error); + fail(failure.code); + } +} + +function createDurableInstallationProvisioner(options) { + exact( + options, + [ + 'stateRoot', 'installationsRoot', 'projectRoot', 'clock', 'idFactory', 'leaseMs', + 'unitRoot', 'supervisor', 'commandPath', 'systemdAnalyze', 'systemctl', 'systemdRuntimeOnly', + 'liveAuthorityResolver', 'runtimeAgentHubSocket', + 'ociAdapter', + ], + ['stateRoot', 'installationsRoot'], + ); + const stateRoot = absolute(options.stateRoot); + const installationsRoot = absolute(options.installationsRoot); + const serviceMode = options.unitRoot !== undefined; + const ociAdapter = options.ociAdapter === undefined ? null : options.ociAdapter; + const unitRoot = serviceMode ? absolute(options.unitRoot) : null; + if (overlaps(stateRoot, installationsRoot) || overlaps(installationsRoot, stateRoot) + || unitRoot && [stateRoot, installationsRoot].some(root => overlaps(root, unitRoot) || overlaps(unitRoot, root))) { + fail('runtime_boundary_violation'); + } + if (serviceMode && options.runtimeAgentHubSocket === undefined) fail('runtime_boundary_violation'); + if (!serviceMode && [ + 'supervisor', 'commandPath', 'systemdAnalyze', 'systemctl', 'systemdRuntimeOnly', 'runtimeAgentHubSocket', + ] + .some(key => options[key] !== undefined)) fail('runtime_boundary_violation'); + if (ociAdapter !== null && [ + 'plan', 'reconcileHostAccount', 'reconcileImage', 'reconcileBridge', 'reconcileContainer', + 'verify', 'commit', 'rollback', + ].some(method => typeof ociAdapter[method] !== 'function')) fail('runtime_boundary_violation'); + const clock = options.clock === undefined ? Date.now : options.clock; + const idFactory = options.idFactory === undefined ? defaultJobId : options.idFactory; + const leaseMs = options.leaseMs === undefined ? DEFAULT_JOB_LEASE_MS : options.leaseMs; + const liveAuthorityResolver = options.liveAuthorityResolver === undefined ? null : options.liveAuthorityResolver; + if (typeof clock !== 'function' || typeof idFactory !== 'function' + || liveAuthorityResolver !== null && typeof liveAuthorityResolver !== 'function' + || !Number.isSafeInteger(leaseMs) || leaseMs < MIN_LEASE_MS || leaseMs > MAX_LEASE_MS) { + fail('runtime_boundary_violation'); + } + const sharedOptions = options.projectRoot === undefined ? {} : { projectRoot: options.projectRoot }; + const { layout, services, supervisor, store } = sanitizedCall(() => { + const selectedLayout = createInstallationLayoutManager({ installationsRoot, ...sharedOptions }); + const selectedServices = serviceMode ? createInstallationServiceManager({ + unitRoot, + ...sharedOptions, + ...(options.commandPath === undefined ? {} : { commandPath: options.commandPath }), + ...(options.systemdAnalyze === undefined ? {} : { systemdAnalyze: options.systemdAnalyze }), + ...(options.runtimeAgentHubSocket === undefined ? {} : { + runtimeAgentHubSocket: options.runtimeAgentHubSocket, + }), + }) : null; + const selectedSupervisor = serviceMode ? (options.supervisor === undefined + ? createSystemdUserSupervisor({ + ...(options.commandPath === undefined ? {} : { commandPath: options.commandPath }), + ...(options.systemctl === undefined ? {} : { systemctl: options.systemctl }), + ...(options.systemdRuntimeOnly === undefined ? {} : { runtimeOnly: options.systemdRuntimeOnly }), + }) + : options.supervisor) : null; + if (serviceMode) { + const methods = [ + 'snapshot', 'reload', 'enable', 'disable', 'start', 'stop', 'resetFailed', + 'restoreState', 'inspect', 'health', + ]; + if (!selectedSupervisor || methods.some(method => typeof selectedSupervisor[method] !== 'function')) { + fail('runtime_boundary_violation'); + } + } + return Object.freeze({ + layout: selectedLayout, + services: selectedServices, + supervisor: selectedSupervisor, + store: createInstallationJobStore({ + stateRoot, + clock, + ...sharedOptions, + ...(serviceMode ? { pipelineId: INSTALLATION_SERVICE_PIPELINE_ID } : {}), + }), + }); + }); + let closed = false; + + function assertOpen() { + if (closed) fail('installation_operation_failed'); + } + + function registerFixture(manifest, authority, fixtureAuthority) { + assertOpen(); + return sanitizedCall(() => store.registerFixture( + manifest, + authority, + fixtureAuthority, + selectedTime(clock), + )); + } + + function registerLive(manifest, authority, registrationAuthority) { + assertOpen(); + return sanitizedCall(() => store.registerLive( + manifest, + authority, + registrationAuthority, + selectedTime(clock), + )); + } + + function request(manifest, authority, operation, requestAuthority, backendValue = null) { + assertOpen(); + return sanitizedCall(() => { + const jobId = idFactory(); + const pipelineId = backendValue === null ? undefined + : backendValue === 'systemd_user' && !serviceMode ? INSTALLATION_JOB_PIPELINE_ID + : pipelineIdForBackend(backendValue); + return store.request( + manifest, + authority, + operation, + requestAuthority, + jobId, + selectedTime(clock), + ...(pipelineId === undefined ? [] : [pipelineId]), + ); + }); + } + + function authorizeLive(manifest, authority, requestAuthority, jobId, liveAuthority) { + assertOpen(); + return sanitizedCall(() => store.authorizeLive( + manifest, + authority, + requestAuthority, + jobId, + liveAuthority, + selectedTime(clock), + )); + } + + function inspect(manifest, authority, requestAuthority) { + assertOpen(); + return sanitizedCall(() => store.current(manifest, authority, requestAuthority)); + } + + function progress(manifest, authority, requestAuthority, jobId) { + assertOpen(); + return sanitizedCall(() => store.progress(manifest, authority, requestAuthority, jobId)); + } + + function liveAuthorityRequest(current, claim) { + return Object.freeze({ + organizationId: current.manifest.organization.id, + runtimeKey: current.manifest.runtime.key, + manifestRevision: current.manifest.revision, + installationRevision: current.revision, + jobId: claim.jobId, + }); + } + + function validateLiveAuthority(authority, current, claim) { + exact(authority, + ['manifestAuthority', 'backend', 'organizationStatus', 'installationState', 'installationRevision', 'currentJobId'], + ['manifestAuthority', 'backend', 'organizationStatus', 'installationState', 'installationRevision', 'currentJobId']); + if (!['pending_owner', 'setup_required', 'active'].includes(authority.organizationStatus) + || authority.installationState !== 'provisioning' || authority.currentJobId !== claim.jobId) { + fail('installation_operation_in_progress'); + } + if (authority.installationRevision !== current.revision) fail('installation_revision_conflict'); + if (authority.backend === 'oci_container_v1' && current.pipelineId !== INSTALLATION_OCI_PIPELINE_ID + || authority.backend === 'native_service_v1' && current.pipelineId !== INSTALLATION_NATIVE_PIPELINE_ID + || authority.backend === 'systemd_user' && ![ + INSTALLATION_JOB_PIPELINE_ID, INSTALLATION_SERVICE_PIPELINE_ID, + ].includes(current.pipelineId) + || !['oci_container_v1', 'native_service_v1', 'systemd_user'].includes(authority.backend)) { + fail('runtime_identity_mismatch'); + } + const manifest = serverInstallationManifest(current.manifest, authority.manifestAuthority); + if (JSON.stringify(manifest) !== JSON.stringify(current.manifest)) fail('runtime_identity_mismatch'); + return authority; + } + + function authoritativeCurrent(current, claim) { + if (current.fixture) return current; + if (liveAuthorityResolver === null) fail('installation_operation_not_allowed'); + const authority = validateLiveAuthority( + liveAuthorityResolver(liveAuthorityRequest(current, claim)), current, claim); + return Object.freeze({ ...current, authority: authority.manifestAuthority }); + } + + function authoritativeMutation(current, claim, mutation) { + if (current.fixture) return mutation(); + if (liveAuthorityResolver === null) fail('installation_operation_not_allowed'); + let invoked = false; + let result; + const authority = liveAuthorityResolver(liveAuthorityRequest(current, claim), () => { + if (invoked) fail('runtime_boundary_violation'); + invoked = true; + result = mutation(); + }); + if (!invoked) fail('runtime_boundary_violation'); + validateLiveAuthority(authority, current, claim); + return result; + } + + function servicePlan(current) { + if (!serviceMode || current.pipelineId !== INSTALLATION_SERVICE_PIPELINE_ID) { + fail('runtime_boundary_violation'); + } + const selectedLayout = layout.derive(current.manifest, current.authority); + return services.plan(current.manifest, current.authority, selectedLayout); + } + + function ociPlan(current, claim) { + if (ociAdapter === null || ![INSTALLATION_OCI_PIPELINE_ID, INSTALLATION_NATIVE_PIPELINE_ID].includes(current.pipelineId)) { + fail('runtime_boundary_violation'); + } + return ociAdapter.plan(current.manifest, current.authority, { fixture: current.fixture, claim }); + } + + let hostMutationDepth = 0; + function mutationGuard(claim, current) { + return mutation => store.mutateClaim(claim, selectedTime(clock), + () => authoritativeMutation(current, claim, () => { + hostMutationDepth += 1; + try { return mutation(); } finally { hostMutationDepth -= 1; } + })); + } + + function dispatchHostRequest(request, dispatch) { + const { authorizeHostRequest } = require('./oci-host-permissions'); + const claim = request.claim; + const authorize = () => { + const current = store.work(claim, selectedTime(clock)); + if (![INSTALLATION_OCI_PIPELINE_ID, INSTALLATION_NATIVE_PIPELINE_ID].includes(current.pipelineId) + || current.cancelRequested && !current.compensating) fail('installation_operation_not_allowed'); + const lease = authorizeHostRequest({ kind: 'provisioning', claim, manifest: current.manifest, + backend: current.pipelineId === INSTALLATION_NATIVE_PIPELINE_ID ? 'native_service_v1' : 'oci_container_v1', stage: current.stage, compensation: current.compensating, + operation: current.operation, installationRevision: current.revision, expiresAt: current.leaseExpiresAt }, request); + return dispatch(lease); + }; + if (hostMutationDepth) return authorize(); + const current = authoritativeCurrent(store.work(claim, selectedTime(clock)), claim); + return mutationGuard(claim, current)(authorize); + } + + function sameSupervisorState(left, right) { + return JSON.stringify(left) === JSON.stringify(right); + } + + function rollbackServices(claim, current, removeCandidate) { + if (!serviceMode || current.pipelineId !== INSTALLATION_SERVICE_PIPELINE_ID + || current.completedStages < 2) return Object.freeze({ selected: null, journal: false }); + const selected = servicePlan(current); + const prior = services.rollbackState(selected); + const installCheckpoint = INSTALLATION_SERVICE_STAGES.indexOf('runtime_service_install') + 1; + if (!prior && current.completedStages >= installCheckpoint) fail('service_installation_failed'); + if (prior) { + const guard = mutationGuard(claim, current); + supervisor.stop(selected, guard); + supervisor.resetFailed(selected, mutationGuard(claim, current)); + supervisor.disable(selected, guard); + services.restoreFiles(selected, mutationGuard(claim, current)); + supervisor.reload(selected, mutationGuard(claim, current)); + supervisor.restoreState(selected, prior, mutationGuard(claim, current)); + const restored = supervisor.snapshot(selected); + if (!sameSupervisorState(restored, prior)) fail('service_installation_failed'); + } + if (removeCandidate) services.removeCandidate(selected, mutationGuard(claim, current)); + return Object.freeze({ selected, journal: Boolean(prior) }); + } + + function rollbackOci(claim, current) { + if (![INSTALLATION_OCI_PIPELINE_ID, INSTALLATION_NATIVE_PIPELINE_ID].includes(current.pipelineId) || ociAdapter === null) { + fail('runtime_boundary_violation'); + } + return ociAdapter.rollback(current.manifest, current.authority, { + fixture: current.fixture, + intent: current.compensationIntent, + claim, + }, mutationGuard(claim, current)); + } + + function executeCompensation(claim, current) { + try { + if ([INSTALLATION_OCI_PIPELINE_ID, INSTALLATION_NATIVE_PIPELINE_ID].includes(current.pipelineId)) rollbackOci(claim, current); + else rollbackServices(claim, current, current.compensationIntent === 'cancelled'); + } catch (rollbackError) { + if (rollbackError?.code === 'installation_operation_in_progress') throw rollbackError; + return store.failCompensation(claim, selectedTime(clock)); + } + return store.finishCompensation(claim, selectedTime(clock)); + } + + function finishAfterError(claim, error) { + const now = selectedTime(clock); + let current; + try { current = authoritativeCurrent(store.work(claim, now), claim); } + catch (stateError) { + if (stateError?.code === 'installation_operation_in_progress') throw stateError; + throw error; + } + const cancellation = current.cancelRequested; + const requiresCompensation = (serviceMode && current.pipelineId === INSTALLATION_SERVICE_PIPELINE_ID + && current.completedStages >= INSTALLATION_SERVICE_STAGES.indexOf('runtime_service_render')) + || [INSTALLATION_OCI_PIPELINE_ID, INSTALLATION_NATIVE_PIPELINE_ID].includes(current.pipelineId); + if (requiresCompensation) { + store.beginCompensation( + claim, + cancellation ? 'cancelled' : 'failed', + error, + selectedTime(clock), + ); + return executeCompensation(claim, + authoritativeCurrent(store.work(claim, selectedTime(clock)), claim)); + } + if (cancellation) return store.finishCancelled(claim, selectedTime(clock)); + return store.finishFailed(claim, error, selectedTime(clock)); + } + + function runNext(workerId) { + assertOpen(); + return sanitizedCall(() => { + const claim = store.claimNext(workerId, selectedTime(clock), leaseMs); + if (!claim) return Object.freeze({ ok: true, status: 'idle' }); + if (claim.terminalJob) return claim.terminalJob; + try { + for (;;) { + const now = selectedTime(clock); + const current = authoritativeCurrent(store.work(claim, now), claim); + if (current.compensating) return executeCompensation(claim, current); + if (current.cancelRequested) { + return finishAfterError(claim, Object.assign(new Error('installation_operation_not_allowed'), { + code: 'installation_operation_not_allowed', + })); + } + if (current.stage === null) { + store.renew(claim, now, [INSTALLATION_OCI_PIPELINE_ID, INSTALLATION_NATIVE_PIPELINE_ID].includes(current.pipelineId) ? 600_000 : leaseMs); + let selected = null; + if ([INSTALLATION_OCI_PIPELINE_ID, INSTALLATION_NATIVE_PIPELINE_ID].includes(current.pipelineId)) { + selected = ociPlan(current, claim); + ociAdapter.verify(selected, claim); + ociAdapter.commit(selected, claim, mutationGuard(claim, current)); + } else { + layout.inspect(current.manifest, current.authority); + } + if (current.pipelineId === INSTALLATION_SERVICE_PIPELINE_ID) { + selected = servicePlan(current); + if (!services.rollbackState(selected)) fail('service_installation_failed'); + services.inspectInstalled(selected); + supervisor.inspect(selected); + store.renew(claim, selectedTime(clock), serviceHealthLeaseMs(selected, leaseMs)); + supervisor.health(selected); + } + if (selected && current.pipelineId === INSTALLATION_SERVICE_PIPELINE_ID) { + services.markVerified(selected, mutationGuard(claim, current)); + } + const verifiedAt = selectedTime(clock); + const finalState = authoritativeCurrent(store.work(claim, verifiedAt), claim); + if (finalState.cancelRequested) { + return finishAfterError(claim, Object.assign(new Error('installation_operation_not_allowed'), { + code: 'installation_operation_not_allowed', + })); + } + if (selected) { + return store.finishSucceeded(claim, verifiedAt); + } + return store.finishSucceeded(claim, verifiedAt); + } + store.renew(claim, now, [INSTALLATION_OCI_PIPELINE_ID, INSTALLATION_NATIVE_PIPELINE_ID].includes(current.pipelineId) ? 600_000 : leaseMs); + let receipt; + if (current.stage === INSTALLATION_JOB_STAGES[0]) { + receipt = layout.materialize( + current.manifest, + current.authority, + mutationGuard(claim, current), + ); + } else if (current.stage === INSTALLATION_JOB_STAGES[1]) { + receipt = layout.inspect(current.manifest, current.authority); + } else if (current.stage === INSTALLATION_SERVICE_STAGES[2]) { + const selected = servicePlan(current); + if (services.rollbackState(selected)) services.finalizeSettled(selected, mutationGuard(claim, current)); + receipt = services.render(selected, mutationGuard(claim, current)); + } else if (current.stage === INSTALLATION_SERVICE_STAGES[3]) { + receipt = services.validate(servicePlan(current)); + } else if (current.stage === INSTALLATION_SERVICE_STAGES[4]) { + const selected = servicePlan(current); + if (services.rollbackState(selected)) fail('service_installation_failed'); + supervisor.reload(selected, mutationGuard(claim, current)); + receipt = services.install(selected, supervisor.snapshot(selected), mutationGuard(claim, current)); + } else if (current.stage === INSTALLATION_SERVICE_STAGES[5]) { + const selected = servicePlan(current); + if (!services.rollbackState(selected)) fail('service_installation_failed'); + supervisor.reload(selected, mutationGuard(claim, current)); + supervisor.enable(selected, mutationGuard(claim, current)); + receipt = supervisor.start(selected, mutationGuard(claim, current)); + } else if (current.stage === INSTALLATION_SERVICE_STAGES[6]) { + const selected = servicePlan(current); + if (!services.rollbackState(selected)) fail('service_installation_failed'); + services.inspectInstalled(selected); + supervisor.inspect(selected); + store.renew(claim, selectedTime(clock), serviceHealthLeaseMs(selected, leaseMs)); + receipt = supervisor.health(selected); + } else if (current.stage === INSTALLATION_OCI_STAGES[0]) { + receipt = ociAdapter.reconcileHostAccount( + current.manifest, + current.authority, + { fixture: current.fixture, claim }, + mutationGuard(claim, current), + ); + } else if (current.stage === INSTALLATION_OCI_STAGES[1]) { + receipt = ociAdapter.reconcileImage(ociPlan(current, claim), claim, mutationGuard(claim, current)); + } else if (current.stage === INSTALLATION_OCI_STAGES[2]) { + receipt = ociAdapter.reconcileBridge(ociPlan(current, claim), claim, mutationGuard(claim, current)); + } else if (current.stage === INSTALLATION_OCI_STAGES[3]) { + receipt = ociAdapter.reconcileContainer(ociPlan(current, claim), claim, mutationGuard(claim, current)); + } else if (current.stage === INSTALLATION_OCI_STAGES[4]) { + receipt = ociAdapter.verify(ociPlan(current, claim), claim); + } else { + fail('runtime_boundary_violation'); + } + const completedAt = selectedTime(clock); + const after = authoritativeCurrent(store.work(claim, completedAt), claim); + if (after.cancelRequested) { + return finishAfterError(claim, Object.assign(new Error('installation_operation_not_allowed'), { + code: 'installation_operation_not_allowed', + })); + } + store.completeStage(claim, current.stage, receipt, completedAt); + } + } catch (error) { + return finishAfterError(claim, error); + } + }); + } + + function health() { + assertOpen(); + return sanitizedCall(() => store.health()); + } + + function close() { + if (closed) return; + store.close(); + closed = true; + } + + return Object.freeze({ + registerFixture, + registerLive, + request, + authorizeLive, + inspect, + progress, + runNext, + dispatchHostRequest, + health, + close, + }); +} + +module.exports = { + DEFAULT_JOB_LEASE_MS, + createDurableInstallationProvisioner, +}; diff --git a/core/core/installations/src/layout.js b/core/core/installations/src/layout.js new file mode 100644 index 0000000..8c11c4a --- /dev/null +++ b/core/core/installations/src/layout.js @@ -0,0 +1,315 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { + PROJECT_ROOT, + MANAGED_INSTALLATION_LAYOUT_VERSION, + MANAGED_INSTALLATION_LAYOUT_TEMPLATE, + MANAGED_INSTALLATION_DIRECTORY_FIELDS, + resolveManagedInstallationRuntimePaths, + managedInstallationRuntimeEnvironment, +} = require('../../../shared/paths/runtime-paths'); +const { serverInstallationManifest } = require('../../../shared/contracts/src'); + +const INSTALLATION_LAYOUT_VERSION = MANAGED_INSTALLATION_LAYOUT_VERSION; +const INSTALLATION_LAYOUT_TEMPLATE = MANAGED_INSTALLATION_LAYOUT_TEMPLATE; +const PRIVATE_DIRECTORY_MODE = 0o700; +const DIRECTORY_FIELDS = MANAGED_INSTALLATION_DIRECTORY_FIELDS; +const RELATIVE_DIRECTORIES = Object.freeze(Object.values(DIRECTORY_FIELDS)); + +const CHILDREN = (() => { + const selected = new Map(); + for (const relative of RELATIVE_DIRECTORIES) { + const parent = path.dirname(relative) === '.' ? '' : path.dirname(relative); + if (!selected.has(parent)) selected.set(parent, new Set()); + selected.get(parent).add(path.basename(relative)); + } + return selected; +})(); + +function fail(code) { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, allowed, required, code) { + if (!plain(value)) fail(code); + const keys = Object.keys(value); + if (keys.some(key => !allowed.includes(key)) || required.some(key => !Object.hasOwn(value, key))) fail(code); +} + +function absolute(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) fail('runtime_boundary_violation'); + return value; +} + +function contains(root, candidate) { + const relative = path.relative(root, candidate); + return relative === '' || (relative !== '..' && !relative.startsWith(`..${path.sep}`) + && !path.isAbsolute(relative)); +} + +function lstatMaybe(target) { + try { return fs.lstatSync(target); } + catch (error) { + if (error?.code === 'ENOENT') return null; + fail('runtime_layout_failed'); + } +} + +function effectiveUid() { + if (typeof process.geteuid !== 'function') fail('runtime_boundary_violation'); + return process.geteuid(); +} + +function privateDirectoryIdentity(target, code = 'runtime_layout_failed', expectedDevice = null) { + const selected = absolute(target); + const info = lstatMaybe(selected); + if (!info || !info.isDirectory() || info.isSymbolicLink() || info.uid !== effectiveUid() + || (info.mode & 0o7777) !== PRIVATE_DIRECTORY_MODE + || (expectedDevice !== null && info.dev !== expectedDevice)) fail(code); + let canonical; + try { canonical = fs.realpathSync(selected); } catch { fail(code); } + if (canonical !== selected) fail(code); + return Object.freeze({ dev: info.dev, ino: info.ino }); +} + +function sameIdentity(left, right) { + return Boolean(left && right && left.dev === right.dev && left.ino === right.ino); +} + +function canonicalProjectRoot(projectRoot) { + const selected = absolute(projectRoot); + const info = lstatMaybe(selected); + if (!info || !info.isDirectory() || info.isSymbolicLink()) fail('runtime_boundary_violation'); + let canonical; + try { canonical = fs.realpathSync(selected); } catch { fail('runtime_boundary_violation'); } + if (canonical !== selected) fail('runtime_boundary_violation'); + return selected; +} + +function configuredRoot(options) { + exact(options, ['installationsRoot', 'projectRoot'], ['installationsRoot'], 'runtime_boundary_violation'); + const installationsRoot = absolute(options.installationsRoot); + const projectRoot = canonicalProjectRoot( + options.projectRoot === undefined ? PROJECT_ROOT : options.projectRoot, + ); + if (installationsRoot === path.parse(installationsRoot).root) fail('runtime_boundary_violation'); + privateDirectoryIdentity(installationsRoot, 'runtime_boundary_violation'); + if (contains(projectRoot, installationsRoot) || contains(installationsRoot, projectRoot)) { + fail('runtime_boundary_violation'); + } + return Object.freeze({ installationsRoot, projectRoot }); +} + +function resolvedLayout(config, manifestValue, authorityValue) { + const manifest = serverInstallationManifest(manifestValue, authorityValue); + if (manifest.runtime.templateId !== INSTALLATION_LAYOUT_TEMPLATE) fail('runtime_boundary_violation'); + const installationRoot = path.join(config.installationsRoot, manifest.runtime.key); + if (path.dirname(installationRoot) !== config.installationsRoot) fail('runtime_boundary_violation'); + const directories = Object.fromEntries(Object.entries(DIRECTORY_FIELDS) + .map(([field, relative]) => [field, path.join(installationRoot, relative)])); + return Object.freeze({ + layoutVersion: INSTALLATION_LAYOUT_VERSION, + templateId: INSTALLATION_LAYOUT_TEMPLATE, + runtimeKey: manifest.runtime.key, + projectRoot: config.projectRoot, + installationRoot, + directories: Object.freeze(directories), + }); +} + +function directoryFor(root, relative) { + return relative ? path.join(root, relative) : root; +} + +function validatePartialLayout(layout, expectedDevice = null) { + const installationIdentity = privateDirectoryIdentity( + layout.installationRoot, + 'runtime_layout_failed', + expectedDevice, + ); + for (const relative of RELATIVE_DIRECTORIES) { + const target = directoryFor(layout.installationRoot, relative); + if (lstatMaybe(target)) { + privateDirectoryIdentity(target, 'runtime_layout_failed', installationIdentity.dev); + } + } + for (const [relative, expected] of CHILDREN) { + const parent = directoryFor(layout.installationRoot, relative); + if (!lstatMaybe(parent)) continue; + privateDirectoryIdentity(parent); + let entries; + try { entries = fs.readdirSync(parent); } catch { fail('runtime_layout_failed'); } + if (entries.some(name => !expected.has(name))) fail('runtime_layout_failed'); + } +} + +function validateCompleteLayout(layout, expectedDevice = null) { + validatePartialLayout(layout, expectedDevice); + const installationIdentity = privateDirectoryIdentity( + layout.installationRoot, + 'runtime_layout_failed', + expectedDevice, + ); + for (const relative of RELATIVE_DIRECTORIES) { + privateDirectoryIdentity( + directoryFor(layout.installationRoot, relative), + 'runtime_layout_failed', + installationIdentity.dev, + ); + } +} + +function directMutation(operation) { + return operation(); +} + +function ensurePrivateDirectory(target, parent, mutate) { + const parentBefore = privateDirectoryIdentity(parent); + let changed = false; + if (!lstatMaybe(target)) { + try { + changed = mutate(() => { + if (lstatMaybe(target)) return false; + fs.mkdirSync(target, { mode: PRIVATE_DIRECTORY_MODE }); + return true; + }) === true; + } catch (error) { + if (error?.code === 'installation_operation_in_progress') throw error; + if (error?.code !== 'EEXIST') fail('runtime_layout_failed'); + } + } + privateDirectoryIdentity(target, 'runtime_layout_failed', parentBefore.dev); + const parentAfter = privateDirectoryIdentity(parent); + if (!sameIdentity(parentBefore, parentAfter)) fail('runtime_layout_failed'); + return changed; +} + +function summary(status, directoryCount, changed) { + return Object.freeze({ + layoutVersion: INSTALLATION_LAYOUT_VERSION, + status, + directoryCount, + changed, + }); +} + +function assertEmptyCleanupAuthority(value) { + exact( + value, + ['fixture', 'installationState', 'retainedData'], + ['fixture', 'installationState', 'retainedData'], + 'runtime_boundary_violation', + ); + if (value.fixture !== true || value.installationState !== 'failed' || value.retainedData !== false) { + fail('runtime_boundary_violation'); + } +} + +function createInstallationLayoutManager(options) { + const config = configuredRoot(options); + const rootIdentity = privateDirectoryIdentity(config.installationsRoot, 'runtime_boundary_violation'); + + function assertRoot() { + const current = privateDirectoryIdentity(config.installationsRoot, 'runtime_boundary_violation'); + if (!sameIdentity(rootIdentity, current)) fail('runtime_boundary_violation'); + } + + function derive(manifestValue, authorityValue) { + assertRoot(); + return resolvedLayout(config, manifestValue, authorityValue); + } + + function runtimePaths(manifestValue, authorityValue) { + return resolveManagedInstallationRuntimePaths(derive(manifestValue, authorityValue)); + } + + function runtimeEnvironment(manifestValue, authorityValue) { + return managedInstallationRuntimeEnvironment(derive(manifestValue, authorityValue)); + } + + function inspect(manifestValue, authorityValue) { + const layout = derive(manifestValue, authorityValue); + validateCompleteLayout(layout, rootIdentity.dev); + assertRoot(); + return summary('verified', RELATIVE_DIRECTORIES.length, false); + } + + function materialize(manifestValue, authorityValue, mutationCapability = directMutation) { + if (typeof mutationCapability !== 'function') fail('runtime_boundary_violation'); + const layout = derive(manifestValue, authorityValue); + const existing = lstatMaybe(layout.installationRoot); + if (existing) validatePartialLayout(layout, rootIdentity.dev); + assertRoot(); + + let changed = false; + if (!existing) { + changed = ensurePrivateDirectory( + layout.installationRoot, + config.installationsRoot, + mutationCapability, + ) || changed; + } + for (const relative of RELATIVE_DIRECTORIES) { + assertRoot(); + const target = directoryFor(layout.installationRoot, relative); + const parent = path.dirname(target); + changed = ensurePrivateDirectory(target, parent, mutationCapability) || changed; + } + validateCompleteLayout(layout, rootIdentity.dev); + assertRoot(); + return summary('verified', RELATIVE_DIRECTORIES.length, changed); + } + + function removeEmpty(manifestValue, authorityValue, cleanupAuthority) { + assertEmptyCleanupAuthority(cleanupAuthority); + const layout = derive(manifestValue, authorityValue); + if (!lstatMaybe(layout.installationRoot)) return summary('removed', 0, false); + validatePartialLayout(layout, rootIdentity.dev); + + for (const relative of RELATIVE_DIRECTORIES) { + const target = directoryFor(layout.installationRoot, relative); + if (!lstatMaybe(target) || CHILDREN.has(relative)) continue; + let entries; + try { entries = fs.readdirSync(target); } catch { fail('runtime_layout_failed'); } + if (entries.length !== 0) fail('runtime_layout_failed'); + } + + const installationIdentity = privateDirectoryIdentity( + layout.installationRoot, + 'runtime_layout_failed', + rootIdentity.dev, + ); + const removalOrder = [...RELATIVE_DIRECTORIES] + .sort((left, right) => right.split('/').length - left.split('/').length); + for (const relative of removalOrder) { + assertRoot(); + const target = directoryFor(layout.installationRoot, relative); + if (!lstatMaybe(target)) continue; + privateDirectoryIdentity(target, 'runtime_layout_failed', installationIdentity.dev); + try { fs.rmdirSync(target); } catch { fail('runtime_layout_failed'); } + } + assertRoot(); + privateDirectoryIdentity(layout.installationRoot, 'runtime_layout_failed', rootIdentity.dev); + try { fs.rmdirSync(layout.installationRoot); } catch { fail('runtime_layout_failed'); } + assertRoot(); + return summary('removed', 0, true); + } + + return Object.freeze({ derive, runtimePaths, runtimeEnvironment, inspect, materialize, removeEmpty }); +} + +module.exports = { + INSTALLATION_LAYOUT_VERSION, + INSTALLATION_LAYOUT_TEMPLATE, + PRIVATE_DIRECTORY_MODE, + RELATIVE_DIRECTORIES, + createInstallationLayoutManager, +}; diff --git a/core/core/installations/src/legacy-pre-update-backups.js b/core/core/installations/src/legacy-pre-update-backups.js new file mode 100644 index 0000000..e203267 --- /dev/null +++ b/core/core/installations/src/legacy-pre-update-backups.js @@ -0,0 +1,52 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +const { atomic, privateJson } = require('./release-delivery-files'); +const { receiptKey } = require('./offsite-policy'); +// Older Core updates used the shared repository rather than a categorized +// archive. Retire only those exact rollout tags after the new full set uploads. +async function retireLegacyPreUpdateBackups({ db, config, record, run, storage, receiptRoot, ownerUid = 0 }) { + if (!db.prepare("SELECT 1 FROM sqlite_schema WHERE name='platform_rollout_backups'").get()) return; + const retired = db.prepare(`SELECT r.id,c.attempt FROM platform_rollouts r JOIN platform_rollout_core c ON c.rollout_id=r.id + WHERE r.status='completed' AND NOT EXISTS (SELECT 1 FROM platform_rollout_backups b WHERE b.rollout_id=r.id)`).all(); + const pending = retired.map(row => { + if (!/^rollout_[a-f0-9]{32}$/.test(row.id) || row.attempt < 0 || row.attempt > 1000) throw Error('backup_retention_failed'); + const marker = path.join(receiptRoot, `${row.id}.pre-update-replaced.json`); + const directory = path.join(config.localRoot, 'backups/platform-core', row.id); + return { ...row, marker, directory, replaced: privateJson(marker, ownerUid, true) }; + }).filter(row => !row.replaced || fs.existsSync(row.directory)); + if (!pending.length) return; + let progress; + if (pending.some(row => !row.replaced)) { + const latest = db.prepare('SELECT rollout_id FROM platform_rollout_backups ORDER BY rowid DESC LIMIT 1').get(); + if (!latest) return; + progress = require('../../accounts/src/rollout-backups').rolloutBackupProgress(db, latest.rollout_id); + if (progress?.status !== 'completed') return; + require('./rollout-backup-proof').rolloutBackupProof(db, latest.rollout_id, record); + } + for (const row of pending) { + const { marker, directory } = row; + if (!row.replaced) { + const tags = new Set(Array.from({ length: row.attempt }, (_, i) => receiptKey(path.join(directory, `attempt-${i + 1}`)))); + const before = run(['snapshots']).flat(); + if (before.some(s => !/^[a-f0-9]{64}$/.test(s.id) || !Array.isArray(s.tags))) throw Error('backup_retention_failed'); + const targets = before.filter(s => s.tags.some(tag => tags.has(tag))); + if (targets.some(s => s.hostname !== 'dispatch' || s.tags.length !== 1)) throw Error('backup_retention_failed'); + const keep = before.filter(s => !targets.includes(s)).map(s => s.id).sort(); + // Prune also resumes an interrupted prior deletion after forget succeeded. + await storage.withDeletionAccess(['data/', 'index/', 'snapshots/'].map(p => config.prefix + '/' + p), async () => { + if (targets.length) run(['forget', ...targets.map(s => s.id)]); + run(['prune', '--max-unused', '0']); + const after = run(['snapshots']).flat().map(s => s.id).sort(); + if (JSON.stringify(after) !== JSON.stringify(keep)) throw Error('backup_retention_failed'); + }); + atomic(marker, { schemaVersion: 1, status: 'replaced', replacement: progress.setId }); + for (const tag of tags) fs.rmSync(path.join(receiptRoot, tag + '.json'), { force: true }); + } + if (fs.existsSync(directory)) { + const stat = fs.lstatSync(directory); + if (!stat.isDirectory() || stat.uid !== config.coreUid || fs.realpathSync(directory) !== directory) throw Error('backup_retention_failed'); + fs.rmSync(directory, { recursive: true, force: true }); + } + } +} +module.exports = { retireLegacyPreUpdateBackups }; diff --git a/core/core/installations/src/lifecycle-reconcile.js b/core/core/installations/src/lifecycle-reconcile.js new file mode 100644 index 0000000..ec5b872 --- /dev/null +++ b/core/core/installations/src/lifecycle-reconcile.js @@ -0,0 +1,74 @@ +'use strict'; + +function fail(code = 'runtime_boundary_violation') { throw Object.assign(new Error(code), { code }); } +function identifier(value) { + if (typeof value !== 'string' || !/^[a-z][a-z0-9_-]{2,95}$/.test(value)) fail(); + return value; +} + +function createInstallationLifecycleReconciler(options) { + if (!options || typeof options !== 'object' || Array.isArray(options) + || Object.keys(options).some(key => !['store', 'authorityFactory', 'runtimeFactory', 'clock', 'concurrency', 'backupOnly'].includes(key)) + || !options.store || typeof options.store.statusLifecycleMismatches !== 'function' + || typeof options.store.lifecycleExecutionCandidates !== 'function' + || typeof options.store.lifecycleExhaustedCandidates !== 'function' + || typeof options.store.lifecycleOutstandingCount !== 'function' + || typeof options.authorityFactory !== 'function' || typeof options.runtimeFactory !== 'function') fail(); + const clock = options.clock || Date.now; + + async function runPending(workerIdValue, limitValue = 20) { + const workerId = identifier(workerIdValue); + if (!Number.isSafeInteger(limitValue) || limitValue < 1 || limitValue > 50) fail('invalid_input'); + let requested = 0; + let completed = 0; + let failed = 0; + const exhausted = options.store.lifecycleExhaustedCandidates(clock(), limitValue); + const mismatches = options.backupOnly ? [] : options.store.statusLifecycleMismatches(limitValue); + for (const mismatch of mismatches) { + const operation = mismatch.organization_status === 'suspended' ? 'suspend' : 'resume'; + const authority = options.authorityFactory( + mismatch.organization_id, 'organization_status_lifecycle', false, + ); + authority.request({ + operation, + idempotencyKey: `organization-status:${operation}:${mismatch.installation_revision}`, + expectedRevision: mismatch.installation_revision, + }); + requested += 1; + } + + const candidates = options.store.lifecycleExecutionCandidates(clock(), limitValue) + .filter(candidate => !options.backupOnly || candidate.operation === 'backup'); + let next = 0; + const concurrency = options.concurrency || 1; + if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 4) fail('invalid_input'); + const results = await Promise.allSettled(Array.from({ length: Math.min(concurrency, candidates.length) }, async () => { + while (next < candidates.length) { + const index = next++; + const candidate = candidates[index]; + const authority = options.authorityFactory( + candidate.organization_id, candidate.authority_scope, false, + ); + const runtime = options.runtimeFactory(candidate.organization_id, authority); + const result = await runtime.run(candidate.id, `${workerId}_${index}`); + if (result.status === 'succeeded') completed += 1; + else failed += 1; + } + })); + const rejected = results.find(result => result.status === 'rejected'); + if (rejected) throw rejected.reason; + const pending = options.store.lifecycleOutstandingCount() > 0; + return Object.freeze({ + requested, + processed: candidates.length, + completed, + failed, + exhausted: exhausted.length, + pending, + }); + } + + return Object.freeze({ runPending }); +} + +module.exports = { createInstallationLifecycleReconciler }; diff --git a/core/core/installations/src/lifecycle.js b/core/core/installations/src/lifecycle.js new file mode 100644 index 0000000..ed061be --- /dev/null +++ b/core/core/installations/src/lifecycle.js @@ -0,0 +1,540 @@ +'use strict'; + +const fs = require('node:fs'); +const { + INSTALLATION_ACTIVATION_EVIDENCE_VERSION, + installationActivationEvidenceDigest, + serverInstallationManifest, +} = require('../../../shared/contracts/src'); +const { PROJECT_ROOT } = require('../../../shared/paths/runtime-paths'); +const { createInstallationLayoutManager } = require('./layout'); +const { createInstallationServiceManager } = require('./services'); +const { createInstallationBackupManager } = require('./backups'); + +function fail(code = 'installation_operation_failed') { throw Object.assign(new Error(code), { code }); } +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} +function exact(value, allowed, required, code = 'runtime_boundary_violation') { + if (!plain(value)) fail(code); + const keys = Object.keys(value); + if (keys.some(key => !allowed.includes(key)) || required.some(key => !Object.hasOwn(value, key))) fail(code); +} +function lifecycleReceipt(status, value = {}) { + const result = { status }; + if (typeof value.changed === 'boolean') result.changed = value.changed; + if (typeof value.syncWasRunning === 'boolean') result.syncWasRunning = value.syncWasRunning; + if (Number.isSafeInteger(value.serviceCount)) result.serviceCount = value.serviceCount; + return Object.freeze(result); +} +function allStopped(supervisor, plan) { + const state = supervisor.snapshot(plan); + if (!Array.isArray(state) || state.some(unit => unit.active)) fail('runtime_health_failed'); + return state; +} +function allRemoved(supervisor, plan) { + const state = supervisor.snapshot(plan); + if (!Array.isArray(state) || state.some(unit => unit.active || unit.enabled)) fail('decommission_failed'); + return state; +} +function activationEvidence(context, raw) { + exact(raw, [ + 'definitionDigest', 'requestDigest', 'previewDigest', 'batchId', 'preparationRunId', + 'target', 'runs', 'publications', 'capturedAt', + ], [ + 'definitionDigest', 'requestDigest', 'previewDigest', 'batchId', 'preparationRunId', + 'target', 'runs', 'publications', 'capturedAt', + ], 'installation_not_ready'); + const payload = Object.freeze({ + schemaVersion: INSTALLATION_ACTIVATION_EVIDENCE_VERSION, + manifestRevision: context.manifest.revision, + jobId: context.job.id, + runtimeKey: context.manifest.runtime.key, + ...raw, + }); + return Object.freeze({ ...payload, evidenceDigest: installationActivationEvidenceDigest(payload) }); +} + +function createManagedInstallationLifecycle(options) { + exact(options, [ + 'authority', 'installationsRoot', 'unitRoot', 'supervisor', 'projectRoot', 'releaseCatalog', + 'projectReleaseId', 'activationRuntimeFactory', 'layoutFactory', 'serviceManagerFactory', + 'backupManagerFactory', 'runtimeAgentHubSocket', 'waitForBackupDeletion', + ], ['authority', 'installationsRoot', 'unitRoot', 'supervisor']); + const authority = options.authority; + const waitForBackupDeletion = options.waitForBackupDeletion || require('./offsite-policy').waitForDspBackupDeletion; + const requiredAuthority = [ + 'claim', 'renew', 'desiredRuntimeState', 'mutate', 'checkpoint', 'succeed', 'failed', + ]; + if (!authority || requiredAuthority.some(method => typeof authority[method] !== 'function')) { + fail('runtime_boundary_violation'); + } + const supervisor = options.supervisor; + if (!supervisor || ['snapshot', 'reload', 'enable', 'disable', 'start', 'stop', 'resetFailed', 'restoreState', 'inspect', 'health'] + .some(method => typeof supervisor[method] !== 'function')) fail('runtime_boundary_violation'); + const projectRoot = options.projectRoot === undefined ? PROJECT_ROOT : options.projectRoot; + const projectReleaseId = options.projectReleaseId === undefined + ? 'dispatch_current_1' : options.projectReleaseId; + if (typeof projectReleaseId !== 'string' || !/^[a-z][a-z0-9_.-]{2,95}$/.test(projectReleaseId)) { + fail('runtime_boundary_violation'); + } + const releaseCatalog = options.releaseCatalog === undefined ? {} : options.releaseCatalog; + if (!plain(releaseCatalog)) fail('runtime_boundary_violation'); + const layoutFactory = options.layoutFactory || (settings => createInstallationLayoutManager(settings)); + const serviceManagerFactory = options.serviceManagerFactory + || (settings => createInstallationServiceManager(settings)); + const backupManagerFactory = options.backupManagerFactory + || (settings => createInstallationBackupManager(settings)); + const activationRuntimeFactory = options.activationRuntimeFactory || (() => fail('installation_operation_not_allowed')); + + function releaseRoot(releaseId, fallback = null) { + const selected = Object.hasOwn(releaseCatalog, releaseId) + ? releaseCatalog[releaseId] + : releaseId === projectReleaseId ? fallback : null; + if (typeof selected !== 'string') fail('installation_operation_not_allowed'); + return selected; + } + + function runtimeContext(claimed) { + const currentRoot = releaseRoot(claimed.manifest.runtime.releaseId, projectRoot); + const layoutManager = layoutFactory({ installationsRoot: options.installationsRoot, projectRoot: currentRoot }); + const layout = layoutManager.derive(claimed.manifest, claimed.manifestAuthority); + const services = serviceManagerFactory({ + unitRoot: options.unitRoot, + projectRoot: currentRoot, + ...(options.runtimeAgentHubSocket === undefined ? {} : { + runtimeAgentHubSocket: options.runtimeAgentHubSocket, + }), + }); + const plan = services.plan(claimed.manifest, claimed.manifestAuthority, layout); + const backupManager = backupManagerFactory({ layout }); + let target = null; + if (claimed.operation === 'upgrade') { + const targetRoot = releaseRoot(claimed.targetManifest.runtime.releaseId); + serverInstallationManifest(claimed.targetManifest, claimed.targetManifestAuthority); + const targetLayoutManager = layoutFactory({ installationsRoot: options.installationsRoot, projectRoot: targetRoot }); + const targetLayout = targetLayoutManager.derive(claimed.targetManifest, claimed.targetManifestAuthority); + const targetServices = serviceManagerFactory({ + unitRoot: options.unitRoot, + projectRoot: targetRoot, + ...(options.runtimeAgentHubSocket === undefined ? {} : { + runtimeAgentHubSocket: options.runtimeAgentHubSocket, + }), + }); + target = Object.freeze({ + projectRoot: targetRoot, + layoutManager: targetLayoutManager, + layout: targetLayout, + services: targetServices, + plan: targetServices.plan(claimed.targetManifest, claimed.targetManifestAuthority, targetLayout), + }); + } + return Object.freeze({ + ...claimed, projectRoot: currentRoot, layoutManager, layout, services, plan, backupManager, target, + }); + } + + function guard(context) { + return mutation => authority.mutate(context.claim, mutation); + } + + function decommissionServiceState(context) { + context.services.finalizeSettled(context.plan, guard(context)); + try { + context.services.inspectInstalled(context.plan); + return 'installed'; + } catch (installedError) { + try { + context.services.inspectAbsent(context.plan); + return 'absent'; + } catch { throw installedError; } + } + } + + function stop(context) { + if (!fs.existsSync(context.layout.installationRoot)) return lifecycleReceipt('absent', { changed: false }); + if (context.operation === 'decommission' && decommissionServiceState(context) === 'absent') { + allRemoved(supervisor, context.plan); + return lifecycleReceipt('absent', { changed: false }); + } + context.services.inspectInstalled(context.plan); + const receipt = supervisor.stop(context.plan, guard(context)); + allStopped(supervisor, context.plan); + return lifecycleReceipt('stopped', receipt); + } + + function start(context) { + context.services.inspectInstalled(context.plan); + supervisor.enable(context.plan, guard(context)); + const receipt = supervisor.start(context.plan, guard(context)); + supervisor.health(context.plan); + return lifecycleReceipt('started', receipt); + } + + function verifyRuntime(context, expectedActive) { + if (!fs.existsSync(context.layout.installationRoot)) { + if (expectedActive) fail('runtime_health_failed'); + return lifecycleReceipt('absent', { changed: false }); + } + context.layoutManager.inspect(context.manifest, context.manifestAuthority); + context.services.inspectInstalled(context.plan); + if (expectedActive) supervisor.health(context.plan); + else allStopped(supervisor, context.plan); + return lifecycleReceipt(expectedActive ? 'healthy' : 'inactive', { + changed: false, serviceCount: context.plan.units.length, + }); + } + + function snapshot(context, selected) { + if (!selected || !fs.existsSync(context.layout.installationRoot)) { + return lifecycleReceipt('absent', { changed: false }); + } + return context.backupManager.snapshot(selected, guard(context)); + } + + function scheduleIntent(context) { + const value = context.operation === 'resume' + ? context.resumeSync + : context.stageReceipts?.inspect_schedule?.syncWasRunning; + if (typeof value !== 'boolean') fail('runtime_boundary_violation'); + return value; + } + + async function inspectSchedule(context) { + if (context.operation === 'decommission' && context.removal?.sync_running !== null && context.removal?.sync_running !== undefined) return lifecycleReceipt('verified', { syncWasRunning: context.removal.sync_running === 1 }); + if (context.startingState !== 'ready') { + return lifecycleReceipt('verified', { changed: false, syncWasRunning: false }); + } + const runtime = activationRuntimeFactory(context); + if (typeof runtime.inspectSchedule !== 'function') fail('runtime_boundary_violation'); + const receipt = await runtime.inspectSchedule(); + if (typeof receipt?.syncWasRunning !== 'boolean') fail('runtime_health_failed'); + return lifecycleReceipt('verified', receipt); + } + + async function quiesceSchedule(context) { + if (context.startingState !== 'ready') { + return lifecycleReceipt('stopped', { changed: false, syncWasRunning: false }); + } + const runtime = activationRuntimeFactory(context); + if (typeof runtime.quiesceSchedule !== 'function') fail('runtime_boundary_violation'); + try { await runtime.quiesceSchedule(scheduleIntent(context)); } + catch (error) { if (context.operation !== 'decommission' || ![0, 1].includes(context.removal?.sync_running)) throw error; } + return lifecycleReceipt('stopped', { changed: false, syncWasRunning: scheduleIntent(context) }); + } + + async function restoreSchedule(context) { + if (context.operation !== 'resume' && context.startingState !== 'ready') { + return lifecycleReceipt('started', { changed: false, syncWasRunning: false }); + } + if (authority.desiredRuntimeState(context.claim) === 'suspended') { + fail('installation_operation_not_allowed'); + } + const runtime = activationRuntimeFactory(context); + if (typeof runtime.restoreSchedule !== 'function') fail('runtime_boundary_violation'); + await runtime.restoreSchedule(scheduleIntent(context)); + return lifecycleReceipt('started', { changed: scheduleIntent(context), syncWasRunning: scheduleIntent(context) }); + } + + async function executeStage(context, stage) { + if (stage === 'verify_unallocated') { + if (fs.existsSync(context.layout.installationRoot)) fail('runtime_boundary_violation'); + return lifecycleReceipt('absent'); + } + if (stage === 'restore_services') { + context.services.render(context.plan, guard(context)); + context.services.validate(context.plan); + context.services.install(context.plan, guard(context)); + supervisor.reload(context.plan, guard(context)); + return lifecycleReceipt('installed', { changed: true }); + } + if (stage === 'inspect_schedule') return inspectSchedule(context); + if (stage === 'quiesce_schedule') return quiesceSchedule(context); + if (stage === 'stop_if_running') { + return context.startingState === 'ready' ? stop(context) : verifyRuntime(context, false); + } + if (stage === 'snapshot') return snapshot(context, context.backup); + if (stage === 'restart_if_needed') { + return context.startingState === 'ready' ? start(context) : lifecycleReceipt('inactive', { changed: false }); + } + if (stage === 'verify_runtime') return verifyRuntime(context, context.startingState === 'ready'); + if (stage === 'restore_schedule') return restoreSchedule(context); + if (stage === 'verify_stopped') return verifyRuntime(context, false); + if (stage === 'safety_snapshot') return snapshot(context, context.safetyBackup); + if (stage === 'restore_snapshot') { + if (!context.sourceBackup) fail('restore_failed'); + return context.backupManager.restore(context.sourceBackup, context.job.id, guard(context)); + } + if (stage === 'verify_restored') { + const receipt = context.backupManager.inspectRestored(context.sourceBackup); + verifyRuntime(context, false); + return receipt; + } + if (stage === 'stop_runtime') return stop(context); + if (stage === 'upgrade_backup' || stage === 'final_backup') { + return snapshot(context, context.backup); + } + if (stage === 'install_release') { + context.services.finalizeSettled(context.plan, guard(context)); + context.target.services.render(context.target.plan, guard(context)); + context.target.services.validate(context.target.plan); + context.target.services.install(context.target.plan, supervisor.snapshot(context.plan), guard(context)); + return lifecycleReceipt('installed', { changed: true, serviceCount: context.target.plan.units.length }); + } + if (stage === 'start_release') { + supervisor.enable(context.target.plan, guard(context)); + const receipt = supervisor.start(context.target.plan, guard(context)); + return lifecycleReceipt('started', receipt); + } + if (stage === 'verify_release') { + context.target.services.inspectInstalled(context.target.plan); + supervisor.health(context.target.plan); + context.target.services.markVerified(context.target.plan, guard(context)); + return Object.freeze({ status: 'verified', changed: false, serviceCount: context.target.plan.units.length, + releaseId: context.targetManifest.runtime.releaseId }); + } + if (stage === 'verify_release_publication') { + const targetContext = Object.freeze({ + ...context, + manifest: context.targetManifest, + manifestAuthority: context.targetManifestAuthority, + projectRoot: context.target.projectRoot, + }); + const activation = activationRuntimeFactory(targetContext); + await activation.verifyInfrastructure(); + const raw = await activation.verifyPublication( + context.priorEvidence.batchId, + context.priorEvidence.preparationRunId, + ); + return Object.freeze({ + status: 'verified', + activationEvidence: activationEvidence(targetContext, raw), + }); + } + if (stage === 'commit_release') { + context.target.services.inspectInstalled(context.target.plan); + supervisor.health(context.target.plan); + if (!context.target.services.rollbackState(context.target.plan)) fail('upgrade_rollback_required'); + return Object.freeze({ status: 'committed', changed: true, serviceCount: context.target.plan.units.length, + releaseId: context.targetManifest.runtime.releaseId }); + } + if (stage === 'start_runtime') return start(context); + if (stage === 'verify_infrastructure') { + const activation = activationRuntimeFactory(context); + await activation.verifyInfrastructure(); + return lifecycleReceipt('verified', { changed: false, serviceCount: context.plan.units.length }); + } + if (stage === 'verify_publication') { + const activation = activationRuntimeFactory(context); + const raw = await activation.verifyPublication( + context.priorEvidence.batchId, + context.priorEvidence.preparationRunId, + ); + return Object.freeze({ status: 'verified', activationEvidence: activationEvidence(context, raw) }); + } + if (stage === 'disable_runtime') { + if (!fs.existsSync(context.layout.installationRoot)) return lifecycleReceipt('absent', { changed: false }); + if (decommissionServiceState(context) === 'absent') { + allRemoved(supervisor, context.plan); + return lifecycleReceipt('absent', { changed: false }); + } + const receipt = supervisor.disable(context.plan, guard(context)); + allRemoved(supervisor, context.plan); + return lifecycleReceipt('disabled', receipt); + } + if (stage === 'remove_services') { + if (!fs.existsSync(context.layout.installationRoot)) return lifecycleReceipt('absent', { changed: false }); + context.services.finalizeSettled(context.plan, guard(context)); + try { + context.services.inspectAbsent(context.plan); + return lifecycleReceipt('absent', { changed: false }); + } catch {} + const receipt = context.services.removeInstalled(context.plan, guard(context)); + supervisor.reload(context.plan, guard(context)); + context.services.inspectAbsent(context.plan); + return lifecycleReceipt('removed', receipt); + } + if (stage === 'verify_retained') { + if (!fs.existsSync(context.layout.installationRoot)) return lifecycleReceipt('absent', { changed: false }); + context.layoutManager.inspect(context.manifest, context.manifestAuthority); + if (context.legacyRemoval) context.services.inspectAbsent(context.plan); + else context.services.inspectInstalled(context.plan); + allRemoved(supervisor, context.plan); + if (context.backup) context.backupManager.inspect(context.backup); + return lifecycleReceipt('retained', { changed: false }); + } + if (stage === 'destroy_runtime') { + // Direct permanent deletion must stop and remove services before erasing data. + { + await executeStage(context, 'stop_runtime'); + await executeStage(context, 'disable_runtime'); + await executeStage(context, 'remove_services'); + } + await waitForBackupDeletion(context.job.id, + context.manifest.organization.id, context.manifest.runtime.key, () => authority.renew(context.claim)); + return context.backupManager.destroy({ + installationState: 'decommissioned', retainedData: true, destructionApproved: true, + }, guard(context)); + } + if (stage === 'verify_destroyed') return context.backupManager.verifyDestroyed(); + fail('runtime_boundary_violation'); + } + + function availableBackup(context, stage, selected) { + const receipt = context.stageReceipts[stage]; + if (!selected || receipt?.status !== 'snapshot') fail('backup_failed'); + return Object.freeze({ + ...selected, + status: 'available', + treeDigest: receipt.treeDigest, + fileCount: receipt.fileCount, + totalBytes: receipt.totalBytes, + }); + } + + async function verifyScheduleIntent(context, expected) { + const runtime = activationRuntimeFactory(context); + if (typeof runtime.inspectSchedule !== 'function') fail('runtime_boundary_violation'); + const receipt = await runtime.inspectSchedule(); + if (receipt?.syncWasRunning !== expected) fail('runtime_health_failed'); + } + + async function finalVerify(context) { + authority.renew(context.claim); + if (context.operation === 'backup') { + context.backupManager.inspect(availableBackup(context, 'snapshot', context.backup)); + verifyRuntime(context, context.startingState === 'ready'); + if (context.startingState === 'ready') await verifyScheduleIntent(context, scheduleIntent(context)); + return; + } + if (context.operation === 'restore') { + context.backupManager.inspectRestored(context.sourceBackup); + verifyRuntime(context, false); + return; + } + if (context.operation === 'upgrade') { + context.target.services.inspectInstalled(context.target.plan); + supervisor.health(context.target.plan); + await verifyScheduleIntent(context, scheduleIntent(context)); + if (authority.desiredRuntimeState(context.claim) !== 'active') fail('installation_operation_not_allowed'); + if (context.target.services.rollbackState(context.target.plan)) { + context.target.services.commit(context.target.plan, guard(context)); + } + if (context.target.services.rollbackState(context.target.plan)) fail('upgrade_rollback_required'); + context.target.services.inspectInstalled(context.target.plan); + supervisor.health(context.target.plan); + return; + } + if (context.operation === 'suspend') { + verifyRuntime(context, false); + return; + } + if (context.operation === 'resume') { + if (context.removal?.installation_state === 'pending') return executeStage(context, 'verify_unallocated'); + verifyRuntime(context, true); + if (context.removal && context.removal.installation_state !== 'ready') return; + await verifyScheduleIntent(context, context.resumeSync); + return; + } + if (context.operation === 'decommission') { + await executeStage(context, 'verify_retained'); + return; + } + if (context.operation === 'destroy') { + context.backupManager.verifyDestroyed(); + return; + } + fail('runtime_boundary_violation'); + } + + function rollbackUpgrade(context, restart) { + if (!context.target) return; + const prior = context.target.services.rollbackState(context.target.plan); + if (prior) { + supervisor.stop(context.target.plan, guard(context)); + supervisor.resetFailed(context.target.plan, guard(context)); + supervisor.disable(context.target.plan, guard(context)); + context.target.services.restoreFiles(context.target.plan, guard(context)); + supervisor.reload(context.target.plan, guard(context)); + } + if (context.stageReceipts?.install_release?.status === 'installed') { + const backup = availableBackup(context, 'upgrade_backup', context.backup); + context.backupManager.restore(backup, `rollback_${context.job.id}`, guard(context)); + context.backupManager.inspectRestored(backup); + } + if (prior) { + supervisor.restoreState(context.target.plan, prior, guard(context)); + context.target.services.finishRollback(context.target.plan, guard(context)); + } + context.services.inspectInstalled(context.plan); + if (restart) { + supervisor.start(context.plan, guard(context)); + supervisor.health(context.plan); + } else { + supervisor.stop(context.plan, guard(context)); + allStopped(supervisor, context.plan); + } + } + + async function compensate(context) { + const restart = authority.desiredRuntimeState(context.claim) === 'active'; + if (context.operation === 'upgrade') { + rollbackUpgrade(context, restart); + if (restart && typeof context.stageReceipts?.inspect_schedule?.syncWasRunning === 'boolean') { + await restoreSchedule(context); + } + return; + } + if (['backup', 'suspend'].includes(context.operation) && context.startingState === 'ready') { + if (restart) start(context); + else stop(context); + if (restart && typeof context.stageReceipts?.inspect_schedule?.syncWasRunning === 'boolean') { + await restoreSchedule(context); + } + return; + } + if (context.operation === 'resume') { + if (context.resumeSync) await quiesceSchedule(context); + supervisor.stop(context.plan, guard(context)); + allStopped(supervisor, context.plan); + return; + } + if (context.operation === 'restore' && context.stageReceipts?.safety_snapshot?.status === 'snapshot') { + const safety = availableBackup(context, 'safety_snapshot', context.safetyBackup); + context.backupManager.restore(safety, `compensate_${context.job.id}`, guard(context)); + context.backupManager.inspectRestored(safety); + verifyRuntime(context, false); + } + return undefined; + } + + async function run(jobId, workerId) { + const context = runtimeContext(authority.claim(jobId, workerId)); + try { + for (let index = context.nextStage; index < context.stages.length; index += 1) { + authority.renew(context.claim); + const stage = context.stages[index]; + const receipt = await executeStage(context, stage); + authority.checkpoint(context.claim, stage, receipt); + context.stageReceipts[stage] = receipt; + } + await finalVerify(context); + return authority.succeed(context.claim); + } catch (error) { + if (error?.code === 'installation_operation_in_progress') throw error; + try { await compensate(context); } + catch (rollbackError) { + if (rollbackError?.code === 'installation_operation_in_progress') throw rollbackError; + const code = context.operation === 'upgrade' + ? 'upgrade_rollback_required' : 'lifecycle_compensation_failed'; + error = Object.assign(new Error(code), { code }); + } + return authority.failed(context.claim, error); + } + } + + return Object.freeze({ run }); +} + +module.exports = { createManagedInstallationLifecycle }; diff --git a/core/core/installations/src/native-deployment.js b/core/core/installations/src/native-deployment.js new file mode 100644 index 0000000..dcc814c --- /dev/null +++ b/core/core/installations/src/native-deployment.js @@ -0,0 +1,100 @@ +'use strict'; +const crypto = require('node:crypto'); +const path = require('node:path'); +const { serverInstallationManifest } = require('../../../shared/contracts/src/installation'); +const { HOST_TENANT_ROOT, HOST_BRIDGE_ROOT, opaqueRuntimeSuffix, runtimeKey: identifier } = require('../../runtime-host-identity'); +const { MANAGED_INSTALLATION_DIRECTORY_FIELDS, managedInstallationRuntimeEnvironment } = require('../../../shared/paths/runtime-paths'); +const BACKEND = 'native_service_v1'; +const ISSUED = new WeakSet(); +function fail() { throw Object.assign(Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); } +function exact(value, keys) { + if (!value || typeof value !== 'object' || Array.isArray(value) + || Object.keys(value).sort().join(',') !== [...keys].sort().join(',')) fail(); +} +const hash = value => `sha256:${crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex')}`; +function releaseDescriptor(value) { + exact(value, ['version', 'backend', 'releaseId', 'channel', 'sourceCommit', 'platform', + 'runtimeAgentProtocol', 'runtimeGatewayProtocol', 'artifactSha256', 'embeddedManifestSha256', 'bridgeManifestSha256']); + if (value.version !== 1 || value.backend !== BACKEND || !/^[a-z][a-z0-9_.-]{2,95}$/.test(value.releaseId) + || !['production', 'fixture'].includes(value.channel) || !/^[a-f0-9]{40}$/.test(value.sourceCommit) + || value.platform !== 'linux/amd64' || value.runtimeAgentProtocol !== 1 || value.runtimeGatewayProtocol !== 1 + || ![value.artifactSha256, value.embeddedManifestSha256, value.bridgeManifestSha256].every(item => /^[a-f0-9]{64}$/.test(item))) fail(); + return Object.freeze({ ...value }); +} +function construct(runtimeKey, manifestRevision, releaseValue, accountValue, deployment) { + identifier(runtimeKey); + const release = releaseDescriptor(releaseValue); + const account = require('./oci-deployment').hostAccount(accountValue, runtimeKey); + exact(deployment, ['version', 'backend', 'channel', 'organizationId', 'runtimeKey', 'manifestRevision', 'releaseId']); + if (deployment.version !== 1 || deployment.backend !== BACKEND || deployment.channel !== release.channel + || identifier(deployment.organizationId) !== deployment.organizationId || deployment.runtimeKey !== runtimeKey + || deployment.manifestRevision !== manifestRevision || deployment.releaseId !== release.releaseId + || !Number.isSafeInteger(manifestRevision) || manifestRevision < 1) fail(); + const suffix = opaqueRuntimeSuffix(runtimeKey), tenantRoot = path.join(HOST_TENANT_ROOT, suffix); + const installationRoot = path.join('/var/lib/dispatch', runtimeKey); + const layout = { layoutVersion: 1, templateId: 'isolated_dsp_v1', runtimeKey, projectRoot: '/opt/dispatch', installationRoot, + directories: Object.fromEntries(Object.entries(MANAGED_INSTALLATION_DIRECTORY_FIELDS).map(([key, relative]) => [key, path.join(installationRoot, relative)])) }; + const base = { + version: 1, backend: BACKEND, runtimeKey, manifestRevision, release, deployment: Object.freeze({ ...deployment }), account, + identity: { suffix, containerName: `dispatch-dsp-${suffix}`, unitName: `dispatch-dsp-${suffix}.service`, bridgeUnitName: `dispatch-runtime-agent-bridge-${suffix}.service` }, + host: { tenantRoot, accountHome: path.join(tenantRoot, 'home'), engineDataRoot: path.join(tenantRoot, 'engine-data'), + engineConfigRoot: path.join(tenantRoot, 'engine-config'), installationRoot: path.join(tenantRoot, 'runtime', runtimeKey), + bridgeRoot: path.join(HOST_BRIDGE_ROOT, suffix), unitPath: `/etc/systemd/system/dispatch-dsp-${suffix}.service`, + bridgeUnitPath: `/etc/systemd/system/dispatch-runtime-agent-bridge-${suffix}.service` }, + guest: { installationRoot, bridgeRoot: '/run/dispatch-agent', environment: { + DISPATCH_MANAGED_RUNTIME: '1', ...managedInstallationRuntimeEnvironment(layout), DISPATCH_RUNTIME_BACKEND: BACKEND, + DISPATCH_RUNTIME_KEY: runtimeKey, DISPATCH_RUNTIME_GATEWAY_SOCKET: path.join(layout.directories.runtimeRoot, 'runtime-gateway.sock'), + DISPATCH_RUNTIME_AGENT_HUB_SOCKET: '/run/dispatch-agent/runtime-agent-hub.sock', + DISPATCH_RUNTIME_AGENT_TOKEN_FILE: path.join(layout.directories.runtimeAgentSecretsRoot, 'registration-token'), + DISPATCH_RUNTIME_AGENT_STATUS_SOCKET: path.join(layout.directories.runtimeRoot, 'runtime-agent-status.sock'), + DISPATCH_CHROME_EXECUTABLE: '/opt/dispatch/dependencies/browser/chrome', + } }, + resources: { id: 'dsp_standard_v1', cpus: '2', memory: '4g', pids: '512', sharedMemory: '512m', temporaryStorage: '512m' }, + security: { user: `${account.uid}:${account.gid}`, readOnlyRoot: true, privateTmp: true, noNewPrivileges: true, browserTransport: 'pipe' }, + }; + for (const value of [base.identity, base.host, base.guest.environment, base.guest, base.resources, base.security]) Object.freeze(value); + const plan = Object.freeze({ ...base, planDigest: hash(base) }); ISSUED.add(plan); return plan; +} +function createPlan(manifestValue, authority, release, account, deployment, channel = 'production') { + const manifest = serverInstallationManifest(manifestValue, authority); + if (manifest.runtime.templateId !== 'isolated_dsp_v1' || manifest.runtime.releaseId !== release.releaseId + || manifest.organization.id !== deployment.organizationId || release.channel !== channel) fail(); + return construct(manifest.runtime.key, manifest.revision, release, account, deployment); +} +function validatePlan(value) { + exact(value, ['version', 'backend', 'runtimeKey', 'manifestRevision', 'release', 'deployment', 'account', 'identity', 'host', 'guest', 'resources', 'security', 'planDigest']); + const plan = construct(value.runtimeKey, value.manifestRevision, value.release, value.account, value.deployment); + if (JSON.stringify(plan) !== JSON.stringify(value)) fail(); + return plan; +} +function requirePlan(value) { if (!ISSUED.has(value)) fail(); return value; } +function artifactRoot(plan) { requirePlan(plan); return `/opt/dispatch-runtime/releases/${plan.release.releaseId}/runtime-artifact`; } +function renderSystemUnit(value) { + const plan = requirePlan(value), root = artifactRoot(plan); + // Private bind aliases preserve runtime paths without an OS image. Every + // service pins its immutable source release for its entire process lifetime. + return [ + '[Unit]', `Description=Dispatch DSP runtime (${plan.identity.suffix})`, `After=network-online.target ${plan.identity.bridgeUnitName}`, `Wants=network-online.target ${plan.identity.bridgeUnitName}`, + 'StartLimitIntervalSec=0', '', '[Service]', 'Type=simple', + `User=${plan.account.name}`, `Group=${plan.account.name}`, 'UMask=0077', + `BindReadOnlyPaths=${root}:/opt/dispatch ${plan.host.bridgeRoot}:/run/dispatch-agent -${root}/dependencies/node/bin/host-files/usr/share/nodejs:/usr/share/nodejs`, + `BindPaths=${plan.host.installationRoot}:${plan.guest.installationRoot}`, + `ReadWritePaths=${plan.host.installationRoot}`, 'WorkingDirectory=/opt/dispatch', + 'Environment=HOME=/tmp/dispatch-home', 'Environment=PATH=/opt/dispatch/dependencies/node/bin:/usr/bin:/bin', + ...Object.entries(plan.guest.environment).map(([key, selected]) => `Environment=${key}=${selected}`), + 'UnsetEnvironment=NODE_OPTIONS LD_PRELOAD LD_LIBRARY_PATH HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy DISPATCH_LOCAL_ROOT DISPATCH_ACCESS_CONTROL_DATABASE_ROOT', + // systemd-analyze verifies the executable before entering BindReadOnlyPaths. + // The immutable release is also visible inside the service namespace. + `ExecStart=${root}/dependencies/node/bin/node --no-warnings /opt/dispatch/runtime/supervisor/src/supervisor.js`, + // The supervisor sends TERM to its children. Sending TERM to the entire + // group here would signal children twice and interrupt their lease cleanup. + 'Restart=on-failure', 'RestartSec=10', 'KillMode=mixed', 'TimeoutStopSec=30', + 'Slice=dispatch-dsp.slice', 'CPUQuota=200%', 'MemoryMax=4G', 'TasksMax=512', 'OOMPolicy=stop', + 'ProtectSystem=strict', 'ProtectHome=true', 'NoNewPrivileges=true', + 'TemporaryFileSystem=/tmp:rw,noexec,nosuid,nodev,size=512M,mode=1777 /dev/shm:rw,noexec,nosuid,nodev,size=512M,mode=1777', + 'InaccessiblePaths=-/run/docker.sock -/run/podman -/run/containerd', + 'ProtectKernelTunables=true', 'ProtectKernelModules=true', 'ProtectControlGroups=true', + 'RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK', '', '[Install]', 'WantedBy=multi-user.target', '', + ].join('\n'); +} +module.exports = { BACKEND, releaseDescriptor, createPlan, validatePlan, requirePlan, artifactRoot, renderSystemUnit }; diff --git a/core/core/installations/src/native-runtime-archive.py b/core/core/installations/src/native-runtime-archive.py new file mode 100644 index 0000000..feae7ac --- /dev/null +++ b/core/core/installations/src/native-runtime-archive.py @@ -0,0 +1,153 @@ +"""Dispatch runtime archive reader. Never delegates extraction to tar/extractall.""" +import errno +import hashlib +import json +import os +import re +import stat +import sys +import tarfile + +MAX_BYTES = 2 * 1024 ** 3 +MAX_FILES = 20000 +MANIFEST = 'runtime-release-manifest.json' + + +def require(condition): + if not condition: + raise ValueError('invalid_native_runtime') + + +def digest_file(filename): + result = hashlib.sha256() + with open(filename, 'rb') as source: + while chunk := source.read(1024 * 1024): + result.update(chunk) + return result.hexdigest() + + +def manifest_entries(data, expected_hash, commit): + require(hashlib.sha256(data).hexdigest() == expected_hash) + manifest = json.loads(data) + require(set(manifest) == {'schemaVersion', 'backend', 'sourceCommit', 'platform', 'files'}) + require(manifest['schemaVersion'] == 1 and manifest['backend'] == 'native_service_v1') + require(manifest['sourceCommit'] == commit and manifest['platform'] == 'linux/amd64') + require(isinstance(manifest['files'], list) and 1 <= len(manifest['files']) <= MAX_FILES) + entries = {} + total = 0 + for entry in manifest['files']: + require(set(entry) == {'path', 'mode', 'size', 'sha256'}) + name = entry['path'] + require(isinstance(name, str) and len(name) <= 240 and re.fullmatch(r'[A-Za-z0-9_./+-]+', name)) + require(all(part not in ('', '.', '..') for part in name.split('/'))) + require(name != MANIFEST and name not in entries) + require(entry['mode'] in ('444', '555') and type(entry['size']) is int and entry['size'] >= 0) + require(re.fullmatch(r'[a-f0-9]{64}', entry['sha256'])) + total += entry['size'] + require(total <= MAX_BYTES) + entries[name] = entry + for name in entries: + parts = name.split('/')[:-1] + while parts: + require('/'.join(parts) not in entries) + parts.pop() + return entries + + +def unpack(archive, root, expected_hash, commit, archive_hash): + require(os.path.isabs(root) and not os.path.lexists(root)) + require(digest_file(archive) == archive_hash) + os.mkdir(root, 0o700) + with tarfile.open(archive, 'r|gz') as source: + first = source.next() + require(first is not None and first.name == MANIFEST and first.isreg() and first.size <= 8 * 1024 ** 2) + data = source.extractfile(first).read() + entries = manifest_entries(data, expected_hash, commit) + with open(os.path.join(root, MANIFEST), 'xb') as target: + target.write(data) + os.chmod(os.path.join(root, MANIFEST), 0o444) + seen = set() + for member in source: + if member is first: + continue + require(member.name in entries and member.name not in seen and member.isreg()) + entry = entries[member.name] + require(member.size == entry['size'] and member.mode == int(entry['mode'], 8)) + target = os.path.join(root, member.name) + os.makedirs(os.path.dirname(target), mode=0o700, exist_ok=True) + result = hashlib.sha256() + with source.extractfile(member) as incoming, open(target, 'xb') as outgoing: + while chunk := incoming.read(1024 * 1024): + result.update(chunk) + outgoing.write(chunk) + outgoing.flush() + os.fsync(outgoing.fileno()) + require(result.hexdigest() == entry['sha256']) + os.chmod(target, int(entry['mode'], 8)) + seen.add(member.name) + require(seen == set(entries)) + for directory, _, _ in os.walk(root, topdown=False): + os.chmod(directory, 0o555) + + +def verify(root, expected_hash, commit): + require(os.path.isabs(root) and os.path.realpath(root) == root) + info = os.lstat(os.path.join(root, MANIFEST)) + require(stat.S_ISREG(info.st_mode) and info.st_size <= 8 * 1024 ** 2) + with open(os.path.join(root, MANIFEST), 'rb') as source: + entries = manifest_entries(source.read(), expected_hash, commit) + seen = set() + for directory, directories, files in os.walk(root, followlinks=False): + for selected in [directory] + [os.path.join(directory, name) for name in directories]: + info = os.lstat(selected) + require(stat.S_ISDIR(info.st_mode) and stat.S_IMODE(info.st_mode) == 0o555 and info.st_uid == 0 and info.st_gid == 0) + for name in files: + filename = os.path.join(directory, name) + relative = os.path.relpath(filename, root) + info = os.lstat(filename) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1 and info.st_uid == 0 and info.st_gid == 0) + if relative == MANIFEST: + require(stat.S_IMODE(info.st_mode) == 0o444) + continue + require(relative in entries) + entry = entries[relative] + require(stat.S_IMODE(info.st_mode) == int(entry['mode'], 8) and info.st_size == entry['size']) + require(digest_file(filename) == entry['sha256']) + seen.add(relative) + require(seen == set(entries)) + + +def pack(root, archive): + require(not os.path.lexists(archive)) + with open(os.path.join(root, MANIFEST), 'rb') as source: + data = source.read() + value = json.loads(data) + entries = manifest_entries(data, hashlib.sha256(data).hexdigest(), value['sourceCommit']) + with tarfile.open(archive, 'x:gz', format=tarfile.USTAR_FORMAT) as target: + for name in [MANIFEST] + sorted(entries): + filename = os.path.join(root, name) + info = os.lstat(filename) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1) + member = tarfile.TarInfo(name) + member.size = info.st_size + member.mode = 0o444 if name == MANIFEST else int(entries[name]['mode'], 8) + with open(filename, 'rb') as source: + target.addfile(member, source) + + +if __name__ == '__main__': + try: + if sys.argv[1] == 'unpack' and len(sys.argv) == 7: + unpack(*sys.argv[2:]) + elif sys.argv[1] == 'verify' and len(sys.argv) == 5: + verify(*sys.argv[2:]) + elif sys.argv[1] == 'pack' and len(sys.argv) == 4: + pack(*sys.argv[2:]) + else: + raise ValueError('invalid_native_runtime') + except OSError as error: + print('archive_disk_full' if error.errno in (errno.ENOSPC, errno.EDQUOT) else 'archive_io_failed', file=sys.stderr) + sys.exit(1) + except Exception: + print('native runtime verification failed', file=sys.stderr) + sys.exit(1) diff --git a/core/core/installations/src/native-runtime-artifact.js b/core/core/installations/src/native-runtime-artifact.js new file mode 100644 index 0000000..2253a47 --- /dev/null +++ b/core/core/installations/src/native-runtime-artifact.js @@ -0,0 +1,21 @@ +'use strict'; +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { releaseDescriptor } = require('./native-deployment'); + +function run(args) { + const result = spawnSync('/usr/bin/python3', ['-I', path.join(__dirname, "./native-runtime-archive.py"), ...args], { + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8' }, encoding: 'utf8', timeout: 300_000, maxBuffer: 4096, + }); + if (result.error || result.status !== 0) throw Object.assign(Error('runtime_identity_mismatch'), { code: 'runtime_identity_mismatch' }); +} +function unpackNativeRuntime(archive, root, value) { + const release = releaseDescriptor(value); + run(['unpack', archive, root, release.embeddedManifestSha256, release.sourceCommit, release.artifactSha256]); +} +function verifyNativeRuntime(root, value) { + const release = releaseDescriptor(value); + run(['verify', root, release.embeddedManifestSha256, release.sourceCommit]); + return true; +} +module.exports = { unpackNativeRuntime, verifyNativeRuntime }; diff --git a/core/core/installations/src/native-runtime-build.js b/core/core/installations/src/native-runtime-build.js new file mode 100644 index 0000000..e2f1ae8 --- /dev/null +++ b/core/core/installations/src/native-runtime-build.js @@ -0,0 +1,80 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { sha } = require('./release-delivery-contract'); +const { hashFileSync } = require('./release-delivery-files'); +const { removeStage } = require('./release-delivery-install'); +const LEGACY_ENTRYPOINTS = require('../../../shared/legacy-entrypoints'); + +function buildNativeRuntime({ projectRoot, archive, sourceCommit, nodeExecutable = process.execPath, + browserRoot = process.env.DISPATCH_BUILD_BROWSER_ROOT || '/opt/google/chrome', consumeStage = null }) { + if (process.platform !== 'linux' || process.arch !== 'x64' || !/^[a-f0-9]{40}$/.test(sourceCommit)) throw Error('unsupported_native_build'); + const stage = fs.mkdtempSync(path.join(path.dirname(archive), 'native-build-')); + function git(args, encoding = 'utf8') { + const result = spawnSync('/usr/bin/git', args, { cwd: projectRoot, encoding, maxBuffer: 32 * 1024 ** 2 }); + if (result.status !== 0) throw Error('native_source_invalid'); + return result.stdout; + } + function write(relative, data, mode) { + const target = path.join(stage, relative); + fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o755 }); + fs.writeFileSync(target, data, { flag: 'wx', mode }); + } + try { + if (git(['status', '--porcelain']).trim() || git(['rev-parse', 'HEAD']).trim() !== sourceCommit) throw Error('clean_checkout_required'); + for (const row of git(['ls-tree', '-r', '-z', 'HEAD']).split('\0').filter(Boolean)) { + const match = /^(100644|100755) blob ([a-f0-9]{40})\t(.+)$/.exec(row); + if (!match) throw Error('unsupported_git_entry'); + const [, mode, object, relative] = match; + if (!/^(runtime|shared|sdk|plugins|compatibility)\//.test(relative) || /\/(tests|examples|docs)\//.test(relative) + || /\.(md|test\.js)$/.test(relative)) continue; + if (relative.startsWith('plugins/') && !/^plugins\/[^/]+\/(backend\/|dispatch-plugin\.json$)/.test(relative)) continue; + if (relative.startsWith('compatibility/') && !/^compatibility\/(cdf|paycom)\//.test(relative)) continue; + write(relative, git(['cat-file', 'blob', object], null), mode === '100755' ? 0o555 : 0o444); + } + for (const [provider, names] of Object.entries(LEGACY_ENTRYPOINTS)) for (const name of names) { + const base = require('../../../shared/plugin-sdk/catalog').plugin(provider) ? `plugins/${provider}/backend` : `compatibility/${provider}`; + const target = `/opt/dispatch/${base}/bin/${name}`; + if (!fs.existsSync(path.join(stage, base, 'bin', name))) throw Error('missing_collector_entrypoint'); + write(`plugins/${provider}/bin/${name}`, `#!/usr/bin/env -S node --no-warnings\n'use strict';\nrequire(${JSON.stringify(target)});\n`, 0o555); + } + const nodeRoot = path.join(stage, 'dependencies/node/bin'); + fs.mkdirSync(path.dirname(nodeRoot), { recursive: true, mode: 0o755 }); + require('./portable-node').bundleNode(nodeExecutable, nodeRoot, '/opt/dispatch/dependencies/node/bin'); + const browserBase = fs.realpathSync(browserRoot); + function copyBrowser(directory, relative = '') { + for (const name of fs.readdirSync(directory).sort()) { + const source = path.join(directory, name), info = fs.lstatSync(source), child = path.join(relative, name); + if (info.isDirectory()) copyBrowser(source, child); + else { + const real = fs.realpathSync(source); + if (!real.startsWith(browserBase + '/') || !fs.statSync(real).isFile()) throw Error('unsafe_browser_dependency'); + write(path.join('dependencies/browser', child), fs.readFileSync(real), fs.statSync(real).mode & 0o111 ? 0o555 : 0o444); + } + } + } + copyBrowser(browserBase); + if (!fs.existsSync(path.join(stage, 'dependencies/browser/chrome'))) throw Error('browser_binary_missing'); + const files = []; + function inventory(directory) { + for (const name of fs.readdirSync(directory).sort()) { + const file = path.join(directory, name), info = fs.lstatSync(file); + if (info.isDirectory()) inventory(file); + else files.push({ path: path.relative(stage, file), mode: (info.mode & 0o777).toString(8), size: info.size, sha256: hashFileSync(file) }); + } + } + inventory(stage); + const bytes = JSON.stringify({ schemaVersion: 1, backend: 'native_service_v1', sourceCommit, platform: 'linux/amd64', files }) + '\n'; + write('runtime-release-manifest.json', bytes, 0o444); + if (consumeStage) return consumeStage(stage, sha(bytes)); + return {artifactSha256:packNativeRuntime(stage, archive), embeddedManifestSha256:sha(bytes)}; + } finally { removeStage(stage); } +} +function packNativeRuntime(stage, archive) { + const result = spawnSync('/usr/bin/python3', ['-I', path.join(__dirname, "./native-runtime-archive.py"), 'pack', stage, archive], + {encoding:'utf8', timeout:300_000, maxBuffer:4096}); + require('./release-build-space').checkArchiveResult(result, 'native_archive_build_failed'); + return hashFileSync(archive); +} +module.exports = {buildNativeRuntime, packNativeRuntime}; diff --git a/core/core/installations/src/oci-adapter.js b/core/core/installations/src/oci-adapter.js new file mode 100644 index 0000000..70bc6c7 --- /dev/null +++ b/core/core/installations/src/oci-adapter.js @@ -0,0 +1,153 @@ +'use strict'; + +const { + OCI_DEPLOYMENT_PLAN_VERSION, + OCI_BACKEND, + createOciDeploymentPlan, + createOciFixtureDeploymentPlan, +} = require('./oci-deployment'); + +function fail(code = 'runtime_boundary_violation') { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, allowed, required = allowed) { + if (!plain(value) || Object.keys(value).some(key => !allowed.includes(key)) + || required.some(key => !Object.hasOwn(value, key))) fail(); +} + +function createOciContainerAdapter(options) { + exact(options, ['hostRegistry', 'hostExecutor', 'releaseResolver', 'credentialPort']); + const { hostRegistry, hostExecutor, releaseResolver, credentialPort } = options; + if (!hostRegistry || ['reserve', 'inspect'].some(method => typeof hostRegistry[method] !== 'function') + || !hostExecutor || [ + 'prepareAccount', 'materializeLayout', 'prepareImage', 'render', 'validate', 'install', + 'start', 'health', 'commit', 'rollback', + ].some(method => typeof hostExecutor[method] !== 'function') + || typeof releaseResolver !== 'function' + || !credentialPort || typeof credentialPort.read !== 'function') fail(); + + function fixtureOption(value) { + exact(value, ['fixture', 'claim']); + if (typeof value.fixture !== 'boolean' || !plain(value.claim)) fail(); + return value; + } + + function receipt(status, changed = false) { + return Object.freeze({ + ociDeploymentPlanVersion: OCI_DEPLOYMENT_PLAN_VERSION, + status, + changed: Boolean(changed), + }); + } + + function plan(manifest, manifestAuthority, optionsValue, destroying = false, withState = false) { + const options = fixtureOption(optionsValue); + const account = hostRegistry.inspect(manifest?.runtime?.key, options.claim); + if (!account || !['reserved', 'active', ...(destroying ? ['retired'] : [])].includes(account.status)) fail('service_installation_failed'); + const release = releaseResolver(manifest.runtime.releaseId, options.fixture); + const deployment = Object.freeze({ + version: 1, + backend: release.backend, + channel: options.fixture ? 'fixture' : 'production', + organizationId: manifest.organization.id, + runtimeKey: manifest.runtime.key, + manifestRevision: manifest.revision, + releaseId: manifest.runtime.releaseId, + }); + const create = options.fixture ? createOciFixtureDeploymentPlan : createOciDeploymentPlan; + const selected = create(manifest, manifestAuthority, release, { + name: account.name, + uid: account.uid, + gid: account.gid, + subuidStart: account.subuidStart, + subgidStart: account.subgidStart, + subidCount: account.subidCount, + }, deployment); + return withState ? Object.freeze({ plan: selected, retired: account.status === 'retired' }) : selected; + } + + function reconcileHostAccount(manifest, manifestAuthority, optionsValue, mutationCapability) { + exact(optionsValue, ['fixture', 'claim']); + if (typeof optionsValue.fixture !== 'boolean' || !plain(optionsValue.claim)) fail(); + let account; + mutationCapability(() => { account = hostRegistry.reserve(manifest.runtime.key, optionsValue.claim); }); + if (!account) fail('service_installation_failed'); + const selected = plan(manifest, manifestAuthority, { + fixture: optionsValue.fixture, claim: optionsValue.claim, + }); + const accountReceipt = hostExecutor.prepareAccount(selected, optionsValue.claim, mutationCapability); + const token = credentialPort.read(selected.runtimeKey); + const layoutReceipt = hostExecutor.materializeLayout(selected, token, optionsValue.claim, mutationCapability); + return receipt('host_account_ready', accountReceipt.changed || layoutReceipt.changed); + } + + function reconcileImage(planValue, claim, mutationCapability) { + if (!plain(claim)) fail(); + const selected = hostExecutor.prepareImage(planValue, claim, mutationCapability); + return receipt('image_ready', selected.changed); + } + + function reconcileBridge(planValue, claim, mutationCapability) { + if (!plain(claim)) fail(); + const rendered = hostExecutor.render(planValue, claim, mutationCapability); + hostExecutor.validate(planValue, claim); + const installed = hostExecutor.install(planValue, claim, mutationCapability); + return receipt('bridge_ready', rendered.changed || installed.changed); + } + + function reconcileContainer(planValue, claim, mutationCapability) { + if (!plain(claim)) fail(); + const selected = hostExecutor.start(planValue, claim, mutationCapability); + return receipt('container_ready', selected.changed); + } + + function verify(planValue, claim) { + if (!plain(claim)) fail(); + hostExecutor.health(planValue, claim); + return receipt('healthy'); + } + + function commit(planValue, claim, mutationCapability) { + if (!plain(claim)) fail(); + return hostExecutor.commit(planValue, claim, mutationCapability); + } + + function rollback(manifest, manifestAuthority, optionsValue, mutationCapability) { + exact(optionsValue, ['fixture', 'intent', 'claim']); + if (typeof optionsValue.fixture !== 'boolean' || !['failed', 'cancelled'].includes(optionsValue.intent) + || !plain(optionsValue.claim)) fail(); + const account = hostRegistry.inspect(manifest.runtime.key, optionsValue.claim); + if (!account) return receipt('rolled_back'); + const selected = plan(manifest, manifestAuthority, { + fixture: optionsValue.fixture, claim: optionsValue.claim, + }); + const rolledBack = hostExecutor.rollback(selected, optionsValue.claim, mutationCapability); + return receipt('rolled_back', rolledBack.changed); + } + + return Object.freeze({ + inspectUnallocated(manifest, manifestAuthority, optionsValue) { + const options = fixtureOption(optionsValue); + require('../../../shared/contracts/src').serverInstallationManifest(manifest, manifestAuthority); + return hostRegistry.inspect(manifest.runtime.key, options.claim) === null; + }, + plan: (manifest, authority, options) => plan(manifest, authority, options), + destructionPlan: (manifest, authority, options) => plan(manifest, authority, options, true), + destructionContext: (manifest, authority, options) => plan(manifest, authority, options, true, true), + reconcileHostAccount, + reconcileImage, + reconcileBridge, + reconcileContainer, + verify, + commit, + rollback, + }); +} + +module.exports = { createOciContainerAdapter }; diff --git a/core/core/installations/src/oci-deployment.js b/core/core/installations/src/oci-deployment.js new file mode 100644 index 0000000..036b7a7 --- /dev/null +++ b/core/core/installations/src/oci-deployment.js @@ -0,0 +1,488 @@ +'use strict'; + +const crypto = require('node:crypto'); +const path = require('node:path'); +const { serverInstallationManifest } = require('../../../shared/contracts/src/installation'); +const { + HOST_TENANT_ROOT, + HOST_BRIDGE_ROOT, + runtimeKey: checkedRuntimeKey, + opaqueRuntimeSuffix, + hostAccountName, +} = require('../../runtime-host-identity'); +const { + MANAGED_INSTALLATION_LAYOUT_VERSION, + MANAGED_INSTALLATION_LAYOUT_TEMPLATE, + MANAGED_INSTALLATION_DIRECTORY_FIELDS, + managedInstallationRuntimeEnvironment, +} = require('../../../shared/paths/runtime-paths'); + +const OCI_DEPLOYMENT_PLAN_VERSION = 1; +const OCI_DEPLOYMENT_AUTHORITY_VERSION = 1; +const OCI_RELEASE_DESCRIPTOR_VERSION = 2; +const OCI_BACKEND = 'oci_container_v1'; +const CONTAINER_UID = 10001; +const CONTAINER_GID = 10001; +const SUBID_COUNT = 65_536; +const MAX_UNIX_SOCKET_PATH_BYTES = 107; + +const SYSTEM_UNIT_ROOT = '/etc/systemd/system'; +const GUEST_STORAGE_ROOT = '/var/lib/dispatch'; +const GUEST_BRIDGE_ROOT = '/run/dispatch-agent'; +const PODMAN = '/usr/bin/podman'; +const RESOURCE_POLICY = Object.freeze({ + id: 'dsp_standard_v1', + cpus: '2', + memory: '4g', + pids: '512', + sharedMemory: '512m', + temporaryStorage: '512m', +}); +const ISSUED_PLANS = new WeakSet(); + +function fail(code = 'runtime_boundary_violation') { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, keys) { + if (!plain(value) || Object.keys(value).sort().join(',') !== [...keys].sort().join(',')) fail(); +} + +function identifier(value) { + try { return checkedRuntimeKey(value); } catch { return fail(); } +} + +function digest(value) { + if (typeof value !== 'string' || !/^sha256:[a-f0-9]{64}$/.test(value)) fail(); + return value; +} + +function opaqueSuffix(runtimeKey) { + try { return opaqueRuntimeSuffix(runtimeKey); } catch { return fail(); } +} + +function releaseDescriptor(value) { + if (value?.backend === 'native_service_v1') return require('./native-deployment').releaseDescriptor(value); + exact(value, [ + 'version', 'backend', 'releaseId', 'channel', 'image', 'imageDigest', 'imageId', 'sourceCommit', 'platform', + 'runtimeAgentProtocol', 'runtimeGatewayProtocol', 'embeddedManifestSha256', + 'imageArchiveSha256', 'bridgeManifestSha256', + ]); + if (value.version !== OCI_RELEASE_DESCRIPTOR_VERSION || value.backend !== OCI_BACKEND + || typeof value.releaseId !== 'string' || !/^[a-z][a-z0-9_.-]{2,95}$/.test(value.releaseId) + || !['production', 'fixture'].includes(value.channel) + || !/^[a-f0-9]{64}$/.test(value.imageId) + || !/^[a-f0-9]{40}$/.test(value.sourceCommit) || value.platform !== 'linux/amd64' + || ![value.embeddedManifestSha256, value.imageArchiveSha256, value.bridgeManifestSha256] + .every(item => /^[a-f0-9]{64}$/.test(item)) + || value.runtimeAgentProtocol !== 1 || value.runtimeGatewayProtocol !== 1) fail(); + const imageDigest = digest(value.imageDigest); + const repository = value.channel === 'production' + ? 'ghcr.io/example-organization/dispatch-runtime' : 'localhost/dispatch-runtime'; + if (value.image !== `${repository}@${imageDigest}`) fail(); + return Object.freeze({ ...value, imageDigest }); +} + +function hostAccount(value, runtimeKey) { + exact(value, ['name', 'uid', 'gid', 'subuidStart', 'subgidStart', 'subidCount']); + if (value.name !== hostAccountName(runtimeKey) + || !Number.isSafeInteger(value.uid) || value.uid < 100 || value.uid > 60_000 + || !Number.isSafeInteger(value.gid) || value.gid < 100 || value.gid > 60_000 + || !Number.isSafeInteger(value.subuidStart) || value.subuidStart < 100_000 + || !Number.isSafeInteger(value.subgidStart) || value.subgidStart < 100_000 + || value.subidCount !== SUBID_COUNT) fail(); + return Object.freeze({ ...value }); +} + +function deploymentAuthority(value, manifest, release, expectedChannel) { + exact(value, [ + 'version', 'backend', 'channel', 'organizationId', 'runtimeKey', 'manifestRevision', 'releaseId', + ]); + if (value.version !== OCI_DEPLOYMENT_AUTHORITY_VERSION || value.backend !== OCI_BACKEND + || value.channel !== expectedChannel || value.organizationId !== manifest.organization.id + || value.runtimeKey !== manifest.runtime.key || value.manifestRevision !== manifest.revision + || value.releaseId !== manifest.runtime.releaseId || value.releaseId !== release.releaseId) { + fail('runtime_identity_mismatch'); + } + return Object.freeze({ ...value }); +} + +function guestLayout(runtimeKey) { + const installationRoot = path.join(GUEST_STORAGE_ROOT, runtimeKey); + const directories = Object.fromEntries(Object.entries(MANAGED_INSTALLATION_DIRECTORY_FIELDS) + .map(([field, relative]) => [field, path.join(installationRoot, relative)])); + return Object.freeze({ + layoutVersion: MANAGED_INSTALLATION_LAYOUT_VERSION, + templateId: MANAGED_INSTALLATION_LAYOUT_TEMPLATE, + runtimeKey, + projectRoot: '/opt/dispatch', + installationRoot, + directories: Object.freeze(directories), + }); +} + +function stableDigest(value) { + return `sha256:${crypto.createHash('sha256').update(JSON.stringify(value), 'utf8').digest('hex')}`; +} + +function createPlan(manifestValue, authorityValue, releaseValue, accountValue, deploymentValue, expectedChannel) { + const manifest = serverInstallationManifest(manifestValue, authorityValue); + const runtimeKey = identifier(manifest.runtime.key); + if (manifest.runtime.templateId !== MANAGED_INSTALLATION_LAYOUT_TEMPLATE) { + fail('runtime_boundary_violation'); + } + const release = releaseDescriptor(releaseValue); + if (release.channel !== expectedChannel) fail('invalid_runtime_release'); + if (release.releaseId !== manifest.runtime.releaseId) fail('runtime_identity_mismatch'); + const deployment = deploymentAuthority(deploymentValue, manifest, release, expectedChannel); + const account = hostAccount(accountValue, runtimeKey); + const suffix = opaqueSuffix(runtimeKey); + const tenantRoot = path.join(HOST_TENANT_ROOT, suffix); + const accountHome = path.join(tenantRoot, 'home'); + const engineDataRoot = path.join(tenantRoot, 'engine-data'); + const engineConfigRoot = path.join(tenantRoot, 'engine-config'); + const installationRoot = path.join(tenantRoot, 'runtime', runtimeKey); + const bridgeRoot = path.join(HOST_BRIDGE_ROOT, suffix); + const layout = guestLayout(runtimeKey); + const environment = Object.freeze({ + DISPATCH_MANAGED_RUNTIME: '1', + ...managedInstallationRuntimeEnvironment(layout), + DISPATCH_RUNTIME_KEY: runtimeKey, + DISPATCH_RUNTIME_GATEWAY_SOCKET: path.join(layout.directories.runtimeRoot, 'runtime-gateway.sock'), + DISPATCH_RUNTIME_AGENT_HUB_SOCKET: path.join(GUEST_BRIDGE_ROOT, 'runtime-agent-hub.sock'), + DISPATCH_RUNTIME_AGENT_TOKEN_FILE: path.join(layout.directories.runtimeAgentSecretsRoot, 'registration-token'), + DISPATCH_RUNTIME_AGENT_STATUS_SOCKET: path.join(layout.directories.runtimeRoot, 'runtime-agent-status.sock'), + DISPATCH_CHROME_EXECUTABLE: '/usr/bin/chromium', + }); + for (const socket of [ + environment.DISPATCH_RUNTIME_GATEWAY_SOCKET, + environment.DISPATCH_RUNTIME_AGENT_HUB_SOCKET, + environment.DISPATCH_RUNTIME_AGENT_STATUS_SOCKET, + path.join(bridgeRoot, 'runtime-agent-hub.sock'), + ]) if (Buffer.byteLength(socket, 'utf8') > MAX_UNIX_SOCKET_PATH_BYTES) fail(); + const base = Object.freeze({ + version: OCI_DEPLOYMENT_PLAN_VERSION, + backend: OCI_BACKEND, + runtimeKey, + manifestRevision: manifest.revision, + release, + deployment, + account, + identity: Object.freeze({ + suffix, + containerName: `dispatch-dsp-${suffix}`, + unitName: `dispatch-dsp-${suffix}.service`, + bridgeUnitName: `dispatch-runtime-agent-bridge-${suffix}.service`, + }), + host: Object.freeze({ + tenantRoot, + accountHome, + engineDataRoot, + engineConfigRoot, + installationRoot, + bridgeRoot, + unitPath: path.join(SYSTEM_UNIT_ROOT, `dispatch-dsp-${suffix}.service`), + bridgeUnitPath: path.join(SYSTEM_UNIT_ROOT, `dispatch-runtime-agent-bridge-${suffix}.service`), + }), + guest: Object.freeze({ + installationRoot: layout.installationRoot, + bridgeRoot: GUEST_BRIDGE_ROOT, + environment, + }), + resources: RESOURCE_POLICY, + security: Object.freeze({ + readOnlyRoot: true, + network: 'pasta', + pidNamespace: 'private', + ipcNamespace: 'private', + utsNamespace: 'private', + user: `${CONTAINER_UID}:${CONTAINER_GID}`, + userNamespace: `keep-id:uid=${CONTAINER_UID},gid=${CONTAINER_GID}`, + capabilities: Object.freeze(['SYS_CHROOT']), + noNewPrivileges: true, + publishPorts: false, + engineSocketMounted: false, + }), + }); + const plan = Object.freeze({ ...base, planDigest: stableDigest(base) }); + ISSUED_PLANS.add(plan); + return plan; +} + +function createOciDeploymentPlan(manifestValue, authorityValue, releaseValue, accountValue, deploymentValue) { + if (releaseValue?.backend === 'native_service_v1') return require('./native-deployment').createPlan(manifestValue, authorityValue, releaseValue, accountValue, deploymentValue); + return createPlan(manifestValue, authorityValue, releaseValue, accountValue, deploymentValue, 'production'); +} + +function createOciFixtureDeploymentPlan(manifestValue, authorityValue, releaseValue, accountValue, deploymentValue) { + if (releaseValue?.backend === 'native_service_v1') return require('./native-deployment').createPlan(manifestValue, authorityValue, releaseValue, accountValue, deploymentValue, 'fixture'); + return createPlan(manifestValue, authorityValue, releaseValue, accountValue, deploymentValue, 'fixture'); +} + +function requirePlan(value) { + if (value?.backend === 'native_service_v1') return require('./native-deployment').requirePlan(value); + if (!ISSUED_PLANS.has(value) || value.version !== OCI_DEPLOYMENT_PLAN_VERSION) fail(); + return value; +} + +function validateOciDeploymentPlan(value) { + if (value?.backend === 'native_service_v1') return require('./native-deployment').validatePlan(value); + exact(value, [ + 'version', 'backend', 'runtimeKey', 'manifestRevision', 'release', 'deployment', 'account', + 'identity', 'host', 'guest', 'resources', 'security', 'planDigest', + ]); + if (value.version !== OCI_DEPLOYMENT_PLAN_VERSION || value.backend !== OCI_BACKEND + || !Number.isSafeInteger(value.manifestRevision) || value.manifestRevision < 1) fail(); + const runtimeKey = identifier(value.runtimeKey); + const release = releaseDescriptor(value.release); + const account = hostAccount(value.account, runtimeKey); + exact(value.deployment, [ + 'version', 'backend', 'channel', 'organizationId', 'runtimeKey', 'manifestRevision', 'releaseId', + ]); + if (value.deployment.version !== OCI_DEPLOYMENT_AUTHORITY_VERSION + || value.deployment.backend !== OCI_BACKEND || value.deployment.channel !== release.channel + || identifier(value.deployment.organizationId) !== value.deployment.organizationId + || value.deployment.runtimeKey !== runtimeKey + || value.deployment.manifestRevision !== value.manifestRevision + || value.deployment.releaseId !== release.releaseId) fail('runtime_identity_mismatch'); + const suffix = opaqueSuffix(runtimeKey); + const tenantRoot = path.join(HOST_TENANT_ROOT, suffix); + const layout = guestLayout(runtimeKey); + const expected = Object.freeze({ + version: OCI_DEPLOYMENT_PLAN_VERSION, + backend: OCI_BACKEND, + runtimeKey, + manifestRevision: value.manifestRevision, + release, + deployment: Object.freeze({ ...value.deployment }), + account, + identity: Object.freeze({ + suffix, + containerName: `dispatch-dsp-${suffix}`, + unitName: `dispatch-dsp-${suffix}.service`, + bridgeUnitName: `dispatch-runtime-agent-bridge-${suffix}.service`, + }), + host: Object.freeze({ + tenantRoot, + accountHome: path.join(tenantRoot, 'home'), + engineDataRoot: path.join(tenantRoot, 'engine-data'), + engineConfigRoot: path.join(tenantRoot, 'engine-config'), + installationRoot: path.join(tenantRoot, 'runtime', runtimeKey), + bridgeRoot: path.join(HOST_BRIDGE_ROOT, suffix), + unitPath: path.join(SYSTEM_UNIT_ROOT, `dispatch-dsp-${suffix}.service`), + bridgeUnitPath: path.join(SYSTEM_UNIT_ROOT, `dispatch-runtime-agent-bridge-${suffix}.service`), + }), + guest: Object.freeze({ + installationRoot: layout.installationRoot, + bridgeRoot: GUEST_BRIDGE_ROOT, + environment: Object.freeze({ + DISPATCH_MANAGED_RUNTIME: '1', + ...managedInstallationRuntimeEnvironment(layout), + DISPATCH_RUNTIME_KEY: runtimeKey, + DISPATCH_RUNTIME_GATEWAY_SOCKET: path.join(layout.directories.runtimeRoot, 'runtime-gateway.sock'), + DISPATCH_RUNTIME_AGENT_HUB_SOCKET: path.join(GUEST_BRIDGE_ROOT, 'runtime-agent-hub.sock'), + DISPATCH_RUNTIME_AGENT_TOKEN_FILE: path.join(layout.directories.runtimeAgentSecretsRoot, 'registration-token'), + DISPATCH_RUNTIME_AGENT_STATUS_SOCKET: path.join(layout.directories.runtimeRoot, 'runtime-agent-status.sock'), + DISPATCH_CHROME_EXECUTABLE: '/usr/bin/chromium', + }), + }), + resources: RESOURCE_POLICY, + security: Object.freeze({ + readOnlyRoot: true, + network: 'pasta', + pidNamespace: 'private', + ipcNamespace: 'private', + utsNamespace: 'private', + user: `${CONTAINER_UID}:${CONTAINER_GID}`, + userNamespace: `keep-id:uid=${CONTAINER_UID},gid=${CONTAINER_GID}`, + capabilities: Object.freeze(['SYS_CHROOT']), + noNewPrivileges: true, + publishPorts: false, + engineSocketMounted: false, + }), + }); + if (JSON.stringify({ ...expected, planDigest: stableDigest(expected) }) !== JSON.stringify(value)) fail(); + const selected = Object.freeze({ ...expected, planDigest: stableDigest(expected) }); + ISSUED_PLANS.add(selected); + return selected; +} + +function podmanArguments(value) { + if (value?.backend === 'native_service_v1') fail(); + const plan = requirePlan(value); + const args = [ + 'run', '--rm', '--replace', '--name', plan.identity.containerName, + '--pull=never', '--http-proxy=false', '--log-driver=journald', `--network=${plan.security.network}`, + `--pid=${plan.security.pidNamespace}`, `--ipc=${plan.security.ipcNamespace}`, `--uts=${plan.security.utsNamespace}`, '--read-only', + '--user', plan.security.user, '--userns', plan.security.userNamespace, + '--cap-drop', 'ALL', '--cap-add', 'SYS_CHROOT', '--security-opt', 'no-new-privileges', + '--memory', RESOURCE_POLICY.memory, '--cpus', RESOURCE_POLICY.cpus, + '--pids-limit', RESOURCE_POLICY.pids, '--shm-size', RESOURCE_POLICY.sharedMemory, + '--tmpfs', `/tmp:rw,noexec,nosuid,nodev,size=${RESOURCE_POLICY.temporaryStorage},mode=1777`, + '--volume', `${plan.host.installationRoot}:${plan.guest.installationRoot}:rw,rprivate,nosuid,nodev`, + '--volume', `${plan.host.bridgeRoot}:${plan.guest.bridgeRoot}:ro,rprivate,nosuid,nodev,noexec`, + '--label', `io.dispatch.runtime-key=${plan.runtimeKey}`, + '--label', `io.dispatch.release-id=${plan.release.releaseId}`, + '--label', `io.dispatch.plan-digest=${plan.planDigest}`, + ]; + for (const [key, selected] of Object.entries(plan.guest.environment).sort(([left], [right]) => left.localeCompare(right))) { + args.push('--env', `${key}=${selected}`); + } + args.push(plan.release.image); + return Object.freeze(args); +} + +function unitWord(value) { + if (typeof value !== 'string' || !/^[A-Za-z0-9_./:@=,+-]+$/.test(value) || /%/.test(value)) fail(); + return value; +} + +function renderOciSystemUnit(value) { + if (value?.backend === 'native_service_v1') return require('./native-deployment').renderSystemUnit(value); + const plan = requirePlan(value); + const command = [PODMAN, ...podmanArguments(plan)].map(unitWord).join(' '); + const stop = [PODMAN, 'stop', '--time', '20', plan.identity.containerName].map(unitWord).join(' '); + const cleanup = [PODMAN, 'rm', '--force', '--ignore', plan.identity.containerName].map(unitWord).join(' '); + const lines = [ + '[Unit]', + `Description=Dispatch isolated DSP runtime (${plan.identity.suffix})`, + 'After=network-online.target', + 'Wants=network-online.target', + 'StartLimitIntervalSec=120', + 'StartLimitBurst=5', + '', + '[Service]', + 'Type=simple', + `User=${unitWord(plan.account.name)}`, + `Group=${unitWord(plan.account.name)}`, + `Environment=HOME=${unitWord(plan.host.accountHome)}`, + `Environment=XDG_DATA_HOME=${unitWord(plan.host.engineDataRoot)}`, + `Environment=XDG_CONFIG_HOME=${unitWord(plan.host.engineConfigRoot)}`, + `Environment=XDG_RUNTIME_DIR=/run/user/${plan.account.uid}`, + 'Environment=PATH=/usr/bin:/bin', + 'UnsetEnvironment=NODE_OPTIONS LD_PRELOAD LD_LIBRARY_PATH HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy DISPATCH_LOCAL_ROOT DISPATCH_ACCESS_CONTROL_DATABASE_ROOT', + `ExecStartPre=${PODMAN} image exists ${unitWord(plan.release.image)}`, + `ExecStart=${command}`, + `ExecStop=${stop}`, + `ExecStopPost=-${cleanup}`, + 'Restart=on-failure', + 'RestartSec=3', + 'KillMode=control-group', + 'TimeoutStartSec=60', + 'TimeoutStopSec=30', + 'UMask=0077', + 'Delegate=yes', + 'Slice=dispatch-dsp.slice', + 'CPUQuota=200%', + 'MemoryMax=4G', + 'TasksMax=512', + 'OOMPolicy=stop', + 'ProtectSystem=strict', + `ReadWritePaths=${unitWord(plan.host.tenantRoot)} /run/user/${plan.account.uid}`, + 'PrivateTmp=true', + 'LockPersonality=true', + 'ProtectKernelTunables=true', + 'ProtectKernelModules=true', + 'ProtectControlGroups=true', + 'RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK', + '', + '[Install]', + 'WantedBy=multi-user.target', + '', + ]; + return `${lines.join('\n')}\n`; +} + +function renderOciBridgeSystemUnit(value, options) { + const plan = requirePlan(value); + exact(options, ['bridgeExecutable', 'centralSocket', 'centralUid', 'controllerUid']); + const bridgeExecutable = unitWord(options.bridgeExecutable); + const centralSocket = unitWord(options.centralSocket); + if (!path.isAbsolute(bridgeExecutable) || !path.isAbsolute(centralSocket) + || !Number.isSafeInteger(options.centralUid) || options.centralUid < 1 + || options.controllerUid !== 0 + || options.centralUid === plan.account.uid || options.controllerUid === plan.account.uid + || options.centralUid === options.controllerUid) fail(); + const lines = [ + '[Unit]', + `Description=Dispatch Runtime Agent bridge (${plan.identity.suffix})`, + 'After=local-fs.target', + ...(plan.backend === 'native_service_v1' ? ['StartLimitIntervalSec=0'] : []), + '', + '[Service]', + 'Type=simple', + ...(plan.backend === 'native_service_v1' ? [`RuntimeDirectory=dispatch-runtime-agents dispatch-runtime-agents/${plan.identity.suffix}`, + 'RuntimeDirectoryMode=0711', 'RuntimeDirectoryPreserve=yes'] : []), + `User=${options.controllerUid}`, + `Group=${options.controllerUid}`, + 'Environment=PATH=/usr/bin:/bin', + `Environment=DISPATCH_RUNTIME_BRIDGE_KEY=${unitWord(plan.runtimeKey)}`, + `Environment=DISPATCH_RUNTIME_BRIDGE_UPSTREAM_SOCKET=${centralSocket}`, + `Environment=DISPATCH_RUNTIME_BRIDGE_TENANT_UID=${plan.account.uid}`, + `Environment=DISPATCH_RUNTIME_BRIDGE_TENANT_GID=${plan.account.gid}`, + `Environment=DISPATCH_RUNTIME_BRIDGE_CENTRAL_UID=${options.centralUid}`, + + `ExecStart=/usr/bin/node --no-warnings ${bridgeExecutable}`, + 'Restart=on-failure', + plan.backend === 'native_service_v1' ? 'RestartSec=5' : 'RestartSec=1', + 'KillMode=control-group', + 'MemoryMax=256M', + 'TasksMax=32', + 'CPUQuota=50%', + 'TimeoutStopSec=15', + 'UMask=0077', + 'NoNewPrivileges=true', + 'PrivateTmp=true', + 'ProtectSystem=strict', + 'ProtectHome=read-only', + 'ProtectKernelTunables=true', + 'ProtectKernelModules=true', + 'ProtectKernelLogs=true', + 'ProtectControlGroups=true', + 'ProtectClock=true', + 'ProtectHostname=true', + 'PrivateDevices=true', + 'RestrictNamespaces=true', + 'RestrictSUIDSGID=true', + 'SystemCallArchitectures=native', + 'CapabilityBoundingSet=CAP_CHOWN CAP_DAC_OVERRIDE CAP_FOWNER', + `ReadOnlyPaths=${unitWord(path.dirname(bridgeExecutable))}`, + `ReadWritePaths=${unitWord(plan.host.bridgeRoot)}`, + 'RestrictAddressFamilies=AF_UNIX', + '', + '[Install]', + 'WantedBy=multi-user.target', + '', + ]; + return `${lines.join('\n')}\n`; +} + +module.exports = { + OCI_DEPLOYMENT_PLAN_VERSION, + OCI_DEPLOYMENT_AUTHORITY_VERSION, + OCI_RELEASE_DESCRIPTOR_VERSION, + OCI_BACKEND, + CONTAINER_UID, + CONTAINER_GID, + SUBID_COUNT, + HOST_TENANT_ROOT, + HOST_BRIDGE_ROOT, + SYSTEM_UNIT_ROOT, + GUEST_STORAGE_ROOT, + GUEST_BRIDGE_ROOT, + RESOURCE_POLICY, + hostAccountName, + releaseDescriptor, + hostAccount, + createOciDeploymentPlan, + createOciFixtureDeploymentPlan, + validateOciDeploymentPlan, + podmanArguments, + renderOciSystemUnit, + renderOciBridgeSystemUnit, +}; diff --git a/core/core/installations/src/oci-helper-input.js b/core/core/installations/src/oci-helper-input.js new file mode 100644 index 0000000..bcefd66 --- /dev/null +++ b/core/core/installations/src/oci-helper-input.js @@ -0,0 +1,31 @@ +'use strict'; + +const fs = require('node:fs'); +const { TextDecoder } = require('node:util'); +const { parseStrictJson } = require('../../../shared/gateway/strict-json'); + +function fail() { + throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); +} + +// Read at most limit + 1 bytes, including when stdin is an unbounded pipe. +// Checking size after readFileSync(0) does not bound memory consumption. +function readHelperRequest(fd, limit) { + if (!Number.isSafeInteger(limit) || limit < 3 || limit > 256 * 1024) fail(); + const buffer = Buffer.alloc(limit + 1); + let length = 0; + while (length <= limit) { + const count = fs.readSync(fd, buffer, length, buffer.length - length, null); + if (!count) break; + length += count; + if (length > limit) fail(); + } + if (length < 3) fail(); + try { + const raw = new TextDecoder('utf-8', { fatal: true }).decode(buffer.subarray(0, length)); + if (!raw.endsWith('\n') || raw.slice(0, -1).includes('\n') || /[\r\0]/.test(raw)) fail(); + return parseStrictJson(raw.slice(0, -1)); + } catch { fail(); } +} + +module.exports = { readHelperRequest }; diff --git a/core/core/installations/src/oci-host-account-registry.js b/core/core/installations/src/oci-host-account-registry.js new file mode 100644 index 0000000..30377f5 --- /dev/null +++ b/core/core/installations/src/oci-host-account-registry.js @@ -0,0 +1,248 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const { INSTALLATION_IDENTIFIER_RE } = require('../../../shared/contracts/src/installation'); +const { hostAccountName } = require('../../runtime-host-identity'); +const { SUBID_COUNT } = require('./oci-deployment'); + +const OCI_HOST_REGISTRY_SCHEMA_VERSION = 1; +const OCI_HOST_REGISTRY_DATABASE = 'oci-host.sqlite3'; +const PRIVATE_DIRECTORY_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; +const ALLOCATIONS_SCHEMA_SQL = `CREATE TABLE allocations ( + runtime_key TEXT PRIMARY KEY, + account_name TEXT NOT NULL UNIQUE, + uid INTEGER NOT NULL UNIQUE CHECK(uid BETWEEN 100 AND 60000), + gid INTEGER NOT NULL UNIQUE CHECK(gid BETWEEN 100 AND 60000), + subuid_start INTEGER NOT NULL UNIQUE CHECK(subuid_start>=100000), + subgid_start INTEGER NOT NULL UNIQUE CHECK(subgid_start>=100000), + subid_count INTEGER NOT NULL CHECK(subid_count=65536), + status TEXT NOT NULL CHECK(status IN ('reserved','active','retired')), + created_at INTEGER NOT NULL CHECK(created_at>=0), + updated_at INTEGER NOT NULL CHECK(updated_at>=created_at) + ) STRICT`; + +function fail(code = 'runtime_boundary_violation') { + throw Object.assign(new Error(code), { code }); +} + +function absolute(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) fail(); + return value; +} + +function directory(target) { + let info; + try { info = fs.lstatSync(target); } catch { fail(); } + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== process.geteuid() + || (info.mode & 0o7777) !== PRIVATE_DIRECTORY_MODE || fs.realpathSync(target) !== target) fail(); + return Object.freeze({ dev: info.dev, ino: info.ino }); +} + +function databaseFile(target, device, allowEmpty = false) { + let info; + try { info = fs.lstatSync(target); } catch { fail(); } + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() || info.nlink !== 1 + || info.dev !== device || (info.mode & 0o7777) !== PRIVATE_FILE_MODE + || (!allowEmpty && info.size < 1) || info.size > 16 * 1024 * 1024 + || fs.realpathSync(target) !== target) fail(); + return Object.freeze({ dev: info.dev, ino: info.ino }); +} + +function runtimeKey(value) { + if (typeof value !== 'string' || !INSTALLATION_IDENTIFIER_RE.test(value) || value === 'local') fail(); + return value; +} + +function positiveInteger(value, minimum, maximum) { + if (!Number.isSafeInteger(value) || value < minimum || value > maximum) fail(); + return value; +} + +function parseRanges(file) { + const result = []; + let raw; + try { raw = fs.readFileSync(file, 'utf8'); } catch { fail(); } + if (Buffer.byteLength(raw, 'utf8') > 1024 * 1024 || raw.includes('\0') || raw.includes('\r')) fail(); + for (const line of raw.split('\n')) { + if (!line) continue; + const parts = line.split(':'); + const start = Number(parts[1]); + const count = Number(parts[2]); + if (parts.length !== 3 || !parts[0] || !Number.isSafeInteger(start) + || !Number.isSafeInteger(count) || start < 0 || count < 1 + || start + count - 1 > 4_294_967_294) fail(); + result.push(Object.freeze({ start, end: start + count - 1 })); + } + return result; +} + +function overlaps(start, count, range) { + const end = start + count - 1; + return start <= range.end && end >= range.start; +} + +function view(row) { + if (!row) return null; + return Object.freeze({ + runtimeKey: row.runtime_key, + name: row.account_name, + uid: row.uid, + gid: row.gid, + subuidStart: row.subuid_start, + subgidStart: row.subgid_start, + subidCount: row.subid_count, + status: row.status, + }); +} + +function createOciHostAccountRegistry(options) { + if (!options || typeof options !== 'object' || Array.isArray(options) + || Object.keys(options).some(key => ![ + 'stateRoot', 'uidMinimum', 'uidMaximum', 'subidMinimum', 'subuidFile', 'subgidFile', + 'identityAvailable', 'clock', + ].includes(key))) fail(); + const stateRoot = absolute(options.stateRoot); + const rootIdentity = directory(stateRoot); + const uidMinimum = positiveInteger(options.uidMinimum === undefined ? 20_000 : options.uidMinimum, 100, 60_000); + const uidMaximum = positiveInteger(options.uidMaximum === undefined ? 59_999 : options.uidMaximum, uidMinimum, 60_000); + const subidMinimum = positiveInteger(options.subidMinimum === undefined ? 1_000_000 : options.subidMinimum, 100_000, 4_000_000_000); + const subuidFile = absolute(options.subuidFile === undefined ? '/etc/subuid' : options.subuidFile); + const subgidFile = absolute(options.subgidFile === undefined ? '/etc/subgid' : options.subgidFile); + const identityAvailable = options.identityAvailable; + const clock = options.clock === undefined ? Date.now : options.clock; + if (typeof identityAvailable !== 'function' || typeof clock !== 'function') fail(); + const file = path.join(stateRoot, OCI_HOST_REGISTRY_DATABASE); + const existing = (() => { try { fs.lstatSync(file); return true; } catch (error) { if (error?.code === 'ENOENT') return false; throw error; } })(); + if (existing) databaseFile(file, rootIdentity.dev, true); + for (const suffix of ['-wal', '-shm']) { + const sidecar = `${file}${suffix}`; + try { fs.lstatSync(sidecar); databaseFile(sidecar, rootIdentity.dev, true); } + catch (error) { if (error?.code !== 'ENOENT') throw error; } + } + let db; + try { + db = new DatabaseSync(file); + if (!existing) fs.chmodSync(file, PRIVATE_FILE_MODE); + databaseFile(file, rootIdentity.dev, !existing); + db.exec(`PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON; + PRAGMA trusted_schema=OFF; PRAGMA busy_timeout=30000;`); + db.exec('BEGIN IMMEDIATE'); + const version = db.prepare('PRAGMA user_version').get().user_version; + const initialSchema = db.prepare('SELECT name FROM sqlite_schema').all(); + if (version === 0 && initialSchema.length === 0) db.exec(`${ALLOCATIONS_SCHEMA_SQL}; PRAGMA user_version=1;`); + if (db.prepare('PRAGMA user_version').get().user_version !== OCI_HOST_REGISTRY_SCHEMA_VERSION + || db.prepare('PRAGMA quick_check(1)').get().quick_check !== 'ok') fail(); + const columns = db.prepare('PRAGMA table_info(allocations)').all().map(row => row.name); + if (JSON.stringify(columns) !== JSON.stringify([ + 'runtime_key', 'account_name', 'uid', 'gid', 'subuid_start', 'subgid_start', 'subid_count', + 'status', 'created_at', 'updated_at', + ])) fail(); + const schema = db.prepare("SELECT sql FROM sqlite_schema WHERE type='table' AND name='allocations'").get()?.sql; + const definitions = db.prepare('SELECT sql FROM sqlite_schema WHERE sql IS NOT NULL').all(); + const normalize = value => String(value).replace(/\s+/g, ' ').trim().replace(/\s*([(),=])\s*/g, '$1'); + if (definitions.length !== 1 || normalize(schema) !== normalize(ALLOCATIONS_SCHEMA_SQL)) fail(); + db.exec('COMMIT'); + } catch (error) { + try { db?.close(); } catch {} + if (error?.code === 'runtime_boundary_violation') throw error; + fail(); + } + let closed = false; + const databaseIdentity = databaseFile(file, rootIdentity.dev); + + function assertOpen() { + if (closed) fail('installation_operation_failed'); + const current = directory(stateRoot); + if (current.dev !== rootIdentity.dev || current.ino !== rootIdentity.ino) fail(); + const currentDatabase = databaseFile(file, rootIdentity.dev); + if (currentDatabase.dev !== databaseIdentity.dev || currentDatabase.ino !== databaseIdentity.ino) fail(); + } + + function inspect(keyValue) { + assertOpen(); + const key = runtimeKey(keyValue); + return view(db.prepare('SELECT * FROM allocations WHERE runtime_key=?').get(key)); + } + + function reserve(keyValue) { + assertOpen(); + const key = runtimeKey(keyValue); + db.exec('BEGIN IMMEDIATE'); + try { + const prior = db.prepare('SELECT * FROM allocations WHERE runtime_key=?').get(key); + if (prior) { + db.exec('COMMIT'); + return view(prior); + } + let uid = null; + const allocatedUids = new Set(db.prepare('SELECT uid FROM allocations').all().map(row => row.uid)); + for (let candidate = uidMinimum; candidate <= uidMaximum; candidate += 1) { + if (!allocatedUids.has(candidate) && identityAvailable(candidate)) { uid = candidate; break; } + } + if (uid === null) fail('service_installation_failed'); + const occupied = [ + ...parseRanges(subuidFile), + ...parseRanges(subgidFile), + ...db.prepare('SELECT subuid_start,subgid_start,subid_count FROM allocations').all() + .flatMap(row => [ + { start: row.subuid_start, end: row.subuid_start + row.subid_count - 1 }, + { start: row.subgid_start, end: row.subgid_start + row.subid_count - 1 }, + ]), + ]; + let subid = Math.ceil(subidMinimum / SUBID_COUNT) * SUBID_COUNT; + while (occupied.some(range => overlaps(subid, SUBID_COUNT, range))) { + subid += SUBID_COUNT; + if (subid + SUBID_COUNT - 1 > 4_294_967_294) fail('service_installation_failed'); + } + const at = clock(); + if (!Number.isSafeInteger(at) || at < 0) fail('installation_operation_failed'); + db.prepare(`INSERT INTO allocations( + runtime_key,account_name,uid,gid,subuid_start,subgid_start,subid_count,status,created_at,updated_at + ) VALUES(?,?,?,?,?,?,?,'reserved',?,?)`).run( + key, hostAccountName(key), uid, uid, subid, subid, SUBID_COUNT, at, at, + ); + const selected = db.prepare('SELECT * FROM allocations WHERE runtime_key=?').get(key); + db.exec('COMMIT'); + return view(selected); + } catch (error) { + try { db.exec('ROLLBACK'); } catch {} + if (error?.code) throw error; + fail('service_installation_failed'); + } + } + + function transition(keyValue, from, to) { + assertOpen(); + const key = runtimeKey(keyValue); + if (!['reserved', 'active'].includes(from) || !['active', 'retired'].includes(to)) fail(); + const at = clock(); + if (!Number.isSafeInteger(at) || at < 0) fail('installation_operation_failed'); + const changed = db.prepare('UPDATE allocations SET status=?,updated_at=? WHERE runtime_key=? AND status=?') + .run(to, at, key, from).changes; + if (changed !== 1) fail('installation_operation_in_progress'); + return inspect(key); + } + + function activate(key) { + const current = inspect(key); + return current?.status === 'active' ? current : transition(key, 'reserved', 'active'); + } + function retire(key) { + const current = inspect(key); + return current?.status === 'retired' ? current : transition(key, 'active', 'retired'); + } + function close() { if (!closed) db.close(); closed = true; } + + return Object.freeze({ reserve, inspect, activate, retire, close }); +} + +module.exports = { + OCI_HOST_REGISTRY_SCHEMA_VERSION, + OCI_HOST_REGISTRY_DATABASE, + ALLOCATIONS_SCHEMA_SQL, + createOciHostAccountRegistry, +}; diff --git a/core/core/installations/src/oci-host-artifact.js b/core/core/installations/src/oci-host-artifact.js new file mode 100644 index 0000000..89ec98d --- /dev/null +++ b/core/core/installations/src/oci-host-artifact.js @@ -0,0 +1,95 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); + +function fail() { throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); } +function rootAncestors(target) { + for (let parent = target; ; parent = path.dirname(parent)) { + const info = fs.lstatSync(parent); + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== 0 || info.gid !== 0 + || (info.mode & 0o022) !== 0 || fs.realpathSync(parent) !== parent) fail(); + if (parent === '/') break; + } +} +function readRootFile(target, mode, limit) { + rootAncestors(path.dirname(target)); + const before = fs.lstatSync(target); + if (!before.isFile() || before.isSymbolicLink() || before.uid !== 0 || before.gid !== 0 + || before.nlink !== 1 || (before.mode & 0o7777) !== mode || before.size < 1 + || before.size > limit || fs.realpathSync(target) !== target) fail(); + const fd = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + const opened = fs.fstatSync(fd); + if (opened.dev !== before.dev || opened.ino !== before.ino || opened.size !== before.size) fail(); + const content = fs.readFileSync(fd); + const after = fs.fstatSync(fd); + const final = fs.lstatSync(target); + if (after.size !== opened.size || after.mtimeMs !== opened.mtimeMs || after.ctimeMs !== opened.ctimeMs + || content.length !== opened.size || final.ino !== opened.ino || final.dev !== opened.dev) fail(); + return content; + } finally { fs.closeSync(fd); } +} +function verifyPreparedHostArtifact(runningHelper, releaseId, manifestSha256, executable = 'dispatch-oci-host-helper') { + if (!['dispatch-oci-host-helper', 'dispatch-oci-host-issuer'].includes(executable)) fail(); + if (typeof releaseId !== 'string' || !/^[a-z][a-z0-9_.-]{2,95}$/.test(releaseId) + || typeof manifestSha256 !== 'string' || !/^[a-f0-9]{64}$/.test(manifestSha256)) fail(); + const control = '/opt/dispatch-control'; + rootAncestors(control); + const releaseRoot = path.join(control, 'releases', releaseId); + const artifactRoot = path.join(releaseRoot, 'host-helper-artifact'); + const helper = path.join(artifactRoot, 'core/installations/bin', executable); + if (fs.realpathSync(runningHelper) !== helper) fail(); + rootAncestors(artifactRoot); + const content = readRootFile(path.join(artifactRoot, 'manifest.json'), 0o444, 64 * 1024); + const hash = value => crypto.createHash('sha256').update(value).digest('hex'); + if (hash(content) !== manifestSha256) fail(); + const manifest = JSON.parse(content.toString('utf8')); + if (!manifest || Object.keys(manifest).sort().join(',') !== 'files,version' || manifest.version !== 1 + || !Array.isArray(manifest.files) || manifest.files.length < 1 || manifest.files.length > 100) fail(); + const files = new Set(['manifest.json']); + const directories = new Set(['']); + for (const entry of manifest.files) { + if (!entry || Object.keys(entry).sort().join(',') !== 'mode,path,sha256' || typeof entry.path !== 'string' + || !/^(?:core|protocol)\/[a-z0-9_./-]+$/.test(entry.path) || !['444', '555'].includes(entry.mode) + || !/^[a-f0-9]{64}$/.test(entry.sha256) || files.has(entry.path) + || path.relative(artifactRoot, path.join(artifactRoot, entry.path)) !== entry.path) fail(); + files.add(entry.path); + for (let parent = path.dirname(entry.path); parent !== '.'; parent = path.dirname(parent)) directories.add(parent); + if (hash(readRootFile(path.join(artifactRoot, entry.path), parseInt(entry.mode, 8), 2 * 1024 * 1024)) !== entry.sha256) fail(); + } + function inspect(relative) { + const directory = path.join(artifactRoot, relative); + const info = fs.lstatSync(directory); + if (!directories.has(relative) || !info.isDirectory() || info.isSymbolicLink() + || info.uid !== 0 || info.gid !== 0 || (info.mode & 0o7777) !== 0o555 + || info.dev !== fs.lstatSync(artifactRoot).dev) fail(); + for (const name of fs.readdirSync(directory)) { + const child = path.join(relative, name); + if (fs.lstatSync(path.join(artifactRoot, child)).isDirectory()) inspect(child); + else if (!files.has(child)) fail(); + } + } + inspect(''); + return Object.freeze({ releaseRoot, artifactRoot }); +} + +// Executing helpers must additionally be bound to the active pointer. Preparation +// verifies the same immutable tree without requiring it to be activated already. +function verifyHostArtifact(runningHelper, releaseId, manifestSha256, executable = 'dispatch-oci-host-helper') { + if (typeof releaseId !== 'string' || !/^[a-z][a-z0-9_.-]{2,95}$/.test(releaseId)) fail(); + const control = '/opt/dispatch-control'; + rootAncestors(control); + const releaseRoot = path.join(control, 'releases', releaseId); + const current = path.join(control, 'current'); + const link = fs.lstatSync(current); + if (!link.isSymbolicLink() || link.uid !== 0 || link.gid !== 0 || link.nlink !== 1 + || fs.readlinkSync(current) !== releaseRoot || fs.realpathSync(current) !== releaseRoot) fail(); + const result = verifyPreparedHostArtifact(runningHelper, releaseId, manifestSha256, executable); + const after = fs.lstatSync(current); + if (after.ino !== link.ino || after.dev !== link.dev || fs.readlinkSync(current) !== releaseRoot) fail(); + return result; +} + +module.exports = { readRootFile, rootAncestors, verifyHostArtifact, verifyPreparedHostArtifact }; diff --git a/core/core/installations/src/oci-host-authority.js b/core/core/installations/src/oci-host-authority.js new file mode 100644 index 0000000..fd057df --- /dev/null +++ b/core/core/installations/src/oci-host-authority.js @@ -0,0 +1,300 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); + +const ISSUER = 'dispatch-access-control'; +const AUDIENCE = 'dispatch-oci-host-v1'; +const DATABASE = 'authority.sqlite3'; +const RETIRED_READS = Object.freeze(['inspect_account', 'verify_destroyed']); +const SCHEMA = [ + `CREATE TABLE leases ( + runtime_key TEXT PRIMARY KEY, + lease_json TEXT NOT NULL, + generation INTEGER NOT NULL CHECK(generation>0), + fence INTEGER NOT NULL CHECK(fence>0), + state TEXT NOT NULL CHECK(state IN ('active','revoked','retired')) + ) STRICT`, + `CREATE TABLE actions ( + id TEXT PRIMARY KEY, + runtime_key TEXT NOT NULL REFERENCES leases(runtime_key), + request_digest TEXT NOT NULL, + generation INTEGER NOT NULL CHECK(generation>0), + fence INTEGER NOT NULL CHECK(fence>0), + state TEXT NOT NULL CHECK(state IN ('issued','running','consumed')) + ) STRICT`, +]; + +function fail() { + throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); +} +function exact(value, keys) { + if (!value || Object.getPrototypeOf(value) !== Object.prototype + || Object.keys(value).sort().join(',') !== [...keys].sort().join(',')) fail(); +} +function identifier(value) { + if (typeof value !== 'string' || !/^[a-z][a-z0-9_-]{2,95}$/.test(value)) fail(); +} +function canonical(value) { + if (value === null || typeof value === 'string' || typeof value === 'boolean') return JSON.stringify(value); + if (typeof value === 'number' && Number.isSafeInteger(value)) return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (!value || Object.getPrototypeOf(value) !== Object.prototype) fail(); + return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`; +} +function requestDigest(request) { + const { authorization, ...body } = request; + const encoded = canonical(body); + if (Buffer.byteLength(encoded) > 256 * 1024) fail(); + return crypto.createHash('sha256').update(encoded).digest('hex'); +} +function validateLease(value, now) { + exact(value, ['version', 'issuer', 'audience', 'organizationId', 'runtimeKey', 'installationRevision', 'manifestRevisions', + 'backend', 'jobKind', 'jobId', 'workerId', 'generation', 'fence', 'expiresAt']); + for (const field of ['organizationId', 'runtimeKey', 'jobId', 'workerId']) identifier(value[field]); + for (const field of ['installationRevision', 'generation', 'fence', 'expiresAt']) { + if (!Number.isSafeInteger(value[field]) || value[field] < 1) fail(); + } + if (!Array.isArray(value.manifestRevisions) || ![1, 2].includes(value.manifestRevisions.length) + || value.manifestRevisions.some(item => !Number.isSafeInteger(item) || item < 1) + || value.manifestRevisions.length === 2 && value.manifestRevisions[1] !== value.manifestRevisions[0] + 1) fail(); + if (value.version !== 1 || value.issuer !== ISSUER || value.audience !== AUDIENCE + || !['provisioning', 'lifecycle'].includes(value.jobKind) + || !['oci_container_v1', 'native_service_v1'].includes(value.backend) || value.runtimeKey === 'local' + || value.expiresAt <= now || value.expiresAt > now + 600_000) fail(); + return value; +} +function protectedDirectory(target, mode) { + const info = fs.lstatSync(target); + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== 0 || info.gid !== 0 + || (info.mode & 0o7777) !== mode || fs.realpathSync(target) !== target) fail(); + return info; +} + +// This API belongs ONLY to the independently protected authority process. It is +// deliberately absent from the sudo helper's wire protocol. The ordinary worker +// cannot issue, renew, revoke, or edit a lease or an action. +function createOciHostAuthority({ root, clock = Date.now }) { + if (process.geteuid() !== 0 || process.getegid() !== 0 || typeof root !== 'string' + || !path.isAbsolute(root) || path.resolve(root) !== root || typeof clock !== 'function') fail(); + const rootInfo = protectedDirectory(root, 0o700); + for (let parent = path.dirname(root); ; parent = path.dirname(parent)) { + const info = fs.lstatSync(parent); + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== 0 || info.gid !== 0 + || (info.mode & 0o022) !== 0 || fs.realpathSync(parent) !== parent) fail(); + if (parent === '/') break; + } + const file = path.join(root, DATABASE); + const entries = fs.readdirSync(root); + if (entries.some(entry => ![DATABASE, `${DATABASE}-wal`, `${DATABASE}-shm`].includes(entry))) fail(); + let created = false; + try { + const fd = fs.openSync(file, fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_WRONLY, 0o600); + fs.closeSync(fd); + created = true; + } catch (error) { if (error.code !== 'EEXIST') throw error; } + function privateFile(target) { + const info = fs.lstatSync(target); + if (!info.isFile() || info.isSymbolicLink() || info.uid !== 0 || info.gid !== 0 + || info.nlink !== 1 || (info.mode & 0o7777) !== 0o600 || info.dev !== rootInfo.dev + || fs.realpathSync(target) !== target) fail(); + return info; + } + const fileInfo = privateFile(file); + for (const entry of entries) privateFile(path.join(root, entry)); + const db = new DatabaseSync(file); + try { + db.exec('PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON; PRAGMA trusted_schema=OFF; PRAGMA busy_timeout=30000;'); + db.exec('BEGIN IMMEDIATE'); + const version = db.prepare('PRAGMA user_version').get().user_version; + const tables = db.prepare("SELECT sql FROM sqlite_schema WHERE type='table' ORDER BY name").all(); + if (version === 0 && tables.length === 0 && (created || fileInfo.size === 0)) { + for (const sql of SCHEMA) db.exec(sql); + db.exec('PRAGMA user_version=1'); + } + const schema = db.prepare("SELECT sql FROM sqlite_schema WHERE sql IS NOT NULL ORDER BY name").all(); + const normalize = value => value.replace(/\s+/g, ' ').trim(); + if (db.prepare('PRAGMA user_version').get().user_version !== 1 + || JSON.stringify(schema.map(row => normalize(row.sql))) !== JSON.stringify([SCHEMA[1], SCHEMA[0]].map(normalize)) + || db.prepare('PRAGMA quick_check').get().quick_check !== 'ok') fail(); + db.exec('COMMIT'); + } catch (error) { try { db.exec('ROLLBACK'); } catch {} db.close(); throw error; } + let closed = false; + function check() { + if (closed) fail(); + const current = protectedDirectory(root, 0o700); + const currentFile = privateFile(file); + if (current.dev !== rootInfo.dev || current.ino !== rootInfo.ino + || currentFile.dev !== fileInfo.dev || currentFile.ino !== fileInfo.ino) fail(); + const now = clock(); + if (!Number.isSafeInteger(now) || now < 0) fail(); + return now; + } + function transaction(callback) { + check(); + db.exec('BEGIN IMMEDIATE'); + try { const result = callback(); db.exec('COMMIT'); return result; } + catch (error) { db.exec('ROLLBACK'); throw error; } + } + function issueLease(value) { + const lease = validateLease(value, check()); + return transaction(() => replaceLease(lease)); + } + function replaceLease(lease) { + if (db.prepare("SELECT 1 FROM actions WHERE runtime_key=? AND state='running'").get(lease.runtimeKey)) fail(); + const prior = db.prepare('SELECT * FROM leases WHERE runtime_key=?').get(lease.runtimeKey); + const priorLease = prior && JSON.parse(prior.lease_json); + if (prior && (prior.state === 'retired' || lease.installationRevision < priorLease.installationRevision + || lease.installationRevision === priorLease.installationRevision + && (lease.generation < prior.generation + || lease.generation === prior.generation && lease.fence <= prior.fence))) fail(); + db.prepare(`INSERT INTO leases VALUES(?,?,?,?, 'active') ON CONFLICT(runtime_key) DO UPDATE SET + lease_json=excluded.lease_json,generation=excluded.generation,fence=excluded.fence,state='active'`) + .run(lease.runtimeKey, canonical(lease), lease.generation, lease.fence); + } + function renewLease(value) { + const lease = validateLease(value, check()); + return transaction(() => { + const current = currentLease(lease.runtimeKey); + if (canonical({ ...current, expiresAt: lease.expiresAt }) !== canonical(lease) + || lease.expiresAt < current.expiresAt) fail(); + db.prepare('UPDATE leases SET lease_json=? WHERE runtime_key=?').run(canonical(lease), lease.runtimeKey); + }); + } + function currentLease(runtimeKey, operation) { + const row = db.prepare('SELECT * FROM leases WHERE runtime_key=?').get(runtimeKey); + if (!row || row.state !== 'active' && !(row.state === 'retired' && RETIRED_READS.includes(operation))) fail(); + return validateLease(JSON.parse(row.lease_json), check()); + } + function bound(request, lease) { + const fields = lease.jobKind === 'provisioning' ? ['jobId', 'workerId', 'generation', 'fence'] + : ['jobId', 'workerId', 'fence']; + exact(request.claim, fields); + for (const field of fields) { + if (request.claim[field] !== lease[field]) fail(); + } + const runtimeKey = request.plan?.runtimeKey ?? request.runtimeKey; + if (runtimeKey !== lease.runtimeKey) fail(); + if (request.plan && (request.plan.backend !== lease.backend + || request.plan.deployment?.organizationId !== lease.organizationId + || !lease.manifestRevisions.includes(request.plan.deployment?.manifestRevision))) fail(); + } + // Call only after deriving the request from the server-owned job/stage, + // manifest, release catalog, account allocation and backup/destruction record. + function issueAction(request) { + return transaction(() => { + const runtimeKey = request.plan?.runtimeKey ?? request.runtimeKey; + if (db.prepare("SELECT 1 FROM actions WHERE runtime_key=? AND state='running'").get(runtimeKey)) fail(); + const lease = currentLease(runtimeKey, request.operation); + bound(request, lease); + const id = crypto.randomBytes(32).toString('hex'); + db.prepare("INSERT INTO actions VALUES(?,?,?,?,?,'issued')") + .run(id, runtimeKey, requestDigest(request), lease.generation, lease.fence); + return id; + }); + } + function revoke(runtimeKey, retired = false) { + identifier(runtimeKey); + if (typeof retired !== 'boolean') fail(); + return transaction(() => { + const row = db.prepare('SELECT state FROM leases WHERE runtime_key=?').get(runtimeKey); + if (!row) fail(); + if (row.state === 'retired') return; + db.prepare('UPDATE leases SET state=? WHERE runtime_key=?').run(retired ? 'retired' : 'revoked', runtimeKey); + }); + } + function validateAction(request, state) { + if (typeof request.authorization !== 'string' || !/^[a-f0-9]{64}$/.test(request.authorization)) fail(); + const row = db.prepare('SELECT * FROM actions WHERE id=?').get(request.authorization); + if (!row || row.state !== state || row.request_digest !== requestDigest(request)) fail(); + const lease = currentLease(row.runtime_key, request.operation); + bound(request, lease); + if (row.generation !== lease.generation || row.fence !== lease.fence) fail(); + } + function execute(request, callback) { + if (typeof callback !== 'function') fail(); + // Persist the in-flight gate before effects. If this process dies, a child + // or systemd job can outlive its SQLite lock. Refuse takeover and new actions + // until a trusted recovery procedure proves those effects have quiesced. + transaction(() => { + validateAction(request, 'issued'); + const row = db.prepare('SELECT runtime_key FROM actions WHERE id=?').get(request.authorization); + if (db.prepare("SELECT 1 FROM actions WHERE runtime_key=? AND state='running'").get(row.runtime_key)) fail(); + db.prepare("UPDATE actions SET state='running' WHERE id=?").run(request.authorization); + }); + return transaction(() => { + validateAction(request, 'running'); + const guard = mutation => { + validateAction(request, 'running'); + const result = mutation(); + if (result && typeof result.then === 'function') fail(); + return result; + }; + Object.defineProperty(guard, 'remainingMs', { value: () => { + validateAction(request, 'running'); + const row = db.prepare('SELECT runtime_key FROM actions WHERE id=?').get(request.authorization); + const remaining = currentLease(row.runtime_key, request.operation).expiresAt - check(); + if (!Number.isSafeInteger(remaining) || remaining < 1) fail(); + return remaining; + } }); + // The write lock serializes takeover, revocation and retirement with every + // host effect. Executor subprocesses remain bounded by their fixed timeouts. + const result = callback(guard); + if (result && typeof result.then === 'function') fail(); + db.prepare("UPDATE actions SET state='consumed' WHERE id=?").run(request.authorization); + return result; + }); + } + // Used only by the protected issuer, after live Access Control validation. + // Each synchronous dispatch revokes its lease on return; it may be renewed + // for another request by the same still-current server claim. + function synchronizeLease(value, operation) { + const lease = validateLease(value, check()); + return transaction(() => { + const prior = db.prepare('SELECT * FROM leases WHERE runtime_key=?').get(lease.runtimeKey); + if (prior?.state === 'retired') { + const before = JSON.parse(prior.lease_json); + if (!RETIRED_READS.includes(operation) || lease.jobKind !== 'lifecycle' + || lease.organizationId !== before.organizationId + || lease.installationRevision < before.installationRevision + || lease.installationRevision === before.installationRevision + && (lease.generation < before.generation + || lease.generation === before.generation && lease.fence < before.fence) + || db.prepare("SELECT 1 FROM actions WHERE runtime_key=? AND state='running'").get(lease.runtimeKey)) fail(); + db.prepare("UPDATE leases SET lease_json=?,generation=?,fence=? WHERE runtime_key=?") + .run(canonical(lease), lease.generation, lease.fence, lease.runtimeKey); + return; + } + if (prior && prior.state !== 'retired') { + const before = JSON.parse(prior.lease_json); + if (canonical({ ...before, expiresAt: lease.expiresAt }) === canonical(lease)) { + if (db.prepare("SELECT 1 FROM actions WHERE runtime_key=? AND state='running'").get(lease.runtimeKey)) fail(); + db.prepare("UPDATE leases SET lease_json=?,state='active' WHERE runtime_key=?") + .run(canonical(lease), lease.runtimeKey); + return; + } + } + return replaceLease(lease); + }); + } + function recover(runtimeKey, quiesce) { + identifier(runtimeKey); + if (typeof quiesce !== 'function') fail(); + return transaction(() => { + const rows = db.prepare("SELECT id FROM actions WHERE runtime_key=? AND state='running'").all(runtimeKey); + if (!rows.length) return false; + // The callback must synchronously prove that the operation cgroup and + // delegated systemd jobs have stopped. An exception retains the gate. + if (quiesce(Object.freeze(rows.map(row => row.id))) !== true) fail(); + db.prepare("UPDATE actions SET state='consumed' WHERE runtime_key=? AND state='running'").run(runtimeKey); + db.prepare("UPDATE leases SET state='revoked' WHERE runtime_key=? AND state!='retired'").run(runtimeKey); + return true; + }); + } + return Object.freeze({ issueLease, renewLease, synchronizeLease, issueAction, revoke, recover, execute, + close() { if (!closed) db.close(); closed = true; } }); +} + +module.exports = { ISSUER, AUDIENCE, createOciHostAuthority, requestDigest }; diff --git a/core/core/installations/src/oci-host-executor.js b/core/core/installations/src/oci-host-executor.js new file mode 100644 index 0000000..057ca71 --- /dev/null +++ b/core/core/installations/src/oci-host-executor.js @@ -0,0 +1,1046 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { registrationToken } = require('../../../shared/agent/protocol'); +const { MANAGED_INSTALLATION_DIRECTORY_FIELDS } = require('../../../shared/paths/runtime-paths'); +const { readRootFile } = require('./oci-host-artifact'); +const { HOST_TENANT_ROOT, HOST_BRIDGE_ROOT } = require('../../runtime-host-identity'); +const { + validateOciDeploymentPlan, + renderOciSystemUnit, + renderOciBridgeSystemUnit, +} = require('./oci-deployment'); + +const MAX_OUTPUT = 128 * 1024; +const COMMANDS = Object.freeze({ + getent: '/usr/bin/getent', + groupadd: '/usr/sbin/groupadd', + groupdel: '/usr/sbin/groupdel', + useradd: '/usr/sbin/useradd', + usermod: '/usr/sbin/usermod', + userdel: '/usr/sbin/userdel', + passwd: '/usr/bin/passwd', + loginctl: '/usr/bin/loginctl', + systemctl: '/usr/bin/systemctl', + systemdRun: '/usr/bin/systemd-run', + systemdAnalyze: '/usr/bin/systemd-analyze', + runuser: '/usr/sbin/runuser', + env: '/usr/bin/env', + podman: '/usr/bin/podman', + install: '/usr/bin/install', + cmp: '/usr/bin/cmp', + dd: '/usr/bin/dd', + rm: '/usr/bin/rm', +}); + +function fail(code = 'service_installation_failed') { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, allowed, required = allowed) { + if (!plain(value) || Object.keys(value).some(key => !allowed.includes(key)) + || required.some(key => !Object.hasOwn(value, key))) { + fail('runtime_boundary_violation'); + } +} + +function absolute(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) fail('runtime_boundary_violation'); + return value; +} + +function lstatMaybe(target) { + try { return fs.lstatSync(target); } catch (error) { + if (error?.code === 'ENOENT') return null; + fail('runtime_boundary_violation'); + } +} + +function sha256File(target, expectedUid = 0, expectedGid = 0, expectedMode = 0o444) { + const before = lstatMaybe(target); + if (!before || !before.isFile() || before.isSymbolicLink() || before.uid !== expectedUid + || before.gid !== expectedGid || before.nlink !== 1 || (before.mode & 0o7777) !== expectedMode + || fs.realpathSync(target) !== target) fail('runtime_boundary_violation'); + const handle = fs.openSync(target, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0)); + try { + const opened = fs.fstatSync(handle); + if (opened.dev !== before.dev || opened.ino !== before.ino || opened.size !== before.size) fail(); + const hash = crypto.createHash('sha256'); + const buffer = Buffer.allocUnsafe(1024 * 1024); + let size = 0; + while (true) { + const count = fs.readSync(handle, buffer, 0, buffer.length, null); + if (!count) break; + hash.update(buffer.subarray(0, count)); + size += count; + } + const after = fs.fstatSync(handle); + if (after.dev !== opened.dev || after.ino !== opened.ino || after.size !== opened.size + || after.mtimeMs !== opened.mtimeMs || size !== opened.size) fail(); + return hash.digest('hex'); + } finally { fs.closeSync(handle); } +} + +function guarded(capability, callback) { + if (typeof capability !== 'function') fail('runtime_boundary_violation'); + return capability(callback); +} + +function defaultExecute(executable, args, options = {}) { + const result = spawnSync(executable, args, { + encoding: 'utf8', + input: options.input, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' }, + timeout: options.timeout || 120_000, + maxBuffer: MAX_OUTPUT, + }); + if (result.error || result.signal || !(options.accepted || [0]).includes(result.status) + || Buffer.byteLength(result.stdout || '', 'utf8') > MAX_OUTPUT + || Buffer.byteLength(result.stderr || '', 'utf8') > MAX_OUTPUT) fail(options.code); + return result; +} + +function createOciHostExecutor(options) { + exact(options, [ + 'registry', 'stateRoot', 'unitRoot', 'releaseRoot', 'centralSocket', 'centralUid', + 'controllerUid', 'execute', 'clock', 'remainingMs', + ], ['registry', 'stateRoot', 'unitRoot', 'releaseRoot', 'centralSocket', 'centralUid', 'controllerUid']); + const registry = options.registry; + if (!registry || ['reserve', 'inspect', 'activate', 'retire'].some(method => typeof registry[method] !== 'function')) { + fail('runtime_boundary_violation'); + } + const stateRoot = absolute(options.stateRoot); + const unitRoot = absolute(options.unitRoot); + if (unitRoot !== '/etc/systemd/system') fail('runtime_boundary_violation'); + const releaseRoot = absolute(options.releaseRoot); + const centralSocket = absolute(options.centralSocket); + const centralUid = options.centralUid; + const controllerUid = options.controllerUid; + const execute = options.execute || defaultExecute; + const clock = options.clock || Date.now; + const remainingMs = options.remainingMs; + if (remainingMs !== undefined && typeof remainingMs !== 'function') fail('runtime_boundary_violation'); + if (!Number.isSafeInteger(centralUid) || centralUid < 1 || controllerUid !== 0 + || centralUid === controllerUid || typeof execute !== 'function' || typeof clock !== 'function') { + fail('runtime_boundary_violation'); + } + for (const [root, mode] of [[stateRoot, 0o700], [unitRoot, 0o755], [releaseRoot, 0o755]]) { + const info = lstatMaybe(root); + if (!info || !info.isDirectory() || info.isSymbolicLink() || info.uid !== process.geteuid() + || (info.mode & 0o7777) !== mode || fs.realpathSync(root) !== root) fail('runtime_boundary_violation'); + } + for (const [root, mode] of [[HOST_TENANT_ROOT, 0o755], [HOST_BRIDGE_ROOT, 0o711]]) { + const info = lstatMaybe(root); + if (!info || !info.isDirectory() || info.isSymbolicLink() || info.uid !== 0 || info.gid !== 0 + || (info.mode & 0o7777) !== mode || fs.realpathSync(root) !== root) fail('runtime_boundary_violation'); + } + const candidatesRoot = path.join(stateRoot, 'candidates'); + const journalsRoot = path.join(stateRoot, 'journals'); + for (const root of [candidatesRoot, journalsRoot]) { + try { fs.mkdirSync(root, { mode: 0o700 }); } catch (error) { if (error?.code !== 'EEXIST') throw error; } + const info = lstatMaybe(root); + if (!info || !info.isDirectory() || info.isSymbolicLink() || info.uid !== process.geteuid() + || (info.mode & 0o7777) !== 0o700 || fs.realpathSync(root) !== root) fail('runtime_boundary_violation'); + } + + function command(executable, args, settings = {}) { + if (!Object.values(COMMANDS).includes(executable) || !Array.isArray(args) + || args.some(value => typeof value !== 'string' || /[\0\r\n]/.test(value))) { + fail('runtime_boundary_violation'); + } + if (remainingMs !== undefined) { + const remaining = remainingMs(); + if (!Number.isSafeInteger(remaining) || remaining < 1) fail('runtime_boundary_violation'); + return execute(executable, args, { ...settings, timeout: Math.min(settings.timeout || 120_000, remaining) }); + } + return execute(executable, args, settings); + } + + function runAs(plan, executable, args, settings = {}) { + const environment = [ + `HOME=${plan.host.accountHome}`, + `XDG_DATA_HOME=${plan.host.engineDataRoot}`, + `XDG_CONFIG_HOME=${plan.host.engineConfigRoot}`, + `XDG_RUNTIME_DIR=/run/user/${plan.account.uid}`, + 'PATH=/usr/bin:/bin', + 'LANG=C.UTF-8', + 'LC_ALL=C.UTF-8', + ]; + return command(COMMANDS.runuser, [ + '--user', plan.account.name, '--', COMMANDS.env, '-i', ...environment, executable, ...args, + ], settings); + } + + function selectedPlan(value, statuses = ['reserved', 'active']) { + const plan = validateOciDeploymentPlan(value); + const allocation = registry.inspect(plan.runtimeKey); + if (!allocation || !statuses.includes(allocation.status) + || JSON.stringify({ + name: allocation.name, uid: allocation.uid, gid: allocation.gid, + subuidStart: allocation.subuidStart, subgidStart: allocation.subgidStart, + subidCount: allocation.subidCount, + }) !== JSON.stringify(plan.account)) fail('runtime_identity_mismatch'); + return plan; + } + + function receipt(plan, status, changed = false) { + return Object.freeze({ + backend: plan.backend, + planVersion: plan.version, + planDigest: plan.planDigest, + status, + changed, + }); + } + + function passwd(name) { + const result = command(COMMANDS.getent, ['passwd', name], { accepted: [0, 2] }); + return result.status === 0 ? result.stdout.trim().split(':') : null; + } + + function group(name) { + const result = command(COMMANDS.getent, ['group', name], { accepted: [0, 2] }); + return result.status === 0 ? result.stdout.trim().split(':') : null; + } + + function exactSubid(file, name, start, count) { + const lines = fs.readFileSync(file, 'utf8').split('\n').filter(line => line.startsWith(`${name}:`)); + if (lines.length === 0) return false; + if (lines.length !== 1 || lines[0] !== `${name}:${start}:${count}`) fail('runtime_boundary_violation'); + return true; + } + + function safeDirectory(target, uid, gid, mode) { + const info = lstatMaybe(target); + if (!info || !info.isDirectory() || info.isSymbolicLink() || info.uid !== uid || info.gid !== gid + || (info.mode & 0o7777) !== mode || fs.realpathSync(target) !== target) fail('runtime_boundary_violation'); + } + + function installDirectory(target, uid, gid, mode, mutationCapability, plan) { + const current = lstatMaybe(target); + if (!current) { + if (plan && uid === plan.account.uid && target !== plan.host.tenantRoot) { + // Tenant-controlled descendants must never be traversed by a privileged + // mkdir/chown. A symlink race is confined to the tenant's own privileges. + guarded(mutationCapability, () => runAs(plan, COMMANDS.install, [ + '-d', '-m', mode.toString(8).padStart(4, '0'), target, + ])); + } else { + const parent = path.dirname(target); + if (parent !== HOST_TENANT_ROOT && parent !== HOST_BRIDGE_ROOT) fail('runtime_boundary_violation'); + safeDirectory(parent, 0, 0, parent === HOST_TENANT_ROOT ? 0o755 : 0o711); + guarded(mutationCapability, () => command(COMMANDS.install, [ + '-d', '-o', String(uid), '-g', String(gid), '-m', mode.toString(8).padStart(4, '0'), target, + ])); + } + } + else if (!current.isDirectory() || current.isSymbolicLink() || current.uid !== uid + || current.gid !== gid || fs.realpathSync(target) !== target) fail('runtime_boundary_violation'); + else if ((current.mode & 0o7777) !== mode) fail('runtime_boundary_violation'); + safeDirectory(target, uid, gid, mode); + } + + function prepareAccount(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + let changed = false; + let selectedGroup = group(plan.account.name); + let selectedPasswd = passwd(plan.account.name); + if (selectedGroup && Number(selectedGroup[2]) !== plan.account.gid + || selectedPasswd && (Number(selectedPasswd[2]) !== plan.account.uid + || Number(selectedPasswd[3]) !== plan.account.gid || selectedPasswd[5] !== plan.host.accountHome + || selectedPasswd[6] !== '/usr/sbin/nologin')) fail('runtime_identity_mismatch'); + if (!selectedGroup) { + guarded(mutationCapability, () => command(COMMANDS.groupadd, [ + '--system', '--gid', String(plan.account.gid), plan.account.name, + ])); + selectedGroup = group(plan.account.name); + changed = true; + } + if (!selectedGroup || Number(selectedGroup[2]) !== plan.account.gid) fail('runtime_identity_mismatch'); + if (!selectedPasswd) { + guarded(mutationCapability, () => command(COMMANDS.useradd, [ + '--system', '--uid', String(plan.account.uid), '--gid', String(plan.account.gid), + '--home-dir', plan.host.accountHome, '--no-create-home', '--shell', '/usr/sbin/nologin', + plan.account.name, + ])); + selectedPasswd = passwd(plan.account.name); + changed = true; + } + if (!selectedPasswd || Number(selectedPasswd[2]) !== plan.account.uid + || Number(selectedPasswd[3]) !== plan.account.gid || selectedPasswd[5] !== plan.host.accountHome + || selectedPasswd[6] !== '/usr/sbin/nologin') fail('runtime_identity_mismatch'); + const password = command(COMMANDS.passwd, ['--status', plan.account.name]).stdout.trim().split(/\s+/); + if (password[0] !== plan.account.name || password[1] !== 'L') { + guarded(mutationCapability, () => command(COMMANDS.passwd, ['--lock', plan.account.name])); + changed = true; + } + for (const [file, start, flag] of (plan.backend === 'native_service_v1' ? [] : [ + ['/etc/subuid', plan.account.subuidStart, '--add-subuids'], + ['/etc/subgid', plan.account.subgidStart, '--add-subgids'], + ])) { + if (!exactSubid(file, plan.account.name, start, plan.account.subidCount)) { + guarded(mutationCapability, () => command(COMMANDS.usermod, [ + flag, `${start}-${start + plan.account.subidCount - 1}`, plan.account.name, + ])); + if (!exactSubid(file, plan.account.name, start, plan.account.subidCount)) fail(); + changed = true; + } + } + for (const root of [plan.host.tenantRoot, plan.host.accountHome, + ...(plan.backend === 'native_service_v1' ? [] : [plan.host.engineDataRoot, plan.host.engineConfigRoot]), + path.dirname(plan.host.installationRoot)]) { + installDirectory(root, plan.account.uid, plan.account.gid, 0o700, mutationCapability, plan); + } + installDirectory(plan.host.bridgeRoot, 0, 0, 0o711, mutationCapability); + if (plan.backend !== 'native_service_v1') { + guarded(mutationCapability, () => command(COMMANDS.loginctl, ['enable-linger', plan.account.name])); + guarded(mutationCapability, () => command(COMMANDS.systemctl, ['start', `user@${plan.account.uid}.service`])); + const runtimeRoot = `/run/user/${plan.account.uid}`; + safeDirectory(runtimeRoot, plan.account.uid, plan.account.gid, 0o700); + } + if (registry.inspect(plan.runtimeKey).status === 'reserved') guarded(mutationCapability, () => registry.activate(plan.runtimeKey)); + return receipt(plan, 'account_ready', changed); + } + + function materializeLayout(planValue, tokenValue, mutationCapability) { + const plan = selectedPlan(planValue); + if (registry.inspect(plan.runtimeKey).status !== 'active') fail('runtime_boundary_violation'); + const token = registrationToken(tokenValue); + installDirectory(plan.host.installationRoot, plan.account.uid, plan.account.gid, 0o700, mutationCapability, plan); + const roots = Object.values(MANAGED_INSTALLATION_DIRECTORY_FIELDS) + .map(relative => path.join(plan.host.installationRoot, relative)) + .sort((left, right) => left.split(path.sep).length - right.split(path.sep).length || left.localeCompare(right)); + for (const root of roots) installDirectory(root, plan.account.uid, plan.account.gid, 0o700, mutationCapability, plan); + const target = path.join(plan.host.installationRoot, 'secrets', 'runtime-agent', 'registration-token'); + const current = lstatMaybe(target); + if (current) { + if (!current.isFile() || current.isSymbolicLink() || current.uid !== plan.account.uid + || current.gid !== plan.account.gid || current.nlink !== 1 || (current.mode & 0o7777) !== 0o600 + || fs.realpathSync(target) !== target + || runAs(plan, COMMANDS.cmp, ['--silent', target, '-'], { + input: `${token}\n`, accepted: [0, 1], + }).status !== 0) fail('runtime_identity_mismatch'); + return receipt(plan, 'layout_ready'); + } + guarded(mutationCapability, () => runAs(plan, COMMANDS.dd, [`of=${target}`, 'conv=excl,fsync', 'oflag=nofollow', 'status=none'], { + input: `${token}\n`, + })); + materializeLayout(plan, token, mutationCapability); + return receipt(plan, 'layout_ready', true); + } + + function releaseDirectory(plan) { + const root = path.join(releaseRoot, plan.release.releaseId); + if (path.dirname(root) !== releaseRoot) fail('runtime_boundary_violation'); + safeDirectory(root, 0, 0, 0o555); + return root; + } + + function verifyBridgeArtifact(plan) { + const root = path.join(releaseDirectory(plan), 'bridge-artifact'); + safeDirectory(root, 0, 0, 0o555); + const manifestFile = path.join(root, 'manifest.json'); + if (sha256File(manifestFile) !== plan.release.bridgeManifestSha256) fail('runtime_identity_mismatch'); + let manifest; + try { manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf8')); } catch { fail('runtime_boundary_violation'); } + if (!plain(manifest) || Object.keys(manifest).sort().join(',') !== 'files,version' + || manifest.version !== 1 || !Array.isArray(manifest.files) || manifest.files.length < 1) fail(); + const expected = []; + for (const entry of manifest.files) { + if (!plain(entry) || Object.keys(entry).sort().join(',') !== 'path,sha256' + || typeof entry.path !== 'string' || !/^[a-zA-Z0-9_.-]+(?:\/[a-zA-Z0-9_.-]+)*$/.test(entry.path) + || !/^[a-f0-9]{64}$/.test(entry.sha256)) fail(); + const target = path.join(root, entry.path); + if (path.relative(root, target) !== entry.path || sha256File(target) !== entry.sha256) fail('runtime_identity_mismatch'); + expected.push(entry.path); + } + if (JSON.stringify([...expected].sort()) !== JSON.stringify(expected)) fail(); + const actual = []; + const actualDirectories = []; + function walk(directory, relative = '') { + safeDirectory(directory, 0, 0, 0o555); + for (const name of fs.readdirSync(directory).sort()) { + const target = path.join(directory, name); + const child = relative ? `${relative}/${name}` : name; + if (child === 'manifest.json') continue; + const info = lstatMaybe(target); + if (!info || info.isSymbolicLink()) fail(); + if (info.isDirectory()) { actualDirectories.push(child); walk(target, child); } + else if (info.isFile()) actual.push(child); + else fail(); + } + } + walk(root); + const expectedDirectories = [...new Set(expected.flatMap(file => { + const segments = file.split('/'); + return segments.slice(0, -1).map((unused, index) => segments.slice(0, index + 1).join('/')); + }))].sort(); + if (JSON.stringify(actual.sort()) !== JSON.stringify(expected) + || JSON.stringify(actualDirectories.sort()) !== JSON.stringify(expectedDirectories)) { + fail('runtime_identity_mismatch'); + } + return root; + } + + function bridgeExecutable(plan) { + const selected = path.join( + verifyBridgeArtifact(plan), 'core', 'agent-bridge', 'src', 'service-cli.js', + ); + if (!lstatMaybe(selected)) fail(); + return selected; + } + + function inspectImage(plan) { + if (plan.backend === 'native_service_v1') return require('./native-runtime-artifact').verifyNativeRuntime( + path.join(releaseDirectory(plan), 'runtime-artifact'), plan.release); + const result = runAs(plan, COMMANDS.podman, ['image', 'inspect', plan.release.image], { accepted: [0, 125] }); + if (result.status !== 0) return null; + let selected; + try { selected = JSON.parse(result.stdout)[0]; } catch { fail(); } + const selectedId = String(selected?.Id || '').replace(/^sha256:/, ''); + if (selectedId !== plan.release.imageId || selected?.Digest !== plan.release.imageDigest + || selected?.Architecture !== 'amd64' || selected?.Os !== 'linux' + || selected?.Config?.User !== '10001:10001' + || !(selected.RepoDigests || []).includes(plan.release.image) + || selected?.Labels?.['org.opencontainers.image.revision'] !== plan.release.sourceCommit) { + fail('runtime_identity_mismatch'); + } + return selected; + } + + function ensureRuntimeManager(plan, mutationCapability) { + if (plan.backend === 'native_service_v1') return; + const runtimeRoot = `/run/user/${plan.account.uid}`; + if (!lstatMaybe(runtimeRoot)) { + const entry = passwd(plan.account.name); + if (!entry || Number(entry[2]) !== plan.account.uid || Number(entry[3]) !== plan.account.gid + || entry[5] !== plan.host.accountHome || entry[6] !== '/usr/sbin/nologin') fail('runtime_identity_mismatch'); + guarded(mutationCapability, () => command(COMMANDS.systemctl, ['start', `user@${plan.account.uid}.service`])); + } + safeDirectory(runtimeRoot, plan.account.uid, plan.account.gid, 0o700); + } + + function prepareImage(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + if (plan.backend === 'native_service_v1') { + inspectImage(plan); + return receipt(plan, 'image_ready'); + } + ensureRuntimeManager(plan, mutationCapability); + let step = 'image_inspect'; + try { + const attestManifest = () => { + step = 'manifest_attestation'; + const manifest = guarded(mutationCapability, () => runAs(plan, COMMANDS.podman, [ + 'run', '--rm', '--name', `${plan.identity.containerName}-manifest`, + '--pull=never', '--network=none', '--read-only', + '--entrypoint', '/usr/bin/sha256sum', plan.release.image, + '/opt/dispatch/runtime-release-manifest.json', + ], { timeout: 120_000 })); + if (manifest.stdout.trim().split(/\s+/)[0] !== plan.release.embeddedManifestSha256) { + fail('runtime_identity_mismatch'); + } + }; + if (inspectImage(plan)) { attestManifest(); return receipt(plan, 'image_ready'); } + const archive = path.join(releaseDirectory(plan), 'runtime-image.tar'); + const info = lstatMaybe(archive); + if (!info || !info.isFile() || info.isSymbolicLink() || info.uid !== 0 || info.gid !== 0 + || info.nlink !== 1 || (info.mode & 0o7777) !== 0o444 || fs.realpathSync(archive) !== archive) fail(); + if (sha256File(archive) !== plan.release.imageArchiveSha256) fail('runtime_identity_mismatch'); + step = 'image_load'; + guarded(mutationCapability, () => runAs(plan, COMMANDS.podman, [ + 'load', '--quiet', '--input', archive, + ], { timeout: 600_000 })); + step = 'loaded_image_inspect'; + if (!inspectImage(plan)) fail('runtime_identity_mismatch'); + attestManifest(); + return receipt(plan, 'image_ready', true); + } catch (error) { error.hostStep = step; throw error; } + } + + function candidatePaths(plan) { + return Object.freeze({ + runtime: path.join(candidatesRoot, plan.identity.unitName), + bridge: path.join(candidatesRoot, plan.identity.bridgeUnitName), + journal: path.join(journalsRoot, `${plan.identity.suffix}.json`), + settled: path.join(journalsRoot, `${plan.identity.suffix}.settled.json`), + }); + } + + function syncDirectory(directory) { + const fd = fs.openSync(directory, fs.constants.O_RDONLY | fs.constants.O_DIRECTORY); + try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); } + } + + function durableWrite(file, content, mode) { + const temporary = `${file}.${crypto.randomBytes(12).toString('hex')}.tmp`; + const fd = fs.openSync(temporary, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL, mode); + try { + fs.fchmodSync(fd, mode); + fs.writeFileSync(fd, content); + fs.fsyncSync(fd); + } finally { fs.closeSync(fd); } + fs.renameSync(temporary, file); + syncDirectory(path.dirname(file)); + } + + function unitContent(plan, name) { + return name === plan.identity.unitName ? renderOciSystemUnit(plan) : renderOciBridgeSystemUnit(plan, { + bridgeExecutable: path.join(releaseRoot, plan.release.releaseId, 'bridge-artifact', + 'core/agent-bridge/src/service-cli.js'), centralSocket, centralUid, controllerUid, + }); + } + + function attestUnit(plan, name, allowAbsent = false, expectedContent = null, recoveryGuard = null) { + const target = path.join(unitRoot, name); + const state = unitState(name); + if (allowAbsent && !lstatMaybe(target) && state.LoadState === 'not-found' + && state.ActiveState === 'inactive' && !state.Job) return state; + if (allowAbsent && recoveryGuard && !lstatMaybe(target) + && state.LoadState === 'loaded' && state.FragmentPath === target + && ['inactive', 'failed'].includes(state.ActiveState) && state.MainPID === '0' + && !state.Job && !state.DropInPaths) { + // The root journal authorizes this interrupted unlink. Reload only after + // proving the cached unit cannot still execute, then attest absence. + guarded(recoveryGuard, () => command(COMMANDS.systemctl, ['daemon-reload'])); + return attestUnit(plan, name, true, expectedContent); + } + const content = readRootFile(target, 0o644, 64 * 1024).toString('utf8'); + if (content !== (expectedContent === null ? unitContent(plan, name) : expectedContent) + || state.LoadState !== 'loaded' || state.FragmentPath !== target + || state.DropInPaths || state.Job) fail('runtime_identity_mismatch'); + if (state.NeedDaemonReload !== 'no') { + if (!recoveryGuard || state.NeedDaemonReload !== 'yes') fail('runtime_identity_mismatch'); + guarded(recoveryGuard, () => command(COMMANDS.systemctl, ['daemon-reload'])); + return attestUnit(plan, name, allowAbsent, expectedContent); + } + return state; + } + + function writeExact(file, content, mode, mutationCapability) { + const current = lstatMaybe(file); + if (current && (!current.isFile() || current.isSymbolicLink() || current.uid !== process.geteuid() + || current.nlink !== 1 || (current.mode & 0o7777) !== mode || fs.readFileSync(file, 'utf8') !== content)) fail(); + if (current) return false; + guarded(mutationCapability, () => durableWrite(file, content, mode)); + return true; + } + + function render(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + prepareImage(plan, mutationCapability); + const paths = candidatePaths(plan); + const runtime = renderOciSystemUnit(plan); + const bridge = renderOciBridgeSystemUnit(plan, { + bridgeExecutable: bridgeExecutable(plan), centralSocket, centralUid, controllerUid, + }); + const runtimeChanged = writeExact(paths.runtime, runtime, 0o600, mutationCapability); + const bridgeChanged = writeExact(paths.bridge, bridge, 0o600, mutationCapability); + const changed = runtimeChanged || bridgeChanged; + return receipt(plan, 'rendered', changed); + } + + function validate(planValue) { + const plan = selectedPlan(planValue); + const paths = candidatePaths(plan); + command(COMMANDS.systemdAnalyze, ['verify', paths.runtime, paths.bridge]); + return receipt(plan, 'validated'); + } + + function unitState(name) { + const result = command(COMMANDS.systemctl, [ + 'show', name, '--property=LoadState,ActiveState,SubState,MainPID,NRestarts,FragmentPath,UnitFileState,User,Group,DropInPaths,NeedDaemonReload,Job', + ], { accepted: [0, 1] }); + const values = {}; + for (const line of result.stdout.trimEnd().split('\n')) { + const split = line.indexOf('='); + if (split > 0) values[line.slice(0, split)] = line.slice(split + 1); + } + return Object.freeze(values); + } + + function journal(plan, mutationCapability) { + const paths = candidatePaths(plan); + const existing = rollbackState(plan); + if (existing) return existing; + const settled = lstatMaybe(paths.settled) + ? selectedPlan(JSON.parse(readRootFile(paths.settled, 0o600, 128 * 1024))) : null; + const units = [plan.identity.bridgeUnitName, plan.identity.unitName].map(name => { + const installed = path.join(unitRoot, name); + const info = lstatMaybe(installed); + const state = attestUnit(settled || plan, name, !info); + if (!['active', 'inactive'].includes(state.ActiveState) + || !['running', 'dead'].includes(state.SubState) + || !['enabled', 'enabled-runtime', 'disabled', ''].includes(state.UnitFileState)) fail(); + return { name, existed: Boolean(info), + content: info ? readRootFile(installed, 0o644, 64 * 1024).toString('base64') : null, + active: state.ActiveState === 'active', enabled: state.UnitFileState || 'not-found' }; + }); + const value = { version: 2, planDigest: plan.planDigest, units }; + guarded(mutationCapability, () => durableWrite(paths.journal, `${JSON.stringify(value)}\n`, 0o600)); + return value; + } + + function install(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + validate(plan); + journal(plan, mutationCapability); + const paths = candidatePaths(plan); + for (const [name, candidate] of [[plan.identity.bridgeUnitName, paths.bridge], [plan.identity.unitName, paths.runtime]]) { + const target = path.join(unitRoot, name); + const content = fs.readFileSync(candidate); + guarded(mutationCapability, () => durableWrite(target, content, 0o644)); + const info = lstatMaybe(target); + if (!info || !info.isFile() || info.isSymbolicLink() || info.uid !== 0 || info.gid !== 0 + || info.nlink !== 1 || (info.mode & 0o7777) !== 0o644 || !fs.readFileSync(target).equals(content)) fail(); + } + guarded(mutationCapability, () => command(COMMANDS.systemctl, ['daemon-reload'])); + return receipt(plan, 'installed', true); + } + + function setActive(planValue, active, mutationCapability) { + const plan = selectedPlan(planValue); + if (active) ensureRuntimeManager(plan, mutationCapability); + const ordered = active + ? [plan.identity.bridgeUnitName, plan.identity.unitName] + : [plan.identity.unitName, plan.identity.bridgeUnitName]; + for (const name of ordered) { + // A stopped target may already have prior definitions during rollback. + const prior = active ? null : rollbackState(plan); + const saved = prior?.units.find(unit => unit.name === name); + const current = lstatMaybe(path.join(unitRoot, name)); + let expected = unitContent(plan, name); + if (saved?.existed && current && fs.readFileSync(path.join(unitRoot, name), 'utf8') + === Buffer.from(saved.content, 'base64').toString('utf8')) expected = Buffer.from(saved.content, 'base64').toString('utf8'); + const state = attestUnit(plan, name, !active, expected, !active && prior ? mutationCapability : null); + if (state.LoadState === 'not-found') continue; + guarded(mutationCapability, () => command(COMMANDS.systemctl, + active ? ['enable', '--now', name] : plan.backend === 'native_service_v1' ? ['disable', '--now', name] : ['stop', name])); + } + return receipt(plan, active ? 'started' : 'stopped', true); + } + + function start(plan, mutationCapability) { return setActive(plan, true, mutationCapability); } + function stop(plan, mutationCapability) { return setActive(plan, false, mutationCapability); } + + function disable(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + const before = [plan.identity.unitName, plan.identity.bridgeUnitName].map(name => attestUnit(plan, name)); + guarded(mutationCapability, () => { + for (const unit of [plan.identity.unitName, plan.identity.bridgeUnitName]) { + command(COMMANDS.systemctl, ['disable', unit], { accepted: [0, 1] }); + } + }); + const after = [plan.identity.unitName, plan.identity.bridgeUnitName].map(unitState); + if (after.some(state => ['enabled', 'enabled-runtime', 'linked', 'linked-runtime'].includes(state.UnitFileState))) { + fail('service_installation_failed'); + } + return receipt(plan, 'disabled', before.some(state => + ['enabled', 'enabled-runtime', 'linked', 'linked-runtime'].includes(state.UnitFileState))); + } + + function inspectState(planValue, expectedActive) { + const plan = selectedPlan(planValue); + if (typeof expectedActive !== 'boolean') fail('runtime_boundary_violation'); + for (const [name, expectedUser] of [[plan.identity.bridgeUnitName, '0'], [plan.identity.unitName, plan.account.name]]) { + const state = attestUnit(plan, name); + const installed = path.join(unitRoot, name); + const active = state.ActiveState === 'active' && state.SubState === 'running' && /^[1-9][0-9]*$/.test(state.MainPID); + const inactive = ['inactive', 'failed'].includes(state.ActiveState) && state.MainPID === '0'; + if (state.LoadState !== 'loaded' || state.FragmentPath !== installed || state.User !== expectedUser + || expectedActive && !active || !expectedActive && !inactive) fail('runtime_health_failed'); + } + if (expectedActive && !inspectImage(plan)) fail('runtime_identity_mismatch'); + return receipt(plan, expectedActive ? 'active' : 'inactive'); + } + + function inspect(planValue) { + return inspectState(planValue, true); + } + + function healthOnce(planValue) { + const plan = selectedPlan(planValue); + inspect(plan); + if (plan.backend === 'native_service_v1') { + const result = nativeProbe(plan, 'health'); + if (result.status !== 0) fail('runtime_health_failed'); + return receipt(plan, 'healthy'); + } + const inspected = runAs(plan, COMMANDS.podman, ['container', 'inspect', plan.identity.containerName]); + let container; + try { container = JSON.parse(inspected.stdout)[0]; } catch { fail('runtime_health_failed'); } + if (String(container?.Image || '').replace(/^sha256:/, '') !== plan.release.imageId + || container?.Name !== plan.identity.containerName + || container?.Config?.User !== plan.security.user + || container?.Config?.Labels?.['io.dispatch.runtime-key'] !== plan.runtimeKey + || container?.Config?.Labels?.['io.dispatch.release-id'] !== plan.release.releaseId + || container?.Config?.Labels?.['io.dispatch.plan-digest'] !== plan.planDigest) fail('runtime_identity_mismatch'); + if (container?.State?.Running !== true) fail('runtime_health_failed'); + const result = runAs(plan, COMMANDS.podman, ['exec', plan.identity.containerName, '/usr/local/bin/node', '--no-warnings', + '/opt/dispatch/runtime/supervisor/src/health.js'], { + accepted: [0, 1, 125], timeout: 60_000, + }); + if (result.status !== 0) fail('runtime_health_failed'); + return receipt(plan, 'healthy'); + } + + function verifyPublication(planValue, payload, mutationCapability) { + const plan = selectedPlan(planValue); + if (!payload || !['capture', 'verify'].includes(payload.mode)) fail('runtime_boundary_violation'); + exact(payload, payload.mode === 'capture' ? ['mode'] : ['mode', 'baseline']); + const { publicationBaseline } = require('../../../shared/contracts/src/publication-baseline'); + if (payload.mode === 'verify') publicationBaseline(payload.baseline); + const input = `${JSON.stringify(payload)}\n`; + if (Buffer.byteLength(input) > 8192) fail('runtime_boundary_violation'); + const executable = ['/usr/local/bin/node', '--no-warnings', '/opt/dispatch/plugins/paycom/backend/bin/dispatch-paycom-publication-continuity']; + let result; + if (plan.backend === 'native_service_v1') { + // Capture runs stopped; verification also runs after resume/start. + // Suspended upgrades remain stopped. Attest both units in the observed state. + const active = payload.mode === 'verify' && unitState(plan.identity.unitName).ActiveState === 'active'; + inspectState(plan, active); + result = guarded(mutationCapability, () => nativeProbe(plan, 'publication', input)); + } else if (payload.mode === 'capture') { + inspectState(plan, false); + ensureRuntimeManager(plan, mutationCapability); + // Recovery may have stopped a missing runtime-dir before Podman could + // remove its persisted probe record. Only this reserved probe name is + // reconciled under the fresh capture action. + guarded(mutationCapability, () => runAs(plan, COMMANDS.podman, + ['rm', '--force', '--ignore', `${plan.identity.containerName}-publication`], { code: 'first_publication_failed' })); + result = guarded(mutationCapability, () => runAs(plan, COMMANDS.podman, [ + 'run', '--rm', '-i', '--name', `${plan.identity.containerName}-publication`, '--pull=never', + '--network=none', '--read-only', '--cap-drop=all', '--security-opt=no-new-privileges', + '--user=10001:10001', '--userns=keep-id:uid=10001,gid=10001', + '--memory=256m', '--cpus=1', '--pids-limit=32', + '--mount', `type=bind,src=${plan.host.installationRoot},dst=${plan.guest.installationRoot},ro=true`, + ...Object.entries(plan.guest.environment).flatMap(([key, value]) => ['--env', `${key}=${value}`]), + '--entrypoint=/usr/local/bin/node', plan.release.image, ...executable.slice(1), + ], { input, timeout: 60_000, code: 'first_publication_failed' })); + } else result = runAs(plan, COMMANDS.podman, ['exec', '-i', plan.identity.containerName, ...executable], + { input, timeout: 60_000, code: 'first_publication_failed' }); + let value; + try { value = JSON.parse(result.stdout); } catch { fail('first_publication_failed'); } + if (payload.mode === 'capture') { + exact(value, ['status', 'publicationBaseline']); + if (value.status !== 'verified') fail('first_publication_failed'); + return Object.freeze({ status: 'verified', publicationBaseline: publicationBaseline(value.publicationBaseline) }); + } + exact(value, ['status', 'publicationBaselineDigest']); + if (value.status !== 'verified' || value.publicationBaselineDigest !== payload.baseline.digest) fail('first_publication_failed'); + return Object.freeze(value); + } + + function nativeProbe(plan, operation, input = '') { + if (plan.backend !== 'native_service_v1' || !['health', 'publication'].includes(operation)) fail('runtime_boundary_violation'); + const source = path.join(releaseDirectory(plan), 'runtime-artifact'); + const executable = operation === 'health' ? 'runtime/supervisor/src/health.js' + : 'plugins/paycom/backend/bin/dispatch-paycom-publication-continuity'; + // A bounded one-shot process with the DSP's identity and read-only data. + // No shell, arbitrary command, shared TCP port, or container engine. + return command(COMMANDS.systemdRun, ['--quiet', '--wait', '--pipe', '--collect', '--service-type=exec', + `--unit=dispatch-probe-${plan.identity.suffix}-${crypto.randomBytes(6).toString('hex')}`, + ...[`User=${plan.account.name}`, `Group=${plan.account.name}`, 'UMask=0077', 'ProtectSystem=strict', 'ProtectHome=true', + 'NoNewPrivileges=true', 'PrivateNetwork=true', 'PrivateTmp=true', 'RuntimeMaxSec=60', 'MemoryMax=256M', 'TasksMax=32', + `BindReadOnlyPaths=${source}:/opt/dispatch -${source}/dependencies/node/bin/host-files/usr/share/nodejs:/usr/share/nodejs ${plan.host.installationRoot}:${plan.guest.installationRoot} ${plan.host.bridgeRoot}:/run/dispatch-agent`, + 'WorkingDirectory=/opt/dispatch'].flatMap(value => ['--property', value]), + ...Object.entries({ ...plan.guest.environment, PATH: '/opt/dispatch/dependencies/node/bin:/usr/bin:/bin', HOME: '/tmp', LANG: 'C.UTF-8' }) + .flatMap(([key, value]) => ['--setenv', `${key}=${value}`]), + path.join(source, 'dependencies/node/bin/node'), '--no-warnings', `/opt/dispatch/${executable}`], + { input, timeout: 65_000, code: operation === 'health' ? 'runtime_health_failed' : 'first_publication_failed' }); + } + + function health(planValue) { + const deadline = clock() + Math.min(90_000, remainingMs ? remainingMs() : 90_000); + for (;;) { + try { return healthOnce(planValue); } + catch (error) { + if (['runtime_boundary_violation', 'runtime_identity_mismatch'].includes(error.code) + || clock() + 1_000 >= deadline) throw error; + if (remainingMs) remainingMs(); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 500); + } + } + } + + function rollbackState(planValue) { + const plan = selectedPlan(planValue); + const file = candidatePaths(plan).journal; + const info = lstatMaybe(file); + if (!info) return null; + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() + || info.nlink !== 1 || (info.mode & 0o7777) !== 0o600) fail(); + const value = JSON.parse(readRootFile(file, 0o600, 192 * 1024)); + exact(value, ['version', 'planDigest', 'units']); + if (value.version !== 2 || value.planDigest !== plan.planDigest || !Array.isArray(value.units) + || value.units.length !== 2) fail(); + const names = [plan.identity.bridgeUnitName, plan.identity.unitName]; + for (const [index, unit] of value.units.entries()) { + exact(unit, ['name', 'existed', 'content', 'active', 'enabled']); + if (unit.name !== names[index] || typeof unit.existed !== 'boolean' || typeof unit.active !== 'boolean' + || !['enabled', 'enabled-runtime', 'disabled', 'not-found'].includes(unit.enabled) + || !unit.existed && (unit.content !== null || unit.active || unit.enabled !== 'not-found') + || unit.existed && (typeof unit.content !== 'string' || unit.content.length > 88_000 + || Buffer.from(unit.content, 'base64').toString('base64') !== unit.content)) fail(); + } + return Object.freeze(value); + } + + function restorePrior(planValue, mutationCapability, activate = true) { + const plan = selectedPlan(planValue); + const prior = rollbackState(plan); + if (!prior) return receipt(plan, 'restored'); + stop(plan, mutationCapability); + for (const unit of prior.units) { + const target = path.join(unitRoot, unit.name); + guarded(mutationCapability, () => { + if (unit.existed) durableWrite(target, Buffer.from(unit.content, 'base64'), 0o644); + else { fs.rmSync(target, { force: true }); syncDirectory(unitRoot); } + }); + } + guarded(mutationCapability, () => command(COMMANDS.systemctl, ['daemon-reload'])); + for (const unit of prior.units) { + guarded(mutationCapability, () => command(COMMANDS.systemctl, ['disable', unit.name], { accepted: [0, 1] })); + guarded(mutationCapability, () => command(COMMANDS.systemctl, ['disable', '--runtime', unit.name], { accepted: [0, 1] })); + if (['enabled', 'enabled-runtime'].includes(unit.enabled)) guarded(mutationCapability, () => command(COMMANDS.systemctl, + unit.enabled === 'enabled-runtime' ? ['enable', '--runtime', unit.name] : ['enable', unit.name])); + if (activate && unit.active) guarded(mutationCapability, () => command(COMMANDS.systemctl, ['start', unit.name])); + } + return receipt(plan, 'restored', true); + } + + function settleCommitted(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + const paths = candidatePaths(plan); + rollbackState(plan); + for (const name of [plan.identity.bridgeUnitName, plan.identity.unitName]) attestUnit(plan, name); + guarded(mutationCapability, () => { + durableWrite(paths.settled, `${JSON.stringify(plan)}\n`, 0o600); + fs.rmSync(paths.journal, { force: true }); + fs.rmSync(paths.runtime, { force: true }); + fs.rmSync(paths.bridge, { force: true }); + syncDirectory(journalsRoot); syncDirectory(candidatesRoot); + }); + return receipt(plan, 'committed', true); + } + + function commit(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + health(plan); + // The following authoritative operation settles this journal only after + // observing the completed prior job. Completion here remains reversible. + return receipt(plan, 'committed', false); + } + + function rollback(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + const changed = Boolean(rollbackState(plan)); + if (changed) restorePrior(plan, mutationCapability); + const paths = candidatePaths(plan); + guarded(mutationCapability, () => { + fs.rmSync(paths.journal, { force: true }); + fs.rmSync(paths.runtime, { force: true }); + fs.rmSync(paths.bridge, { force: true }); + syncDirectory(journalsRoot); syncDirectory(candidatesRoot); + }); + return receipt(plan, 'rolled_back', changed); + } + + function priorPlanWithoutJournal(plan) { + const file = candidatePaths(plan).settled; + const prior = selectedPlan(JSON.parse(readRootFile(file, 0o600, 128 * 1024))); + if (prior.runtimeKey !== plan.runtimeKey || prior.deployment.manifestRevision + 1 !== plan.deployment.manifestRevision) fail(); + return prior; + } + + function rollbackStopped(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + const prior = rollbackState(plan); + if (!prior) { + // Installation cannot mutate unit files before its durable journal. With + // no journal, only exact definitions from the settled prior release are + // acceptable; this also handles failure during image/render preparation. + const unchanged = priorPlanWithoutJournal(plan); + stop(unchanged, mutationCapability); + inspectState(unchanged, false); + return receipt(plan, 'restored', false); + } + restorePrior(plan, mutationCapability, false); + for (const unit of prior.units) { + const state = attestUnit(plan, unit.name, !unit.existed, unit.existed ? Buffer.from(unit.content, 'base64').toString('utf8') : null); + if (state.ActiveState !== 'inactive' || state.SubState !== 'dead' + || (state.UnitFileState || 'not-found') !== unit.enabled) fail(); + } + return receipt(plan, 'restored', true); + } + + function startPrior(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + ensureRuntimeManager(plan, mutationCapability); + const journal = rollbackState(plan); + const unchanged = journal ? null : priorPlanWithoutJournal(plan); + for (const name of [plan.identity.bridgeUnitName, plan.identity.unitName]) { + const saved = journal?.units.find(unit => unit.name === name); + if (saved && !saved.existed) fail(); + const state = attestUnit(unchanged || plan, name, false, + saved ? Buffer.from(saved.content, 'base64').toString('utf8') : null); + const enabled = saved?.enabled || state.UnitFileState; + const native = plan.backend === 'native_service_v1'; + if (state.UnitFileState !== enabled && !(native && state.UnitFileState === 'enabled')) fail(); + guarded(mutationCapability, () => command(COMMANDS.systemctl, native ? ['enable', '--now', name] : ['start', name])); + if (unitState(name).UnitFileState !== (native ? 'enabled' : enabled)) fail(); + } + return receipt(plan, 'started', true); + } + + function settleRollback(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + rollbackState(plan); + const paths = candidatePaths(plan); + guarded(mutationCapability, () => { + fs.rmSync(paths.journal, { force: true }); + fs.rmSync(paths.runtime, { force: true }); + fs.rmSync(paths.bridge, { force: true }); + syncDirectory(journalsRoot); syncDirectory(candidatesRoot); + }); + return receipt(plan, 'rolled_back', true); + } + + function removeServices(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + journal(plan, mutationCapability); + stop(plan, mutationCapability); + for (const name of [plan.identity.unitName, plan.identity.bridgeUnitName]) { + guarded(mutationCapability, () => command(COMMANDS.systemctl, ['disable', name], { accepted: [0, 1] })); + guarded(mutationCapability, () => { fs.rmSync(path.join(unitRoot, name), { force: true }); syncDirectory(unitRoot); }); + } + guarded(mutationCapability, () => command(COMMANDS.systemctl, ['daemon-reload'])); + return receipt(plan, 'removed', true); + } + + function inspectRemoved(planValue) { + const plan = selectedPlan(planValue); + for (const name of [plan.identity.unitName, plan.identity.bridgeUnitName]) { + if (lstatMaybe(path.join(unitRoot, name)) || unitState(name).LoadState !== 'not-found') { + fail('decommission_failed'); + } + } + return receipt(plan, 'absent'); + } + + function settleRemoved(planValue, mutationCapability) { + const plan = selectedPlan(planValue); + inspectRemoved(plan); + const paths = candidatePaths(plan); + guarded(mutationCapability, () => { + fs.rmSync(paths.journal, { force: true }); + fs.rmSync(paths.runtime, { force: true }); + fs.rmSync(paths.bridge, { force: true }); + syncDirectory(journalsRoot); syncDirectory(candidatesRoot); + }); + return receipt(plan, 'retained', true); + } + + function destroyAccount(planValue, mutationCapability) { + const plan = selectedPlan(planValue, ['active', 'retired']); + if (registry.inspect(plan.runtimeKey).status === 'retired') { + verifyDestroyed(plan); + return receipt(plan, 'destroyed'); + } + const existingPasswd = passwd(plan.account.name); + const existingGroup = group(plan.account.name); + if (existingPasswd && (Number(existingPasswd[2]) !== plan.account.uid + || Number(existingPasswd[3]) !== plan.account.gid || existingPasswd[5] !== plan.host.accountHome + || existingPasswd[6] !== '/usr/sbin/nologin') + || existingGroup && Number(existingGroup[2]) !== plan.account.gid) fail('runtime_identity_mismatch'); + const hasSubuid = exactSubid('/etc/subuid', plan.account.name, plan.account.subuidStart, plan.account.subidCount); + const hasSubgid = exactSubid('/etc/subgid', plan.account.name, plan.account.subgidStart, plan.account.subidCount); + if (!existingPasswd && (hasSubuid || hasSubgid)) fail('runtime_identity_mismatch'); + for (const name of [plan.identity.unitName, plan.identity.bridgeUnitName]) { + const state = unitState(name); + if (state.LoadState !== 'not-found') fail('destruction_failed'); + } + if (existingPasswd && plan.backend !== 'native_service_v1') guarded(mutationCapability, () => runAs(plan, COMMANDS.podman, ['system', 'reset', '--force'], { + accepted: [0, 125], timeout: 300_000, + })); + guarded(mutationCapability, () => command(COMMANDS.loginctl, ['disable-linger', plan.account.name], { accepted: [0, 1] })); + guarded(mutationCapability, () => command(COMMANDS.systemctl, ['stop', `user@${plan.account.uid}.service`, + `user-runtime-dir@${plan.account.uid}.service`], { accepted: [0, 1] })); + if (existingPasswd && hasSubuid) guarded(mutationCapability, () => command(COMMANDS.usermod, [ + '--del-subuids', `${plan.account.subuidStart}-${plan.account.subuidStart + plan.account.subidCount - 1}`, + plan.account.name, + ], { accepted: [0, 6] })); + if (existingPasswd && hasSubgid) guarded(mutationCapability, () => command(COMMANDS.usermod, [ + '--del-subgids', `${plan.account.subgidStart}-${plan.account.subgidStart + plan.account.subidCount - 1}`, + plan.account.name, + ], { accepted: [0, 6] })); + if (existingPasswd) guarded(mutationCapability, () => command(COMMANDS.userdel, [plan.account.name])); + if (group(plan.account.name)) guarded(mutationCapability, () => command(COMMANDS.groupdel, [plan.account.name], { accepted: [0, 6] })); + guarded(mutationCapability, () => command(COMMANDS.rm, [ + '--recursive', '--force', '--one-file-system', plan.host.tenantRoot, + ])); + guarded(mutationCapability, () => command(COMMANDS.rm, [ + '--recursive', '--force', '--one-file-system', plan.host.bridgeRoot, + ])); + if (passwd(plan.account.name) || group(plan.account.name) + || fs.readFileSync('/etc/subuid', 'utf8').split('\n').some(line => line.startsWith(`${plan.account.name}:`)) + || fs.readFileSync('/etc/subgid', 'utf8').split('\n').some(line => line.startsWith(`${plan.account.name}:`)) + || lstatMaybe(plan.host.tenantRoot) || lstatMaybe(plan.host.bridgeRoot)) { + fail('destruction_failed'); + } + guarded(mutationCapability, () => registry.retire(plan.runtimeKey)); + return receipt(plan, 'destroyed', true); + } + + function verifyDestroyed(planValue) { + const plan = selectedPlan(planValue, ['retired']); + const allocation = registry.inspect(plan.runtimeKey); + if (!allocation || allocation.status !== 'retired' || passwd(plan.account.name) + || group(plan.account.name) || lstatMaybe(plan.host.tenantRoot) + || fs.readFileSync('/etc/subuid', 'utf8').split('\n').some(line => line.startsWith(`${plan.account.name}:`)) + || fs.readFileSync('/etc/subgid', 'utf8').split('\n').some(line => line.startsWith(`${plan.account.name}:`)) + || lstatMaybe(plan.host.bridgeRoot) || lstatMaybe(path.join(unitRoot, plan.identity.unitName)) + || lstatMaybe(path.join(unitRoot, plan.identity.bridgeUnitName))) fail('destruction_failed'); + return receipt(plan, 'absent'); + } + + return Object.freeze({ + prepareAccount, + materializeLayout, + prepareImage, + render, + validate, + install, + start, + stop, + disable, + inspect, + inspectState, + health, + verifyPublication, + rollbackState, + restorePrior, + commit, + settleCommitted, + rollback, + rollbackStopped, + startPrior, + settleRollback, + removeServices, + inspectRemoved, + settleRemoved, + destroyAccount, + verifyDestroyed, + }); +} + +module.exports = { COMMANDS, createOciHostExecutor }; diff --git a/core/core/installations/src/oci-host-helper-client.js b/core/core/installations/src/oci-host-helper-client.js new file mode 100644 index 0000000..67cd33b --- /dev/null +++ b/core/core/installations/src/oci-host-helper-client.js @@ -0,0 +1,117 @@ +'use strict'; + +const { spawnSync } = require('node:child_process'); +const { OCI_HOST_HELPER_PROTOCOL_VERSION } = require('./oci-host-helper'); + +const DEFAULT_OCI_HOST_HELPER = '/opt/dispatch-control/current/host-helper-artifact/core/installations/bin/dispatch-oci-host-helper'; +const MAX_HELPER_BYTES = 256 * 1024; + +function fail(code = 'service_installation_failed') { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function createOciHostHelperClient(options = {}) { + if (!plain(options) || Object.keys(options).some(key => !['helper', 'sudo', 'execute', 'authorizeRequest', 'requestPort'].includes(key))) { + fail('runtime_boundary_violation'); + } + const helper = options.helper || DEFAULT_OCI_HOST_HELPER; + const sudo = options.sudo || '/usr/bin/sudo'; + const execute = options.execute || spawnSync; + const authorizeRequest = options.authorizeRequest; + if (helper !== DEFAULT_OCI_HOST_HELPER || sudo !== '/usr/bin/sudo' || typeof execute !== 'function' + || (typeof authorizeRequest !== 'function' && typeof options.requestPort !== 'function')) { + fail('runtime_boundary_violation'); + } + + function rpc(operation, claim, payload = {}) { + if (!plain(claim) || !plain(payload)) fail('runtime_boundary_violation'); + const request = { version: OCI_HOST_HELPER_PROTOCOL_VERSION, operation, claim, ...payload }; + if (Buffer.byteLength(JSON.stringify(request), 'utf8') > MAX_HELPER_BYTES) fail('runtime_boundary_violation'); + if (options.requestPort) return options.requestPort(request); + const authorization = authorizeRequest(request); + if (typeof authorization !== 'string' || !/^[a-f0-9]{64}$/.test(authorization)) fail('runtime_boundary_violation'); + request.authorization = authorization; + const input = `${JSON.stringify(request)}\n`; + if (Buffer.byteLength(input, 'utf8') > MAX_HELPER_BYTES) fail('runtime_boundary_violation'); + const result = execute(sudo, ['-n', helper], { + input, + encoding: 'utf8', + timeout: 600_000, + maxBuffer: MAX_HELPER_BYTES, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' }, + }); + if (result.error || result.signal || result.status !== 0 || typeof result.stdout !== 'string' + || !result.stdout.endsWith('\n') || result.stdout.slice(0, -1).includes('\n') + || Buffer.byteLength(result.stdout, 'utf8') > MAX_HELPER_BYTES) fail(); + let response; + try { response = JSON.parse(result.stdout.slice(0, -1)); } catch { fail(); } + if (!plain(response) || typeof response.ok !== 'boolean') fail(); + if (!response.ok) fail(typeof response.status === 'string' ? response.status : 'service_installation_failed'); + if (Object.keys(response).sort().join(',') !== 'ok,result') fail(); + return response.result; + } + + const hostRegistry = Object.freeze({ + reserve: (runtimeKey, claim) => rpc('reserve_account', claim, { runtimeKey }), + inspect: (runtimeKey, claim) => rpc('inspect_account', claim, { runtimeKey }), + }); + + function mutate(operation, plan, claim, capability, payload = {}) { + if (typeof capability !== 'function') fail('runtime_boundary_violation'); + return capability(() => rpc(operation, claim, { plan, ...payload })); + } + + const hostExecutor = Object.freeze({ + prepareAccount: (plan, claim, capability) => mutate('prepare_account', plan, claim, capability), + materializeLayout: (plan, token, claim, capability) => + mutate('materialize_layout', plan, claim, capability, { token }), + prepareImage: (plan, claim, capability) => mutate('prepare_image', plan, claim, capability), + render: (plan, claim, capability) => mutate('render', plan, claim, capability), + validate: (plan, claim) => rpc('validate', claim, { plan }), + install: (plan, claim, capability) => mutate('install', plan, claim, capability), + start: (plan, claim, capability) => mutate('start', plan, claim, capability), + stop: (plan, claim, capability) => mutate('stop', plan, claim, capability), + disable: (plan, claim, capability) => mutate('disable', plan, claim, capability), + inspect: (plan, claim) => rpc('inspect', claim, { plan }), + inspectInactive: (plan, claim) => rpc('inspect_inactive', claim, { plan }), + health: (plan, claim) => rpc('health', claim, { plan }), + verifyPublication: (plan, payload, claim) => rpc('verify_publication', claim, { plan, payload }), + commit: (plan, claim, capability) => mutate('commit', plan, claim, capability), + settleCommitted: (plan, claim, capability) => mutate('settle_committed', plan, claim, capability), + rollback: (plan, claim, capability) => mutate('rollback', plan, claim, capability), + rollbackStopped: (plan, claim, capability) => mutate('rollback_stopped', plan, claim, capability), + startPrior: (plan, claim, capability) => mutate('start_prior', plan, claim, capability), + settleRollback: (plan, claim, capability) => mutate('settle_rollback', plan, claim, capability), + removeServices: (plan, claim, capability) => mutate('remove_services', plan, claim, capability), + inspectRemoved: (plan, claim) => rpc('inspect_removed', claim, { plan }), + settleRemoved: (plan, claim, capability) => mutate('settle_removed', plan, claim, capability), + destroyAccount: (plan, claim, capability) => mutate('destroy_account', plan, claim, capability), + verifyDestroyed: (plan, claim) => rpc('verify_destroyed', claim, { plan }), + }); + + function createBackupManager(plan, claim) { + const query = (operation, payload) => rpc(operation, claim, { plan, payload }); + const change = (operation, payload, capability) => { + if (typeof capability !== 'function') fail('runtime_boundary_violation'); + return capability(() => query(operation, payload)); + }; + return Object.freeze({ + snapshot: (spec, capability) => change('backup_snapshot', { spec }, capability), + inspect: spec => query('backup_inspect', { spec }), + restore: (source, operationId, capability) => + change('backup_restore', { source, operationId }, capability), + inspectRestored: source => query('backup_inspect_restored', { source }), + destroy: (authority, capability) => change('backup_destroy', { authority }, capability), + verifyDestroyed: () => query('backup_verify_destroyed', {}), + }); + } + + return Object.freeze({ hostRegistry, hostExecutor, createBackupManager }); +} + +module.exports = { DEFAULT_OCI_HOST_HELPER, createOciHostHelperClient }; diff --git a/core/core/installations/src/oci-host-helper.js b/core/core/installations/src/oci-host-helper.js new file mode 100644 index 0000000..b179275 --- /dev/null +++ b/core/core/installations/src/oci-host-helper.js @@ -0,0 +1,309 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { createOciHostAccountRegistry } = require('./oci-host-account-registry'); +const { createOciHostExecutor } = require('./oci-host-executor'); +const { validateOciDeploymentPlan } = require('./oci-deployment'); +const { MANAGED_INSTALLATION_DIRECTORY_FIELDS } = require('../../../shared/paths/runtime-paths'); +const { hostAccountName } = require('../../runtime-host-identity'); +const { createOciHostAuthority } = require('./oci-host-authority'); +const { readRootFile, verifyHostArtifact } = require('./oci-host-artifact'); + +const OCI_HOST_HELPER_PROTOCOL_VERSION = 2; +const OCI_HOST_HELPER_CONFIG = '/etc/dispatch/oci-host.json'; +const CONTROL_RELEASE_ROOT = '/opt/dispatch-control/releases'; +const OCI_HOST_HELPER_RELATIVE = 'host-helper-artifact/core/installations/bin/dispatch-oci-host-helper'; +const OCI_TENANT_BACKUP_HELPER_RELATIVE = 'host-helper-artifact/core/installations/bin/dispatch-oci-tenant-backup-helper'; +const OCI_HOST_HELPER_OPERATIONS = Object.freeze([ + 'reserve_account', + 'inspect_account', + 'prepare_account', + 'materialize_layout', + 'prepare_image', + 'render', + 'validate', + 'install', + 'start', + 'stop', + 'disable', + 'inspect', + 'inspect_inactive', + 'health', + 'verify_publication', + 'commit', + 'settle_committed', + 'rollback', + 'rollback_stopped', + 'start_prior', + 'settle_rollback', + 'remove_services', + 'inspect_removed', + 'settle_removed', + 'destroy_account', + 'verify_destroyed', + 'backup_snapshot', + 'backup_inspect', + 'backup_restore', + 'backup_inspect_restored', + 'backup_destroy', + 'backup_verify_destroyed', +]); + +function fail(code = 'service_installation_failed') { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, allowed, required = allowed) { + if (!plain(value) || Object.keys(value).some(key => !allowed.includes(key)) + || required.some(key => !Object.hasOwn(value, key))) fail('runtime_boundary_violation'); +} + +function absolute(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) fail('runtime_boundary_violation'); + return value; +} + +function claim(value) { + exact(value, ['jobId', 'workerId', 'fence', 'generation'], ['jobId', 'workerId', 'fence']); + if (![value.jobId, value.workerId].every(item => typeof item === 'string' && /^[a-z][a-z0-9_-]{2,95}$/.test(item)) + || !Number.isSafeInteger(value.fence) || value.fence < 1 + || Object.hasOwn(value, 'generation') && (!Number.isSafeInteger(value.generation) || value.generation < 1)) fail('runtime_boundary_violation'); + return value; +} + +function loadConfig(file = OCI_HOST_HELPER_CONFIG) { + const selected = absolute(file); + let value; + try { value = JSON.parse(readRootFile(selected, 0o600, 16 * 1024).toString('utf8')); } catch { fail('runtime_boundary_violation'); } + exact(value, ['stateRoot', 'authorityRoot', 'unitRoot', 'releaseRoot', 'centralSocket', 'centralUid', 'controllerUid', + 'controlReleaseId', 'helperManifestSha256', 'authorityUid', 'helperCallerUid', 'helperCallerGid']); + for (const key of ['authorityUid', 'helperCallerUid', 'helperCallerGid']) { + if (!Number.isSafeInteger(value[key]) || value[key] < 1) fail('runtime_boundary_violation'); + } + if (value.authorityUid === value.helperCallerUid) fail('runtime_boundary_violation'); + return Object.freeze({ + stateRoot: absolute(value.stateRoot), + authorityRoot: absolute(value.authorityRoot), + unitRoot: absolute(value.unitRoot), + releaseRoot: absolute(value.releaseRoot), + centralSocket: absolute(value.centralSocket), + centralUid: value.centralUid, + controllerUid: value.controllerUid, + controlReleaseId: value.controlReleaseId, + helperManifestSha256: value.helperManifestSha256, + authorityUid: value.authorityUid, + helperCallerUid: value.helperCallerUid, + helperCallerGid: value.helperCallerGid, + }); +} + +function identityAvailable(value, remainingMs) { + for (const database of ['passwd', 'group']) { + const result = spawnSync('/usr/bin/getent', [database, String(value)], { + encoding: 'utf8', timeout: Math.min(5_000, remainingMs()), maxBuffer: 16 * 1024, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' }, + }); + if (result.error || result.signal || ![0, 2].includes(result.status)) fail(); + if (result.status === 0) return false; + } + return true; +} + +function namedIdentityOccupied(name, remainingMs) { + for (const database of ['passwd', 'group']) { + const result = spawnSync('/usr/bin/getent', [database, name], { + encoding: 'utf8', timeout: Math.min(5_000, remainingMs()), maxBuffer: 16 * 1024, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' }, + }); + if (result.error || result.signal || ![0, 2].includes(result.status)) fail(); + if (result.status === 0) return true; + } + return false; +} + +function tenantLayout(plan) { + return Object.freeze({ + installationRoot: plan.host.installationRoot, + directories: Object.freeze(Object.fromEntries(Object.entries(MANAGED_INSTALLATION_DIRECTORY_FIELDS) + .map(([field, relative]) => [field, path.join(plan.host.installationRoot, relative)]))), + }); +} + +function tenantBackup(plan, operation, payload, helperPath, timeout) { + if (!Number.isSafeInteger(timeout) || timeout < 1) fail('runtime_boundary_violation'); + const helper = fs.lstatSync(helperPath); + if (!helper.isFile() || helper.isSymbolicLink() || helper.uid !== 0 || helper.gid !== 0 + || helper.nlink !== 1 || (helper.mode & 0o7777) !== 0o555 + || fs.realpathSync(helperPath) !== helperPath) fail('runtime_boundary_violation'); + const request = `${JSON.stringify({ version: 1, operation, layout: tenantLayout(plan), payload })}\n`; + if (Buffer.byteLength(request, 'utf8') > 128 * 1024) fail('runtime_boundary_violation'); + const result = spawnSync('/usr/sbin/runuser', [ + '--user', plan.account.name, '--', '/usr/bin/env', '-i', + `HOME=${plan.host.tenantRoot}`, 'PATH=/usr/bin:/bin', 'LANG=C.UTF-8', 'LC_ALL=C.UTF-8', + '/usr/bin/node', '--no-warnings', helperPath, + ], { input: request, encoding: 'utf8', timeout: Math.min(600_000, timeout), maxBuffer: 256 * 1024, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' } }); + if (result.error || result.signal || typeof result.stdout !== 'string' || !result.stdout.endsWith('\n') + || result.stdout.slice(0, -1).includes('\n')) fail('backup_failed'); + let response; + try { response = JSON.parse(result.stdout.slice(0, -1)); } catch { fail('backup_failed'); } + if (!plain(response) || typeof response.ok !== 'boolean') fail('backup_failed'); + if (!response.ok) fail(typeof response.status === 'string' ? response.status : 'backup_failed'); + if (result.status !== 0 || Object.keys(response).sort().join(',') !== 'ok,result') fail('backup_failed'); + return response.result; +} + +function createOciHostHelper(options = {}) { + exact(options, ['configFile', 'clock'], []); + if (typeof process.geteuid !== 'function' || process.geteuid() !== 0 || process.getegid() !== 0) { + fail('runtime_boundary_violation'); + } + const config = loadConfig(options.configFile === undefined ? OCI_HOST_HELPER_CONFIG : options.configFile); + if (process.env.SUDO_UID !== String(config.helperCallerUid)) fail('runtime_boundary_violation'); + const runningHelper = fs.realpathSync(process.argv[1]); + const relativeHelper = path.relative(CONTROL_RELEASE_ROOT, runningHelper); + const segments = relativeHelper.split(path.sep); + if (segments.length < 2 || !/^[a-z][a-z0-9_.-]{2,95}$/.test(segments[0]) + || segments.slice(1).join('/') !== OCI_HOST_HELPER_RELATIVE) fail('runtime_boundary_violation'); + const controlReleaseRoot = path.join(CONTROL_RELEASE_ROOT, segments[0]); + verifyHostArtifact(runningHelper, config.controlReleaseId, config.helperManifestSha256); + const tenantBackupHelper = path.join(controlReleaseRoot, OCI_TENANT_BACKUP_HELPER_RELATIVE); + const clock = options.clock || Date.now; + const authority = createOciHostAuthority({ root: config.authorityRoot, clock }); + let registry; + let executor; + let currentGuard; + function initializeRegistry() { + if (registry) return; + registry = createOciHostAccountRegistry({ + stateRoot: config.stateRoot, + identityAvailable: value => identityAvailable(value, currentGuard.remainingMs), + clock: () => { currentGuard.remainingMs(); return clock(); }, + }); + } + function initializeExecutor() { + if (executor) return; + initializeRegistry(); + executor = createOciHostExecutor({ + registry, + stateRoot: config.stateRoot, + unitRoot: config.unitRoot, + releaseRoot: config.releaseRoot, + centralSocket: config.centralSocket, + centralUid: config.centralUid, + controllerUid: config.controllerUid, + clock, + remainingMs: () => currentGuard.remainingMs(), + }); + } + + function execute(requestValue) { + exact(requestValue, ['version', 'operation', 'claim', 'authorization', 'runtimeKey', 'plan', 'token', 'payload'], + ['version', 'operation', 'claim', 'authorization']); + if (requestValue.version !== OCI_HOST_HELPER_PROTOCOL_VERSION + || !OCI_HOST_HELPER_OPERATIONS.includes(requestValue.operation)) fail('runtime_boundary_violation'); + claim(requestValue.claim); + return authority.execute(requestValue, mutate => executeAuthorized(requestValue, mutate)); + } + + function executeAuthorized(requestValue, mutate) { + currentGuard = mutate; + const operation = requestValue.operation; + const accountOperation = ['reserve_account', 'inspect_account'].includes(operation); + const expectedKeys = accountOperation ? 'authorization,claim,operation,runtimeKey,version' + : operation === 'materialize_layout' ? 'authorization,claim,operation,plan,token,version' + : (operation.startsWith('backup_') || operation === 'verify_publication') ? 'authorization,claim,operation,payload,plan,version' + : 'authorization,claim,operation,plan,version'; + if (Object.keys(requestValue).sort().join(',') !== expectedKeys) fail('runtime_boundary_violation'); + if (!accountOperation) validateOciDeploymentPlan(requestValue.plan); + mutate(accountOperation ? initializeRegistry : initializeExecutor); + if (operation === 'reserve_account') { + if (!registry.inspect(requestValue.runtimeKey) + && namedIdentityOccupied(hostAccountName(requestValue.runtimeKey), mutate.remainingMs)) { + fail('runtime_boundary_violation'); + } + return mutate(() => registry.reserve(requestValue.runtimeKey)); + } + if (operation === 'inspect_account') { + return registry.inspect(requestValue.runtimeKey); + } + const backupOperations = Object.freeze({ + backup_snapshot: 'snapshot', backup_inspect: 'inspect', backup_restore: 'restore', + backup_inspect_restored: 'inspect_restored', backup_destroy: 'destroy', + backup_verify_destroyed: 'verify_destroyed', + }); + const plan = validateOciDeploymentPlan(requestValue.plan); + if (Object.hasOwn(backupOperations, operation)) { + if (!plain(requestValue.payload)) fail('runtime_boundary_violation'); + if (operation === 'backup_destroy' || operation === 'backup_verify_destroyed') { + exact(requestValue.payload, operation === 'backup_destroy' ? ['authority'] : []); + if (operation === 'backup_destroy') { + const approval = requestValue.payload.authority; + exact(approval, ['installationState', 'retainedData', 'destructionApproved']); + if (approval.installationState !== 'decommissioned' || approval.retainedData !== true + || approval.destructionApproved !== true) fail('runtime_boundary_violation'); + } + // A crash after tenant deletion must not require running a helper as an + // account that no longer exists. The authorized action still binds this + // exact destruction/absence check and the server-derived plan. + try { fs.lstatSync(plan.host.installationRoot); } + catch (error) { + if (error.code !== 'ENOENT') throw error; + return Object.freeze({ status: operation === 'backup_destroy' ? 'destroyed' : 'absent', changed: false }); + } + } + return mutate(() => tenantBackup(plan, backupOperations[operation], requestValue.payload, tenantBackupHelper, mutate.remainingMs())); + } + const methods = Object.freeze({ + prepare_account: () => executor.prepareAccount(plan, mutate), + materialize_layout: () => executor.materializeLayout(plan, requestValue.token, mutate), + prepare_image: () => executor.prepareImage(plan, mutate), + render: () => executor.render(plan, mutate), + validate: () => executor.validate(plan), + install: () => executor.install(plan, mutate), + start: () => executor.start(plan, mutate), + stop: () => executor.stop(plan, mutate), + disable: () => executor.disable(plan, mutate), + inspect: () => executor.inspect(plan), + inspect_inactive: () => executor.inspectState(plan, false), + health: () => executor.health(plan), + verify_publication: () => executor.verifyPublication(plan, requestValue.payload, mutate), + commit: () => executor.commit(plan, mutate), + settle_committed: () => executor.settleCommitted(plan, mutate), + rollback: () => executor.rollback(plan, mutate), + rollback_stopped: () => executor.rollbackStopped(plan, mutate), + start_prior: () => executor.startPrior(plan, mutate), + settle_rollback: () => executor.settleRollback(plan, mutate), + remove_services: () => executor.removeServices(plan, mutate), + inspect_removed: () => executor.inspectRemoved(plan), + settle_removed: () => executor.settleRemoved(plan, mutate), + destroy_account: () => executor.destroyAccount(plan, mutate), + verify_destroyed: () => executor.verifyDestroyed(plan), + }); + const selected = methods[operation]; + if (!selected) fail('runtime_boundary_violation'); + return selected(); + } + + function close() { try { registry?.close(); } finally { authority.close(); } } + return Object.freeze({ execute, close }); +} + +module.exports = { + OCI_HOST_HELPER_PROTOCOL_VERSION, + OCI_HOST_HELPER_CONFIG, + CONTROL_RELEASE_ROOT, + OCI_HOST_HELPER_RELATIVE, + OCI_TENANT_BACKUP_HELPER_RELATIVE, + OCI_HOST_HELPER_OPERATIONS, + loadConfig, + createOciHostHelper, +}; diff --git a/core/core/installations/src/oci-host-issuer.js b/core/core/installations/src/oci-host-issuer.js new file mode 100644 index 0000000..163bc38 --- /dev/null +++ b/core/core/installations/src/oci-host-issuer.js @@ -0,0 +1,146 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { createOciHostAuthority } = require('./oci-host-authority'); +const { loadConfig, OCI_HOST_HELPER_OPERATIONS } = require('./oci-host-helper'); +const { verifyHostArtifact, readRootFile } = require('./oci-host-artifact'); +const { opaqueRuntimeSuffix } = require('../../runtime-host-identity'); +const { createOciHostAccountRegistry } = require('./oci-host-account-registry'); + +const ISSUER_COMMAND = '/opt/dispatch-control/current/host-helper-artifact/core/installations/bin/dispatch-oci-host-issuer'; +const HELPER_COMMAND = '/opt/dispatch-control/current/host-helper-artifact/core/installations/bin/dispatch-oci-host-helper'; +const ENV = Object.freeze({ PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' }); +function fail() { throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); } +function command(file, args, options = {}) { + const result = spawnSync(file, args, { encoding: 'utf8', env: ENV, timeout: 30_000, + maxBuffer: 256 * 1024, ...options }); + if (result.error || result.signal || ![0, ...(options.accepted || [])].includes(result.status)) fail(); + return result; +} +function state(unit) { + const result = command('/usr/bin/systemctl', ['show', unit, + '--property=LoadState,ActiveState,SubState,Job,ControlGroup,FragmentPath,DropInPaths'], { accepted: [1] }); + return Object.fromEntries(result.stdout.trim().split('\n').map(line => { + const split = line.indexOf('='); return [line.slice(0, split), line.slice(split + 1)]; + })); +} +function stopped(unit) { + const value = state(unit); + if (value.Job || !['inactive', 'failed'].includes(value.ActiveState) + || !['dead', 'failed'].includes(value.SubState)) fail(); + if (value.ControlGroup) { + if (!/^\/(?:system|dispatch-dsp).slice\/dispatch-[a-z0-9-]+\.service$/.test(value.ControlGroup)) fail(); + try { + const events = fs.readFileSync(`/sys/fs/cgroup${value.ControlGroup}/cgroup.events`, 'utf8'); + if (!/^populated 0$/m.test(events)) fail(); + } catch (error) { if (error.code !== 'ENOENT') throw error; } + } +} +function recoverHost(runtimeKey, ids, config) { + // These IDs came from the root ledger, never from a caller-selected unit. + for (const id of ids) { + if (!/^[a-f0-9]{64}$/.test(id)) fail(); + const unit = `dispatch-host-action-${id}.service`; + command('/usr/bin/systemctl', ['stop', unit], { accepted: [5] }); + stopped(unit); + } + const suffix = opaqueRuntimeSuffix(runtimeKey); + for (const unit of [`dispatch-dsp-${suffix}.service`, `dispatch-runtime-agent-bridge-${suffix}.service`]) { + const value = state(unit); + if (value.LoadState !== 'not-found') { + const expected = `/etc/systemd/system/${unit}`; + if (value.FragmentPath !== expected || value.DropInPaths) fail(); + readRootFile(expected, 0o644, 64 * 1024); + command('/usr/bin/systemctl', ['stop', unit]); + } + stopped(unit); + } + const registry = createOciHostAccountRegistry({ stateRoot: config.stateRoot, identityAvailable: () => false }); + let account; + try { account = registry.inspect(runtimeKey); } finally { registry.close(); } + if (account && ['reserved', 'active'].includes(account.status)) { + const passwd = command('/usr/bin/getent', ['passwd', account.name], { accepted: [2] }); + if (passwd.status === 0) { + const fields = passwd.stdout.trim().split(':'); + const root = `/var/lib/dispatch/tenants/${suffix}`; + if (Number(fields[2]) !== account.uid || Number(fields[3]) !== account.gid + || fields[5] !== `${root}/home` || fields[6] !== '/usr/sbin/nologin') fail(); + let runtimeDirectory; + try { runtimeDirectory = fs.lstatSync(`/run/user/${account.uid}`); } + catch (error) { if (error.code !== 'ENOENT') throw error; } + if (runtimeDirectory && (!runtimeDirectory.isDirectory() || runtimeDirectory.uid !== account.uid + || runtimeDirectory.gid !== account.gid || (runtimeDirectory.mode & 0o7777) !== 0o700)) fail(); + if (account.status === 'active' && runtimeDirectory) command('/usr/sbin/runuser', ['--user', account.name, '--', '/usr/bin/env', '-i', + `HOME=${root}/home`, `XDG_DATA_HOME=${root}/engine-data`, `XDG_CONFIG_HOME=${root}/engine-config`, + `XDG_RUNTIME_DIR=/run/user/${account.uid}`, 'PATH=/usr/bin:/bin', '/usr/bin/podman', + 'rm', '--force', '--ignore', `dispatch-dsp-${suffix}`, `dispatch-dsp-${suffix}-manifest`, `dispatch-dsp-${suffix}-publication`]); + } + // Destruction can remove passwd/runtime-dir before its action receipt. + // Quiescence still covers the reserved numeric allocation in that case. + command('/usr/bin/loginctl', ['terminate-user', String(account.uid)], { accepted: [1] }); + command('/usr/bin/systemctl', ['stop', `user@${account.uid}.service`, `user-runtime-dir@${account.uid}.service`]); + // No process in this allocation may survive recovery, including a + // rootless container or user-manager scope delegated outside the action. + for (const entry of fs.readdirSync('/proc')) { + if (!/^[1-9][0-9]*$/.test(entry)) continue; + let status; + try { status = fs.readFileSync(`/proc/${entry}/status`, 'utf8'); } + catch (error) { if (['ENOENT', 'ESRCH'].includes(error.code)) continue; throw error; } + if (/^State:\s+Z\b/m.test(status)) continue; + const uids = /^Uid:\s+([0-9\s]+)$/m.exec(status)?.[1].trim().split(/\s+/).map(Number) || []; + if (uids.some(uid => uid === account.uid || uid >= account.subuidStart && uid < account.subuidStart + account.subidCount)) fail(); + } + // Leave the manager stopped. Only a subsequent authorized image/start + // action may restart tenant work after the old running gate is closed. + } + return true; +} + +function dispatchAuthorized(request) { + if (process.geteuid() !== 0 || process.getegid() !== 0) fail(); + const config = loadConfig(); + if (process.env.SUDO_UID !== String(config.authorityUid)) fail(); + verifyHostArtifact(fs.realpathSync(process.argv[1]), config.controlReleaseId, + config.helperManifestSha256, 'dispatch-oci-host-issuer'); + if (!request || Object.keys(request).sort().join(',') !== 'lease,request,version' || request.version !== 1 + || !request.request || request.request.authorization !== undefined + || !OCI_HOST_HELPER_OPERATIONS.includes(request.request.operation)) fail(); + const authority = createOciHostAuthority({ root: config.authorityRoot }); + const runtimeKey = request.lease?.runtimeKey; + let issued = false; + try { + authority.recover(runtimeKey, ids => recoverHost(runtimeKey, ids, config)); + authority.synchronizeLease(request.lease, request.request.operation); + issued = true; + const authorization = authority.issueAction(request.request); + const unit = `dispatch-host-action-${authorization}.service`; + const remaining = request.lease.expiresAt - Date.now(); + if (!Number.isSafeInteger(remaining) || remaining < 1 || remaining > 600_000) fail(); + const result = command('/usr/bin/systemd-run', [ + '--quiet', '--wait', '--pipe', '--collect', `--unit=${unit}`, + '--property=Type=exec', '--property=KillMode=control-group', '--property=TimeoutStopSec=10s', + `--property=RuntimeMaxSec=${remaining}ms`, `--property=User=${config.helperCallerUid}`, + `--property=Group=${config.helperCallerGid}`, '--property=UMask=0077', + '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin', 'LANG=C.UTF-8', 'LC_ALL=C.UTF-8', + '/usr/bin/sudo', '-n', HELPER_COMMAND, + ], { input: `${JSON.stringify({ ...request.request, authorization })}\n`, + timeout: remaining + 20_000, accepted: [1] }); + stopped(unit); + let response; + try { response = JSON.parse(result.stdout); } catch { fail(); } + if (!response || typeof response.ok !== 'boolean') fail(); + if (response.ok && Object.keys(response).sort().join(',') !== 'ok,result') fail(); + if (!response.ok) { + try { authority.recover(runtimeKey, ids => recoverHost(runtimeKey, ids, config)); } + catch { /* The durable running gate remains closed for the next recovery. */ } + } + if (response.ok && request.request.operation === 'verify_destroyed') authority.revoke(runtimeKey, true); + return response; + } finally { + try { if (issued) authority.revoke(runtimeKey); } finally { authority.close(); } + } +} + +module.exports = { ISSUER_COMMAND, dispatchAuthorized }; diff --git a/core/core/installations/src/oci-host-permissions.js b/core/core/installations/src/oci-host-permissions.js new file mode 100644 index 0000000..41d9ea2 --- /dev/null +++ b/core/core/installations/src/oci-host-permissions.js @@ -0,0 +1,71 @@ +'use strict'; + +const { ISSUER, AUDIENCE } = require('./oci-host-authority'); +const PROVISIONING = Object.freeze({ + runtime_oci_host_account: ['reserve_account', 'prepare_account', 'materialize_layout'], + runtime_oci_image_reconcile: ['prepare_image'], + runtime_oci_bridge_reconcile: ['render', 'validate', 'install'], + runtime_oci_container_reconcile: ['start'], + runtime_oci_verify: ['health'], + final: ['health', 'commit'], + compensation: ['rollback'], +}); +const LIFECYCLE = Object.freeze({ + capture_publication: ['verify_publication'], + inspect_schedule: [], quiesce_schedule: [], restore_schedule: [], + stop_if_running: ['stop', 'inspect_inactive'], stop_runtime: ['stop', 'inspect_inactive'], + snapshot: ['backup_snapshot'], safety_snapshot: ['backup_snapshot'], upgrade_backup: ['backup_snapshot', 'backup_inspect'], + final_backup: ['backup_snapshot'], + restart_if_needed: ['start', 'health'], start_runtime: ['start', 'health'], + verify_runtime: ['health', 'inspect_inactive'], verify_stopped: ['inspect_inactive'], + restore_snapshot: ['backup_inspect', 'backup_restore'], verify_restored: ['backup_inspect_restored', 'inspect_inactive'], + install_release: ['prepare_image', 'render', 'validate', 'install'], + start_release: ['start', 'health'], verify_release: ['health'], verify_release_publication: ['verify_publication'], + verify_stopped_release: ['inspect_inactive'], + commit_release: ['health', 'inspect_inactive'], verify_infrastructure: ['health'], verify_publication: ['verify_publication'], + disable_runtime: ['disable', 'inspect_inactive'], remove_services: ['remove_services', 'inspect_removed'], + restore_services: ['render', 'validate', 'install'], + verify_unallocated: [], + verify_retained: ['inspect_inactive', 'inspect_removed', 'backup_inspect'], + destroy_runtime: ['stop', 'inspect_inactive', 'disable', 'remove_services', 'settle_removed', 'verify_destroyed', 'inspect_removed', 'backup_destroy', 'backup_verify_destroyed', 'destroy_account'], + verify_destroyed: ['verify_destroyed', 'backup_verify_destroyed'], + final: ['health', 'inspect_inactive', 'backup_inspect_restored', 'commit', 'inspect_removed', + 'backup_inspect', 'settle_removed', 'verify_destroyed'], +}); +const COMPENSATION = Object.freeze({ + upgrade: ['rollback_stopped', 'inspect_inactive', 'backup_restore', 'backup_inspect_restored', + 'start', 'start_prior', 'health', 'settle_rollback'], + backup: ['start', 'stop', 'health', 'inspect_inactive'], + suspend: ['start', 'stop', 'health', 'inspect_inactive'], + resume: ['stop', 'inspect_inactive'], + restore: ['backup_restore', 'backup_inspect_restored', 'inspect_inactive'], + decommission: ['stop', 'disable', 'inspect_inactive'], destroy: [], +}); +function fail() { throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); } +function authorizeHostRequest(snapshot, request) { + const { manifest, claim, kind, stage, operation, compensation, expiresAt, installationRevision } = snapshot; + if (!['oci_container_v1', 'native_service_v1'].includes(snapshot.backend) || request.version !== 2 + || Object.keys(request.claim).sort().join(',') !== Object.keys(claim).sort().join(',') + || Object.keys(claim).some(key => request.claim[key] !== claim[key]) + || (request.plan?.runtimeKey ?? request.runtimeKey) !== manifest.runtime.key) fail(); + const allowed = kind === 'provisioning' ? PROVISIONING[compensation ? 'compensation' : stage || 'final'] + : compensation ? COMPENSATION[operation] : LIFECYCLE[stage || 'final']; + if (!allowed || request.operation !== 'inspect_account' && !allowed.includes(request.operation) + && !(snapshot.canSettle && request.operation === 'settle_committed')) fail(); + if (request.plan) { + const plan = request.plan; + const target = operation === 'upgrade' && plan.deployment?.manifestRevision === manifest.revision + 1; + if (plan.backend !== snapshot.backend || plan.deployment?.organizationId !== manifest.organization.id + || plan.deployment?.manifestRevision !== manifest.revision + (target ? 1 : 0) + || plan.release?.releaseId !== (target ? snapshot.targetReleaseId : manifest.runtime.releaseId)) fail(); + if (target && !compensation && !['install_release', 'start_release', 'verify_release', + 'verify_stopped_release', 'verify_release_publication', 'restore_schedule', 'commit_release', null].includes(stage)) fail(); + } + return Object.freeze({ version: 1, issuer: ISSUER, audience: AUDIENCE, + organizationId: manifest.organization.id, runtimeKey: manifest.runtime.key, + installationRevision, manifestRevisions: operation === 'upgrade' ? [manifest.revision, manifest.revision + 1] : [manifest.revision], + backend: snapshot.backend, jobKind: kind, jobId: claim.jobId, workerId: claim.workerId, + generation: kind === 'provisioning' ? claim.generation : installationRevision, + fence: claim.fence, expiresAt }); +} +module.exports = { authorizeHostRequest }; diff --git a/core/core/installations/src/oci-lifecycle.js b/core/core/installations/src/oci-lifecycle.js new file mode 100644 index 0000000..f4a4ecd --- /dev/null +++ b/core/core/installations/src/oci-lifecycle.js @@ -0,0 +1,496 @@ +'use strict'; + +const { + INSTALLATION_ACTIVATION_EVIDENCE_VERSION, + installationActivationEvidenceDigest, + serverInstallationManifest, +} = require('../../../shared/contracts/src'); + +function fail(code = 'installation_operation_failed') { + throw Object.assign(new Error(code), { code }); +} +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} +function lifecycleReceipt(status, value = {}) { + const receipt = { status }; + for (const field of ['changed', 'syncWasRunning', 'serviceCount']) { + if (Object.hasOwn(value, field)) receipt[field] = value[field]; + } + return Object.freeze(receipt); +} +function activationEvidence(context, raw) { + if (!plain(raw)) fail('installation_not_ready'); + const payload = Object.freeze({ + schemaVersion: INSTALLATION_ACTIVATION_EVIDENCE_VERSION, + manifestRevision: context.targetManifest?.runtime ? context.targetManifest.revision : context.manifest.revision, + jobId: context.job.id, + runtimeKey: context.manifest.runtime.key, + definitionDigest: raw.definitionDigest, + requestDigest: raw.requestDigest, + previewDigest: raw.previewDigest, + batchId: raw.batchId, + preparationRunId: raw.preparationRunId, + target: raw.target, + runs: raw.runs, + publications: raw.publications, + capturedAt: raw.capturedAt, + }); + return Object.freeze({ ...payload, evidenceDigest: installationActivationEvidenceDigest(payload) }); +} + +function createOciInstallationLifecycle(options) { + if (!plain(options) || Object.keys(options).filter(key => key !== 'offsitePolicy').sort().join(',') !== [ + 'adapter', 'authority', 'backupManagerFactory', 'hostExecutor', 'runtimeFactory', + ].sort().join(',')) fail('runtime_boundary_violation'); + const { authority, adapter, hostExecutor, backupManagerFactory, runtimeFactory } = options; + const offsitePolicy = options.offsitePolicy || require('./offsite-policy'); + if (!authority || ['claim', 'renew', 'desiredRuntimeState', 'mutate', 'checkpoint', 'succeed', 'failed'] + .some(method => typeof authority[method] !== 'function') + || !adapter || typeof adapter.plan !== 'function' + || !hostExecutor || [ + 'start', 'stop', 'disable', 'health', 'inspectInactive', 'render', 'validate', 'install', + 'commit', 'rollback', 'rollbackStopped', 'settleRollback', 'removeServices', 'inspectRemoved', 'settleRemoved', + 'destroyAccount', 'verifyDestroyed', 'verifyPublication', + ].some(method => typeof hostExecutor[method] !== 'function') + || typeof backupManagerFactory !== 'function' || typeof runtimeFactory !== 'function') { + fail('runtime_boundary_violation'); + } + + function guard(context) { return callback => authority.mutate(context.claim, callback); } + + function runtimeContext(claimed) { + if (!['oci_container_v1', 'native_service_v1'].includes(claimed.backend)) fail('runtime_identity_mismatch'); + serverInstallationManifest(claimed.manifest, claimed.manifestAuthority); + const createPlan = claimed.operation === 'destroy' ? adapter.destructionPlan : adapter.plan; + if (typeof createPlan !== 'function') fail('runtime_boundary_violation'); + const destruction = claimed.operation === 'destroy' && typeof adapter.destructionContext === 'function' + ? adapter.destructionContext(claimed.manifest, claimed.manifestAuthority, { fixture: false, claim: claimed.claim }) : null; + const plan = destruction ? destruction.plan + : createPlan(claimed.manifest, claimed.manifestAuthority, { fixture: false, claim: claimed.claim }); + if (claimed.nextStage === 0 && !claimed.stageReceipts?.__compensating + && claimed.operation !== 'destroy' && (['ready', 'suspended'].includes(claimed.startingState) + || claimed.backend === 'native_service_v1' && ['waiting_for_owner', 'waiting_for_provider_auth'].includes(claimed.startingState)) + && typeof hostExecutor.settleCommitted === 'function') { + hostExecutor.settleCommitted(plan, claimed.claim, guard(claimed)); + } + const backupManager = backupManagerFactory(plan, claimed.claim); + if (!backupManager || ['snapshot', 'inspect', 'restore', 'inspectRestored', 'destroy'] + .some(method => typeof backupManager[method] !== 'function')) fail('runtime_boundary_violation'); + let target = null; + if (claimed.operation === 'upgrade') { + serverInstallationManifest(claimed.targetManifest, claimed.targetManifestAuthority); + target = Object.freeze({ + plan: adapter.plan(claimed.targetManifest, claimed.targetManifestAuthority, { + fixture: false, claim: claimed.claim, + }), + }); + } + return { ...claimed, plan, backupManager, target, retired: destruction?.retired === true }; + } + + function scheduleIntent(context) { + const value = context.operation === 'resume' ? context.resumeSync + : context.stageReceipts?.inspect_schedule?.syncWasRunning; + if (typeof value !== 'boolean') fail('runtime_boundary_violation'); + return value; + } + + function runtime(context, target = false) { + const selectedPlan = target ? context.target?.plan : context.plan; + if (!selectedPlan) fail('runtime_boundary_violation'); + const selected = runtimeFactory(selectedPlan, context); + if (!selected || ['inspectSchedule', 'quiesceSchedule', 'restoreSchedule', 'verifyInfrastructure', 'verifyPublication'] + .some(method => typeof selected[method] !== 'function')) fail('runtime_boundary_violation'); + return selected; + } + + function stopped(context) { + hostExecutor.inspectInactive(context.plan, context.claim); + return lifecycleReceipt('inactive', { changed: false, serviceCount: 2 }); + } + + function stop(context) { + const changed = hostExecutor.stop(context.plan, context.claim, guard(context))?.changed === true; + hostExecutor.inspectInactive(context.plan, context.claim); + return lifecycleReceipt('stopped', { changed, serviceCount: 2 }); + } + + function start(context, target = false) { + const plan = target ? context.target.plan : context.plan; + const changed = hostExecutor.start(plan, context.claim, guard(context))?.changed === true; + hostExecutor.health(plan, context.claim); + return lifecycleReceipt('started', { changed, serviceCount: 2 }); + } + + function availableBackup(context, stage, selected) { + const receipt = context.stageReceipts?.[stage]; + if (!selected || receipt?.status !== 'snapshot') fail('backup_failed'); + return Object.freeze({ + ...selected, + status: 'available', + treeDigest: receipt.treeDigest, + fileCount: receipt.fileCount, + totalBytes: receipt.totalBytes, + }); + } + + function verifyPublication(context, target = false) { + const baseline = context.stageReceipts?.capture_publication?.publicationBaseline; + if (!baseline) fail('first_publication_failed'); + return hostExecutor.verifyPublication(target ? context.target.plan : context.plan, + { mode: 'verify', baseline }, context.claim); + } + + function capturePublication(context) { + return hostExecutor.verifyPublication(context.plan, { mode: 'capture' }, context.claim).publicationBaseline; + } + + async function executeStage(context, stage) { + if (stage === 'restore_services') { + hostExecutor.render(context.plan, context.claim, guard(context)); + hostExecutor.validate(context.plan, context.claim); + hostExecutor.install(context.plan, context.claim, guard(context)); + return lifecycleReceipt('installed', { changed: true }); + } + if (stage === 'inspect_schedule') { + if (context.operation === 'decommission' && context.removal?.sync_running !== null && context.removal?.sync_running !== undefined) return lifecycleReceipt('verified', { syncWasRunning: context.removal.sync_running === 1 }); + if (context.withoutPaycom || context.startingState !== 'ready') return lifecycleReceipt('verified', { changed: false, syncWasRunning: false }); + const value = await runtime(context).inspectSchedule(); + return lifecycleReceipt('verified', { changed: false, syncWasRunning: value.syncWasRunning }); + } + if (stage === 'quiesce_schedule') { + if (context.withoutPaycom || context.startingState !== 'ready') return lifecycleReceipt('stopped', { changed: false, syncWasRunning: false }); + try { await runtime(context).quiesceSchedule(scheduleIntent(context)); } + catch (error) { + // An interrupted backup may already have stopped the runtime. Removal + // still proceeds to the fenced host stop and disabled-state verification. + if (context.operation !== 'decommission' || ![0, 1].includes(context.removal?.sync_running)) throw error; + } + return lifecycleReceipt('stopped', { changed: false, syncWasRunning: scheduleIntent(context) }); + } + if (stage === 'stop_if_running') return context.startingState !== 'suspended' ? stop(context) : stopped(context); + if (stage === 'snapshot') return context.backupManager.snapshot(context.backup, guard(context)); + if (stage === 'restart_if_needed') return context.startingState !== 'suspended' + ? start(context) : lifecycleReceipt('inactive', { changed: false, serviceCount: 2 }); + if (stage === 'restore_schedule') { + if (!context.withoutPaycom && (context.operation === 'resume' || context.startingState === 'ready')) { + if (authority.desiredRuntimeState(context.claim) === 'suspended') fail('installation_operation_not_allowed'); + await runtime(context, context.operation === 'upgrade').restoreSchedule(scheduleIntent(context)); + } + return lifecycleReceipt('started', { changed: scheduleIntent(context), syncWasRunning: scheduleIntent(context) }); + } + if (stage === 'verify_runtime') { + if (context.startingState !== 'suspended') { + hostExecutor.health(context.plan, context.claim); + if (context.startingState === 'ready') await runtime(context).verifyInfrastructure(); + return lifecycleReceipt('healthy', { changed: false, serviceCount: 2 }); + } + return stopped(context); + } + if (stage === 'verify_stopped') return stopped(context); + if (stage === 'safety_snapshot') { + const receipt = context.backupManager.snapshot(context.safetyBackup, guard(context)); + const { offsiteRequired, waitForOffsiteBackup } = offsitePolicy; + if (context.plan.backend === 'native_service_v1' || context.requireOffsiteSafety || offsiteRequired()) await waitForOffsiteBackup(require('node:path').join(context.plan.host.installationRoot, 'backups', context.safetyBackup.id), + receipt.treeDigest, () => authority.renew(context.claim), { required: context.plan.backend === 'native_service_v1' || context.requireOffsiteSafety, recoveryRequired: context.plan.backend === 'native_service_v1' }); + return receipt; + } + if (stage === 'restore_snapshot') { + if (!context.sourceBackup) fail('restore_failed'); + return context.backupManager.restore(context.sourceBackup, context.job.id, guard(context)); + } + if (stage === 'verify_restored') { + const receipt = context.backupManager.inspectRestored(context.sourceBackup); + stopped(context); + return receipt; + } + if (stage === 'stop_runtime') return stop(context); + if (stage === 'capture_publication') return Object.freeze({ status: 'verified', publicationBaseline: capturePublication(context) }); + if (stage === 'upgrade_backup' && context.stageReceipts.__preUpdateBackup === context.backup?.id) { + context.backupManager.inspect(context.backup); + return { status: 'snapshot', changed: false, treeDigest: context.backup.treeDigest, + fileCount: context.backup.fileCount, totalBytes: context.backup.totalBytes }; + } + if (stage === 'upgrade_backup' || stage === 'final_backup') { + const receipt = context.backupManager.snapshot(context.backup, guard(context)); + const { offsiteRequired, waitForOffsiteBackup } = offsitePolicy; + const native = context.plan.backend === 'native_service_v1'; + if (native || offsiteRequired()) await waitForOffsiteBackup(require('node:path').join(context.plan.host.installationRoot, 'backups', context.backup.id), + receipt.treeDigest, () => authority.renew(context.claim), { required: native, recoveryRequired: native }); + return receipt; + } + if (stage === 'install_release') { + hostExecutor.render(context.target.plan, context.claim, guard(context)); + hostExecutor.validate(context.target.plan, context.claim); + hostExecutor.install(context.target.plan, context.claim, guard(context)); + return lifecycleReceipt('installed', { changed: true, serviceCount: 2 }); + } + if (stage === 'start_release') return start(context, true); + if (stage === 'verify_stopped_release') { + hostExecutor.inspectInactive(context.target.plan, context.claim); + return Object.freeze({ status: 'verified', changed: false, serviceCount: 2, releaseId: context.targetManifest.runtime.releaseId }); + } + if (stage === 'verify_release') { + hostExecutor.health(context.target.plan, context.claim); + await runtime(context, true).verifyInfrastructure(); + return Object.freeze({ status: 'verified', changed: false, serviceCount: 2, releaseId: context.targetManifest.runtime.releaseId }); + } + if (stage === 'verify_release_publication') { + const proof = verifyPublication(context, true); + const raw = context.startingState === 'suspended' && context.plan.backend === 'native_service_v1' + ? { ...context.priorEvidence, capturedAt: new Date().toISOString() } + : await runtime(context, true).verifyPublication(context.priorEvidence, context.stageReceipts.capture_publication.publicationBaseline.target); + return Object.freeze({ status: 'verified', publicationBaselineDigest: proof.publicationBaselineDigest, activationEvidence: activationEvidence(context, raw) }); + } + if (stage === 'commit_release') { + if (context.startingState === 'suspended') hostExecutor.inspectInactive(context.target.plan, context.claim); + else hostExecutor.health(context.target.plan, context.claim); + return Object.freeze({ status: 'committed', changed: true, serviceCount: 2, + releaseId: context.targetManifest.runtime.releaseId }); + } + if (stage === 'start_runtime') return start(context); + if (stage === 'verify_infrastructure') { + hostExecutor.health(context.plan, context.claim); + await runtime(context).verifyInfrastructure(); + return lifecycleReceipt('verified', { changed: false, serviceCount: 2 }); + } + if (stage === 'verify_publication') { + const proof = verifyPublication(context); + const raw = await runtime(context).verifyPublication(context.priorEvidence, context.stageReceipts.capture_publication.publicationBaseline.target); + return Object.freeze({ status: 'verified', publicationBaselineDigest: proof.publicationBaselineDigest, activationEvidence: activationEvidence(context, raw) }); + } + if (stage === 'disable_runtime') { + const value = hostExecutor.disable(context.plan, context.claim, guard(context)); + hostExecutor.inspectInactive(context.plan, context.claim); + return lifecycleReceipt('disabled', { changed: value?.changed === true, serviceCount: 2 }); + } + if (stage === 'remove_services') { + const value = hostExecutor.removeServices(context.plan, context.claim, guard(context)); + hostExecutor.inspectRemoved(context.plan, context.claim); + return lifecycleReceipt('removed', { changed: value?.changed === true, serviceCount: 2 }); + } + if (stage === 'verify_retained') { + if (context.legacyRemoval) hostExecutor.inspectRemoved(context.plan, context.claim); + else hostExecutor.inspectInactive(context.plan, context.claim); + if (context.backup) context.backupManager.inspect(availableBackup(context, 'final_backup', context.backup)); + return lifecycleReceipt('retained', { changed: false }); + } + if (stage === 'destroy_runtime') { + if (!context.retired) { + let removed = false; + try { hostExecutor.inspectRemoved(context.plan, context.claim); removed = true; } catch {} + if (!removed) { + stop(context); + hostExecutor.disable(context.plan, context.claim, guard(context)); + hostExecutor.removeServices(context.plan, context.claim, guard(context)); + } + hostExecutor.inspectRemoved(context.plan, context.claim); + hostExecutor.settleRemoved(context.plan, context.claim, guard(context)); + } + await offsitePolicy.waitForDspBackupDeletion(context.job.id, + context.manifest.organization.id, context.manifest.runtime.key, () => authority.renew(context.claim), { required: context.backend === 'native_service_v1' }); + if (context.retired) { + hostExecutor.verifyDestroyed(context.plan, context.claim); + return lifecycleReceipt('destroyed', { changed: false }); + } + context.backupManager.destroy({ + installationState: 'decommissioned', retainedData: true, destructionApproved: true, + }, guard(context)); + hostExecutor.destroyAccount(context.plan, context.claim, guard(context)); + return lifecycleReceipt('destroyed', { changed: true }); + } + if (stage === 'verify_destroyed') { + hostExecutor.verifyDestroyed(context.plan, context.claim); + return lifecycleReceipt('absent', { changed: false }); + } + fail('runtime_boundary_violation'); + } + + async function finalVerify(context) { + authority.renew(context.claim); + if (context.operation === 'backup') { + context.backupManager.inspect(availableBackup(context, 'snapshot', context.backup)); + if (context.startingState !== 'suspended') hostExecutor.health(context.plan, context.claim); + else hostExecutor.inspectInactive(context.plan, context.claim); + return; + } + if (context.operation === 'restore') { + context.backupManager.inspectRestored(context.sourceBackup); + hostExecutor.inspectInactive(context.plan, context.claim); + return; + } + if (context.operation === 'upgrade') { + if (context.plan.backend === 'native_service_v1' && context.startingState === 'suspended') { + hostExecutor.inspectInactive(context.target.plan, context.claim); + if (authority.desiredRuntimeState(context.claim) !== 'suspended') fail('installation_operation_not_allowed'); + return; + } + hostExecutor.health(context.target.plan, context.claim); + if (context.withoutPaycom || context.plan.backend === 'native_service_v1' && ['waiting_for_owner', 'waiting_for_provider_auth'].includes(context.startingState)) { + if (authority.desiredRuntimeState(context.claim) !== 'active') fail('installation_operation_not_allowed'); + hostExecutor.commit(context.target.plan, context.claim, guard(context)); + return; + } + const expected = scheduleIntent(context); + const actual = await runtime(context, true).inspectSchedule(); + if (actual.syncWasRunning !== expected) fail('runtime_health_failed'); + if (authority.desiredRuntimeState(context.claim) !== 'active') fail('installation_operation_not_allowed'); + hostExecutor.commit(context.target.plan, context.claim, guard(context)); + hostExecutor.health(context.target.plan, context.claim); + return; + } + if (context.operation === 'suspend') return hostExecutor.inspectInactive(context.plan, context.claim); + if (context.operation === 'resume') { + hostExecutor.health(context.plan, context.claim); + if (context.removal?.legacy_services) hostExecutor.commit(context.plan, context.claim, guard(context)); + if (context.removal && context.removal.installation_state !== 'ready') return; + const actual = context.withoutPaycom ? { syncWasRunning: false } : await runtime(context).inspectSchedule(); + if ((!context.removal || context.removal.installation_state === 'ready') && actual.syncWasRunning !== context.resumeSync) fail('runtime_health_failed'); + return; + } + if (context.operation === 'decommission') { + if (context.legacyRemoval) { + hostExecutor.inspectRemoved(context.plan, context.claim); + if (context.backup) context.backupManager.inspect(availableBackup(context, 'final_backup', context.backup)); + hostExecutor.settleRemoved(context.plan, context.claim, guard(context)); + return; + } + hostExecutor.inspectInactive(context.plan, context.claim); + return; + } + if (context.operation === 'destroy') return hostExecutor.verifyDestroyed(context.plan, context.claim); + fail('runtime_boundary_violation'); + } + + async function compensate(context) { + const restart = authority.desiredRuntimeState(context.claim) === 'active'; + if (context.operation === 'upgrade') { + if (!context.stageReceipts?.__compensationRestored) { + hostExecutor.rollbackStopped(context.target.plan, context.claim, guard(context)); + hostExecutor.inspectInactive(context.plan, context.claim); + if (context.stageReceipts?.upgrade_backup?.status === 'snapshot') { + const backup = availableBackup(context, 'upgrade_backup', context.backup); + context.backupManager.restore(backup, `rollback_${context.job.id}`, guard(context)); + context.backupManager.inspectRestored(backup); + } + if (typeof authority.checkpointCompensationRestore !== 'function') fail('upgrade_rollback_required'); + authority.checkpointCompensationRestore(context.claim); + context.stageReceipts.__compensationRestored = true; + } + if (restart) { + if (typeof hostExecutor.startPrior === 'function') { + hostExecutor.startPrior(context.target.plan, context.claim, guard(context)); + hostExecutor.health(context.plan, context.claim); + } else start(context); + if (!context.withoutPaycom && context.startingState === 'ready' && typeof context.stageReceipts?.inspect_schedule?.syncWasRunning === 'boolean') { + await runtime(context).restoreSchedule(scheduleIntent(context)); + } + } else hostExecutor.inspectInactive(context.plan, context.claim); + return; + } + if (['backup', 'suspend'].includes(context.operation) && context.startingState !== 'suspended') { + if (restart) { + start(context); + if (!context.withoutPaycom && context.startingState === 'ready' && typeof context.stageReceipts?.inspect_schedule?.syncWasRunning === 'boolean') { + await runtime(context).restoreSchedule(scheduleIntent(context)); + } + } else stop(context); + return; + } + if (context.operation === 'resume') { + hostExecutor.stop(context.plan, context.claim, guard(context)); + hostExecutor.inspectInactive(context.plan, context.claim); + return; + } + if (context.operation === 'restore' && context.stageReceipts?.safety_snapshot?.status === 'snapshot') { + const safety = availableBackup(context, 'safety_snapshot', context.safetyBackup); + context.backupManager.restore(safety, `compensate_${context.job.id}`, guard(context)); + context.backupManager.inspectRestored(safety); + hostExecutor.inspectInactive(context.plan, context.claim); + return; + } + if (context.operation === 'decommission') { + hostExecutor.stop(context.plan, context.claim, guard(context)); + hostExecutor.disable(context.plan, context.claim, guard(context)); + hostExecutor.inspectInactive(context.plan, context.claim); + } + } + + function unallocated(claimed) { + return (['decommission', 'destroy'].includes(claimed.operation) || claimed.operation === 'resume' && claimed.removal?.installation_state === 'pending') + && typeof adapter.inspectUnallocated === 'function' + && adapter.inspectUnallocated(claimed.manifest, claimed.manifestAuthority, + { fixture: false, claim: claimed.claim }); + } + + async function removeUnallocated(claimed) { + if (claimed.operation === 'destroy') await offsitePolicy.waitForDspBackupDeletion(claimed.job.id, + claimed.manifest.organization.id, claimed.manifest.runtime.key, () => authority.renew(claimed.claim), { required: claimed.backend === 'native_service_v1' }); + // A never-allocated DSP has nothing to back up or retire. Ask the protected + // registry again before completion; never create an account merely to remove it. + const receipts = { + verify_unallocated: { status: 'absent' }, + inspect_schedule: { status: 'verified', syncWasRunning: false }, + quiesce_schedule: { status: 'stopped', syncWasRunning: false }, + stop_runtime: { status: 'absent' }, final_backup: { status: 'absent' }, + disable_runtime: { status: 'absent' }, remove_services: { status: 'absent' }, + verify_retained: { status: 'absent' }, destroy_runtime: { status: 'destroyed', changed: false }, + verify_destroyed: { status: 'absent' }, + }; + for (const stage of claimed.stages.slice(claimed.nextStage)) { + authority.renew(claimed.claim); + if (!unallocated(claimed) || !receipts[stage]) fail('runtime_boundary_violation'); + authority.checkpoint(claimed.claim, stage, receipts[stage]); + } + if (!unallocated(claimed)) fail('runtime_boundary_violation'); + return authority.succeed(claimed.claim); + } + + async function run(jobId, workerId) { + const claimed = authority.claim(jobId, workerId); + let context; + try { + if (unallocated(claimed)) return await removeUnallocated(claimed); + context = runtimeContext(claimed); + if (context.operation === 'upgrade' && context.nextStage === 0 && !context.stageReceipts?.__compensating) offsitePolicy.assertOffsiteReady(); + if (context.stageReceipts?.__compensating) fail(context.stageReceipts.__compensationFailure || 'installation_operation_failed'); + for (let index = context.nextStage; index < context.stages.length; index += 1) { + authority.renew(context.claim); + const stage = context.stages[index]; + const finishStage = authority.startStage?.(context.claim, stage) || (() => {}); + let receipt; + try { receipt = await executeStage(context, stage); finishStage(); } + catch (error) { finishStage(error); throw error; } + authority.checkpoint(context.claim, stage, receipt); + context.stageReceipts[stage] = receipt; + } + await finalVerify(context); + return authority.succeed(context.claim); + } catch (error) { + if (error?.code === 'installation_operation_in_progress') throw error; + if (!context) return authority.failed(claimed.claim, error); + try { + if (typeof authority.beginCompensation === 'function') authority.beginCompensation(context.claim, error); + if (!context.stageReceipts?.__compensated) { + await compensate(context); + if (typeof authority.completeCompensation === 'function') authority.completeCompensation(context.claim); + } + if (context.operation === 'upgrade') hostExecutor.settleRollback(context.target.plan, context.claim, guard(context)); + } + catch (rollbackError) { + if (rollbackError?.code === 'installation_operation_in_progress') throw rollbackError; + error = Object.assign(new Error(context.operation === 'upgrade' + ? 'upgrade_rollback_required' : 'lifecycle_compensation_failed'), { + code: context.operation === 'upgrade' ? 'upgrade_rollback_required' : 'lifecycle_compensation_failed', + }); + } + return authority.failed(context.claim, error); + } + } + + return Object.freeze({ run }); +} + +module.exports = { createOciInstallationLifecycle }; diff --git a/core/core/installations/src/oci-protected-client.js b/core/core/installations/src/oci-protected-client.js new file mode 100644 index 0000000..48bb85a --- /dev/null +++ b/core/core/installations/src/oci-protected-client.js @@ -0,0 +1,30 @@ +'use strict'; + +const { spawnSync } = require('node:child_process'); +const { createOciHostHelperClient } = require('./oci-host-helper-client'); +const { ISSUER_COMMAND } = require('./oci-host-issuer'); + +// This client runs as the trusted Access Control Unix identity. Only the +// separate host caller has sudo permission for the execution helper. +function createProtectedOciHostClient({ dispatchRequest }) { + if (typeof dispatchRequest !== 'function') throw new TypeError('runtime_boundary_violation'); + return createOciHostHelperClient({ requestPort: request => dispatchRequest(request, lease => { + const input = `${JSON.stringify({ version: 1, lease, request })}\n`; + if (Buffer.byteLength(input) > 256 * 1024) throw new Error('runtime_boundary_violation'); + const result = spawnSync('/usr/bin/sudo', ['-n', ISSUER_COMMAND], { + input, encoding: 'utf8', timeout: 650_000, maxBuffer: 256 * 1024, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' }, + }); + let response; + if (!result.error && !result.signal && typeof result.stdout === 'string' + && result.stdout.endsWith('\n') && !result.stdout.slice(0, -1).includes('\n')) { + try { response = JSON.parse(result.stdout); } catch {} + } + if (result.status !== 0 || !response?.ok || Object.keys(response).sort().join(',') !== 'ok,result') { + const code = response?.status || 'service_installation_failed'; + throw Object.assign(new Error(code), { code, hostStep: response?.step }); + } + return response.result; + }) }); +} +module.exports = { createProtectedOciHostClient }; diff --git a/core/core/installations/src/oci-runtime-agent-credential.js b/core/core/installations/src/oci-runtime-agent-credential.js new file mode 100644 index 0000000..4b68924 --- /dev/null +++ b/core/core/installations/src/oci-runtime-agent-credential.js @@ -0,0 +1,102 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { INSTALLATION_IDENTIFIER_RE } = require('../../../shared/contracts/src/installation'); +const { registrationToken } = require('../../../shared/agent/protocol'); + +function fail(code = 'runtime_boundary_violation') { + throw Object.assign(new Error(code), { code }); +} + +function fsync(target, directory = false) { + const flags = fs.constants.O_RDONLY | (directory ? fs.constants.O_DIRECTORY : 0); + const handle = fs.openSync(target, flags); + try { fs.fsyncSync(handle); } finally { fs.closeSync(handle); } +} + +function createOciRuntimeAgentCredentialPort({ credentialRoot } = {}) { + if (typeof credentialRoot !== 'string' || !path.isAbsolute(credentialRoot) + || path.resolve(credentialRoot) !== credentialRoot || /[\0\r\n]/.test(credentialRoot)) fail(); + const root = fs.lstatSync(credentialRoot); + if (!root.isDirectory() || root.isSymbolicLink() || root.uid !== process.geteuid() + || (root.mode & 0o7777) !== 0o700 || fs.realpathSync(credentialRoot) !== credentialRoot) fail(); + const identity = Object.freeze({ dev: root.dev, ino: root.ino }); + + function checkedRoot() { + const current = fs.lstatSync(credentialRoot); + if (!current.isDirectory() || current.isSymbolicLink() || current.uid !== process.geteuid() + || (current.mode & 0o7777) !== 0o700 || fs.realpathSync(credentialRoot) !== credentialRoot + || current.dev !== identity.dev || current.ino !== identity.ino) fail(); + } + + function file(runtimeKey) { + if (typeof runtimeKey !== 'string' || runtimeKey === 'local' || !INSTALLATION_IDENTIFIER_RE.test(runtimeKey)) fail(); + return path.join(credentialRoot, `${runtimeKey}.token`); + } + + function read(runtimeKey) { + checkedRoot(); + const selected = file(runtimeKey); + const info = fs.lstatSync(selected); + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() || info.nlink !== 1 + || info.dev !== identity.dev || (info.mode & 0o7777) !== 0o600 + || fs.realpathSync(selected) !== selected || info.size < 32 || info.size > 256) fail(); + const raw = fs.readFileSync(selected, 'utf8'); + if (!raw.endsWith('\n') || raw.slice(0, -1).includes('\n') || raw.includes('\r') || raw.includes('\0')) fail(); + return registrationToken(raw.slice(0, -1)); + } + + function issue(runtimeKey, { rotate = false } = {}) { + if (typeof rotate !== 'boolean') fail(); + checkedRoot(); + const selected = file(runtimeKey); + let token; + let tokenChanged = false; + try { + token = read(runtimeKey); + if (rotate) token = null; + } catch (error) { + if (error?.code !== 'ENOENT') throw error; + token = null; + } + if (token === null) { + token = crypto.randomBytes(32).toString('base64url'); + const temporary = path.join(credentialRoot, `.${runtimeKey}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`); + try { + fs.writeFileSync(temporary, `${token}\n`, { mode: 0o600, flag: 'wx' }); + fsync(temporary); + fs.renameSync(temporary, selected); + fsync(credentialRoot, true); + } finally { try { fs.rmSync(temporary, { force: true }); } catch {} } + tokenChanged = true; + } + token = read(runtimeKey); + return Object.freeze({ + runtimeKey, + tokenHash: crypto.createHash('sha256').update(token, 'utf8').digest('hex'), + changed: tokenChanged, + tokenChanged, + }); + } + + function revoke(runtimeKey, expectedTokenHash = null) { + if (expectedTokenHash !== null && !/^[a-f0-9]{64}$/.test(expectedTokenHash)) fail(); + const selected = file(runtimeKey); + let token; + try { token = read(runtimeKey); } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } + const digest = crypto.createHash('sha256').update(token, 'utf8').digest('hex'); + if (expectedTokenHash !== null && digest !== expectedTokenHash) return false; + fs.unlinkSync(selected); + fsync(credentialRoot, true); + return true; + } + + return Object.freeze({ issue, revoke, read }); +} + +module.exports = { createOciRuntimeAgentCredentialPort }; diff --git a/core/core/installations/src/oci-runtime-lifecycle-port.js b/core/core/installations/src/oci-runtime-lifecycle-port.js new file mode 100644 index 0000000..54d1046 --- /dev/null +++ b/core/core/installations/src/oci-runtime-lifecycle-port.js @@ -0,0 +1,90 @@ +'use strict'; + +const { PAYCOM_SYNC_ID } = require('../../../shared/paycom-activation'); + +function fail(code = 'runtime_health_failed') { + throw Object.assign(new Error(code), { code }); +} + +function successful(value, statuses, code = 'runtime_health_failed') { + if (!value?.ok || !statuses.includes(value.status)) fail(code); + return value; +} + +function createOciRuntimeLifecyclePort(options) { + if (!options || typeof options !== 'object' || Array.isArray(options) + || Object.keys(options).some(key => !['client', 'clock'].includes(key)) + || !options.client?.sync || typeof options.client.sync.status !== 'function' + || typeof options.client.sync.stop !== 'function' || typeof options.client.sync.start !== 'function' + || !options.client?.collections || typeof options.client.collections.health !== 'function' + || !options.client?.workforce || typeof options.client.workforce.day !== 'function' + || !options.client?.system || typeof options.client.system.status !== 'function' + || typeof options.client.health !== 'function') fail('runtime_boundary_violation'); + const client = options.client; + const clock = options.clock || Date.now; + if (typeof clock !== 'function') fail('runtime_boundary_violation'); + + async function inspectSchedule() { + const result = successful(await client.sync.status(PAYCOM_SYNC_ID), ['found']); + if (!['running', 'stopped'].includes(result.data?.desiredState)) fail(); + return Object.freeze({ syncWasRunning: result.data.desiredState === 'running' }); + } + + async function quiesceSchedule(syncWasRunning) { + if (typeof syncWasRunning !== 'boolean') fail('runtime_boundary_violation'); + const before = successful(await client.sync.status(PAYCOM_SYNC_ID), ['found']); + if (before.data?.desiredState === 'running') { + const stopped = successful(await client.sync.stop(PAYCOM_SYNC_ID, { drain: true, waitMs: 120_000 }), ['stopped']); + if (stopped.data?.desiredState !== 'stopped' || stopped.data?.activity !== 'idle' + || stopped.data?.activeRun !== null || stopped.data?.queuedRunCount !== 0) fail(); + } else if (before.data?.desiredState !== 'stopped') fail(); + const manager = successful(await client.collections.health(), ['ready']); + if (manager.data?.counts?.queued !== 0 || manager.data?.counts?.running !== 0) fail(); + return Object.freeze({ syncWasRunning }); + } + + async function restoreSchedule(syncWasRunning) { + if (typeof syncWasRunning !== 'boolean') fail('runtime_boundary_violation'); + const before = successful(await client.sync.status(PAYCOM_SYNC_ID), ['found']); + if (syncWasRunning && before.data?.desiredState === 'stopped') { + const started = successful(await client.sync.start(PAYCOM_SYNC_ID), ['started']); + if (started.data?.sync?.desiredState !== 'running') fail(); + } else if (!syncWasRunning && before.data?.desiredState === 'running') { + const stopped = successful(await client.sync.stop(PAYCOM_SYNC_ID, { drain: true, waitMs: 120_000 }), ['stopped']); + if (stopped.data?.desiredState !== 'stopped' || stopped.data?.activity !== 'idle' + || stopped.data?.activeRun !== null || stopped.data?.queuedRunCount !== 0) fail(); + } else if (before.data?.desiredState !== (syncWasRunning ? 'running' : 'stopped')) fail(); + return Object.freeze({ syncWasRunning }); + } + + async function verifyInfrastructure() { + successful(await client.health(), ['ready']); + successful(await client.system.status(), ['ready', 'degraded']); + return Object.freeze({ status: 'verified' }); + } + + async function verifyPublication(priorEvidence, currentTarget = priorEvidence?.target) { + if (!priorEvidence || typeof priorEvidence !== 'object' || typeof priorEvidence.target !== 'string') { + fail('first_publication_failed'); + } + await verifyInfrastructure(); + successful(await client.workforce.day({ date: currentTarget, limit: 1, offset: 0 }), ['found']); + return Object.freeze({ + definitionDigest: priorEvidence.definitionDigest, + requestDigest: priorEvidence.requestDigest, + previewDigest: priorEvidence.previewDigest, + batchId: priorEvidence.batchId, + preparationRunId: priorEvidence.preparationRunId, + target: priorEvidence.target, + runs: priorEvidence.runs, + publications: priorEvidence.publications, + capturedAt: new Date(clock()).toISOString(), + }); + } + + return Object.freeze({ + inspectSchedule, quiesceSchedule, restoreSchedule, verifyInfrastructure, verifyPublication, + }); +} + +module.exports = { createOciRuntimeLifecyclePort }; diff --git a/core/core/installations/src/offsite-backup.js b/core/core/installations/src/offsite-backup.js new file mode 100644 index 0000000..2346aab --- /dev/null +++ b/core/core/installations/src/offsite-backup.js @@ -0,0 +1,137 @@ +'use strict'; +// Root-side exporter. Only completed, consistent snapshots are eligible. Restic +// encrypts before uploading; every new upload is downloaded and restored first. +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { spawnSync } = require('node:child_process'); +const { DatabaseSync } = require('node:sqlite'); +const { privateJson, atomic } = require('./release-delivery-files'); +const { receiptKey, RECEIPTS } = require('./offsite-policy'); +const CONFIG = '/etc/dispatch/offsite-backup.json'; +const WORK = '/var/lib/dispatch-backup'; +const fail = code => { throw Object.assign(new Error(code), { code }); }; +const digest = value => crypto.createHash('sha256').update(value).digest('hex'); +function checkPath(file, uid, directory = false) { + const stat = fs.lstatSync(file); + if (stat.uid !== uid || stat.isSymbolicLink() || (stat.mode & 0o077) || fs.realpathSync(file) !== file + || (directory ? !stat.isDirectory() : !stat.isFile() || stat.nlink !== 1 || stat.size > 2 * 1024 ** 3)) fail('unsafe_backup_storage'); + return stat; +} +function loadConfig(file = CONFIG) { + const c = privateJson(file, 0); + if (c.schemaVersion !== 1 || !/^[a-f0-9]{32}$/.test(c.accountId) + || !/^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/.test(c.bucket) + || !/^[a-z][a-z0-9_-]{2,63}$/.test(c.prefix) + || !path.isAbsolute(c.localRoot || '') || fs.realpathSync(c.localRoot) !== c.localRoot + || !Number.isSafeInteger(c.coreUid) || c.coreUid < 1 || c.retention !== 'retain-all') fail('offsite_config_invalid'); + const credentials = privateJson('/etc/dispatch/offsite-backup-credentials.json', 0); + if (!/^[a-f0-9]{32}$/.test(credentials.accessKeyId) || !/^[a-f0-9]{64}$/.test(credentials.secretAccessKey)) fail('offsite_config_invalid'); + const password = '/etc/dispatch/offsite-backup-password'; + const stat = checkPath(password, 0); + if (stat.size < 32 || stat.size > 4096) fail('offsite_config_invalid'); + return { ...c, environment: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8', + AWS_ACCESS_KEY_ID: credentials.accessKeyId, AWS_SECRET_ACCESS_KEY: credentials.secretAccessKey, + AWS_DEFAULT_REGION: 'auto', RESTIC_PASSWORD_FILE: password, + RESTIC_REPOSITORY: `s3:https://${c.accountId}.r2.cloudflarestorage.com/${c.bucket}/${c.prefix}` } }; +} +function createRestic(environment, { binary = '/usr/bin/restic', timeout = 240_000 } = {}) { + return (args, cwd) => { + const result = spawnSync(binary, ['--no-cache', '--json', ...args], { cwd, env: environment, + encoding: 'utf8', timeout, maxBuffer: 2 * 1024 * 1024, input: '' }); + // Do not forward output/errors that could include object names or credentials. + if (result.status !== 0 || result.error || result.signal) fail('offsite_transfer_failed'); + return result.stdout.trim().split('\n').filter(Boolean).map(line => { try { return JSON.parse(line); } catch { return null; } }); + }; +} +function tree(root, uid, destination) { + checkPath(root, uid, true); + const entries = []; let size = 0; + function visit(directory, relative, to) { + if (relative.split('/').length > 64 || entries.length > 100000) fail('backup_too_large'); + const beforeDir = checkPath(directory, uid, true); + if (to) fs.mkdirSync(to, { mode: 0o700 }); + for (const name of fs.readdirSync(directory).sort()) { + const file = path.join(directory, name), rel = relative ? relative + '/' + name : name; + const stat = fs.lstatSync(file); + if (stat.isDirectory()) { entries.push({ path: rel, type: 'directory' }); visit(file, rel, to && path.join(to, name)); continue; } + checkPath(file, uid); size += stat.size; + if (size > 8 * 1024 ** 3 || entries.length >= 100000) fail('backup_too_large'); + const fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + let out; const hash = crypto.createHash('sha256'); let bytes = 0; + try { + const opened = fs.fstatSync(fd); + if (opened.ino !== stat.ino || opened.dev !== stat.dev || opened.uid !== uid || opened.nlink !== 1) fail('unsafe_backup_storage'); + if (to) out = fs.openSync(path.join(to, name), 'wx', 0o600); + const buffer = Buffer.alloc(1024 * 1024); let count; + while ((count = fs.readSync(fd, buffer, 0, buffer.length, null))) { + hash.update(buffer.subarray(0, count)); bytes += count; + if (bytes > stat.size) fail('backup_changed'); + if (out !== undefined) { let written = 0; while (written < count) written += fs.writeSync(out, buffer, written, count - written); } + } + const after = fs.fstatSync(fd); + if (bytes !== stat.size || after.mtimeMs !== stat.mtimeMs || after.ino !== stat.ino) fail('backup_changed'); + if (out !== undefined) fs.fsyncSync(out); + } finally { fs.closeSync(fd); if (out !== undefined) fs.closeSync(out); } + entries.push({ path: rel, type: 'file', size: bytes, sha256: hash.digest('hex') }); + } + const afterDir = checkPath(directory, uid, true); + if (afterDir.ino !== beforeDir.ino || afterDir.mtimeMs !== beforeDir.mtimeMs) fail('backup_changed'); + } + visit(root, '', destination); + entries.sort((a, b) => a.path.localeCompare(b.path)); + return { entries, size, digest: digest(JSON.stringify(entries)) }; +} +function verifySnapshot(directory, uid) { + const scanned = tree(directory, uid); + const manifestFile = path.join(directory, 'manifest.json'); + if (checkPath(manifestFile, uid).size > 32 * 1024 * 1024) fail('backup_too_large'); + const manifest = JSON.parse(fs.readFileSync(manifestFile)); + if (manifest.kind === 'core' && [1,3].includes(manifest.version)) { + const file = scanned.entries.find(entry => entry.path === 'access-control-before.sqlite3'); + if (!file || file.sha256 !== manifest.sha256 || file.size !== manifest.size) fail('backup_corrupt'); + const db = new DatabaseSync(path.join(directory, file.path), { readOnly: true }); + try { if(manifest.scope==='core') require('../../accounts/src/core-backup').verifyCoreDatabase(db); if (db.prepare('PRAGMA quick_check').get().quick_check !== 'ok' || db.prepare('PRAGMA foreign_key_check').all().length) fail('backup_corrupt'); } + finally { db.close(); } + if(manifest.version===3){const entries=scanned.entries.filter(e=>e.path!=='manifest.json');if(manifest.scope!=='core'||JSON.stringify(entries)!==JSON.stringify(manifest.entries)||digest(JSON.stringify(entries))!==manifest.treeDigest)fail('backup_corrupt');return {digest:manifest.treeDigest,tree:scanned};} + return { digest: manifest.sha256, tree: scanned }; + } + if (![1, 2].includes(manifest.version) || !Array.isArray(manifest.entries)) fail('backup_corrupt'); + const entries = scanned.entries.filter(e => e.path.startsWith('payload/') && !['payload/data', 'payload/state', ...(manifest.version === 2 ? ['payload/config', 'payload/auth-secrets'] : [])].includes(e.path)) + .map(e => ({ ...e, path: e.path.slice(8) })); + if (JSON.stringify(entries) !== JSON.stringify(manifest.entries) || digest(JSON.stringify(entries)) !== manifest.treeDigest) fail('backup_corrupt'); + return { digest: manifest.treeDigest, tree: scanned }; +} +function exportSnapshot({ source, uid, workRoot = WORK, receiptRoot = RECEIPTS, run, config = null, + recoveryCapture = config ? require('./host-recovery-bundle').captureHostRecovery : null }) { + checkPath(workRoot, process.geteuid(), true); + const before = verifySnapshot(source, uid); + const receiptFile = path.join(receiptRoot, receiptKey(source) + '.json'); + // Receipts contain only opaque hashes and time, so Core can check them without R2 keys. + const existing = require('./offsite-policy').publicRootJson(receiptFile, true, process.geteuid()); + if (existing?.status === 'verified' && existing.digest === before.digest && (!recoveryCapture || existing.recoveryDigest)) return existing; + const space = fs.statfsSync(workRoot); + if (space.bavail * space.bsize < before.tree.size * 2 + 64 * 1024 * 1024) fail('offsite_backup_space_unavailable'); + const work = fs.mkdtempSync(path.join(workRoot, 'transfer-')); fs.chmodSync(work, 0o700); + try { + const copied = tree(source, uid, path.join(work, 'snapshot')); + if (copied.digest !== before.tree.digest) fail('backup_changed'); + const isCore = JSON.parse(fs.readFileSync(path.join(source, 'manifest.json'))).kind === 'core'; + // Managed DSP archives are exported by backup-archives with their metadata. + if (recoveryCapture && !isCore) fail('backup_metadata_missing'); + const recovery = recoveryCapture ? recoveryCapture({ config, destination: path.join(work, 'recovery'), + snapshotSource: source, kind: 'core' }) : null; + if (recovery) atomic(path.join(work, 'recovery-proof.json'), recovery); + const lines = run(['backup', '--host', 'dispatch', '--tag', receiptKey(source), '--', 'snapshot', + ...(recovery ? ['recovery', 'recovery-proof.json'] : [])], work); + const snapshotId = lines.find(line => line?.message_type === 'summary')?.snapshot_id; + if (!/^[a-f0-9]{64}$/.test(snapshotId)) fail('offsite_transfer_failed'); + const receipt = { schemaVersion: 1, status: 'verified', digest: before.digest, verification: 'upload', snapshotId, verifiedAt: Date.now(), + ...(recovery ? { recoveryDigest: recovery.sha256, organizationIds: recovery.organizationIds, + ...(recovery.organizationInventoryVersion === 1 ? { organizationInventoryVersion: 1 } : {}) } : {}) }; + atomic(receiptFile, receipt, 0o644); + fs.chmodSync(receiptFile, 0o644); + return receipt; + } finally { fs.rmSync(work, { recursive: true, force: true }); } +} +module.exports = { CONFIG, WORK, loadConfig, createRestic, tree, verifySnapshot, exportSnapshot }; diff --git a/core/core/installations/src/offsite-policy.js b/core/core/installations/src/offsite-policy.js new file mode 100644 index 0000000..4ee432a --- /dev/null +++ b/core/core/installations/src/offsite-policy.js @@ -0,0 +1,63 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const POLICY = '/etc/dispatch/offsite-backup-policy.json'; +const RECEIPTS = '/var/lib/dispatch-backup-receipts'; +const fail = code => { throw Object.assign(new Error(code), { code }); }; +function publicRootJson(file, optional = false, uid = 0, maxBytes = 16384) { + let stat; + try { stat = fs.lstatSync(file); } catch (error) { if (optional && error.code === 'ENOENT') return null; throw error; } + if (!stat.isFile() || stat.uid !== uid || stat.nlink !== 1 || (stat.mode & 0o022) || stat.size > maxBytes + || fs.realpathSync(file) !== file) fail('offsite_backup_unavailable'); + return JSON.parse(fs.readFileSync(file)); +} +function receiptKey(directory) { return crypto.createHash('sha256').update(directory).digest('hex'); } +function hasVerifiedReceipt(directory, digest, { root = RECEIPTS, uid = 0, recoveryRequired = false } = {}) { + const receipt = publicRootJson(path.join(root, receiptKey(directory) + '.json'), true, uid); + return Boolean(receipt && receipt.schemaVersion === 1 && receipt.status === 'verified' + && receipt.digest === digest && /^[a-f0-9]{64}$/.test(receipt.snapshotId) + && Number.isSafeInteger(receipt.verifiedAt) && receipt.verifiedAt > 0 + && (!recoveryRequired || /^[a-f0-9]{64}$/.test(receipt.recoveryDigest))); +} +function createOffsitePolicy({ policyFile = POLICY, receiptRoot = RECEIPTS, uid = 0, + clock = Date.now, sleep = ms => new Promise(resolve => setTimeout(resolve, ms)) } = {}) { + function offsiteRequired() { + const policy = publicRootJson(policyFile, true, uid); + if (!policy) return false; + if (policy.schemaVersion !== 1 || policy.required !== true) fail('offsite_backup_unavailable'); + return true; + } + function assertOffsiteReady() { + if (!offsiteRequired()) return; + const status = publicRootJson(path.join(receiptRoot, 'status.json'), true, uid); + if (!status || status.status !== 'verified' || !Number.isSafeInteger(status.checkedAt) + || clock() - status.checkedAt > 300_000 || status.checkedAt > clock() + 5000) fail('offsite_backup_unavailable'); + } + async function waitForOffsiteBackup(directory, digest, renew = () => {}, { required = false, recoveryRequired = false } = {}) { + if (!required && !offsiteRequired()) return; + if (!/^[a-f0-9]{64}$/.test(digest)) fail('offsite_backup_unavailable'); + require('./worker-notify').exportReady(); + const deadline = clock() + (recoveryRequired ? 3600000 : 300000); + while (!hasVerifiedReceipt(directory, digest, { root: receiptRoot, uid, recoveryRequired })) { + renew(); + if (clock() >= deadline) fail('offsite_backup_unavailable'); + await sleep(2000); + } + } + async function waitForDspBackupDeletion(jobId, organizationId, runtimeKey, renew = () => {}, { required = false } = {}) { + if (!required && !offsiteRequired()) return; + if (!/^[a-z][a-z0-9_-]{2,95}$/.test(jobId)) fail('offsite_backup_unavailable'); + const deadline = clock() + 3_600_000; + while (true) { + const proof = publicRootJson(path.join(receiptRoot, `deleted-${jobId}.json`), true, uid); + if (proof?.schemaVersion === 1 && proof.status === 'destroyed' && proof.jobId === jobId + && proof.organizationId === organizationId && proof.runtimeKey === runtimeKey) return; + renew(); + if (clock() >= deadline) fail('offsite_backup_unavailable'); + await sleep(2000); + } + } + return { offsiteRequired, assertOffsiteReady, waitForOffsiteBackup, waitForDspBackupDeletion }; +} +module.exports = { ...createOffsitePolicy(), createOffsitePolicy, hasVerifiedReceipt, receiptKey, publicRootJson, RECEIPTS, POLICY }; diff --git a/core/core/installations/src/operation-timing.js b/core/core/installations/src/operation-timing.js new file mode 100644 index 0000000..fd6f883 --- /dev/null +++ b/core/core/installations/src/operation-timing.js @@ -0,0 +1,39 @@ +'use strict'; +const crypto = require('node:crypto'); +const initialized = new WeakSet(); +function initialize(db) { + if (initialized.has(db)) return; + db.exec(`CREATE TABLE IF NOT EXISTS operation_stage_timings ( + id TEXT PRIMARY KEY, job_id TEXT NOT NULL, attempt INTEGER NOT NULL, + stage TEXT NOT NULL, started_at INTEGER NOT NULL, finished_at INTEGER, + duration_ms INTEGER, status TEXT NOT NULL, failure_code TEXT + ) STRICT; CREATE INDEX IF NOT EXISTS operation_stage_timings_job ON operation_stage_timings(job_id, started_at)`); + initialized.add(db); +} +function start(database, { jobId, attempt, stage }, clock = Date.now) { + const getDb = typeof database === 'function' ? database : () => database; + const db = getDb(); + if (!db) return () => {}; + if (!/^[a-z][a-z0-9_-]{2,95}$/.test(jobId) || !Number.isSafeInteger(attempt) || attempt < 0 + || !/^[a-z][a-z0-9_]{1,63}$/.test(stage)) throw Error('invalid_operation_timing'); + initialize(db); + const id = crypto.randomUUID(), startedAt = clock(); + db.prepare("INSERT INTO operation_stage_timings VALUES(?,?,?,?,?,NULL,NULL,'running',NULL)").run(id, jobId, attempt, stage, startedAt); + return error => { + const endedAt = clock(); + const code = error ? require('../../../shared/contracts/src').installationFailure(error).code : null; + getDb().prepare('UPDATE operation_stage_timings SET finished_at=?,duration_ms=?,status=?,failure_code=? WHERE id=?') + .run(endedAt, Math.max(0, endedAt - startedAt), error ? 'failed' : 'succeeded', code, id); + }; +} +function wait(db, jobId, reason, waiting, clock = Date.now) { + if (!/^[a-z][a-z0-9_-]{2,95}$/.test(jobId) || !/^[a-z][a-z0-9_]{1,63}$/.test(reason)) throw Error('invalid_operation_timing'); + initialize(db); + const row = db.prepare("SELECT id,started_at FROM operation_stage_timings WHERE job_id=? AND stage=? AND status='waiting'").get(jobId, reason); + if (waiting && !row) db.prepare("INSERT INTO operation_stage_timings VALUES(?,?,0,?,?,NULL,NULL,'waiting',NULL)").run(crypto.randomUUID(), jobId, reason, clock()); + if (!waiting && row) { + const endedAt = clock(); + db.prepare("UPDATE operation_stage_timings SET finished_at=?,duration_ms=?,status='succeeded' WHERE id=?").run(endedAt, Math.max(0, endedAt-row.started_at), row.id); + } +} +module.exports = { initialize, start, wait }; diff --git a/core/core/installations/src/owner-onboarding.js b/core/core/installations/src/owner-onboarding.js new file mode 100644 index 0000000..79b4a73 --- /dev/null +++ b/core/core/installations/src/owner-onboarding.js @@ -0,0 +1,91 @@ +'use strict'; + +const { createOnboardingStore } = require('../../accounts/src/onboarding-store'); +const { managedInstallationContext } = require('../../accounts/src/installation-authority'); +const crypto = require('node:crypto'); +const { MAX_PROVIDER_EVIDENCE_AGE_MS } = require('../../accounts/src/installation-activation'); +const { providerEvidence } = require('./activation'); +const { setupFailure } = require('../../../shared/contracts/src/paycom-setup'); +function fail(code = 'installation_not_ready') { throw Object.assign(new Error(code), { code }); } +function createOwnerOnboardingWorker({ store, invoke, backends = ['oci_container_v1', 'native_service_v1'], clock = Date.now, + testProvider = null, + delay = ms => new Promise(resolve => setTimeout(resolve, ms)) }) { + const requests = createOnboardingStore(store, clock); + async function run(id, workerId) { + const pending = requests.get(id); + if (!pending || !backends.includes(store.installationBackend(pending.organization_id))) fail(); + const row = requests.claim(id, workerId); + try { + const selected = managedInstallationContext(store, row.organization_id); + function guard() { + if (!require('../../accounts/src/plugins').available(store, row.organization_id, 'paycom')) fail('plugin_disabled'); + const current = managedInstallationContext(store, row.organization_id); + if (!backends.includes(current.backend) + || !['ready', 'waiting_for_provider_auth'].includes(current.installation.status) + || !['active', 'setup_required'].includes(current.organization.status) + || !current.ownerActive + || current.manifest.revision !== row.manifest_revision + || JSON.stringify(current.manifest) !== JSON.stringify(selected.manifest) + || current.installation.revision !== selected.installation.revision + || store.activeLifecycleJob(row.organization_id) + || store.runningActivationJob(row.organization_id) + || store.db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(row.organization_id)) fail(); + requests.renew(row); + } + // A claimed retry gets a fresh identity; polling keeps that identity stable. + const requestId = `setup_${crypto.createHash('sha256').update(`${row.id}:${row.fence}`).digest('hex').slice(0, 32)}`; + const deadline = clock() + 180_000; + let command = 'start'; + let step = 'test'; + for (;;) { + guard(); + const result = step === 'test' && testProvider ? await testProvider(selected.manifest.runtime.key) + : await invoke(selected.manifest.runtime.key, 'paycom.setup', { + command, requestId, step, manifest: selected.manifest, + manifestAuthority: selected.manifestAuthority, parameters: {}, + }); + guard(); + if (!result?.ok && result?.status === 'execution_capacity_wait' && selected.backend === 'directory_service_v1') { + requests.defer(row); + return { status: 'queued' }; + } + if (!result?.ok) fail(setupFailure(result?.status)); + if (result.status === 'succeeded') { + if (step === 'sync') { + if (result.data?.syncId !== 'paycom-main-workforce' || result.data.intervalSeconds !== 3600 + || result.data.desiredState !== 'running') fail('runtime_health_failed'); + requests.finish(row); + return { status: 'succeeded' }; + } + const evidence = providerEvidence(result.data); + const testedAt = Date.parse(evidence.testedAt); + if (testedAt > clock() + 60_000 || clock() - testedAt > MAX_PROVIDER_EVIDENCE_AGE_MS) fail('provider_auth_required'); + step = 'sync'; + command = 'start'; + continue; + } + if (result.status !== 'running' || clock() >= deadline) fail('provider_setup_failed'); + command = 'status'; + await delay(1000); + } + } catch (error) { + if (error?.code === 'installation_operation_in_progress') throw error; + requests.finish(row, setupFailure(error?.code)); + return { status: 'failed' }; + } + } + async function runPending(workerId, limit = 20) { + const candidates = requests.candidates(limit, backends); + let completed = 0; + let failed = 0; + for (const [index, row] of candidates.entries()) { + try { + const result = await run(row.id, `${workerId}_${index}`); + if (result.status === 'succeeded') completed += 1; else if (result.status === 'failed') failed += 1; + } catch { failed += 1; } + } + return { processed: candidates.length, completed, failed }; + } + return { run, runPending }; +} +module.exports = { createOwnerOnboardingWorker }; diff --git a/core/core/installations/src/parallel-backup-exports.js b/core/core/installations/src/parallel-backup-exports.js new file mode 100644 index 0000000..39378c2 --- /dev/null +++ b/core/core/installations/src/parallel-backup-exports.js @@ -0,0 +1,46 @@ +'use strict'; +const { Worker } = require('node:worker_threads'); +const path = require('node:path'); +// Each archive has its own repository and receipt. The parent scan retains the +// global deletion lock; only independent uploads run in these bounded workers. +async function parallelBackupExports(rows, { concurrency = 3, execute = upload, discover = async () => [], + pollMs = 500, maxExports = 1000 } = {}) { + if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 3) throw Error('invalid_backup_concurrency'); + const results = new Map(), seen = new Set(), queue = [], active = new Set(); + function admit(items) { + for (const row of items) if (!seen.has(row.id) && seen.size < maxExports) { seen.add(row.id); queue.push(row); } + } + admit(rows); + let discoveryError; + while (true) { + // Refresh even while a slow export runs: idle slots must accept snapshots + // that became ready after this scan started. Deletion stays in the parent. + try { admit(await discover()); } catch (error) { discoveryError = error; } + while (!discoveryError && queue.length && active.size < concurrency) { + const row = queue.shift(); + const task = Promise.resolve().then(() => execute(row.id)).then( + value => results.set(row.id, {ok:true,value}), () => results.set(row.id, {ok:false}) + ).finally(() => active.delete(task)); + active.add(task); + } + if (!active.size) break; + let timer; + await Promise.race([...active, new Promise(resolve => { timer = setTimeout(resolve, pollMs); })]); + clearTimeout(timer); + } + // Never release the parent's deletion lock with orphan upload workers. + await Promise.all(active); + if (discoveryError) throw discoveryError; + return results; +} +function upload(backupId) { + return new Promise((resolve, reject) => { + const worker = new Worker(path.join(__dirname, "./backup-export-worker.js"), { workerData: { backupId }, stdout: true, stderr: true }); + worker.stdout.resume(); worker.stderr.resume(); + let result; + worker.on('message', value => { result = value; }); + worker.on('error', () => reject(Error('backup_upload_failed'))); + worker.on('exit', code => code === 0 && result?.ok ? resolve(result) : reject(Error('backup_upload_failed'))); + }); +} +module.exports = { parallelBackupExports }; diff --git a/core/core/installations/src/platform-backup-worker.js b/core/core/installations/src/platform-backup-worker.js new file mode 100644 index 0000000..65247f6 --- /dev/null +++ b/core/core/installations/src/platform-backup-worker.js @@ -0,0 +1,492 @@ +'use strict'; +const fs = require('node:fs'), + path = require('node:path'); +const { DatabaseSync, backup } = require('node:sqlite'); +const { createPlatformBackups } = require('../../accounts/src/platform-backups'); +const { + createAccessInstallationLifecycleAuthority, +} = require('../../accounts/src/installation-lifecycle'); +const { + restoreDspMetadata, + checkDspMetadata, +} = require('../../accounts/src/backup-metadata'); +const { atomic, hashFileSync } = require('./release-delivery-files'); +function createPlatformBackupWorker({ + store, + localRoot, + archive, + clock = Date.now, + restartCore = null, +}) { + const db = store.db; + const manager = createPlatformBackups({ store, enabled: true, clock, archive }); + const update = (row, status, phase, jobId = null, failure = null) => + db + .prepare( + 'UPDATE platform_backup_requests SET status=?,phase=?,job_id=?,failure_code=?,updated_at=? WHERE id=?', + ) + .run(status, phase, jobId, failure, clock(), row.id); + function queue(row, operation, phase, extra = {}) { + update(row, 'running', phase); + const authority = createAccessInstallationLifecycleAuthority({ + store, + organizationId: row.organization_id, + authorityScope: 'platform_backups', + actorUserId: row.actor_user_id, + clock, + }); + const job = authority.request({ + operation, + idempotencyKey: `${row.id}:${phase}${JSON.parse(row.input_json).attempt ? ':' + JSON.parse(row.input_json).attempt : ''}`, + expectedRevision: store.installationControl(row.organization_id).revision, + ...extra, + }); + update(row, 'running', phase, job.id); + } + function deletionPending() { + return !!db + .prepare( + `SELECT 1 FROM installation_lifecycle_jobs j JOIN installations i ON i.organization_id=j.organization_id + WHERE j.operation='destroy' AND (j.status IN ('queued','running') + OR (j.status='succeeded' AND i.backend='native_service_v1'))`, + ) + .get(); + } + async function coreSnapshot(row) { + const root = path.join(localRoot, 'backups', 'scheduled-core'); + fs.mkdirSync(root, { recursive: true, mode: 0o700 }); + const directory = path.join(root, row.id), + temp = path.join(root, `.creating-${row.id}`); + if (!fs.existsSync(directory)) { + fs.rmSync(temp, { recursive: true, force: true }); + fs.mkdirSync(temp, { mode: 0o700 }); + const file = path.join(temp, 'access-control-before.sqlite3'); + // SQLite backup runs asynchronously. Give it its own connection so other + // DSP requests can commit on the live connection while it copies. + const source = new DatabaseSync(path.join(localRoot, 'data/access-control/access-control.sqlite3'), { readOnly: true }); + try { await backup(source, file); } finally { source.close(); } + fs.chmodSync(file, 0o600); + require('../../accounts/src/core-backup').sanitizeCoreDatabase(file); + const probe = new DatabaseSync(file, { readOnly: true }); + try { + if ( + probe.prepare('PRAGMA quick_check').get().quick_check !== 'ok' || + probe.prepare('PRAGMA foreign_key_check').all().length + ) + throw Error('backup_failed'); + } finally { + probe.close(); + } + require('./core-backup-files').capture(localRoot, path.join(temp, 'core-files')); + const inventory = require('./offsite-backup').tree(temp, process.geteuid()); + atomic(path.join(temp, 'manifest.json'), { + version: 3, + entries: inventory.entries, + treeDigest: inventory.digest, + kind: 'core', + scope: 'core', + sha256: hashFileSync(file), + size: fs.statSync(file).size, + }); + fs.renameSync(temp, directory); + } + store.transaction(() => { + if (deletionPending()) throw Error('backup_waiting_for_deletion'); + const saved = new DatabaseSync(path.join(directory, 'access-control-before.sqlite3'), { + readOnly: true, + }); + try { + if ( + saved + .prepare('SELECT id FROM organizations') + .all() + .some((org) => !store.organization(org.id)) + ) + throw Error('backup_changed'); + } finally { + saved.close(); + } + const input = JSON.parse(row.input_json), + now = row.created_at; + db.prepare( + "INSERT OR IGNORE INTO platform_backup_records VALUES(?,NULL,'core',?,?,?,?,NULL)", + ).run( + row.id, + JSON.stringify({ schemaVersion: 2, scope: 'core', name: 'Platform Core' }), + input.retentionDays, + now, + input.retentionDays === null ? null : now + input.retentionDays * 86400000, + ); + require('../../accounts/src/backup-categories').recordCategory(db, row.id, + require('../../accounts/src/backup-categories').categoryForRequest(row)); + update(row, 'running', 'uploading'); + store.afterCommit?.(() => require('./worker-notify').exportReady(localRoot)); + }); + } + function failure(row, code = 'backup_operation_failed') { + update(row, 'failed', 'failed', row.job_id, code); + } + function advance(row) { + if (db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(row.organization_id)) + return failure(row, 'installation_operation_not_allowed'); + const input = JSON.parse(row.input_json), + control = store.installationControl(row.organization_id); + if (row.phase === 'queued') { + if (!control || store.activeLifecycleJob(row.organization_id)) return; + if (row.kind === 'backup') return queue(row, 'backup', 'backing_up'); + const proof = archive().backups?.[input.backupId]; + if (!proof?.localReady) { + if (clock() - row.created_at > 3600000) failure(row, 'backup_download_timeout'); + return; + } + const record = db + .prepare('SELECT * FROM platform_backup_records WHERE id=?') + .get(input.backupId); + if ( + !record || + proof.status !== 'verified' || + proof.metadataDigest !== + require('node:crypto').createHash('sha256').update(record.metadata_json).digest('hex') + ) + throw Error('backup_unverified'); + checkDspMetadata(store, row.organization_id, JSON.parse(record.metadata_json)); + store.updateOrganizationStatus(row.organization_id, 'suspended', clock()); + return queue( + row, + control.status === 'ready' ? 'suspend' : 'restore', + control.status === 'ready' ? 'stopping' : 'restoring', + control.status === 'ready' ? {} : { backupId: input.backupId }, + ); + } + const job = row.job_id ? store.lifecycleJob(row.job_id) : null; + if (job?.status === 'running' && job.attempt >= job.max_attempts && job.lease_expires_at <= clock()) { + failure(row, 'backup_worker_interrupted'); + return; + } + if (job && ['queued', 'running'].includes(job.status)) return; + if (row.kind === 'core' && input.action === 'restore' && row.phase === 'uploading') { + const proofs = archive().backups || {}, + original = proofs[input.backupId], + safety = proofs[row.id]; + if ( + original?.status !== 'verified' || + !original.localReady || + safety?.status !== 'verified' + ) { + if (clock() - row.updated_at > 3600000) failure(row, 'backup_verification_timeout'); + return; + } + const source = path.join(localRoot, 'backups/scheduled-core', input.backupId); + const rollback = path.join(localRoot, 'backups/scheduled-core', row.id); + const { verifySnapshot } = require('./offsite-backup'); + verifySnapshot(source, process.geteuid()); + verifySnapshot(rollback, process.geteuid()); + const saved = new DatabaseSync(path.join(source, 'access-control-before.sqlite3'), {readOnly:true}); + try { + input.restoredOwnerIds = saved.prepare('SELECT id FROM users').all().filter(owner => !store.userById(owner.id)).map(owner => owner.id); + } finally { saved.close(); } + db.prepare('UPDATE platform_backup_requests SET input_json=? WHERE id=?').run(JSON.stringify(input), row.id); + const restore = (directory, compensate = false) => { + require('./core-backup-files').restore(localRoot, path.join(directory, 'core-files')); + require('../../accounts/src/core-backup').restoreCoreDatabase( + store, + path.join(directory, 'access-control-before.sqlite3'), + clock(), + {removeOwnerIds: compensate ? input.restoredOwnerIds : []}, + ); + }; + try { + restore(source); + update( + row, + restartCore ? 'running' : 'completed', + restartCore ? 'verifying_core' : 'completed', + ); + } catch { + try { + restore(rollback, true); + failure(row, 'restore_recovered_previous'); + } catch { + failure(row, 'restore_recovery_required'); + } + } + return; + } + if (row.phase === 'uploading') { + const id = job?.backup_id || row.id; + const proof = archive().backups?.[id]; + require('./operation-timing').wait(db, row.id, 'waiting_for_upload', proof?.status !== 'verified' && proof?.status !== 'failed', clock); + // A resumed request must wait for a fresh exporter result, rather than + // immediately pausing again on the catalog error that caused the retry. + if (proof?.status === 'failed' && proof.checkedAt >= row.updated_at) return failure(row, 'backup_upload_failed'); + if (proof?.status === 'verified') + update(row, 'completed', 'completed', row.job_id); + else if (clock() - row.updated_at > 3600000) failure(row, 'backup_verification_timeout'); + return; + } + if (!job) return failure(row); + if (job.status === 'failed') { + if (row.phase === 'starting') { + const restoreJob = store.lifecycleJobByRequest( + row.organization_id, + 'platform_backups', + `${row.id}:restoring`, + ); + if (restoreJob?.safety_backup_id) { + store.updateOrganizationStatus(row.organization_id, 'suspended', clock()); + return queue(row, 'restore', 'recovering', { backupId: restoreJob.safety_backup_id }); + } + } + if ( + row.phase === 'restoring' && + input.wasRunning && + control.status === 'suspended' && + job.failure_code !== 'lifecycle_compensation_failed' + ) { + store.updateOrganizationStatus(row.organization_id, 'active', clock()); + return queue(row, 'resume', 'restarting_previous'); + } + return failure(row, job.failure_code); + } + if (row.phase === 'backing_up') return update(row, 'running', 'uploading', job.id); + if (row.phase === 'stopping') + return queue(row, 'restore', 'restoring', { backupId: input.backupId }); + if (['restoring', 'recovering'].includes(row.phase)) { + const backupId = + row.phase === 'restoring' + ? input.backupId + : JSON.parse(JSON.parse(job.stage_receipts_json).__request).backupId; + const record = db.prepare('SELECT * FROM platform_backup_records WHERE id=?').get(backupId); + if (!record) throw Error('backup_metadata_missing'); + restoreDspMetadata(store, row.organization_id, JSON.parse(record.metadata_json), clock()); + if (input.wasRunning) { + store.updateOrganizationStatus(row.organization_id, 'active', clock()); + queue(row, 'resume', row.phase === 'restoring' ? 'starting' : 'restarting_previous'); + } else if (row.phase === 'recovering') failure(row, 'restore_recovered_previous'); + else update(row, 'completed', 'completed', job.id); + return; + } + if (row.phase === 'starting') return update(row, 'completed', 'completed', job.id); + if (row.phase === 'restarting_previous') return failure(row, 'restore_recovered_previous'); + } + async function tick() { + // Commit the full-system schedule only after every restore component has + // completed. Core-only recovery never changes the full-system schedule. + for (const group of db + .prepare( + "SELECT DISTINCT json_extract(input_json,'$.restoreSet') AS id FROM platform_backup_requests WHERE json_extract(input_json,'$.restoreSet') IS NOT NULL AND phase='completed'", + ) + .all()) { + const members = db + .prepare( + "SELECT * FROM platform_backup_requests WHERE json_extract(input_json,'$.restoreSet')=?", + ) + .all(group.id); + if (!members.every((r) => r.status === 'completed')) continue; + const input = JSON.parse(members[0].input_json); + if (!input.systemSchedule) continue; + store.transaction(() => { + const settings = require('../../accounts/src/backup-schedule').backupSettings( + input.systemSchedule, + ); + db.prepare( + "UPDATE backup_scope_settings SET settings_json=?,revision=revision+1,updated_at=? WHERE scope='system'", + ).run(JSON.stringify(settings), clock()); + for (const member of members) { + const value = JSON.parse(member.input_json); + delete value.systemSchedule; + db.prepare('UPDATE platform_backup_requests SET input_json=? WHERE id=?').run( + JSON.stringify(value), + member.id, + ); + } + }); + } + // Root decides which shared archives contained a deleted DSP. Reconcile + // only matching deletion receipts; unrelated Core recovery points remain. + for (const request of db + .prepare("SELECT * FROM backup_deletions WHERE status='queued'") + .all()) { + if (archive().deletions?.[request.id]?.status === 'failed') { + db.prepare( + "UPDATE backup_deletions SET status='failed',failure_code='backup_deletion_failed' WHERE id=?", + ).run(request.id); + continue; + } + const proof = archive().backups?.[request.backup_id]; + if (proof?.status !== 'destroyed') continue; + const record = db + .prepare('SELECT * FROM platform_backup_records WHERE id=?') + .get(request.backup_id); + if ( + !record || + proof.metadataDigest !== + require('node:crypto').createHash('sha256').update(record.metadata_json).digest('hex') + ) + continue; + store.transaction(() => { + db.prepare('UPDATE platform_backup_records SET deleted_at=? WHERE id=?').run( + clock(), + record.id, + ); + db.prepare( + "UPDATE installation_backups SET status='destroyed',destroyed_at=?,tree_digest=NULL,file_count=NULL,total_bytes=NULL,completed_at=NULL WHERE id=?", + ).run(clock(), record.id); + db.prepare("UPDATE backup_deletions SET status='completed' WHERE id=?").run(request.id); + for (const set of db.prepare("SELECT * FROM backup_sets WHERE status!='deleted'").all()) { + const members = JSON.parse(set.members_json); + if (!members.some((m) => m.backupId === record.id)) continue; + const allDeleted = members.every(m => !db.prepare('SELECT 1 FROM platform_backup_records WHERE id=? AND deleted_at IS NULL').get(m.backupId)); + db.prepare('UPDATE backup_sets SET status=? WHERE id=?').run( + allDeleted ? 'deleting' : 'incomplete', + set.id, + ); + } + }); + } + for (const [id, proof] of Object.entries(archive().backups || {})) { + if (proof.status !== 'destroyed') continue; + const row = db + .prepare( + "SELECT metadata_json FROM platform_backup_records WHERE id=? AND kind='core' AND deleted_at IS NULL", + ) + .get(id); + if ( + !db.prepare('SELECT 1 FROM backup_deletions WHERE backup_id=?').get(id) && + row && + proof.metadataDigest === + require('node:crypto').createHash('sha256').update(row.metadata_json).digest('hex') + ) + db.prepare( + "UPDATE platform_backup_records SET deleted_at=?,metadata_json='{}' WHERE id=?", + ).run(clock(), id); + } + for (const set of db.prepare("SELECT id FROM backup_sets WHERE status='deleting'").all()) + if (archive().sets?.[set.id]?.status === 'deleted') + db.prepare("UPDATE backup_sets SET status='deleted' WHERE id=?").run(set.id); + for (const set of db + .prepare("SELECT * FROM backup_sets WHERE status IN ('pending','verified','incomplete')") + .all()) { + const members = JSON.parse(set.members_json).map((member) => { + const request = db + .prepare('SELECT * FROM platform_backup_requests WHERE id=?') + .get(member.requestId); + const job = request?.job_id ? store.lifecycleJob(request.job_id) : null; + return { + ...member, + backupId: + request ? job?.backup_id || (request.kind === 'core' ? request.id : null) : member.backupId || null, + status: request?.status || 'failed', + }; + }); + const missing = members.some( + (m) => + m.status === 'completed' && + (!db + .prepare('SELECT 1 FROM platform_backup_records WHERE id=? AND deleted_at IS NULL') + .get(m.backupId) || + archive().backups?.[m.backupId]?.status === 'expired'), + ); + const status = + set.status === 'incomplete' || missing || members.some((m) => m.status === 'failed') + ? 'incomplete' + : members.every((m) => m.status === 'completed') + ? 'verified' + : 'pending'; + db.prepare('UPDATE backup_sets SET members_json=?,status=? WHERE id=?').run( + JSON.stringify(members), + status, + set.id, + ); + } + manager.schedule(); + require('../../accounts/src/backup-categories').replacePreUpdateBackups(store, archive(), clock()); + const rollout = db.prepare("SELECT * FROM platform_rollouts WHERE status!='completed'").get(); + const rows = rollout + ? db.prepare("SELECT * FROM platform_backup_requests WHERE status IN ('queued','running') AND json_extract(input_json,'$.rolloutId')=? ORDER BY created_at,id").all(rollout.id) + : db.prepare("SELECT * FROM platform_backup_requests WHERE status IN ('queued','running') ORDER BY CASE status WHEN 'running' THEN 0 ELSE 1 END,created_at,COALESCE(json_extract(input_json,'$.position'),0),id LIMIT 1").all(); + if (rollout?.status === 'paused') return; + await Promise.all(rows.map(processRow)); + } + async function processRow(selected) { + let row; + store.transaction(() => { + row = db.prepare("SELECT * FROM platform_backup_requests WHERE id=? AND status IN ('queued','running')").get(selected.id); + if (row && JSON.parse(row.input_json).restoreSet) { + const input = JSON.parse(row.input_json); + const siblings = db + .prepare( + "SELECT * FROM platform_backup_requests WHERE json_extract(input_json,'$.restoreSet')=? AND id!=?", + ) + .all(input.restoreSet, row.id); + if (siblings.some((r) => r.status === 'failed')) { + failure(row, 'full_system_restore_incomplete'); + row = null; + return; + } + } + if (row?.kind === 'core' && deletionPending()) row = null; + if (!row) return; + if (row.kind === 'core' && row.phase === 'queued') update(row, 'running', 'snapshotting'); + if (row.kind !== 'core' || row.phase === 'uploading') { + try { + store.transaction(() => advance(row)); + } catch (error) { + // If data was restored but metadata could not commit, return both to + // the safety snapshot before reopening access. + const job = row.job_id ? store.lifecycleJob(row.job_id) : null; + if (row.phase === 'restoring' && job?.status === 'succeeded' && job.safety_backup_id) { + try { + store.transaction(() => + queue(row, 'restore', 'recovering', { backupId: job.safety_backup_id }), + ); + } catch { + failure(row, 'restore_recovery_required'); + } + } else failure(row, error.code || 'backup_operation_failed'); + } + } + }); + if (row?.kind === 'core' && ['verifying_core', 'recovering_core'].includes(row.phase) && restartCore) { + try { + if (row.phase === 'recovering_core') throw Error('resume_core_compensation'); + await restartCore(); + update(row, 'completed', 'completed'); + } catch { + // Persist compensation intent before changing files. A worker crash + // during rollback must never report the requested restore as successful. + update(row, 'running', 'recovering_core'); + try { + const safety = path.join(localRoot, 'backups/scheduled-core', row.id); + require('./offsite-backup').verifySnapshot(safety, process.geteuid()); + require('./core-backup-files').restore(localRoot, path.join(safety, 'core-files')); + require('../../accounts/src/core-backup').restoreCoreDatabase( + store, + path.join(safety, 'access-control-before.sqlite3'), + clock(), + {removeOwnerIds: JSON.parse(row.input_json).restoredOwnerIds || []}, + ); + await restartCore(); + failure(row, 'restore_recovered_previous'); + } catch { + failure(row, 'restore_recovery_required'); + } + } + } + if (row?.kind === 'core' && ['queued', 'snapshotting'].includes(row.phase)) { + try { + await coreSnapshot(row); + } catch (error) { + for (const name of [row.id, `.creating-${row.id}`]) + fs.rmSync(path.join(localRoot, 'backups/scheduled-core', name), { + recursive: true, + force: true, + }); + if (error.message === 'backup_waiting_for_deletion') update(row, 'queued', 'queued'); + else failure(row, 'backup_failed'); + } + } + } + return { tick }; +} +module.exports = { createPlatformBackupWorker }; diff --git a/core/core/installations/src/platform-core-update.js b/core/core/installations/src/platform-core-update.js new file mode 100644 index 0000000..270b656 --- /dev/null +++ b/core/core/installations/src/platform-core-update.js @@ -0,0 +1,120 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const crypto = require('node:crypto'); +const { spawnSync } = require('node:child_process'); +function fail() { throw new Error('core_update_failed'); } +const sha = data => crypto.createHash('sha256').update(data).digest('hex'); + +// Only deployment-owned, immutable artifacts can execute outside the dashboard. +function verifyCoreArtifact(releaseId, release) { + const root = release.core.artifactPath; + if (root !== `/opt/dispatch-platform/releases/${releaseId}/core-artifact`) fail(); + for (let parent = root; ; parent = path.dirname(parent)) { + const stat = fs.lstatSync(parent); + if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== 0 || (stat.mode & 0o022)) fail(); + if (parent === '/') break; + } + const read = (file, mode) => { + const stat = fs.lstatSync(file); + if (!stat.isFile() || stat.isSymbolicLink() || stat.uid !== 0 || stat.nlink !== 1 + || (stat.mode & 0o7777) !== mode || stat.size > 16 * 1024 * 1024) fail(); + return fs.readFileSync(file); + }; + const bytes = read(path.join(root, 'manifest.json'), 0o444); + if (bytes.length > 256 * 1024 || sha(bytes) !== release.core.manifestSha256) fail(); + const manifest = JSON.parse(bytes); + if (manifest.schemaVersion !== 1 || manifest.releaseId !== releaseId || manifest.sourceCommit !== release.sourceCommit + || !Array.isArray(manifest.files) || manifest.files.length < 2 || manifest.files.length > 2000) fail(); + const expected = new Set(['manifest.json']); + for (const item of manifest.files) { + if (typeof item.path !== 'string' || !/^[a-zA-Z0-9_.\/-]+$/.test(item.path) + || item.path.split('/').some(part => !part || part === '.' || part === '..') + || !['444', '555'].includes(item.mode) || !/^[a-f0-9]{64}$/.test(item.sha256) || expected.has(item.path)) fail(); + const file = path.join(root, item.path); + for (let parent = path.dirname(file); parent !== root; parent = path.dirname(parent)) { + const stat = fs.lstatSync(parent); + if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== 0 || (stat.mode & 0o022)) fail(); + } + if (sha(read(file, Number.parseInt(item.mode, 8))) !== item.sha256) fail(); + if (['apply', 'verify'].includes(item.path) && item.mode !== '555') fail(); + expected.add(item.path); + } + const visit = directory => { + for (const name of fs.readdirSync(directory)) { + const file = path.join(directory, name); const stat = fs.lstatSync(file); + if (stat.isSymbolicLink()) fail(); + if (stat.isDirectory()) { + if (stat.uid !== 0 || (stat.mode & 0o022)) fail(); + visit(file); + } else if (!expected.has(path.relative(root, file))) fail(); + } + }; + visit(root); + if (!expected.has('apply') || !expected.has('verify')) fail(); +} + +function executeCoreStage(action, releaseId, release, rolloutId, attempt) { + verifyCoreArtifact(releaseId, release); + const result = spawnSync(path.join(release.core.artifactPath, action), [], { + input: JSON.stringify({ protocolVersion: 1, action, releaseId, version: release.version, + sourceCommit: release.sourceCommit, rolloutId, attempt }) + '\n', + encoding: 'utf8', timeout: action === 'apply' ? 5_400_000 : 120_000, maxBuffer: 16 * 1024, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8', HOME: os.homedir(), + XDG_RUNTIME_DIR: `/run/user/${process.geteuid()}`, DBUS_SESSION_BUS_ADDRESS: `unix:path=/run/user/${process.geteuid()}/bus` }, + }); + if (result.error || result.signal || result.status !== 0) fail(); + const receipt = JSON.parse(result.stdout); + if (receipt.ok !== true || receipt.releaseId !== releaseId || receipt.sourceCommit !== release.sourceCommit + || receipt.version !== release.version) fail(); +} + +// The external updater holds its process lock across both stages. Each stage is +// persisted before execution; replay after interruption is deliberately idempotent. +function createPlatformCoreUpdater({ store, platformReleases, execute = executeCoreStage, clock = Date.now }) { + let db = store.db; + async function run() { + const row = db.prepare(`SELECT r.id,r.release_id,r.status AS rollout_status,c.status,c.release_json,c.attempt + FROM platform_rollouts r JOIN platform_rollout_core c ON c.rollout_id=r.id + WHERE r.status!='completed' AND c.status!='succeeded' ORDER BY r.created_at LIMIT 1`).get(); + if (!row || row.status === 'failed' || (row.rollout_status === 'paused' && row.status === 'queued')) return { status: 'idle' }; + const backups = db.prepare("SELECT 1 FROM sqlite_schema WHERE name='platform_rollout_backups'").get() + ? require('../../accounts/src/rollout-backups').rolloutBackupProgress(db, row.id) : null; + require('./operation-timing').wait(db, row.id, 'waiting_for_backups', Boolean(backups && backups.status !== 'completed'), clock); + if (backups && backups.status !== 'completed') return { status: 'waiting_for_backups' }; + const release = JSON.parse(row.release_json); + let stage = row.status === 'verifying' ? 'verify' : 'apply'; + try { + if (!platformReleases[row.release_id] || JSON.stringify(platformReleases[row.release_id]) !== row.release_json) fail(); + let attempt = row.attempt; + if (row.status !== 'verifying') { + attempt += 1; + db.prepare("UPDATE platform_rollout_core SET status='updating',attempt=?,updated_at=? WHERE rollout_id=?") + .run(attempt, clock(), row.id); + const finishApply = require('./operation-timing').start(() => store.db, { jobId: row.id, attempt, stage: 'core_apply' }, clock); + try { await execute('apply', row.release_id, release, row.id, attempt); store.refresh?.(); finishApply(); } + catch (error) { store.refresh?.(); finishApply(error); throw error; } + store.refresh?.(); db = store.db; + db.prepare("UPDATE platform_rollout_core SET status='verifying',updated_at=? WHERE rollout_id=?").run(clock(), row.id); + } + stage = 'verify'; + const finishVerify = require('./operation-timing').start(() => store.db, { jobId: row.id, attempt, stage: 'core_verify' }, clock); + try { await execute('verify', row.release_id, release, row.id, attempt); store.refresh?.(); finishVerify(); } + catch (error) { store.refresh?.(); finishVerify(error); throw error; } + store.refresh?.(); db = store.db; + db.prepare("UPDATE platform_rollout_core SET status='succeeded',failure_code=NULL,updated_at=? WHERE rollout_id=?").run(clock(), row.id); + return { status: 'core_verified' }; + } catch { + store.refresh?.(); db = store.db; + store.transaction(() => { + db.prepare("UPDATE platform_rollout_core SET status='failed',failure_code=?,updated_at=? WHERE rollout_id=?").run(stage === 'verify' ? 'core_verification_failed' : 'core_apply_failed', clock(), row.id); + db.prepare("UPDATE platform_rollouts SET status='paused',updated_at=? WHERE id=?").run(clock(), row.id); + }); + return { status: 'core_update_failed' }; + } + } + return { run }; +} +module.exports = { verifyCoreArtifact, executeCoreStage, createPlatformCoreUpdater }; diff --git a/core/core/installations/src/platform-release-catalog.js b/core/core/installations/src/platform-release-catalog.js new file mode 100644 index 0000000..c1f6627 --- /dev/null +++ b/core/core/installations/src/platform-release-catalog.js @@ -0,0 +1,61 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { VERSION: RELEASE_VERSION } = require('../../../shared/release-version'); +const ID = /^[a-z][a-z0-9_.-]{2,95}$/; +const SHA = /^[a-f0-9]{64}$/; +const VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-[0-9A-Za-z.-]+)?$/; +function fail() { throw Object.assign(new Error('platform_release_invalid'), { code: 'platform_release_invalid' }); } +function exact(value, keys) { + if (!value || typeof value !== 'object' || Array.isArray(value) + || Object.keys(value).sort().join(',') !== [...keys].sort().join(',')) fail(); +} +function short(value, maximum) { + if (typeof value !== 'string' || !value.trim() || value.length > maximum || /[\x00-\x1f\x7f]/.test(value)) fail(); + return value; +} +function platformRelease(id, value, runtime) { + exact(value, ['version', 'publishedAt', 'sourceCommit', 'runtimeImageDigest', 'changelog', 'core']); + if (!ID.test(id) || !(VERSION.test(value.version) || RELEASE_VERSION.test(value.version)) || value.version.length > 80 + || !/^[a-f0-9]{40}$/.test(value.sourceCommit) + || !/^sha256:[a-f0-9]{64}$/.test(value.runtimeImageDigest) + || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(value.publishedAt) + || !Number.isFinite(Date.parse(value.publishedAt))) fail(); + if (!runtime || runtime.sourceCommit !== value.sourceCommit + || (runtime.backend === 'native_service_v1' ? `sha256:${runtime.artifactSha256}` : runtime.imageDigest) !== value.runtimeImageDigest) fail(); + exact(value.core, ['artifactPath', 'manifestSha256']); + if (typeof value.core.artifactPath !== 'string' + || value.core.artifactPath !== `/opt/dispatch-platform/releases/${id}/core-artifact` + || !SHA.test(value.core.manifestSha256)) fail(); + if (!Array.isArray(value.changelog) || value.changelog.length < 1 || value.changelog.length > 100) fail(); + const changelog = value.changelog.map(item => { + exact(item, ['kind', 'title', 'description']); + if (!['added', 'improved', 'fixed', 'removed', 'changed'].includes(item.kind)) fail(); + short(item.title, 160); + if (item.description !== '') short(item.description, 600); + return { kind: item.kind, title: item.title, description: item.description }; + }); + return Object.freeze({ version: value.version, publishedAt: value.publishedAt, sourceCommit: value.sourceCommit, + runtimeImageDigest: value.runtimeImageDigest, changelog, + core: { artifactPath: value.core.artifactPath, manifestSha256: value.core.manifestSha256 } }); +} +function loadPlatformReleaseCatalog(file, runtimes) { + if (file === undefined) return Object.freeze({}); + if (typeof file !== 'string' || !path.isAbsolute(file) || path.resolve(file) !== file) fail(); + const stat = fs.lstatSync(file); + if (!stat.isFile() || stat.isSymbolicLink() || stat.uid !== process.geteuid() || stat.nlink !== 1 + || (stat.mode & 0o7777) !== 0o600 || fs.realpathSync(file) !== file || stat.size > 256 * 1024) fail(); + const catalog = JSON.parse(fs.readFileSync(file, 'utf8')); + exact(catalog, ['schemaVersion', 'releases']); + if (catalog.schemaVersion !== 1 || !catalog.releases || typeof catalog.releases !== 'object' || Array.isArray(catalog.releases)) fail(); + const result = {}; + const versions = new Set(); + for (const [id, release] of Object.entries(catalog.releases)) { + const checked = platformRelease(id, release, runtimes[id]); + if (versions.has(checked.version)) fail(); + versions.add(checked.version); result[id] = checked; + } + return Object.freeze(result); +} +module.exports = { platformRelease, loadPlatformReleaseCatalog }; diff --git a/core/core/installations/src/platform-update-store.js b/core/core/installations/src/platform-update-store.js new file mode 100644 index 0000000..fb2ddde --- /dev/null +++ b/core/core/installations/src/platform-update-store.js @@ -0,0 +1,38 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); + +// Stable updater protocol, independent of the application's schema version. +// Releases must preserve these tables while migrating other Core data in place. +function openPlatformUpdateStore(root) { + const file = path.join(root, 'access-control.sqlite3'); + let db; + function open() { + const info = fs.lstatSync(file); + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() || info.nlink !== 1 + || (info.mode & 0o7777) !== 0o600 || fs.realpathSync(file) !== file) throw new Error('unsafe_access_storage'); + db = new DatabaseSync(file); + db.exec('PRAGMA foreign_keys=ON; PRAGMA trusted_schema=OFF; PRAGMA busy_timeout=3000; PRAGMA synchronous=FULL;'); + for (const [table, columns] of Object.entries({ + platform_rollouts: ['id', 'release_id', 'actor_user_id', 'idempotency_key', 'status', 'created_at', 'updated_at'], + platform_rollout_core: ['rollout_id', 'status', 'release_json', 'attempt', 'failure_code', 'updated_at'], + })) { + if (JSON.stringify(db.prepare(`PRAGMA table_info(${table})`).all().map(row => row.name)) !== JSON.stringify(columns)) { + db.close(); throw new Error('platform_update_protocol_incompatible'); + } + } + } + open(); + return { + get db() { return db; }, + refresh() { db.close(); open(); }, + close() { db.close(); }, + transaction(fn) { + db.exec('BEGIN IMMEDIATE'); + try { const result = fn(); db.exec('COMMIT'); return result; } + catch (error) { db.exec('ROLLBACK'); throw error; } + }, + }; +} +module.exports = { openPlatformUpdateStore }; diff --git a/core/core/installations/src/portable-node.js b/core/core/installations/src/portable-node.js new file mode 100644 index 0000000..c4932af --- /dev/null +++ b/core/core/installations/src/portable-node.js @@ -0,0 +1,69 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const BUILTINS = ['cjs-module-lexer/lexer.js', 'cjs-module-lexer/dist/lexer.js', 'undici/undici-fetch.js', + 'acorn/dist/acorn.js', 'acorn-walk/dist/walk.js', 'minimatch/dist/cjs/index.bundle.js']; +// Carry Node's loader and libc too. Private RPATHs keep these dependencies out +// of every other program's library search on the restored host. +function bundleNode(executable, destination, runtimeRoot = '/usr/local/lib/dispatch-node') { + if (fs.existsSync(destination) || !path.isAbsolute(destination)) throw Error('portable_node_invalid'); + const source = fs.realpathSync(executable); + const interpreter = spawnSync('/usr/bin/patchelf', ['--print-interpreter', source], { encoding: 'utf8' }); + if (interpreter.status !== 0 || !path.isAbsolute(interpreter.stdout.trim())) throw Error('node_loader_unavailable'); + const loaderSource = fs.realpathSync(interpreter.stdout.trim()); + const listed = spawnSync(loaderSource, ['--list', source], { encoding: 'utf8', env: { PATH: '/usr/bin:/bin', LANG: 'C' } }); + if (listed.status !== 0 || /not found/.test(listed.stdout)) throw Error('node_dependencies_unavailable'); + fs.mkdirSync(destination, { mode: 0o755 }); + const target = path.join(destination, 'node'), libraries = path.join(destination, 'lib'); + fs.mkdirSync(libraries, { mode: 0o755 }); + fs.copyFileSync(source, target); fs.chmodSync(target, 0o755); + const external = path.join(destination, 'host-files/usr/share/nodejs'); + fs.mkdirSync(external, { recursive: true, mode: 0o755 }); + for (const line of listed.stdout.split('\n')) { + const match = /^\s*(\S+) => (\/[^\s]+) \(/.exec(line); + if (!match || path.isAbsolute(match[1]) && fs.realpathSync(match[2]) === loaderSource) continue; + const library = path.join(libraries, match[1]); + if (path.basename(library) !== match[1] || fs.existsSync(library)) throw Error('node_dependencies_invalid'); + fs.copyFileSync(fs.realpathSync(match[2]), library); fs.chmodSync(library, 0o755); + } + for (const file of [target, ...fs.readdirSync(libraries).map(name => path.join(libraries, name))]) { + const bytes = fs.readFileSync(file); + for (const relative of BUILTINS) { + const builtin = '/usr/share/nodejs/' + relative; + if (!bytes.includes(Buffer.from(builtin + '\0'))) continue; + const output = path.join(external, relative); + fs.mkdirSync(path.dirname(output), { recursive: true, mode: 0o755 }); + fs.copyFileSync(builtin, output); fs.chmodSync(output, 0o444); + } + } + const loader = path.join(libraries, 'ld-linux-x86-64.so.2'); + fs.copyFileSync(loaderSource, loader); fs.chmodSync(loader, 0o555); + for (const file of [target, ...fs.readdirSync(libraries).filter(name => name !== 'ld-linux-x86-64.so.2').map(name => path.join(libraries, name))]) { + const rpath = file === target ? '$ORIGIN/lib' : '$ORIGIN'; + const current = spawnSync('/usr/bin/patchelf', ['--print-rpath', file], { encoding: 'utf8' }); + if (current.status !== 0) throw Error('node_dependency_packaging_failed'); + if (current.stdout.trim() !== rpath && spawnSync('/usr/bin/patchelf', ['--set-rpath', rpath, file]).status !== 0) throw Error('node_dependency_packaging_failed'); + fs.chmodSync(file, file === target ? 0o755 : 0o555); + } + if (!/^\/[A-Za-z0-9_./-]+$/.test(runtimeRoot)) throw Error('node_runtime_path_invalid'); + const targetInterpreter = path.join(runtimeRoot, 'lib/ld-linux-x86-64.so.2'); + if (interpreter.stdout.trim() !== targetInterpreter && spawnSync('/usr/bin/patchelf', ['--set-interpreter', targetInterpreter, target]).status !== 0) throw Error('node_loader_packaging_failed'); + fs.chmodSync(target, 0o555); + // Backup workers use umask 0077. Shared executable directories must remain + // traversable by Core and DSP service accounts after recovery. Normalize + // copied ownership too: root must own executables restored by its worker. + function directories(directory) { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const file = path.join(directory, entry.name); + if (entry.isDirectory()) directories(file); + else fs.chownSync(file, process.geteuid(), process.getegid()); + } + fs.chownSync(directory, process.geteuid(), process.getegid()); + fs.chmodSync(directory, 0o755); + } + directories(destination); + const probe = spawnSync(loader, ['--library-path', libraries, target, '--version'], { encoding: 'utf8', env: { PATH: '/usr/bin:/bin' } }); + if (probe.status !== 0 || !/^v[0-9]+\.[0-9]+\.[0-9]+\n$/.test(probe.stdout)) throw Error('portable_node_unusable'); + return probe.stdout.trim(); +} +module.exports = { bundleNode, BUILTINS }; diff --git a/core/core/installations/src/prepare-release-dependencies.py b/core/core/installations/src/prepare-release-dependencies.py new file mode 100644 index 0000000..4d00177 --- /dev/null +++ b/core/core/installations/src/prepare-release-dependencies.py @@ -0,0 +1,44 @@ +"""Fetch the repository-pinned Chrome archive into a new disposable build directory.""" +import hashlib +import json +from pathlib import Path +import shutil +import sys +import tempfile +import urllib.request +import zipfile + + +def prepare(destination): + pins = json.loads((Path(__file__).resolve().parent.parent / 'runtime-dependencies.json').read_text()) + destination = Path(destination) + if not destination.is_absolute() or destination.exists(): + raise ValueError('new_absolute_destination_required') + url = f"https://storage.googleapis.com/chrome-for-testing-public/{pins['chrome']}/linux64/chrome-linux64.zip" + destination.parent.mkdir(parents=True, exist_ok=True) + if shutil.disk_usage(destination.parent).free < 2 * 1024**3: + raise ValueError('browser_build_insufficient_space') + with tempfile.TemporaryDirectory(prefix='dispatch-browser-', dir=destination.parent) as stage: + archive = Path(stage) / 'chrome.zip' + with urllib.request.urlopen(url, timeout=120) as incoming, archive.open('xb') as outgoing: + shutil.copyfileobj(incoming, outgoing) + with archive.open('rb') as source: + checksum = hashlib.file_digest(source, 'sha256').hexdigest() + if checksum != pins['chromeArchiveSha256']: + raise ValueError('browser_checksum_failed') + with zipfile.ZipFile(archive) as source: + # The archive is authenticated above; still restrict paths and links. + for item in source.infolist(): + if not item.filename.startswith('chrome-linux64/') or any(x in ('..', '') for x in item.filename.rstrip('/').split('/')) or (item.external_attr >> 16) & 0o170000 == 0o120000: + raise ValueError('browser_archive_invalid') + source.extractall(stage) + for item in source.infolist(): + file = Path(stage) / item.filename + file.chmod(0o755 if file.is_dir() or (item.external_attr >> 16) & 0o111 else 0o644) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.move(str(Path(stage) / 'chrome-linux64'), destination) + print(destination) + + +if __name__ == '__main__': + prepare(sys.argv[1]) diff --git a/core/core/installations/src/r2-backup-storage.js b/core/core/installations/src/r2-backup-storage.js new file mode 100644 index 0000000..68c5103 --- /dev/null +++ b/core/core/installations/src/r2-backup-storage.js @@ -0,0 +1,321 @@ +'use strict'; +// Root-only storage administration. Fixed hosts, bounded responses, no redirects +// and no credentials in argv, logs, tenant environments or public receipts. +const https = require('node:https'), + crypto = require('node:crypto'), + fs = require('node:fs'); +const { privateJson, atomic } = require('./release-delivery-files'); +function fail() { + throw Object.assign(Error('backup_storage_unavailable'), { code: 'backup_storage_unavailable' }); +} +function http(url, method, headers, body = '') { + return new Promise((resolve, reject) => { + const req = https.request(url, { method, headers, timeout: 30000 }, (res) => { + let bytes = 0; + const chunks = []; + res.on('data', (chunk) => { + bytes += chunk.length; + if (bytes > 4 * 1024 * 1024) req.destroy(Error()); + else chunks.push(chunk); + }); + res.on('end', () => + resolve({ status: res.statusCode, body: Buffer.concat(chunks).toString('utf8') }), + ); + }); + req.on('timeout', () => req.destroy(Error())); + req.on('error', () => + reject( + Object.assign(Error('backup_storage_unavailable'), { code: 'backup_storage_unavailable' }), + ), + ); + req.end(body); + }); +} +function readCredentials() { + const credential = privateJson('/etc/dispatch/offsite-backup-credentials.json', 0); + const managementFile = '/etc/dispatch/cloudflare-r2-management-token'; + const file = fs.lstatSync(managementFile); + if ( + file.uid !== 0 || + !file.isFile() || + file.nlink !== 1 || + file.mode & 0o077 || + fs.realpathSync(managementFile) !== managementFile + ) + fail(); + const token = fs.readFileSync(managementFile, 'utf8').trim(); + if (!/^[!-~]{20,512}$/.test(token)) fail(); + return { credential, token }; +} +function createR2BackupStorage(config, { request = http, credentials = readCredentials(), journalFile = '/var/lib/dispatch-backup/deletion-locks.json', ownerUid = 0 } = {}) { + const { credential, token } = credentials; + async function control(method, suffix, body) { + const response = await request( + `https://api.cloudflare.com/client/v4/accounts/${config.accountId}/r2/buckets/${config.bucket}${suffix}`, + method, + { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json' }, + body ? JSON.stringify(body) : '', + ); + let value; + try { + value = JSON.parse(response.body); + } catch { + fail(); + } + if (response.status < 200 || response.status >= 300 || value.success !== true) fail(); + return value.result; + } + async function ensureLocks() { + await restoreDeletionLocks(); + const current = await control('GET', '/lock'), + rules = current?.rules || []; + let changed = false; + for (const days of [null, 7, 30, 90, 365]) { + const tier = days === null ? 'all' : String(days), + id = `dispatch-archives-${tier}`; + const desired = { + id, + enabled: true, + prefix: `archives/${tier}/`, + condition: + days === null ? { type: 'Indefinite' } : { type: 'Age', maxAgeSeconds: days * 86400 }, + }; + const existing = rules.find((r) => r.id === id); + if (existing) { + if ( + JSON.stringify(existing.condition) !== JSON.stringify(desired.condition) || + existing.prefix !== desired.prefix || + !existing.enabled + ) + fail(); + } else { + rules.push(desired); + changed = true; + } + } + const artifactRule = { id: 'dispatch-recovery-artifacts', enabled: true, prefix: 'recovery-artifacts/', condition: { type: 'Indefinite' } }; + const existingArtifactRule = rules.find(rule => rule.id === artifactRule.id); + if (existingArtifactRule && (existingArtifactRule.prefix !== artifactRule.prefix || !existingArtifactRule.enabled + || existingArtifactRule.condition?.type !== 'Indefinite')) fail(); + if (!existingArtifactRule) { rules.push(artifactRule); changed = true; } + if (changed) await control('PUT', '/lock', { rules }); + const confirmed = await control('GET', '/lock'); + for (const rule of rules.filter((r) => r.id.startsWith('dispatch-archives-') || r.id === 'dispatch-recovery-artifacts')) { + const actual = confirmed.rules.find((r) => r.id === rule.id); + if ( + !actual?.enabled || + actual.prefix !== rule.prefix || + JSON.stringify(actual.condition) !== JSON.stringify(rule.condition) + ) + fail(); + } + } + const hash = (b) => crypto.createHash('sha256').update(b).digest('hex'); + const encode = (s) => + encodeURIComponent(s).replace( + /[!'()*]/g, + (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`, + ); + async function s3(method, key = '', query = {}) { + const host = `${config.accountId}.r2.cloudflarestorage.com`, + uri = `/${config.bucket}/${key.split('/').map(encode).join('/')}`; + const queryString = Object.entries(query) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => `${encode(k)}=${encode(v)}`) + .join('&'); + const stamp = new Date().toISOString().replace(/[:-]|\.\d{3}/g, ''), + date = stamp.slice(0, 8), + scope = `${date}/auto/s3/aws4_request`; + const headers = `host:${host}\nx-amz-content-sha256:${hash('')}\nx-amz-date:${stamp}\n`, + signed = 'host;x-amz-content-sha256;x-amz-date'; + const canonical = [method, uri, queryString, headers, signed, hash('')].join('\n'); + const sign = (k, v) => crypto.createHmac('sha256', k).update(v).digest(); + const keyBytes = sign( + sign(sign(sign(`AWS4${credential.secretAccessKey}`, date), 'auto'), 's3'), + 'aws4_request', + ); + const signature = crypto + .createHmac('sha256', keyBytes) + .update(`AWS4-HMAC-SHA256\n${stamp}\n${scope}\n${hash(canonical)}`) + .digest('hex'); + const auth = `AWS4-HMAC-SHA256 Credential=${credential.accessKeyId}/${scope}, SignedHeaders=${signed}, Signature=${signature}`; + return request(`https://${host}${uri}${queryString ? '?' + queryString : ''}`, method, { + Authorization: auth, + 'x-amz-date': stamp, + 'x-amz-content-sha256': hash(''), + }); + } + async function removeExpired(record, now = Date.now()) { + if ( + ![7, 30, 90, 365].includes(record.retentionDays) || + !Number.isSafeInteger(record.expiresAt) || + now < record.expiresAt || + !/^(backup|breq)_[a-f0-9]{32}$/.test(record.id) + ) + fail(); + return removePermanent(record); + } + async function listSets() { + const ids=[];let continuation; + for(let page=0;page<1000;page++) { + const response=await s3('GET','',{'list-type':'2',prefix:'sets/',delimiter:'/', 'encoding-type':'url','max-keys':'1000',...(continuation?{'continuation-token':continuation}:{})}); + if(response.status!==200 || !response.body.includes('\s*([^<]*)<\/Prefix>\s*<\/CommonPrefixes>/g)){ + const prefix=decodeURIComponent(match[1]);if(!/^sets\/breq_[a-f0-9]{32}\/$/.test(prefix))fail();ids.push(prefix.split('/')[1]); + } + if(!/\s*true\s*<\/IsTruncated>/.test(response.body))return ids; + continuation=/([^<]+)<\/NextContinuationToken>/.exec(response.body)?.[1].replaceAll('&','&'); + if(!continuation)fail(); + } + fail(); + } + async function listArchives() { + const result = []; + for (const retentionDays of [null, 7, 30, 90, 365]) { + const tier = retentionDays === null ? 'all' : String(retentionDays), prefix = `archives/${tier}/`; + let continuation; + for (let page = 0; page < 1000; page++) { + const response = await s3('GET', '', { 'list-type': '2', prefix, delimiter: '/', 'encoding-type': 'url', 'max-keys': '1000', + ...(continuation ? { 'continuation-token': continuation } : {}) }); + if (response.status !== 200 || !response.body.includes('\s*([^<]*)<\/Prefix>\s*<\/CommonPrefixes>/g)) { + const selected = decodeURIComponent(match[1]), id = selected.slice(prefix.length, -1); + if (!selected.startsWith(prefix) || !selected.endsWith('/') || !/^(backup|breq)_[a-f0-9]{32}$/.test(id)) fail(); + result.push({ id, retentionDays }); + } + if (!/\s*true\s*<\/IsTruncated>/.test(response.body)) break; + continuation = /([^<]+)<\/NextContinuationToken>/.exec(response.body)?.[1].replaceAll('&', '&'); + if (!continuation || page === 999) fail(); + } + } + return result; + } + // Count encrypted objects, including repository metadata, rather than the + // uncompressed recovery payload. Listing is read-only and fully paginated. + async function usage() { + const archives = {}, sets = {}; + let legacyBytes = 0, artifactBytes = 0; + const add = (a, b) => { const n = a + b; if (!Number.isSafeInteger(n)) fail(); return n; }; + for (const prefix of ['archives/', 'sets/', 'recovery-artifacts/', ...(config.prefix ? [`${config.prefix}/`] : [])]) { + const tokens = new Set(), keys = new Set(); + let continuation; + for (let page = 0; page < 1000; page++) { + const response = await s3('GET', '', {'list-type':'2', prefix, 'encoding-type':'url', 'max-keys':'1000', ...(continuation ? {'continuation-token':continuation} : {})}); + const body = response.body; + if (response.status !== 200 || !body.includes('')) fail(); + const contents = [...body.matchAll(/([\s\S]*?)<\/Contents>/g)]; + if ((body.match(//g) || []).length !== contents.length) fail(); + for (const [, item] of contents) { + const rawKey = /([^<]*)<\/Key>/.exec(item)?.[1], rawSize = /(\d+)<\/Size>/.exec(item)?.[1]; + if (!rawKey || rawSize === undefined) fail(); + const key = decodeURIComponent(rawKey), size = Number(rawSize); + if (!key.startsWith(prefix) || keys.has(key) || !Number.isSafeInteger(size) || size < 0) fail(); + keys.add(key); + if (prefix === 'archives/') { + const match = /^archives\/(?:all|7|30|90|365)\/((?:backup|breq)_[a-f0-9]{32})\/[a-z0-9/]+$/.exec(key); + if (!match) fail(); + archives[match[1]] = add(archives[match[1]] || 0, size); + } else if (prefix === 'sets/') { + const match = /^sets\/(breq_[a-f0-9]{32})\/[a-z0-9/]+$/.exec(key); + if (!match) fail(); + sets[match[1]] = add(sets[match[1]] || 0, size); + } else if (prefix === 'recovery-artifacts/') { + if (!/^recovery-artifacts\/[a-f0-9]{64}\/[a-z0-9/]+$/.test(key)) fail(); + artifactBytes = add(artifactBytes, size); + } else legacyBytes = add(legacyBytes, size); + } + const truncated = /\s*(true|false)\s*<\/IsTruncated>/.exec(body)?.[1]; + if (!truncated) fail(); + if (truncated === 'false') break; + continuation = /([^<]+)<\/NextContinuationToken>/.exec(body)?.[1] + .replaceAll('&', '&').replaceAll('<', '<').replaceAll('>', '>').replaceAll('"', '"').replaceAll(''', "'"); + if (!continuation || tokens.has(continuation) || page === 999) fail(); + tokens.add(continuation); + } + } + return {archives, sets, legacyBytes, artifactBytes}; + } + async function removePermanent(record) { + if (![null, 7, 30, 90, 365].includes(record.retentionDays) + || !/^(backup|breq)_[a-f0-9]{32}$/.test(record.id)) fail(); + const prefix = `archives/${record.retentionDays === null ? 'all' : record.retentionDays}/${record.id}/`; + return removePrefix(prefix); + } + async function removeSet(id) { + if(!/^breq_[a-f0-9]{32}$/.test(id))fail(); + return removePrefix(`sets/${id}/`); + } + async function removePrefix(prefix) { + // Re-list the first page after each batch, avoiding deletion-pagination races. + for (let page = 0; page < 1000; page++) { + const response = await s3('GET', '', { + 'list-type': '2', + prefix, + 'encoding-type': 'url', + 'max-keys': '1000', + }); + if (response.status !== 200 || !response.body.includes('')) fail(); + const keys = [...response.body.matchAll(/([^<]*)<\/Key>/g)].map((m) => + decodeURIComponent(m[1]), + ); + if ( + keys.some( + (k) => + !k.startsWith(prefix) || + !/^(archives\/(all|7|30|90|365)\/(backup|breq)_[a-f0-9]{32}|sets\/breq_[a-f0-9]{32})\/[a-z0-9/]+$/.test(k), + ) + ) + fail(); + if (!keys.length) { + if (response.body.includes('') || /\s*true\s*<\/IsTruncated>/.test(response.body)) fail(); + return; + } + for (const key of keys) { + const deleted = await s3('DELETE', key); + if (deleted.status !== 204) fail(); + } + } + fail(); + } + const canonical = value => value && typeof value === 'object' + ? JSON.stringify(Object.keys(value).sort().map(key => [key, canonical(value[key])])) : JSON.stringify(value); + const deletionPrefixes = () => [ + ...['all', 7, 30, 90, 365].map(tier => `archives/${tier}/`), + ...['data/', 'index/', 'snapshots/'].map(part => `${config.prefix}/${part}`), + ]; + async function restoreDeletionLocks() { + if (!fs.existsSync(journalFile)) return; + const journal = privateJson(journalFile, ownerUid); + if (journal.accountId !== config.accountId || journal.bucket !== config.bucket + || !Array.isArray(journal.rules) || journal.rules.some(rule => !deletionPrefixes().includes(rule.prefix))) fail(); + const current = await control('GET', '/lock'); + const rules = current.rules.filter(rule => !journal.rules.some(saved => saved.id === rule.id)); + rules.push(...journal.rules); + await control('PUT', '/lock', { rules }); + const restored = await control('GET', '/lock'); + if (journal.rules.some(rule => !restored.rules.some(actual => canonical(actual) === canonical(rule)))) fail(); + fs.unlinkSync(journalFile); + } + async function withDeletionAccess(prefixes, callback) { + if (!Array.isArray(prefixes) || prefixes.some(prefix => !deletionPrefixes().includes(prefix))) fail(); + await restoreDeletionLocks(); + const current = await control('GET', '/lock'); + if (!Array.isArray(current.rules)) fail(); + // A broader administrator-defined lock is not ours to weaken. + if (current.rules.some(rule => rule.enabled && prefixes.some(prefix => prefix.startsWith(rule.prefix || '') + && !prefixes.includes(rule.prefix)))) fail(); + const removed = current.rules.filter(rule => prefixes.includes(rule.prefix)); + atomic(journalFile, { accountId: config.accountId, bucket: config.bucket, rules: removed }); + try { + await control('PUT', '/lock', { rules: current.rules.filter(rule => !prefixes.includes(rule.prefix)) }); + const unlocked = await control('GET', '/lock'); + if (unlocked.rules.some(rule => rule.enabled && prefixes.includes(rule.prefix))) fail(); + return await callback(); + } finally { + // The durable journal also restores locks before the next scan after a crash. + await restoreDeletionLocks(); + } + } + return { usage, removeSet, listSets, ensureLocks, removeExpired, removePermanent, withDeletionAccess, restoreDeletionLocks, listArchives }; +} +module.exports = { createR2BackupStorage }; diff --git a/core/core/installations/src/recover-archive-catalog.js b/core/core/installations/src/recover-archive-catalog.js new file mode 100644 index 0000000..c9fcc8d --- /dev/null +++ b/core/core/installations/src/recover-archive-catalog.js @@ -0,0 +1,138 @@ +"use strict"; +const fs = require("node:fs"), + path = require("node:path"); +const { DatabaseSync } = require("node:sqlite"); +const { tree, verifySnapshot } = require("./offsite-backup"); +const { recoveryRoots } = require("./host-recovery-bundle"); +const capsule = require("./recovery-capsule"); +const hash = (value) => + require("node:crypto").createHash("sha256").update(value).digest("hex"); +// A restored database predates its own archive and may predate newer backups. +// Rediscover those encrypted repositories before permitting further deletion. +async function recoverArchiveCatalog({ + config, + storage, + runFor, + workRoot, + ownerUid, + record, + save, + clock, +}) { + const marker = path.join(workRoot, "rediscover.json"); + if (!fs.existsSync(marker)) return; + const rows = await storage.listArchives(); + for (const row of rows) { + if ( + !/^(backup|breq)_[a-f0-9]{32}$/.test(row.id) || + ![null, 7, 30, 90, 365].includes(row.retentionDays) + ) + throw Error("backup_catalog_invalid"); + if (record(row.id)) continue; + const work = fs.mkdtempSync(path.join(workRoot, "rediscover-")); + try { + const run = runFor(row.id, row.retentionDays), + snapshots = run(["snapshots"]).flat(); + if (snapshots.length !== 1 || !/^[a-f0-9]{64}$/.test(snapshots[0].id)) + throw Error("backup_catalog_invalid"); + run(["restore", snapshots[0].id, "--target", work, "--verify"]); + const bundle = path.join(work, "bundle"), + info = JSON.parse(fs.readFileSync(path.join(bundle, "dsp.json"))); + if ( + info.schemaVersion !== 1 || + info.id !== row.id || + !["core", "dsp"].includes(info.kind) || + info.retentionDays !== row.retentionDays + ) + throw Error("backup_catalog_invalid"); + const checked = verifySnapshot(path.join(bundle, "snapshot"), ownerUid), + total = tree(bundle, ownerUid); + const proof = JSON.parse( + fs.readFileSync(path.join(bundle, "recovery-proof.json")), + ), + manifest = JSON.parse( + fs.readFileSync(path.join(bundle, "recovery/recovery.json")), + ); + require('./recovery-artifacts').hydrate(path.join(bundle, 'recovery'), proof.sha256, + require('./recovery-artifacts').resticReader(config)); + capsule.verify( + path.join(bundle, "recovery"), + proof.sha256, + recoveryRoots(manifest.metadata, manifest.roots), + ); + const organizationId = + info.kind === "core" ? null : info.metadata.organizationId; + if ( + organizationId !== null && + !/^[a-z][a-z0-9_-]{2,95}$/.test(organizationId) + ) + throw Error("backup_catalog_invalid"); + const createdAt = Date.parse(snapshots[0].time); + if (!Number.isSafeInteger(createdAt)) + throw Error("backup_catalog_invalid"); + const snapshot = JSON.parse( + fs.readFileSync(path.join(bundle, "snapshot/manifest.json")), + ); + const metadataJson = JSON.stringify(info.metadata); + const db = new DatabaseSync( + path.join( + config.localRoot, + "data/access-control/access-control.sqlite3", + ), + ); + try { + db.exec("PRAGMA foreign_keys=ON; PRAGMA busy_timeout=3000"); + if ( + organizationId === null || + db + .prepare("SELECT 1 FROM organizations WHERE id=?") + .get(organizationId) + ) + db.prepare( + "INSERT OR IGNORE INTO platform_backup_records VALUES(?,?,?,?,?,?,?,NULL)", + ).run( + row.id, + organizationId, + info.kind, + metadataJson, + row.retentionDays, + createdAt, + row.retentionDays === null + ? null + : createdAt + row.retentionDays * 86400000, + ); + } finally { + db.close(); + } + save(row.id, { + schemaVersion: 1, + id: row.id, + organizationId, + kind: info.kind, + status: "verified", + metadataDigest: hash(metadataJson), + metadataJson, + digest: checked.digest, + bundleDigest: total.digest, + snapshotId: snapshots[0].id, + size: total.size, + retentionDays: row.retentionDays, + verifiedAt: clock(), + expiresAt: + row.retentionDays === null + ? null + : createdAt + row.retentionDays * 86400000, + format: snapshot.version, + trigger: snapshot.purpose || "scheduled", + deletedAt: null, + recoveryDigest: proof.sha256, + recoveryArtifacts: [...new Map(manifest.entries.filter(entry => entry.artifact).map(entry => [entry.artifact.digest, entry.artifact])).values()], + organizationIds: proof.organizationIds, + }); + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } + } + fs.unlinkSync(marker); +} +module.exports = { recoverArchiveCatalog }; diff --git a/core/core/installations/src/recovery-artifacts.js b/core/core/installations/src/recovery-artifacts.js new file mode 100644 index 0000000..686bd7a --- /dev/null +++ b/core/core/installations/src/recovery-artifacts.js @@ -0,0 +1,180 @@ +'use strict'; +// Immutable release payloads have their own encrypted, indefinitely retained +// repositories. Backup deletion and release cleanup never delete this prefix. +const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto'); +const capsule = require('./recovery-capsule'); +const sha = value => crypto.createHash('sha256').update(value).digest('hex'); +const fail = () => { throw Error('recovery_artifact_invalid'); }; +const releaseRoot = value => typeof value === 'string' && /^\/opt\/dispatch-(platform|runtime|control|updater|release-delivery)\/releases\/[a-z0-9][a-z0-9_.-]{2,95}$/.test(value); +const hex = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); +function inventory(root, ownerUid) { + const entries = []; + function visit(file) { + const s = fs.lstatSync(file); + if (s.uid !== ownerUid || !s.isSymbolicLink() && s.mode & 0o022 || fs.realpathSync(path.dirname(file)) !== path.dirname(file)) fail(); + entries.push([path.relative(root, file), s.ino, s.size, s.mtimeMs, s.ctimeMs, s.mode]); + if (s.isDirectory()) for (const name of fs.readdirSync(file).sort()) visit(path.join(file, name)); + else if (!s.isFile() && !s.isSymbolicLink()) fail(); + } + visit(root); return sha(JSON.stringify(entries)); +} +function createArtifactStore(config, { cacheRoot = '/var/lib/dispatch-backup/recovery-artifacts', ownerUid = 0, + runFactory = environment => require('./offsite-backup').createRestic(environment) } = {}) { + fs.mkdirSync(cacheRoot, { recursive: true, mode: 0o700 }); + const stat = fs.lstatSync(cacheRoot); + if (!stat.isDirectory() || stat.uid !== ownerUid || stat.mode & 0o077 || fs.realpathSync(cacheRoot) !== cacheRoot) fail(); + function prepare(source, target = source) { + if (!releaseRoot(target)) fail(); + const fingerprint = inventory(source, ownerUid); + const key = sha(JSON.stringify([config.accountId, config.bucket, target, fingerprint])); + const directory = path.join(cacheRoot, key), file = path.join(directory, 'receipt.json'); + if (fs.existsSync(file)) { + const receipt = JSON.parse(fs.readFileSync(file)); + if (!hex(receipt.digest) || !hex(receipt.snapshotId) || receipt.root !== target) fail(); + capsule.verify(path.join(directory, 'artifact'), receipt.digest, new Set([target])); + return { ...receipt, directory: path.join(directory, 'artifact') }; + } + // The production caller holds a per-root flock across capture and upload. + fs.rmSync(directory, { recursive: true, force: true }); + fs.mkdirSync(directory, { mode: 0o700 }); + const artifact = path.join(directory, 'artifact'); + const proof = capsule.capture(artifact, [{ source, target }], { kind: 'release' }, new Set([ownerUid])); + capsule.verify(artifact, proof.sha256, new Set([target])); + if (inventory(source, ownerUid) !== fingerprint) fail(); + const run = runFactory({ ...config.environment, + RESTIC_REPOSITORY: `s3:https://${config.accountId}.r2.cloudflarestorage.com/${config.bucket}/recovery-artifacts/${proof.sha256}` }); + let snapshots; + try { snapshots = run(['--no-lock', 'snapshots']).flat(); } + catch { run(['--no-lock', 'init', '--repository-version', '2']); snapshots = []; } + let snapshotId = snapshots.at(-1)?.id; + if (!snapshots.length) snapshotId = run(['--no-lock', 'backup', '--host', 'dispatch', '--', 'artifact'], directory) + .find(item => item?.message_type === 'summary')?.snapshot_id; + if (!hex(snapshotId)) fail(); + // A new shared dependency is read back once before any backup can refer to it. + const check = path.join(directory, 'check'); + run(['--no-lock', 'restore', snapshotId, '--target', check, '--verify']); + capsule.verify(path.join(check, 'artifact'), proof.sha256, new Set([target])); + fs.rmSync(check, { recursive: true, force: true }); + const receipt = { root: target, digest: proof.sha256, snapshotId }; + require('./release-delivery-files').atomic(file, receipt); + return { ...receipt, directory: artifact }; + } + return { prepare }; +} +function prepareRelease(config, source) { + if (process.geteuid() !== 0 || !releaseRoot(source)) fail(); + const { spawnSync } = require('node:child_process'); + const cacheRoot = '/var/lib/dispatch-backup/recovery-artifacts'; + fs.mkdirSync(cacheRoot, { recursive: true, mode: 0o700 }); + const result = spawnSync('/usr/bin/flock', ['--shared', path.join(cacheRoot, 'maintenance.lock'), + '/usr/bin/flock', '--wait', '600', path.join(cacheRoot, sha(source) + '.lock'), + '/usr/bin/node', '--no-warnings', __filename, source], { encoding: 'utf8', timeout: 900000, maxBuffer: 4096, + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8' } }); + if (result.status !== 0 || result.error) fail(); + return JSON.parse(result.stdout); +} +// Append authenticated inventories without copying their large payload files. +// Hydration below reconstructs the original capsule before ordinary verification. +function append(directory, proof, artifacts) { + if (!artifacts.length) { + const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'recovery.json'))); + capsule.verify(directory, proof.sha256, new Set(manifest.roots)); + return proof; + } + const original = fs.readFileSync(path.join(directory, 'recovery.json')); + if (sha(original) !== proof.sha256) fail(); + const manifest = JSON.parse(original); + for (const artifact of artifacts) { + const inventory = capsule.verify(artifact.directory, artifact.digest, new Set([artifact.root])); + if (manifest.roots.some(root => root === artifact.root || root.startsWith(artifact.root + '/') || artifact.root.startsWith(root + '/'))) fail(); + manifest.roots.push(artifact.root); + for (const entry of inventory.entries) { + const payload = `files/${String(manifest.entries.length).padStart(8, '0')}`; + manifest.entries.push(entry.type === 'file' ? { ...entry, payload, + artifact: { root: artifact.root, digest: artifact.digest, snapshotId: artifact.snapshotId, payload: entry.payload } } : entry); + } + } + const bytes = JSON.stringify(manifest) + '\n'; + require('./release-delivery-files').atomic(path.join(directory, 'recovery.json'), bytes); + capsule.verify(directory, sha(bytes), new Set(manifest.roots), entry => { + const a = artifacts.find(a => a.digest === entry.artifact.digest && a.root === entry.artifact.root); + if (!a || !/^files\/[0-9]{8}$/.test(entry.artifact.payload)) fail(); + return path.join(a.directory, entry.artifact.payload); + }); + return { ...proof, sha256: sha(bytes), files: manifest.entries.filter(e => e.type === 'file').length, + size: manifest.entries.reduce((sum, e) => sum + (e.size || 0), 0) }; +} +function hydrate(directory, expectedDigest, run, workRoot = path.dirname(directory)) { + const file = path.join(directory, 'recovery.json'); + for (const dir of [directory, path.join(directory, 'files')]) { + if (!fs.lstatSync(dir).isDirectory() || fs.realpathSync(dir) !== dir) fail(); + } + const info = fs.lstatSync(file); + if (!info.isFile() || info.nlink !== 1 || info.size > 128 * 1024 ** 2) fail(); + const bytes = fs.readFileSync(file); + if (!hex(expectedDigest) || sha(bytes) !== expectedDigest) throw Error('recovery_capsule_invalid'); + const manifest = JSON.parse(bytes), downloaded = new Map(); + if (manifest.schemaVersion !== 1 || !Array.isArray(manifest.entries) || manifest.entries.length > 1000000) fail(); + const work = fs.mkdtempSync(path.join(workRoot, 'artifact-restore-')); + try { + for (const entry of manifest.entries) { + if (!entry.artifact) continue; + const a = entry.artifact; + if (entry.type !== 'file' || !releaseRoot(a.root) || !hex(a.digest) || !hex(a.snapshotId) + || !/^files\/[0-9]{8}$/.test(a.payload) || !/^files\/[0-9]{8}$/.test(entry.payload) + || !entry.path.startsWith(a.root + '/')) fail(); + const key = `${a.digest}:${a.snapshotId}`; + if (!downloaded.has(key)) { + const target = path.join(work, String(downloaded.size)); + run(`recovery-artifacts/${a.digest}`, ['--no-lock', 'restore', a.snapshotId, '--target', target, '--verify']); + const artifact = path.join(target, 'artifact'); + const inventory = capsule.verify(artifact, a.digest, new Set([a.root])); + downloaded.set(key, { directory: artifact, entries: new Map(inventory.entries.map(e => [e.path, e])) }); + } + const cached = downloaded.get(key), original = cached.entries.get(entry.path); + if (!original || original.payload !== a.payload || ['sha256', 'size', 'uid', 'gid', 'mode'].some(k => entry[k] !== original[k])) fail(); + const destination = path.join(directory, entry.payload); + if (fs.existsSync(destination)) { + const actual = capsule.streamFile(destination); + if (actual.sha256 !== entry.sha256 || actual.size !== entry.size) fail(); + } else capsule.streamFile(path.join(cached.directory, a.payload), destination); + } + } finally { fs.rmSync(work, { recursive: true, force: true }); } +} +function pruneLocalCache(cacheRoot = '/var/lib/dispatch-backup/recovery-artifacts', ownerUid = 0, locked = false) { + if (ownerUid === 0 && !locked && fs.existsSync(cacheRoot)) { + const result = require('node:child_process').spawnSync('/usr/bin/flock', ['--nonblock', '--conflict-exit-code', '75', + path.join(cacheRoot, 'maintenance.lock'), '/usr/bin/node', '--no-warnings', __filename, '--prune-cache'], + {stdio:'ignore', env:{PATH:'/usr/bin:/bin'}, timeout:30000}); + if (![0,75].includes(result.status)) fail(); + return; + } + // Called under the exporter lock with no artifact workers running. Remote + // dependencies remain retained; obsolete local copies are just a cache. + if (!fs.existsSync(cacheRoot)) return; + const parent = fs.lstatSync(cacheRoot); + if (!parent.isDirectory() || parent.uid !== ownerUid || parent.mode & 0o077 || fs.realpathSync(cacheRoot) !== cacheRoot) fail(); + for (const name of fs.readdirSync(cacheRoot)) { + if (!hex(name)) continue; + const directory = path.join(cacheRoot, name), info = fs.lstatSync(directory); + if (!info.isDirectory() || info.uid !== ownerUid || info.mode & 0o077 || fs.realpathSync(directory) !== directory) fail(); + const receiptFile = path.join(directory, 'receipt.json'); + if (!fs.existsSync(receiptFile)) { fs.rmSync(directory, { recursive: true }); continue; } + const receipt = require('./release-delivery-files').privateJson(receiptFile, ownerUid); + if (!releaseRoot(receipt.root) || !hex(receipt.digest) || !hex(receipt.snapshotId)) fail(); + if (!fs.existsSync(receipt.root)) fs.rmSync(directory, { recursive: true }); + } +} +function resticReader(config) { + return (repository, args) => require('./offsite-backup').createRestic({ ...config.environment, + RESTIC_REPOSITORY: `s3:https://${config.accountId}.r2.cloudflarestorage.com/${config.bucket}/${repository}` })(args); +} +if (require.main === module) { + try { + if (process.geteuid() !== 0 || process.argv.length !== 3) fail(); + if (process.argv[2] === '--prune-cache') { pruneLocalCache(undefined, 0, true); process.exit(0); } + const result = createArtifactStore(require('./offsite-backup').loadConfig()).prepare(process.argv[2]); + process.stdout.write(JSON.stringify(result) + '\n'); + } catch { process.stderr.write('recovery_artifact_invalid\n'); process.exitCode = 1; } +} +module.exports = { createArtifactStore, prepareRelease, append, hydrate, resticReader, releaseRoot, pruneLocalCache }; diff --git a/core/core/installations/src/recovery-capsule.js b/core/core/installations/src/recovery-capsule.js new file mode 100644 index 0000000..06167e9 --- /dev/null +++ b/core/core/installations/src/recovery-capsule.js @@ -0,0 +1,251 @@ +'use strict'; +// Portable file inventory for encrypted recovery bundles. Payload files are +// private regular files; original ownership/modes are applied only on restore. +const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto'); +const MAX_FILES = 1000000; +const fail = () => { throw Object.assign(Error('recovery_capsule_invalid'), { code: 'recovery_capsule_invalid' }); }; +const hash = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); +function absolute(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value || /[\0\r\n]/.test(value)) fail(); + return value; +} +function relative(value) { + if (typeof value !== 'string' || value.length > 4096 || value.startsWith('/') || value.split('/').some(p => !p || p === '.' || p === '..') || /[\0\r\n]/.test(value)) fail(); + return value; +} +function streamFile(source, target) { + const fd = fs.openSync(source, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + let out; + try { + const before = fs.fstatSync(fd); + if (!before.isFile() || before.nlink !== 1) fail(); + if (target) out = fs.openSync(target, 'wx', 0o600); + const checksum = crypto.createHash('sha256'), buffer = Buffer.allocUnsafe(1024 * 1024); + let size = 0, length; + while ((length = fs.readSync(fd, buffer, 0, buffer.length, null))) { + checksum.update(buffer.subarray(0, length)); size += length; + if (size > before.size) fail(); + if (out !== undefined) { + let offset = 0; + while (offset < length) offset += fs.writeSync(out, buffer, offset, length - offset); + } + } + const after = fs.fstatSync(fd); + if (before.ino !== after.ino || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || size !== before.size) fail(); + if (out !== undefined) fs.fsyncSync(out); + return { size, sha256: checksum.digest('hex') }; + } finally { fs.closeSync(fd); if (out !== undefined) fs.closeSync(out); } +} +function isDatabase(file) { + let fd; + try { + fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + if (!fs.fstatSync(fd).isFile()) return false; + const header = Buffer.alloc(16); + return fs.readSync(fd, header, 0, 16, 0) === 16 && header.toString() === 'SQLite format 3\0'; + } catch (error) { if (error.code === 'ENOENT' || error.code === 'ELOOP') return false; throw error; } + finally { if (fd !== undefined) fs.closeSync(fd); } +} +function snapshotDatabase(source, target, sealedSource = false) { + // Opening a WAL database read-only can still create -wal/-shm beside it. + // Lifecycle snapshots have a fixed tree digest: never let SQLite open those + // source files directly. Include committed WAL pages in the private copy. + let work; + const original = source; + const copied = []; + if (sealedSource) { + work = fs.mkdtempSync(path.join(path.dirname(target), '.sqlite-read-')); + source = path.join(work, 'database'); + try { + for (const suffix of ['', '-wal']) { + if (suffix && !fs.existsSync(original + suffix)) continue; + copied.push([suffix, streamFile(original + suffix, source + suffix)]); + } + for (const [suffix, proof] of copied) { + if (JSON.stringify(streamFile(original + suffix)) !== JSON.stringify(proof)) fail(); + } + } catch (error) { fs.rmSync(work, { recursive: true, force: true }); throw error; } + } + const script = `import sqlite3,sys,urllib.parse,os +source,target=sys.argv[1:] +src=sqlite3.connect('file:'+urllib.parse.quote(source)+'?mode=ro',uri=True,timeout=30) +src.execute('PRAGMA trusted_schema=OFF') +dst=sqlite3.connect(target) +try: + src.backup(dst) + if dst.execute('PRAGMA quick_check').fetchone()[0]!='ok': raise RuntimeError('database_corrupt') +finally: + dst.close(); src.close() +os.chmod(target,0o600) +with open(target,'rb') as f: os.fsync(f.fileno()) +`; + try { + const result = require('node:child_process').spawnSync('/usr/bin/python3', ['-I', '-c', script, source, target], + { timeout: 120000, maxBuffer: 1024, encoding: 'utf8' }); + if (result.status !== 0 || result.error) fail(); + return streamFile(target); + } finally { if (work) fs.rmSync(work, { recursive: true, force: true }); } +} +function capture(destination, roots, metadata, allowedOwners) { + absolute(destination); + if (fs.existsSync(destination) || !Array.isArray(roots) || !roots.length || !(allowedOwners instanceof Set)) fail(); + const entries = [], targets = new Set(); + fs.mkdirSync(destination, { mode: 0o700 }); + fs.mkdirSync(path.join(destination, 'files'), { mode: 0o700 }); + for (const selected of roots) { + const source = absolute(selected.source), target = absolute(selected.target); + const excludeContents = new Set((selected.excludeContents || []).map(relative)); + const exclude = new Set((selected.exclude || []).map(relative)); + const overrides = Object.entries(selected.overrides || {}).map(([suffix, replacement]) => [relative(suffix), absolute(replacement)]); + if (source === destination || destination.startsWith(source + '/') || [...targets].some(root => + target === root || target.startsWith(root + '/') || root.startsWith(target + '/'))) fail(); + targets.add(target); + function visit(file, suffix) { + if (exclude.has(suffix)) return; + const override = overrides.find(([prefix]) => suffix === prefix || suffix.startsWith(prefix + '/')); + if (override) file = path.join(override[1], suffix.slice(override[0].length)); + // Each database is copied using SQLite's backup API, including committed + // WAL pages. Its transient sidecars must not overwrite that clean copy. + if (/-(wal|shm)$/.test(file) && isDatabase(file.replace(/-(wal|shm)$/, ''))) return; + if (entries.length >= MAX_FILES || fs.realpathSync(path.dirname(file)) !== path.dirname(file)) fail(); + const info = fs.lstatSync(file), name = suffix ? path.join(target, suffix) : target; + if (!allowedOwners.has(info.uid) || (info.mode & 0o7000)) fail(); + const entry = { path: name, uid: info.uid, gid: info.gid, mode: info.mode & 0o777 }; + if (info.isSymbolicLink()) { + const link = fs.readlinkSync(file); + if (/\0/.test(link)) fail(); + entries.push({ ...entry, type: 'link', link }); + } else if (info.isDirectory()) { + entries.push({ ...entry, type: 'directory' }); + const names = () => fs.readdirSync(file).sort().filter(name => !/-(wal|shm)$/.test(name) + || !isDatabase(path.join(file, name.replace(/-(wal|shm)$/, '')))); + const beforeNames = names(); + if (!excludeContents.has(suffix)) for (const child of beforeNames) visit(path.join(file, child), suffix ? `${suffix}/${child}` : child); + const after = fs.lstatSync(file); + // SQLite may create/remove its own WAL index while backing up a closed + // WAL database. Require the non-transient directory inventory to match. + if (after.ino !== info.ino || JSON.stringify(beforeNames) !== JSON.stringify(names())) fail(); + } else if (info.isFile()) { + const payload = `files/${String(entries.length).padStart(8, '0')}`; + const content = isDatabase(file) ? snapshotDatabase(file, path.join(destination, payload), Boolean(override)) + : streamFile(file, path.join(destination, payload)); + entries.push({ ...entry, type: 'file', payload, ...content }); + } else fail(); // Sockets/PIDs and device nodes are recreated by supervision. + } + visit(source, ''); + } + const manifest = { schemaVersion: 1, metadata, roots: [...targets], entries }; + const bytes = JSON.stringify(manifest) + '\n'; + fs.writeFileSync(path.join(destination, 'recovery.json'), bytes, { flag: 'wx', mode: 0o600 }); + return { sha256: hash(bytes), files: entries.filter(e => e.type === 'file').length, + size: entries.reduce((sum, e) => sum + (e.size || 0), 0) }; +} +function verify(directory, expectedDigest, allowedRoots, resolvePayload = null) { + absolute(directory); + if (!(allowedRoots instanceof Set) || !/^[a-f0-9]{64}$/.test(expectedDigest)) fail(); + for (const selected of [directory, path.join(directory, 'files')]) { + if (!fs.lstatSync(selected).isDirectory() || fs.realpathSync(selected) !== selected) fail(); + } + const manifestFile = path.join(directory, 'recovery.json'), info = fs.lstatSync(manifestFile); + if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || info.size > 128 * 1024 ** 2) fail(); + const bytes = fs.readFileSync(manifestFile); + if (hash(bytes) !== expectedDigest) fail(); + const manifest = JSON.parse(bytes); + if (manifest.schemaVersion !== 1 || !Array.isArray(manifest.roots) || !Array.isArray(manifest.entries) + || !manifest.entries.length || manifest.entries.length > MAX_FILES) fail(); + if (!manifest.roots.length || new Set(manifest.roots).size !== manifest.roots.length + || manifest.roots.some(root => !allowedRoots.has(absolute(root)) + || manifest.roots.some(other => other !== root && root.startsWith(other + '/')))) fail(); + const paths = new Map(), payloads = new Set(), localPayloads = new Set(); + for (const entry of manifest.entries) { + absolute(entry.path); + if (!manifest.roots.some(root => entry.path === root || entry.path.startsWith(root + '/')) || paths.has(entry.path) + || !Number.isSafeInteger(entry.uid) || entry.uid < 0 || !Number.isSafeInteger(entry.gid) || entry.gid < 0 + || !Number.isSafeInteger(entry.mode) || entry.mode < 0 || entry.mode > 0o777) fail(); + paths.set(entry.path, entry); + if (entry.type === 'file') { + relative(entry.payload); + if (!/^files\/[0-9]{8}$/.test(entry.payload) || payloads.has(entry.payload)) fail(); + const local = path.join(directory, entry.payload); + const external = resolvePayload && entry.artifact ? resolvePayload(entry) : null; + const actual = streamFile(external || local); + if (!external) localPayloads.add(entry.payload); + if (actual.size !== entry.size || actual.sha256 !== entry.sha256) fail(); + payloads.add(entry.payload); + } else if (entry.type === 'link') { + if (typeof entry.link !== 'string' || /[\0\r\n]/.test(entry.link)) fail(); + const target = path.resolve(path.dirname(entry.path), entry.link); + if (!manifest.roots.some(root => target === root || target.startsWith(root + '/'))) fail(); + } else if (entry.type !== 'directory') fail(); + } + for (const entry of paths.values()) { + let parent = path.dirname(entry.path); + while (parent !== '/') { + if (paths.has(parent) && paths.get(parent).type !== 'directory') fail(); + parent = path.dirname(parent); + } + } + if (manifest.roots.some(root => !paths.has(root))) fail(); + const actualPayloads = fs.readdirSync(path.join(directory, 'files')).map(name => `files/${name}`); + if (actualPayloads.length !== localPayloads.size || actualPayloads.some(name => !localPayloads.has(name))) fail(); + return manifest; +} +function materialize(directory, stagingRoot, expectedDigest, allowedRoots) { + const manifest = verify(directory, expectedDigest, allowedRoots); + absolute(stagingRoot); + if (fs.existsSync(stagingRoot)) fail(); + fs.mkdirSync(stagingRoot, { mode: 0o700 }); + for (const entry of manifest.entries.filter(e => e.type !== 'link')) { + const file = path.join(stagingRoot, entry.path.slice(1)); + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); + if (entry.type === 'directory') fs.mkdirSync(file, { recursive: true, mode: 0o700 }); + else streamFile(path.join(directory, entry.payload), file); + } + // Create links last so payload placement never traverses a restored symlink. + for (const entry of manifest.entries.filter(e => e.type === 'link')) { + const file = path.join(stagingRoot, entry.path.slice(1)); + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o700 }); fs.symlinkSync(entry.link, file); + } + return manifest; +} +function installFresh(directory, expectedDigest, allowedRoots) { + if (process.geteuid() !== 0) fail(); + const manifest = verify(directory, expectedDigest, allowedRoots); + // Never use this restore primitive to overwrite a running installation. The + // operator restores onto a clean host, or removes a stopped fixture first. + for (const root of manifest.roots) { + try { fs.lstatSync(root); fail(); } catch (error) { if (error.code !== 'ENOENT') throw error; } + let parent = path.dirname(root); + while (!fs.existsSync(parent)) parent = path.dirname(parent); + if (fs.realpathSync(parent) !== parent) fail(); + } + const stage = fs.mkdtempSync('/var/tmp/dispatch-recovery-'); + fs.rmdirSync(stage); + try { + materialize(directory, stage, expectedDigest, allowedRoots); + for (const entry of [...manifest.entries].sort((a, b) => b.path.length - a.path.length)) { + const file = path.join(stage, entry.path.slice(1)); + fs.lchownSync(file, entry.uid, entry.gid); + if (entry.type !== 'link') fs.chmodSync(file, entry.mode); + } + for (const root of manifest.roots) { + const parents = []; + for (let parent = path.dirname(root); !fs.existsSync(parent); parent = path.dirname(parent)) parents.push(parent); + fs.mkdirSync(path.dirname(root), { recursive: true, mode: 0o755 }); + // Shared release ancestors must be traversable even under umask 0077. + // Existing parents and the archived private roots retain their own modes. + for (const parent of parents) fs.chmodSync(parent, 0o755); + // cp preserves symlinks and ownership across filesystems; rename is only + // possible when the restored root shares the staging filesystem. + const source = path.join(stage, root.slice(1)); + try { fs.renameSync(source, root); } + catch (error) { + if (error.code !== 'EXDEV') throw error; + const result = require('node:child_process').spawnSync('/usr/bin/cp', ['--archive', '--no-target-directory', '--', source, root], { timeout: 600000 }); + if (result.status !== 0) fail(); + } + } + return manifest; + } finally { fs.rmSync(stage, { recursive: true, force: true }); } +} +module.exports = { capture, verify, materialize, installFresh, streamFile }; diff --git a/core/core/installations/src/recovery-kit-runner.js b/core/core/installations/src/recovery-kit-runner.js new file mode 100644 index 0000000..77ca935 --- /dev/null +++ b/core/core/installations/src/recovery-kit-runner.js @@ -0,0 +1,86 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +const { spawnSync } = require('node:child_process'); +async function main(kit, args) { + if (process.geteuid() !== 0) throw Error('recovery_requires_root'); + const c = JSON.parse(fs.readFileSync(path.join(kit, 'storage.json'))); + if (!/^[a-f0-9]{32}$/.test(c.accountId) || !/^[a-z0-9][a-z0-9-]{1,61}[a-z0-9]$/.test(c.bucket) + || !/^[a-z][a-z0-9_-]{2,63}$/.test(c.prefix) || !/^[a-f0-9]{32}$/.test(c.credential?.accessKeyId) + || !/^[a-f0-9]{64}$/.test(c.credential?.secretAccessKey)) throw Error('recovery_kit_invalid'); + function run(repository, parameters) { + const result = spawnSync(path.join(kit, 'lib/ld-linux-x86-64.so.2'), ['--library-path', path.join(kit, 'lib'), path.join(kit, 'restic'), '--no-cache', '--json', ...parameters], { encoding: 'utf8', + timeout: 3600000, maxBuffer: 8 * 1024 ** 2, env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8', + AWS_ACCESS_KEY_ID: c.credential.accessKeyId, AWS_SECRET_ACCESS_KEY: c.credential.secretAccessKey, + AWS_DEFAULT_REGION: 'auto', RESTIC_PASSWORD_FILE: path.join(kit, 'password'), + RESTIC_REPOSITORY: `s3:https://${c.accountId}.r2.cloudflarestorage.com/${c.bucket}/${repository}` } }); + if (result.status !== 0 || result.error) throw Error('recovery_download_failed'); + return result.stdout; + } + if (args.length === 1 && args[0] === 'verify-kit') { + require('./host-recovery-bundle').supportedHost(); + if (!fs.statSync(path.join(kit, 'password')).size || !fs.statSync(path.join(kit, 'restic')).size) throw Error('recovery_kit_invalid'); + const probe = spawnSync(path.join(kit, 'lib/ld-linux-x86-64.so.2'), ['--library-path', path.join(kit, 'lib'), path.join(kit, 'restic'), 'version'], { encoding: 'utf8', timeout: 10000 }); + if (probe.status !== 0 || !/^restic /.test(probe.stdout)) throw Error('recovery_kit_invalid'); + return { status: 'local_kit_verified' }; + } + if (args.length === 1 && args[0] === 'list') { + const storage = require('./r2-backup-storage').createR2BackupStorage(c, { credentials: { credential: c.credential, token: null } }); + const result = []; + for(const id of await storage.listSets()) {try {const manifest=JSON.parse(run(`sets/${id}`,['dump','latest','system.json']));result.push({id,kind:'system',fullPlatform:true,status:manifest.status,createdAt:new Date(manifest.createdAt).toISOString()});}catch{result.push({id,kind:'system',status:'unavailable'});}} + for (const row of await storage.listArchives()) { + try { + const repository = `archives/${row.retentionDays === null ? 'all' : row.retentionDays}/${row.id}`; + const metadata = JSON.parse(run(repository, ['dump', 'latest', 'bundle/dsp.json'])); + const snapshots = JSON.parse(run(repository, ['snapshots'])); + const recoveryProof = JSON.parse(run(repository, ['dump', 'latest', 'bundle/recovery-proof.json'])); + result.push({ ...row, kind: metadata.kind, fullPlatform: metadata.kind === 'core' && metadata.metadata?.scope !== 'core' && /^[a-f0-9]{64}$/.test(recoveryProof.sha256), createdAt: snapshots.at(-1)?.time }); + } catch { result.push({ ...row, status: 'unavailable' }); } + } + for (const snapshot of JSON.parse(run(c.prefix, ['snapshots']))) { + try { + const proof = JSON.parse(run(c.prefix, ['dump', snapshot.id, 'recovery-proof.json'])); + if (/^[a-f0-9]{64}$/.test(proof.sha256)) result.push({ id: 'platform-core', snapshotId: snapshot.id, createdAt: snapshot.time, fullPlatform: true }); + } catch {} // Legacy data-only snapshots and connection canaries are not full-host recovery points. + } + return result; + } + if(args[0]==='restore-system'&&args.length===2&&/^breq_[a-f0-9]{32}$/.test(args[1])) { + const set=JSON.parse(run(`sets/${args[1]}`,['dump','latest','system.json'])); + if(set.schemaVersion!==1||set.id!==args[1]||set.kind!=='system'||set.status!=='verified'||!Array.isArray(set.components)||set.components.some(c=>c.deleted))throw Error('system_backup_incomplete'); + const work=fs.mkdtempSync('/var/tmp/dispatch-system-restore-'); + try{ + const components=[]; + for(const [index,component] of set.components.entries()){ + if(!/^(breq|backup)_[a-f0-9]{32}$/.test(component.id)||![null,7,30,90,365].includes(component.retentionDays))throw Error('system_backup_invalid'); + const directory=path.join(work,String(index)); + run(`archives/${component.retentionDays===null?'all':component.retentionDays}/${component.id}`,['restore','latest','--target',directory,'--verify']); + const bundle=path.join(directory,'bundle'),info=JSON.parse(fs.readFileSync(path.join(bundle,'dsp.json'))),proof=JSON.parse(fs.readFileSync(path.join(bundle,'recovery-proof.json'))); + if(info.id!==component.id||info.kind!==component.kind||component.kind==='dsp'&&info.metadata.organizationId!==component.organizationId)throw Error('system_backup_invalid'); + require('./recovery-artifacts').hydrate(path.join(bundle, 'recovery'), proof.sha256, run); + components.push({...component,directory:bundle,digest:proof.sha256}); + } + const assembled=require('./assemble-system-recovery').assembleSystemRecovery({components,destination:path.join(work,'assembled'),systemSchedule:set.systemSchedule}); + return await require('./host-recovery-bundle').restoreHostRecovery(assembled); + }finally{fs.rmSync(work,{recursive:true,force:true});} + } + if (args[0] !== 'restore' || args.length < 3 || args.length > 4) throw Error('usage: list | restore BACKUP_ID RETENTION | restore platform-core SNAPSHOT_ID'); + const [_, id, tier, selectedSnapshot = 'latest'] = args; + if (id !== 'platform-core' && !/^(backup|breq)_[a-f0-9]{32}$/.test(id) + || id !== 'platform-core' && !['all', '7', '30', '90', '365'].includes(tier) + || id === 'platform-core' && !/^[a-f0-9]{64}$/.test(tier) + || selectedSnapshot !== 'latest' && !/^[a-f0-9]{64}$/.test(selectedSnapshot)) throw Error('recovery_selection_invalid'); + const work = fs.mkdtempSync('/var/tmp/dispatch-full-restore-'); fs.chmodSync(work, 0o700); + try { + run(id === 'platform-core' ? c.prefix : `archives/${tier}/${id}`, + ['restore', id === 'platform-core' ? tier : selectedSnapshot, '--target', work, '--verify']); + const base = id === 'platform-core' ? work : path.join(work, 'bundle'); + const proof = JSON.parse(fs.readFileSync(path.join(base, 'recovery-proof.json'))); + if (id !== 'platform-core') { + const metadata = JSON.parse(fs.readFileSync(path.join(base, 'dsp.json'))); + if (metadata.id !== id || metadata.kind !== 'core') throw Error('select_a_full_platform_backup'); + } + require('./recovery-artifacts').hydrate(path.join(base, 'recovery'), proof.sha256, run); + return require('./host-recovery-bundle').restoreHostRecovery({ directory: path.join(base, 'recovery'), digest: proof.sha256 }); + } finally { fs.rmSync(work, { recursive: true, force: true }); } +} +module.exports = { main }; diff --git a/core/core/installations/src/recovery-prewarm.js b/core/core/installations/src/recovery-prewarm.js new file mode 100644 index 0000000..17073ab --- /dev/null +++ b/core/core/installations/src/recovery-prewarm.js @@ -0,0 +1,47 @@ +'use strict'; +// Only immutable release payloads are warmed. Mutable snapshots and credentials +// remain captured afresh by the ordinary backup pipeline before each rollout. +const fs = require('node:fs'); +const path = require('node:path'); +const {atomic} = require('./release-delivery-files'); +const {releaseRoot, prepareRelease} = require('./recovery-artifacts'); +const BASES = ['platform','runtime','control','updater','release-delivery'].map(name => `/opt/dispatch-${name}/releases`); +function candidates(bases = BASES) { + const roots = []; + for (const base of bases) { + if (!fs.existsSync(base)) continue; + const parent = fs.lstatSync(base); + if (parent.uid !== 0 || !parent.isDirectory() || parent.mode & 0o022 || fs.realpathSync(base) !== base) throw Error('unsafe_recovery_root'); + for (const entry of fs.readdirSync(base, {withFileTypes:true})) { + const root = path.join(base, entry.name); + // Installation stages are not complete immutable release roots yet. + if (entry.isDirectory() && !entry.name.endsWith('.pending') && releaseRoot(root)) { + const stat = fs.lstatSync(root); + if (stat.uid === 0 && (stat.mode & 0o222) === 0) roots.push(root); + } + } + } + return roots; +} +function prewarm({roots = candidates(), prepare = prepareRelease, report = () => {}, clock = Date.now, + config = require('./offsite-backup').loadConfig()} = {}) { + let prepared = 0, failed = 0; + const startedAt = clock(), stages = []; + report({ schemaVersion: 1, status: 'running', startedAt, checkedAt: clock(), prepared, failed, stages }); + for (const root of roots) { + const start = clock(); + let status = 'verified'; + try { prepare(config, root); prepared++; } catch { failed++; status = 'failed'; } + stages.push({ component: root.split('/')[2], releaseId: path.basename(root), status, durationMs: clock() - start }); + report({ schemaVersion: 1, status: 'running', startedAt, checkedAt: clock(), prepared, failed, stages: stages.slice(-100) }); + } + report({ schemaVersion: 1, status: failed ? 'attention' : 'ready', startedAt, checkedAt: clock(), + durationMs: clock() - startedAt, prepared, failed, stages: stages.slice(-100) }); + return {status:failed ? 'recovery_prewarm_incomplete' : 'recovery_prewarmed', prepared, failed}; +} +function install(executable, localRoot) { + atomic('/etc/systemd/system/dispatch-recovery-prewarm.service', `[Unit]\nDescription=Prepare immutable Dispatch recovery files ahead of backups\nAfter=network-online.target\n\n[Service]\nType=oneshot\nUMask=0077\nNice=10\nIOSchedulingClass=idle\nExecStart=/usr/bin/node --no-warnings ${executable} prewarm\nTimeoutStartSec=1h\n`, 0o644); + atomic('/etc/systemd/system/dispatch-recovery-prewarm.timer', '[Unit]\nDescription=Keep Dispatch recovery files prepared\n\n[Timer]\nOnBootSec=2min\nOnUnitInactiveSec=15min\n\n[Install]\nWantedBy=timers.target\n', 0o644); + if (localRoot) atomic('/etc/systemd/system/dispatch-recovery-prewarm.path', `[Unit]\nDescription=Prepare immutable recovery files during release preflight\n\n[Path]\nPathChanged=${localRoot}/run/release-preflight\nUnit=dispatch-recovery-prewarm.service\n\n[Install]\nWantedBy=multi-user.target\n`, 0o644); +} +module.exports = {candidates, prewarm, install}; diff --git a/core/core/installations/src/release-build-space.js b/core/core/installations/src/release-build-space.js new file mode 100644 index 0000000..0f0167e --- /dev/null +++ b/core/core/installations/src/release-build-space.js @@ -0,0 +1,39 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const GiB = 1024 ** 3; +function treeBytes(directory) { + const stat = fs.statSync(directory); + if (!stat.isDirectory()) return stat.size; + return fs.readdirSync(directory).reduce((total, name) => { + const child = path.join(directory, name); + // Browser symlinks are validated when bundled. Count their file bytes here. + if (fs.lstatSync(child).isSymbolicLink()) return total + fs.statSync(child).size; + return total + treeBytes(child); + }, 0); +} +function preflight(output, {format, preparedDirectory, browserRoot = process.env.DISPATCH_BUILD_BROWSER_ROOT || '/opt/google/chrome', statfs = fs.statfsSync} = {}) { + // Keep scratch beside the output, on the filesystem the caller selected. + // Reserve expanded files, archives and 1 GiB of headroom; this is an estimate, + // so archive errors still distinguish exhaustion if another build fills disk. + const sourceBytes = treeBytes(preparedDirectory || browserRoot) + fs.statSync(process.execPath).size; + const requiredBytes = sourceBytes * (format === 'both' ? 5 : 4) + GiB; + const available = statfs(path.dirname(output)); + const availableBytes = available.bavail * available.bsize; + if (availableBytes < requiredBytes) throw Object.assign(Error('release_build_insufficient_space'), {requiredBytes, availableBytes}); + return {requiredBytes, availableBytes}; +} +function checkArchiveResult(result, fallback) { + if (!result.error && result.status === 0) return; + const code = result.error?.code === 'ETIMEDOUT' ? 'release_archive_timeout' + : /archive_disk_full/.test(result.stderr || '') ? 'release_archive_disk_full' + : result.signal ? 'release_archive_interrupted' : fallback; + throw Object.assign(Error(code), {code}); +} +async function withOutput(output, work) { + // mkdir without recursive is the ownership claim, including concurrent calls. + fs.mkdirSync(output, {mode:0o700}); + try { return await work(); } + catch (error) { require('./release-delivery-install').removeStage(output); throw error; } +} +module.exports = {preflight, checkArchiveResult, withOutput}; diff --git a/core/core/installations/src/release-catalog.js b/core/core/installations/src/release-catalog.js new file mode 100644 index 0000000..3874eeb --- /dev/null +++ b/core/core/installations/src/release-catalog.js @@ -0,0 +1,55 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { releaseDescriptor } = require('./oci-deployment'); + +function fail() { throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); } + +function loadPrivateReleaseCatalog(fileValue) { + if (fileValue === undefined) return Object.freeze({}); + if (typeof fileValue !== 'string' || !path.isAbsolute(fileValue) || path.resolve(fileValue) !== fileValue + || /[\0\r\n]/.test(fileValue)) fail(); + const info = fs.lstatSync(fileValue); + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() || info.nlink !== 1 + || (info.mode & 0o7777) !== 0o600 || fs.realpathSync(fileValue) !== fileValue + || info.size < 3 || info.size > 64 * 1024) fail(); + let parsed; + try { parsed = JSON.parse(fs.readFileSync(fileValue, 'utf8')); } catch { fail(); } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) fail(); + const catalog = {}; + for (const [releaseId, root] of Object.entries(parsed)) { + if (!/^[a-z][a-z0-9_.-]{2,95}$/.test(releaseId) || typeof root !== 'string' + || !path.isAbsolute(root) || path.resolve(root) !== root || fs.realpathSync(root) !== root) fail(); + const rootInfo = fs.lstatSync(root); + if (!rootInfo.isDirectory() || rootInfo.isSymbolicLink() || rootInfo.uid !== process.geteuid() + || (rootInfo.mode & 0o022) !== 0) fail(); + catalog[releaseId] = root; + } + return Object.freeze(catalog); +} + +function loadPrivateOciReleaseCatalog(fileValue) { + if (fileValue === undefined) return Object.freeze({}); + if (typeof fileValue !== 'string' || !path.isAbsolute(fileValue) || path.resolve(fileValue) !== fileValue + || /[\0\r\n]/.test(fileValue)) fail(); + const info = fs.lstatSync(fileValue); + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() || info.nlink !== 1 + || (info.mode & 0o7777) !== 0o600 || fs.realpathSync(fileValue) !== fileValue + || info.size < 3 || info.size > 256 * 1024) fail(); + let parsed; + try { parsed = JSON.parse(fs.readFileSync(fileValue, 'utf8')); } catch { fail(); } + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed) + || Object.keys(parsed).sort().join(',') !== 'releases,schemaVersion' + || parsed.schemaVersion !== 1 || !parsed.releases || typeof parsed.releases !== 'object' + || Array.isArray(parsed.releases)) fail(); + const result = {}; + for (const [releaseId, value] of Object.entries(parsed.releases)) { + const selected = releaseDescriptor(value); + if (selected.releaseId !== releaseId || selected.channel !== 'production') fail(); + result[releaseId] = selected; + } + return Object.freeze(result); +} + +module.exports = { loadPrivateReleaseCatalog, loadPrivateOciReleaseCatalog }; diff --git a/core/core/installations/src/release-delivery-build.js b/core/core/installations/src/release-delivery-build.js new file mode 100644 index 0000000..96bc84e --- /dev/null +++ b/core/core/installations/src/release-delivery-build.js @@ -0,0 +1,168 @@ +'use strict'; +const fs=require('node:fs'); +const path=require('node:path'); +const {spawnSync}=require('node:child_process'); +const {PROJECT_ROOT}=require('../../../shared/paths/runtime-paths'); +const {sha,bundle,identity,releaseManifest}=require('./release-delivery-contract'); +const {hashFile}=require('./release-delivery-files'); +const {authoring, markdown}=require('./release-notes'); +const {removeStage}=require('./release-delivery-install'); +const formats = require('./release-formats'); +const legacy = formats.format('legacy'), split = formats.format('split'); +function command(executable,args,options={}) { + const result=spawnSync(executable,args,{cwd:PROJECT_ROOT,encoding:'utf8',timeout:600_000,maxBuffer:32*1024*1024,...options}); + if(result.status!==0||result.error) { + // Build logs contain tool diagnostics; credentials are supplied only to the later publication step. + if(result.stderr)process.stderr.write(result.stderr.toString().slice(-8192)); + throw new Error(`release_build_command_failed:${path.basename(executable)}:${args.find(arg=>['load','save','run','rm','unshare','inspect','tag'].includes(arg))||'source'}`); + } + return result.stdout; +} +function portableCore(output, popup = null) { + if(command('/usr/bin/git',['status','--porcelain']).trim())throw Error('clean_checkout_required'); + const commit=command('/usr/bin/git',['rev-parse','HEAD']).trim(); + const files=[]; + for(const row of command('/usr/bin/git',['ls-tree','-r','-z','HEAD']).split('\0').filter(Boolean)) { + const match=/^(\d+) blob ([a-f0-9]{40})\t(.+)$/.exec(row);if(!match)throw Error('unsupported_git_entry'); + const [,mode,object,relative]=match; + if(!(/^(core|dashboard|protocol)\//.test(relative)||['bin/dispatch-dashboard','bin/dispatch-access-admin'].includes(relative)) + ||/\/(tests|examples|docs)\//.test(relative))continue; + if(!['100644','100755'].includes(mode))throw Error('unsupported_git_mode'); + const data=Buffer.from(command('/usr/bin/git',['cat-file','blob',object],{encoding:null})); + files.push({path:`code/${relative}`,mode:mode==='100755'?'555':'444',sha256:sha(data),data:data.toString('base64')}); + } + files.push(...require('./release-frontend').buildFrontend(PROJECT_ROOT, commit, path.dirname(output))); + if(popup) { + require('../../accounts/src/release-popup').validatePopup(popup,{releaseId:popup.releaseId,version:popup.version,sourceCommit:commit}); + const data=Buffer.from(JSON.stringify(popup)+'\n'); + const relative='code/dashboard/release-popup.json'; + if(files.some(file=>file.path===relative))throw Error('reserved_release_popup_file'); + files.push({path:relative,mode:'444',sha256:sha(data),data:data.toString('base64')}); + } + const temporary=fs.mkdtempSync(path.join(path.dirname(output),'helper-build-')); + try { + require('./create-host-helper-artifact').main([path.join(temporary,'host-helper-artifact')]); + collect(temporary,'host-helper-artifact',files); + } finally {removeStage(temporary);} + const value=bundle({schemaVersion:1,kind:'core',sourceCommit:commit,files},'core',commit); + fs.writeFileSync(output,JSON.stringify(value)+'\n',{flag:'wx',mode:0o644});return commit; +} +function collect(root,relative,files) { + const file=path.join(root,relative),stat=fs.lstatSync(file); + if(stat.isDirectory())for(const name of fs.readdirSync(file).sort())collect(root,path.join(relative,name),files); + else { + if(!stat.isFile()||stat.isSymbolicLink()||stat.nlink!==1)throw Error('unsafe_bundle_file'); + const data=fs.readFileSync(file);files.push({path:relative,mode:stat.mode&0o111?'555':'444',sha256:sha(data),data:data.toString('base64')}); + } +} +// Both formats share one source snapshot and one native dependency inventory. +async function build(version, notesFile, output, format = 'legacy', preparedDirectory = null) { + if (format !== 'both') formats.format(format); + if (format === 'both' && preparedDirectory) throw Error('both_formats_cannot_reuse'); + const started = Date.now(); + process.umask(0o022); + if (!path.isAbsolute(output) || path.resolve(output) !== output || output === PROJECT_ROOT || output.startsWith(PROJECT_ROOT + '/') || fs.existsSync(output)) throw Error('invalid_output'); + if (command('/usr/bin/git', ['status', '--porcelain']).trim()) throw Error('clean_checkout_required'); + const commit = command('/usr/bin/git', ['rev-parse', 'HEAD']).trim(), id = identity(version, commit); + const authored = authoring(JSON.parse(fs.readFileSync(notesFile, 'utf8'))); + const popup = authored.popup ? {schemaVersion:1, releaseId:id, version, sourceCommit:commit, ...authored.popup} : null; + const names = format === 'both' ? ['legacy', 'split'] : [format]; + require('./release-build-space').preflight(output, {format, preparedDirectory}); + return require('./release-build-space').withOutput(output, async () => { + const directories = Object.fromEntries(names.map(name => [name, format === 'both' ? path.join(output, name) : output])); + if (format === 'both') for (const directory of Object.values(directories)) fs.mkdirSync(directory, {mode:0o700}); + const parts = {}; + if (preparedDirectory) { + parts[format] = await reuseComponents(preparedDirectory, output, commit, popup, format); + } else { + const base = directories.legacy || directories.split; + portableCore(path.join(base, legacy.assets.core), popup); + const bridgeRoot = path.join(output, 'bridge-build'); fs.mkdirSync(bridgeRoot); + require('./create-bridge-artifact').main([path.join(bridgeRoot, 'bridge-artifact')]); + const files = []; collect(bridgeRoot, 'bridge-artifact', files); + fs.writeFileSync(path.join(base, legacy.assets.bridge), JSON.stringify(bundle({schemaVersion:1, kind:'bridge', sourceCommit:commit, files}, 'bridge', commit)) + '\n'); + const bridgeHash = sha(fs.readFileSync(path.join(bridgeRoot, 'bridge-artifact/manifest.json'))); removeStage(bridgeRoot); + const dependencies = directories.split ? require('./release-dependencies').verify() : null; + const native = require('./native-runtime-build'); + native.buildNativeRuntime({projectRoot:PROJECT_ROOT, archive:path.join(base, legacy.assets.runtime), sourceCommit:commit, + consumeStage(stage, embeddedManifestSha256) { + if (directories.legacy) parts.legacy = {bridgeHash, artifact: { + embeddedManifestSha256, artifactSha256:native.packNativeRuntime(stage, path.join(base, legacy.assets.runtime))}}; + if (directories.split) parts.split = {bridgeHash, dependencies, ...splitStage(stage, embeddedManifestSha256, base, directories.split, commit)}; + }}); + if (!directories.legacy) for (const kind of ['core', 'bridge']) fs.unlinkSync(path.join(base, legacy.assets[kind])); + } + const results = {}; + for (const name of names) results[name] = await finish(version, commit, id, authored, directories[name], name, parts[name], started); + if (format === 'both') fs.writeFileSync(path.join(output, 'build-metrics.json'), JSON.stringify({durationMs:Date.now()-started, format, nativeBuildCount:1}) + '\n'); + return format === 'both' ? {version, sourceCommit:commit, formats:results} : results[format]; + }); +} +function splitStage(stage, embeddedManifestSha256, base, output, commit) { + const packageRoot = path.join(output, 'package-build'); fs.mkdirSync(packageRoot); + try { + const appRoot = path.join(packageRoot, 'app'), dependencyRoot = path.join(packageRoot, 'dependencies'); + fs.mkdirSync(appRoot); fs.mkdirSync(dependencyRoot); + for (const kind of ['core', 'bridge']) { + const root = path.join(appRoot, kind); fs.mkdirSync(root); + require('./release-delivery-install').writeBundle(JSON.parse(fs.readFileSync(path.join(base, legacy.assets[kind]))), root); + } + fs.renameSync(path.join(stage, 'dependencies'), path.join(dependencyRoot, 'dependencies')); + fs.renameSync(stage, path.join(appRoot, 'runtime')); + const {pack, runtimeIdentity} = require('./release-package'); + const assets = { + app:pack(appRoot, path.join(output, split.assets.app), 'app', commit), + dependencies:pack(dependencyRoot, path.join(output, split.assets.dependencies), 'dependencies'), + }; + return {assets, artifact:{embeddedManifestSha256, artifactSha256:runtimeIdentity(assets)}}; + } finally { removeStage(packageRoot); } +} +async function finish(version, commit, id, authored, output, format, parts, started) { + const selected = formats.format(format), {changelog:notes, notes:presentation, github} = authored; + const {artifact, bridgeHash, dependencies = null} = parts, assets = parts.assets || {}; + const runtime = {version:1, backend:'native_service_v1', releaseId:id, channel:'production', sourceCommit:commit, + platform:'linux/amd64', runtimeAgentProtocol:1, runtimeGatewayProtocol:1, ...artifact, bridgeManifestSha256:bridgeHash}; + if (format === 'legacy') for (const [key, name] of Object.entries(selected.assets)) { + const file = path.join(output, name); assets[key] = {name, size:fs.statSync(file).size, sha256:await hashFile(file)}; + } + const richNotes = presentation ? {schemaVersion:1, releaseId:id, sourceCommit:commit, ...presentation} : null; + const manifest = releaseManifest({schemaVersion:selected.schemaVersion, version, releaseId:id, sourceCommit:commit, changelog:notes, assets, runtime, + ...(format === 'split' ? {notes:richNotes, dependencies} : {})}); + const bytes = JSON.stringify(manifest, null, 2) + '\n'; + if (Buffer.byteLength(bytes) > 256*1024) throw Error('release_manifest_too_large'); + fs.writeFileSync(path.join(output, formats.manifest), bytes); + if (selected.notesSidecar && richNotes) fs.writeFileSync(path.join(output, formats.notes), JSON.stringify(richNotes, null, 2) + '\n'); + fs.writeFileSync(path.join(output, formats.changelog), markdown(version, notes, presentation, github)); + if (selected.checksums) { + const sums = []; + for (const name of [...Object.values(assets).map(a => a.name), formats.manifest, ...(richNotes ? [formats.notes] : [])]) sums.push(`${await hashFile(path.join(output, name))} ${name}`); + fs.writeFileSync(path.join(output, selected.checksums), sums.join('\n') + '\n'); + } + fs.writeFileSync(path.join(output, 'build-metrics.json'), JSON.stringify({durationMs:Date.now()-started, format, assets}) + '\n'); + return manifest; +} +async function reuseComponents(directory, output, commit, popup, format = 'legacy') { + const manifest = releaseManifest(JSON.parse(fs.readFileSync(path.join(directory, formats.manifest)))); + if (manifest.sourceCommit !== commit || manifest.runtime.backend !== 'native_service_v1') throw Error('verified_source_mismatch'); + if (manifest.schemaVersion !== formats.format(format).schemaVersion) throw Error('verified_format_mismatch'); + for (const asset of Object.values(manifest.assets)) { + const file = path.join(directory, asset.name), stat = fs.lstatSync(file); + if (!stat.isFile() || stat.isSymbolicLink() || stat.size !== asset.size || await hashFile(file) !== asset.sha256) + throw Error('verified_asset_mismatch'); + } + if (format === 'split') return require('./release-reuse-split').reuse(directory, output, manifest, popup); + const core = bundle(JSON.parse(fs.readFileSync(path.join(directory, legacy.assets.core))), 'core', commit); + bundle(JSON.parse(fs.readFileSync(path.join(directory, legacy.assets.bridge))), 'bridge', commit); + core.files = core.files.filter(file => file.path !== 'code/dashboard/release-popup.json'); + if (popup) { + require('../../accounts/src/release-popup').validatePopup(popup, {releaseId: popup.releaseId, version: popup.version, sourceCommit: commit}); + const bytes = Buffer.from(JSON.stringify(popup) + '\n'); + core.files.push({path:'code/dashboard/release-popup.json', mode:'444', sha256:sha(bytes), data:bytes.toString('base64')}); + } + bundle(core, 'core', commit); + fs.writeFileSync(path.join(output, legacy.assets.core), JSON.stringify(core) + '\n', {flag:'wx'}); + for (const name of [legacy.assets.bridge, legacy.assets.runtime]) fs.copyFileSync(path.join(directory, name), path.join(output, name), fs.constants.COPYFILE_EXCL); + return {artifact: {artifactSha256: manifest.runtime.artifactSha256, embeddedManifestSha256: manifest.runtime.embeddedManifestSha256}, + bridgeHash: manifest.runtime.bridgeManifestSha256}; +} +module.exports={build,portableCore,command,collect,reuseComponents}; diff --git a/core/core/installations/src/release-delivery-config.js b/core/core/installations/src/release-delivery-config.js new file mode 100644 index 0000000..9aa2933 --- /dev/null +++ b/core/core/installations/src/release-delivery-config.js @@ -0,0 +1,15 @@ +'use strict'; +const path=require('node:path'); +const { exact, fail }=require('./release-delivery-contract'); +const CONFIG='/etc/dispatch/release-delivery.json'; +const TOKEN='/etc/dispatch/release-delivery-token'; +const STATE='/var/lib/dispatch-release-delivery'; +function configuration(value) { + exact(value,['uid','gid','localRoot','unitRoot','publicOrigin','port']); + if(!Number.isSafeInteger(value.uid)||value.uid<100||!Number.isSafeInteger(value.gid)||value.gid<100 + ||!Number.isInteger(value.port)||value.port<1024||value.port>65535||value.publicOrigin!=='https://dispatch.example.test')fail('release_config_invalid'); + for(const key of ['localRoot','unitRoot'])if(typeof value[key]!=='string'||!/^\/[A-Za-z0-9_./-]+$/.test(value[key]) + ||path.resolve(value[key])!==value[key]||value[key]==='/')fail('release_config_invalid'); + return value; +} +module.exports={configuration,CONFIG,TOKEN,STATE}; diff --git a/core/core/installations/src/release-delivery-contract.js b/core/core/installations/src/release-delivery-contract.js new file mode 100644 index 0000000..efbc37c --- /dev/null +++ b/core/core/installations/src/release-delivery-contract.js @@ -0,0 +1,72 @@ +'use strict'; +const crypto = require('node:crypto'); +const { releaseDescriptor } = require('./oci-deployment'); +const { platformRelease } = require('./platform-release-catalog'); +const { VERSION } = require('../../../shared/release-version'); +const COMMIT = /^[a-f0-9]{40}$/; +const SHA = /^[a-f0-9]{64}$/; +const MAX_BUNDLE = 32 * 1024 * 1024; +const formats = require('./release-formats'); +const ASSETS = Object.freeze({ ...formats.format('legacy').assets, runtime: formats.ociRuntime }); +const sha = bytes => crypto.createHash('sha256').update(bytes).digest('hex'); +function fail(code = 'release_invalid') { throw Object.assign(new Error(code), { code }); } +function exact(value, keys) { + if (!value || typeof value !== 'object' || Array.isArray(value) + || Object.keys(value).sort().join(',') !== [...keys].sort().join(',')) fail(); +} +function identity(version, commit) { + if (typeof version !== 'string' || version.length > 60 || !VERSION.test(version) || !COMMIT.test(commit)) fail(); + return `dispatch_${version.replace('+', '_')}`; +} +function releaseManifest(value) { + const split = value?.schemaVersion === 2; + exact(value, ['schemaVersion', 'version', 'releaseId', 'sourceCommit', 'changelog', 'assets', 'runtime', ...(split ? ['notes', 'dependencies'] : [])]); + if (![1, 2].includes(value.schemaVersion) || value.releaseId !== identity(value.version, value.sourceCommit)) fail(); + exact(value.assets, split ? ['app', 'dependencies'] : Object.keys(ASSETS)); + const runtime = releaseDescriptor(value.runtime); + const assets = split ? formats.format('split').assets + : runtime.backend === 'native_service_v1' ? formats.format('legacy').assets : ASSETS; + if (split) { + if (runtime.backend !== 'native_service_v1') fail(); + exact(value.dependencies, ['node', 'chrome']); + if (!/^\d+\.\d+\.\d+$/.test(value.dependencies.node) || !/^\d+\.\d+\.\d+\.\d+$/.test(value.dependencies.chrome)) fail(); + if (value.notes !== null) require('./release-notes').releaseNotes(value.notes, value); + } + for (const [kind, name] of Object.entries(assets)) { + const asset = value.assets[kind]; exact(asset, ['name', 'size', 'sha256', ...(split ? ['unpackedSize'] : [])]); + if (split && (!Number.isSafeInteger(asset.unpackedSize) || asset.unpackedSize < 1 || asset.unpackedSize > (kind === 'app' ? 128 * 1024 ** 2 : 2 * 1024 ** 3))) fail(); + if (asset.name !== name || !Number.isSafeInteger(asset.size) || asset.size < 1 + || asset.size > (['runtime', 'dependencies'].includes(kind) ? 2 * 1024 ** 3 : kind === 'app' ? 128 * 1024 ** 2 : MAX_BUNDLE) || !SHA.test(asset.sha256)) fail(); + } + if (runtime.releaseId !== value.releaseId || runtime.sourceCommit !== value.sourceCommit + || runtime.channel !== 'production' || (runtime.artifactSha256 || runtime.imageArchiveSha256) !== (split ? require('./release-package').runtimeIdentity(value.assets) : value.assets.runtime.sha256)) fail(); + platformRelease(value.releaseId, { version: value.version, sourceCommit: value.sourceCommit, + publishedAt: '2026-01-01T00:00:00.000Z', runtimeImageDigest: runtime.imageDigest || `sha256:${runtime.artifactSha256}`, changelog: value.changelog, + core: { artifactPath: `/opt/dispatch-platform/releases/${value.releaseId}/core-artifact`, manifestSha256: '0'.repeat(64) } }, runtime); + return value; +} +function bundle(value, kind, commit) { + exact(value, ['schemaVersion', 'kind', 'sourceCommit', 'files']); + if (value.schemaVersion !== 1 || value.kind !== kind || value.sourceCommit !== commit || !COMMIT.test(commit) + || !Array.isArray(value.files) || value.files.length < 1 || value.files.length > 2000) fail(); + const names = new Set(); let total = 0; + for (const entry of value.files) { + exact(entry, ['path', 'mode', 'sha256', 'data']); + if (typeof entry.path !== 'string' || entry.path.length > 240 || !/^[A-Za-z0-9_./-]+$/.test(entry.path) + || entry.path.split('/').some(p => !p || p === '.' || p === '..') || names.has(entry.path) + || !['444', '555'].includes(entry.mode) || !SHA.test(entry.sha256) || typeof entry.data !== 'string') fail(); + const allowed = kind === 'core' ? /^(code\/(core|host|dashboard|shared|sdk|plugins)\/|code\/bin\/dispatch-(dashboard|access-admin)$|host-helper-artifact\/)/ + : /^bridge-artifact\//; + if (!allowed.test(entry.path)) fail(); + const bytes = Buffer.from(entry.data, 'base64'); total += bytes.length; + if (total > MAX_BUNDLE || bytes.toString('base64') !== entry.data || sha(bytes) !== entry.sha256) fail(); + names.add(entry.path); + } + // A file must never also be an ancestor directory of another entry. + for (const name of names) { + const parts = name.split('/'); parts.pop(); + while (parts.length) { if (names.has(parts.join('/'))) fail(); parts.pop(); } + } + return value; +} +module.exports = { ASSETS, MAX_BUNDLE, VERSION, COMMIT, sha, fail, exact, identity, releaseManifest, bundle }; diff --git a/core/core/installations/src/release-delivery-files.js b/core/core/installations/src/release-delivery-files.js new file mode 100644 index 0000000..fe9b225 --- /dev/null +++ b/core/core/installations/src/release-delivery-files.js @@ -0,0 +1,42 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const fail = code => { throw Object.assign(new Error(code), { code }); }; +function atomic(file, value, mode = 0o600) { + const tmp = `${file}.new-${process.pid}-${crypto.randomBytes(6).toString('hex')}`; + const fd = fs.openSync(tmp, 'wx', mode); + // Root workers run with umask 0077. Apply the requested mode before publishing + // public receipts so the unprivileged Core can read their verified results. + try { fs.writeFileSync(fd, typeof value === 'string' ? value : JSON.stringify(value) + '\n'); fs.fchmodSync(fd, mode); fs.fsyncSync(fd); } + finally { fs.closeSync(fd); } + fs.renameSync(tmp, file); + const dir = fs.openSync(path.dirname(file), 'r'); try { fs.fsyncSync(dir); } finally { fs.closeSync(dir); } +} +function privateJson(file, uid, optional = false) { + let stat; + try { stat = fs.lstatSync(file); } catch (error) { if (optional && error.code === 'ENOENT') return null; throw error; } + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.uid !== uid || (stat.mode & 0o7777) !== 0o600 + || stat.size > 256 * 1024 || fs.realpathSync(file) !== file) fail('unsafe_release_storage'); + return JSON.parse(fs.readFileSync(file, 'utf8')); +} +async function hashFile(file) { + const hash = crypto.createHash('sha256'); + for await (const chunk of fs.createReadStream(file)) hash.update(chunk); + return hash.digest('hex'); +} +function hashFileSync(file) { + const hash=crypto.createHash('sha256'),buffer=Buffer.alloc(1024*1024),fd=fs.openSync(file,'r'); + try { let length; while ((length=fs.readSync(fd,buffer,0,buffer.length,null))>0) hash.update(buffer.subarray(0,length)); } + finally { fs.closeSync(fd); } + return hash.digest('hex'); +} +function trustedDirectory(directory, uid = 0) { + const stat = fs.lstatSync(directory); + if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== uid || (stat.mode & 0o022) + || fs.realpathSync(directory) !== directory) fail('unsafe_release_storage'); +} +function rootParents(directory) { + for (let current = directory; ; current = path.dirname(current)) { trustedDirectory(current); if (current === '/') break; } +} +module.exports = { atomic, privateJson, hashFile, hashFileSync, trustedDirectory, rootParents }; diff --git a/core/core/installations/src/release-delivery-github.js b/core/core/installations/src/release-delivery-github.js new file mode 100644 index 0000000..64935a0 --- /dev/null +++ b/core/core/installations/src/release-delivery-github.js @@ -0,0 +1,110 @@ +'use strict'; +const fs = require('node:fs'); +const crypto = require('node:crypto'); +const { fail } = require('./release-delivery-contract'); +const API = 'https://api.github.com/repos/example-organization/dispatch-platform'; +const DOWNLOAD_HOSTS = new Set(['release-assets.githubusercontent.com', 'objects.githubusercontent.com']); +function createGitHubReleaseSource({ token, fetcher = fetch }) { + if (typeof token !== 'string' || !token.trim() || /\s/.test(token)) fail('github_credentials_invalid'); + async function response(route, accept, timeout, headers = {}) { + let url = `${API}${route}`; + const signal = AbortSignal.timeout(timeout); + for (let redirects = 0; redirects < 4; redirects += 1) { + const target = new URL(url); + if (target.protocol !== 'https:' || target.username || target.password || target.port + || (target.hostname !== 'api.github.com' && !DOWNLOAD_HOSTS.has(target.hostname))) fail('github_redirect_invalid'); + const result = await fetcher(url, { redirect: 'manual', signal, headers: { + ...headers, Accept: accept, 'User-Agent': 'Dispatch-Release-Delivery', 'X-GitHub-Api-Version': '2022-11-28', + ...(target.hostname === 'api.github.com' ? { Authorization: `Bearer ${token}` } : {}), + } }); + if ([301, 302, 303, 307, 308].includes(result.status)) { + const location = result.headers.get('location'); await result.body?.cancel(); + if (!location) fail('github_redirect_invalid'); url = new URL(location, url).href; continue; + } + if (!result.ok) { await result.body?.cancel(); fail(result.status === 401 || result.status === 403 ? 'github_access_failed' : 'github_unavailable'); } + return result; + } + fail('github_redirect_invalid'); + } + async function json(route) { + const result = await response(route, 'application/vnd.github+json', 30_000); + const chunks = []; let length = 0; + for await (const chunk of result.body) { length += chunk.length; if (length > 4 * 1024 ** 2) fail('github_response_invalid'); chunks.push(chunk); } + try { return JSON.parse(Buffer.concat(chunks).toString('utf8')); } catch { fail('github_response_invalid'); } + } + async function download(asset, file, expected, onProgress = () => {}) { + if (!Number.isSafeInteger(asset.id) || asset.id < 1 || asset.state !== 'uploaded' || asset.size !== expected.size + || asset.digest !== `sha256:${expected.sha256}`) fail('release_asset_invalid'); + const receipt = `${file}.identity`; + const identity = JSON.stringify({ id: asset.id, size: expected.size, sha256: expected.sha256 }); + function safe(filename) { + const stat = fs.lstatSync(filename); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.uid !== process.geteuid() || stat.mode & 0o077) fail('unsafe_release_storage'); + return stat; + } + let offset = 0; + if (fs.existsSync(file)) { + const stat = safe(file); + const matching = fs.existsSync(receipt) && safe(receipt).size < 1024 && fs.readFileSync(receipt, 'utf8') === identity; + if (matching && stat.size <= expected.size) offset = stat.size; + else { fs.unlinkSync(file); } + } + require('./release-delivery-files').atomic(receipt, identity); + if (offset === expected.size) { + if (await require('./release-delivery-files').hashFile(file) === expected.sha256) { fs.unlinkSync(receipt); onProgress(offset); return; } + fs.unlinkSync(file); offset = 0; + } + const result = await response(`/releases/assets/${asset.id}`, 'application/octet-stream', 600_000, + offset ? { Range: `bytes=${offset}-`, 'Accept-Encoding': 'identity' } : { 'Accept-Encoding': 'identity' }); + if (result.status === 206) { + if (!offset || result.headers.get('content-range') !== `bytes ${offset}-${expected.size - 1}/${expected.size}`) { + await result.body?.cancel(); fs.rmSync(file, { force: true }); fs.rmSync(receipt, { force: true }); fail('release_asset_invalid'); + } + } else if (result.status === 200) { offset = 0; } + else { await result.body?.cancel(); fail('release_asset_invalid'); } + if (fs.existsSync(file)) safe(file); + const handle = await fs.promises.open(file, fs.constants.O_CREAT | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW | (offset ? fs.constants.O_APPEND : fs.constants.O_TRUNC), 0o600); + const hash = crypto.createHash('sha256'); let size = offset; + try { + if (offset) for await (const chunk of fs.createReadStream(file, { end: offset - 1 })) hash.update(chunk); + onProgress(size); + for await (const chunk of result.body) { + size += chunk.length; if (size > expected.size) fail('release_asset_invalid'); + hash.update(chunk); + let written = 0; + while (written < chunk.length) { const item = await handle.write(chunk, written); if (!item.bytesWritten) fail('release_download_failed'); written += item.bytesWritten; } + onProgress(size); + } + if (size !== expected.size) fail('release_download_incomplete'); + if (hash.digest('hex') !== expected.sha256) fail('release_checksum_failed'); + await handle.sync(); + fs.unlinkSync(receipt); + } catch (error) { + if (['release_asset_invalid', 'release_checksum_failed'].includes(error.code)) { + fs.rmSync(file, { force: true }); fs.rmSync(receipt, { force: true }); + } + throw error; + } finally { await handle.sync(); await handle.close(); } + } + + return { + supportsResume: true, + async list() { + const result = []; + for (let page = 1; page <= 10; page += 1) { + const values = await json(`/releases?per_page=100&page=${page}`); + if (!Array.isArray(values)) fail('github_response_invalid'); + result.push(...values); if (values.length < 100) break; + } + return result; + }, + async verifyCommit(version, commit) { + let ref = (await json(`/git/ref/tags/${encodeURIComponent(version)}`)).object; + for (let depth = 0; ref?.type === 'tag' && depth < 4; depth += 1) ref = (await json(`/git/tags/${ref.sha}`)).object; + if (ref?.type !== 'commit' || ref.sha !== commit) fail('release_commit_mismatch'); + const comparison = await json(`/compare/${commit}...main`); + if (!['ahead', 'identical'].includes(comparison.status)) fail('release_commit_mismatch'); + }, download, + }; +} +module.exports = { createGitHubReleaseSource }; diff --git a/core/core/installations/src/release-delivery-install.js b/core/core/installations/src/release-delivery-install.js new file mode 100644 index 0000000..a4165ec --- /dev/null +++ b/core/core/installations/src/release-delivery-install.js @@ -0,0 +1,128 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { bundle, sha, fail, identity } = require('./release-delivery-contract'); +const { rootParents, atomic, hashFile, hashFileSync } = require('./release-delivery-files'); +const { finishCoreArtifact } = require('./core-artifact-layout'); +const { verifyCoreArtifact } = require('./platform-core-update'); +const { verifyPreparedHostArtifact } = require('./oci-host-artifact'); +function writeBundle(value, root) { + for (const item of value.files) { + const file = path.join(root, item.path); + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o755 }); + fs.writeFileSync(file, Buffer.from(item.data, 'base64'), { mode: parseInt(item.mode, 8), flag: 'wx' }); + } +} +function tree(root) { + const files = []; + function visit(directory) { + for (const name of fs.readdirSync(directory).sort()) { + const file = path.join(directory, name); const stat = fs.lstatSync(file); + if (stat.isSymbolicLink()) fail('unsafe_release_storage'); + if (stat.isDirectory()) visit(file); + else { + if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== process.geteuid()) fail('unsafe_release_storage'); + files.push({ path: path.relative(root, file), mode: stat.mode & 0o7777, hash: hashFileSync(file) }); + } + } + } + visit(root); return files; +} +function seal(root) { + for (const name of fs.readdirSync(root)) { const file = path.join(root, name); if (fs.lstatSync(file).isDirectory()) seal(file); } + fs.chmodSync(root, 0o555); +} +function removeStage(root) { + if (!fs.existsSync(root)) return; + function writable(dir) { fs.chmodSync(dir, 0o700); for (const name of fs.readdirSync(dir)) { const file = path.join(dir, name); if (fs.lstatSync(file).isDirectory()) writable(file); } } + writable(root); fs.rmSync(root, { recursive: true }); +} +function installTree(source, target) { + rootParents(path.dirname(target)); + if (fs.existsSync(target)) { + rootParents(target); + if (JSON.stringify(tree(source)) !== JSON.stringify(tree(target))) fail('immutable_release_conflict'); + return; + } + const temporary = `${target}.pending`; + // A prior interruption can leave only this daemon-owned preparation directory. + if (fs.existsSync(temporary)) { rootParents(temporary); removeStage(temporary); } + fs.cpSync(source, temporary, { recursive: true, errorOnExist: true, force: false }); + seal(temporary); fs.renameSync(temporary, target); + const fd=fs.openSync(path.dirname(target),'r'); try { fs.fsyncSync(fd); } finally { fs.closeSync(fd); } +} +async function prepareRelease({ config, manifest, directory, publishedAt, configureSandbox = installBrowserSandboxProfile }) { + if (process.geteuid() !== 0) fail('release_installer_requires_root'); + const id = identity(manifest.version, manifest.sourceCommit); + const stage = path.join(directory, 'prepared'); + let packages = null; + try { + packages = manifest.schemaVersion === 2 ? require('./release-package').preparePackages(directory, manifest) : null; + const core = packages?.core || bundle(JSON.parse(fs.readFileSync(path.join(directory, manifest.assets.core.name))), 'core', manifest.sourceCommit); + const bridge = packages?.bridge || bundle(JSON.parse(fs.readFileSync(path.join(directory, manifest.assets.bridge.name))), 'bridge', manifest.sourceCommit); + const bridgeManifest = bridge.files.find(item => item.path === 'bridge-artifact/manifest.json'); + if (!bridgeManifest || sha(Buffer.from(bridgeManifest.data, 'base64')) !== manifest.runtime.bridgeManifestSha256) fail('release_checksum_failed'); + removeStage(stage); fs.mkdirSync(stage, { mode: 0o700 }); + const coreRoot=path.join(stage,'core'); fs.mkdirSync(coreRoot); writeBundle(core, coreRoot); + const platformRoot=path.join(stage,'platform'); fs.mkdirSync(platformRoot); + const artifact=path.join(platformRoot,'core-artifact'); fs.mkdirSync(artifact); + fs.renameSync(path.join(coreRoot,'code'), path.join(artifact,'code')); + const helperRoot=path.join(coreRoot,'host-helper-artifact'); + const helperHash=sha(fs.readFileSync(path.join(helperRoot,'manifest.json'))); + const deployment={releaseId:id,version:manifest.version,sourceCommit:manifest.sourceCommit,helperManifestSha256:helperHash, + localRoot:config.localRoot,unitRoot:config.unitRoot,publicOrigin:config.publicOrigin,port:config.port}; + const coreHash=finishCoreArtifact(artifact,deployment); + const runtimeRoot=path.join(stage,'runtime'); fs.mkdirSync(runtimeRoot); writeBundle(bridge,runtimeRoot); + const native = manifest.runtime.backend === 'native_service_v1'; + if (packages) { + fs.renameSync(packages.runtime, path.join(runtimeRoot, 'runtime-artifact')); + } else if (native) { + require('./native-runtime-artifact').unpackNativeRuntime(path.join(directory, manifest.assets.runtime.name), + path.join(runtimeRoot, 'runtime-artifact'), manifest.runtime); + } else { + fs.copyFileSync(path.join(directory,manifest.assets.runtime.name),path.join(runtimeRoot,'runtime-image.tar')); + fs.chownSync(path.join(runtimeRoot,'runtime-image.tar'),0,0); + fs.chmodSync(path.join(runtimeRoot,'runtime-image.tar'),0o444); + } + seal(coreRoot); seal(platformRoot); seal(runtimeRoot); + rootParents('/opt'); + for(const base of ['/opt/dispatch-platform/releases','/opt/dispatch-control/releases','/opt/dispatch-runtime/releases']) { + fs.mkdirSync(base,{recursive:true,mode:0o755}); rootParents(base); + } + installTree(platformRoot,`/opt/dispatch-platform/releases/${id}`); + installTree(coreRoot,`/opt/dispatch-control/releases/${id}`); + installTree(runtimeRoot,`/opt/dispatch-runtime/releases/${id}`); + const release={version:manifest.version,publishedAt,sourceCommit:manifest.sourceCommit,runtimeImageDigest:manifest.runtime.imageDigest || `sha256:${manifest.runtime.artifactSha256}`, + changelog:manifest.changelog,core:{artifactPath:`/opt/dispatch-platform/releases/${id}/core-artifact`,manifestSha256:coreHash}}; + verifyCoreArtifact(id,release); + verifyPreparedHostArtifact(`/opt/dispatch-control/releases/${id}/host-helper-artifact/core/installations/bin/dispatch-oci-host-issuer`,id,helperHash,'dispatch-oci-host-issuer'); + if (native) require('./native-runtime-artifact').verifyNativeRuntime(`/opt/dispatch-runtime/releases/${id}/runtime-artifact`, manifest.runtime); + else if(await hashFile(`/opt/dispatch-runtime/releases/${id}/runtime-image.tar`)!==manifest.runtime.imageArchiveSha256)fail('release_checksum_failed'); + if (native) configureSandbox(); + // The bootstrap installs this daemon once; it grants only this verified release's fixed, argument-free switch command. + const command=`/opt/dispatch-platform/releases/${id}/core-artifact/switch-host`; + const policy=[command, `/opt/dispatch-platform/releases/${id}/core-artifact/prepare-backup`].map(selected => + `Defaults!${selected} env_reset,!setenv,secure_path="/usr/bin:/bin"\nDefaults!${selected} env_delete += "NODE_OPTIONS NODE_PATH LD_PRELOAD LD_LIBRARY_PATH"\n#${config.uid} ALL=(root) NOPASSWD: NOSETENV: ${selected} ""\n`).join(''); + rootParents('/etc/sudoers.d'); + const sudoFile=`/etc/sudoers.d/dispatch-release-${id.slice('dispatch_'.length).replaceAll('.','_')}`; + const check=path.join(directory,'sudoers-check'); atomic(check,policy,0o440); + const checked=spawnSync('/usr/sbin/visudo',['-cf',check],{encoding:'utf8'}); + if(checked.status!==0)fail('release_permissions_failed'); + if(fs.existsSync(sudoFile)&&fs.readFileSync(sudoFile,'utf8')!==policy)fail('immutable_release_conflict'); + if(!fs.existsSync(sudoFile))atomic(sudoFile,policy,0o440); + return release; + } finally { removeStage(stage); if (packages) removeStage(packages.stage); } +} +function installBrowserSandboxProfile() { + if (!fs.existsSync('/sys/module/apparmor')) return; + rootParents('/etc/apparmor.d'); + // Chromium still runs its own user-namespace and seccomp sandbox. This named + // profile permits that namespace on Ubuntu hosts restricting unconfined userns. + const profile = 'abi ,\ninclude \nprofile dispatch-native-chrome /opt/{dispatch/dependencies/browser,dispatch-runtime/releases/*/runtime-artifact/dependencies/browser}/chrome flags=(unconfined) {\n userns,\n}\n'; + const file = '/etc/apparmor.d/dispatch-native-chrome'; + atomic(file, profile, 0o644); + const result = spawnSync('/usr/sbin/apparmor_parser', ['-r', file], { encoding: 'utf8', timeout: 30_000, maxBuffer: 4096 }); + if (result.error || result.status !== 0) fail('browser_sandbox_unavailable'); +} +module.exports = { prepareRelease, writeBundle, installTree, removeStage, tree, seal, installBrowserSandboxProfile }; diff --git a/core/core/installations/src/release-delivery-publish-github.js b/core/core/installations/src/release-delivery-publish-github.js new file mode 100644 index 0000000..b9a7467 --- /dev/null +++ b/core/core/installations/src/release-delivery-publish-github.js @@ -0,0 +1,47 @@ +'use strict'; +const fs=require('node:fs'); +const path=require('node:path'); +const {spawnSync}=require('node:child_process'); +const {releaseManifest}=require('./release-delivery-contract'); +const {hashFile}=require('./release-delivery-files'); +const {releaseNotes,NAME,LIMIT}=require('./release-notes'); +const formats=require('./release-formats'); +const REPO='example-organization/dispatch-platform'; +function gh(args){const r=spawnSync('gh',args,{encoding:'utf8',timeout:600_000,maxBuffer:1024*1024});if(r.status!==0)throw Error('github_release_command_failed');return r.stdout;} +async function publish(directory,run=gh){ + const manifest=releaseManifest(JSON.parse(fs.readFileSync(path.join(directory,formats.manifest)))); + const selected=Object.values(formats.formats).find(value=>value.schemaVersion===manifest.schemaVersion); + const files=[...Object.values(manifest.assets).map(a=>a.name),formats.manifest,...(selected.checksums?[selected.checksums]:[])]; + const notesFile=path.join(directory,NAME); + if(selected.notesSidecar && fs.existsSync(notesFile)){ + if(fs.statSync(notesFile).size>LIMIT)throw Error('release_notes_invalid'); + releaseNotes(JSON.parse(fs.readFileSync(notesFile,'utf8')),manifest);files.push(NAME); + } + for(const asset of Object.values(manifest.assets)){ + const file=path.join(directory,asset.name); + if(fs.statSync(file).size!==asset.size||await hashFile(file)!==asset.sha256)throw Error('local_asset_mismatch'); + } + // Only this build's draft is resumable. Published versions and existing tags are never replaced. + const releases=JSON.parse(run(['api',`repos/${REPO}/releases?per_page=100`])); + const existing=releases.find(r=>r.tag_name===manifest.version); + if(existing&&(!existing.draft||existing.target_commitish!==manifest.sourceCommit))throw Error('release_exists'); + if(!existing) { + const refs=JSON.parse(run(['api',`repos/${REPO}/git/matching-refs/tags/${manifest.version}`])); + if(refs.some(r=>r.ref===`refs/tags/${manifest.version}`))throw Error('tag_exists'); + run(['release','create',manifest.version,'--repo',REPO,'--draft','--target',manifest.sourceCommit,'--title',`Dispatch ${manifest.version}`,'--notes-file',path.join(directory,formats.changelog)]); + } + let remote=JSON.parse(run(['release','view',manifest.version,'--repo',REPO,'--json','assets'])); + for(const name of files){ + const file=path.join(directory,name),digest=`sha256:${await hashFile(file)}`; + const asset=remote.assets.find(a=>a.name===name); + if(asset){if(asset.digest!==digest||asset.state!=='uploaded')throw Error('draft_asset_conflict');} + else run(['release','upload',manifest.version,file,'--repo',REPO]); + } + remote=JSON.parse(run(['release','view',manifest.version,'--repo',REPO,'--json','assets'])); + if(remote.assets.length!==files.length)throw Error('unexpected_release_assets'); + for(const name of files){const asset=remote.assets.find(a=>a.name===name);if(!asset||asset.state!=='uploaded'||asset.digest!==`sha256:${await hashFile(path.join(directory,name))}`)throw Error('upload_verification_failed');} + run(['release','edit',manifest.version,'--repo',REPO,'--notes-file',path.join(directory,formats.changelog),'--draft=false','--latest']); + console.log(`https://github.com/${REPO}/releases/tag/${manifest.version}`); +} +if(require.main===module)publish(process.argv[2]).catch(error=>{console.error(error.message);process.exitCode=1;}); +module.exports={publish}; diff --git a/core/core/installations/src/release-delivery-publish.js b/core/core/installations/src/release-delivery-publish.js new file mode 100644 index 0000000..58dcab4 --- /dev/null +++ b/core/core/installations/src/release-delivery-publish.js @@ -0,0 +1,54 @@ +'use strict'; +const fs=require('node:fs'); +const path=require('node:path'); +const { atomic, privateJson }=require('./release-delivery-files'); +const { fail }=require('./release-delivery-contract'); +const { loadPrivateOciReleaseCatalog }=require('./release-catalog'); +const { loadPlatformReleaseCatalog, platformRelease }=require('./platform-release-catalog'); +const { saveReleaseNotes }=require('./release-notes'); +const { saveReleaseHistory }=require('./release-history'); +const { releaseDescriptor }=require('./oci-deployment'); +function publish(config, input) { + if(process.geteuid()!==config.uid||process.getegid()!==config.gid||config.uid===0)fail('release_publisher_identity'); + const root=path.join(config.localRoot,'config'); + if(input.action==='history') { + const runtimes=loadPrivateOciReleaseCatalog(path.join(root,'oci-releases.json')); + saveReleaseHistory(config.localRoot,loadPlatformReleaseCatalog(path.join(root,'platform-releases.json'),runtimes));return; + } + if(input.action==='history_entry') { + if(input.notes)require('./release-notes').releaseNotes(input.notes,{...input.release,releaseId:input.releaseId}); + saveReleaseHistory(config.localRoot,{[input.releaseId]:input.release}); + if(input.notes)saveReleaseNotes(config.localRoot,input.notes,{...input.release,releaseId:input.releaseId});return; + } + if(input.action==='status') { atomic(path.join(root,'release-delivery-status.json'),input.status); return; } + if(input.action==='notes') { + const catalog=privateJson(path.join(root,'platform-releases.json'),config.uid); + const release=catalog.releases?.[input.notes?.releaseId]; + if(!release)fail(); + saveReleaseNotes(config.localRoot,input.notes,{...release,releaseId:input.notes.releaseId});return; + } + if(input.action!=='publish')fail(); + const id=input.runtime.releaseId; + releaseDescriptor(input.runtime); platformRelease(id,input.release,input.runtime); + const runtimeFile=path.join(root,'oci-releases.json'), platformFile=path.join(root,'platform-releases.json'); + const runtimes=privateJson(runtimeFile,config.uid)||{}; + const platforms=privateJson(platformFile,config.uid)||{}; + loadPrivateOciReleaseCatalog(runtimeFile); loadPlatformReleaseCatalog(platformFile,runtimes.releases); + for(const [catalog,value] of [[runtimes,input.runtime],[platforms,input.release]]) { + if(catalog.releases[id]&&JSON.stringify(catalog.releases[id])!==JSON.stringify(value))fail('immutable_release_conflict'); + catalog.releases[id]=value; + } + for(const [other,value] of Object.entries(platforms.releases))if(other!==id&&value.version===input.release.version)fail('immutable_release_conflict'); + // Archive failures are retried by history sync and must not hide a valid update. + try{saveReleaseHistory(config.localRoot,platforms.releases);}catch{} + // Keep both catalogs readable if retained release history reaches the reader's bound. + if([runtimes,platforms].some(value=>Buffer.byteLength(JSON.stringify(value)+'\n')>256*1024))fail('release_catalog_full'); + atomic(runtimeFile,runtimes); // Existing platform references remain valid until the second rename. + atomic(platformFile,platforms); + loadPlatformReleaseCatalog(platformFile,loadPrivateOciReleaseCatalog(runtimeFile)); +} +if(require.main===module) { + let raw='';process.stdin.setEncoding('utf8');process.stdin.on('data',chunk=>{raw+=chunk;if(Buffer.byteLength(raw)>512*1024)process.exit(1);}); + process.stdin.on('end',()=>{try{const {config,input}=JSON.parse(raw);publish(config,input);}catch{process.stderr.write('release_catalog_publish_failed\n');process.exitCode=1;}}); +} +module.exports={publish}; diff --git a/core/core/installations/src/release-delivery-watch.js b/core/core/installations/src/release-delivery-watch.js new file mode 100644 index 0000000..ea89836 --- /dev/null +++ b/core/core/installations/src/release-delivery-watch.js @@ -0,0 +1,184 @@ +'use strict'; +const {manifest: MANIFEST} = require('./release-formats'); +const fs=require('node:fs'); +const path=require('node:path'); +const { atomic, privateJson, hashFile }=require('./release-delivery-files'); +const { VERSION, releaseManifest, fail }=require('./release-delivery-contract'); +const { compareVersions }=require('../../../shared/release-version'); +const { releaseNotes, NAME, LIMIT }=require('./release-notes'); +const SAFE_ERRORS=new Set(['github_access_failed','github_unavailable','release_checksum_failed','release_commit_mismatch', + 'release_asset_invalid','release_package_invalid','release_download_incomplete','release_storage_full','release_invalid','immutable_release_conflict','release_permissions_failed','release_catalog_publish_failed']); +function createReleaseWatcher({ root, source, prepare, publish, publishNotes=async()=>{}, status, retryRequest=()=>null, clock=Date.now, target = null }) { + if (target && (typeof target.version !== 'string' || !VERSION.test(target.version) || !/^[a-f0-9]{40}$/.test(target.sourceCommit))) fail(); + let progress = null; + function report(stage, extra = {}) { + const now = clock(); + progress = { ...progress, stage, stageStartedAt: progress?.stage === stage ? progress.stageStartedAt : now, updatedAt: now, ...extra }; + atomic(path.join(root, 'preparation-progress.json'), progress); + } + const stateFile=path.join(root,'state.json'); + function save(state){atomic(stateFile,state);} + function notesAsset(release, item) { + const matches=release.assets.filter(asset=>asset.name===NAME); + if(matches.length>1)fail('release_asset_invalid'); + const asset=matches[0]; + if(asset&&(!/^sha256:[a-f0-9]{64}$/.test(asset.digest)||asset.size<1||asset.size>LIMIT))fail('release_asset_invalid'); + const fingerprint=asset?.digest||null; + if(Object.hasOwn(item,'notesFingerprint')&&item.notesFingerprint!==fingerprint)fail('immutable_release_conflict'); + item.notesFingerprint=fingerprint; + return asset; + } + async function fetchNotes(asset, manifest, directory) { + if (manifest.schemaVersion === 2) { if (asset) fail('release_asset_invalid'); return manifest.notes; } + if(!asset)return null; + const file=await cached(asset,{name:NAME,size:asset.size,sha256:asset.digest.slice(7)},directory); + return releaseNotes(JSON.parse(fs.readFileSync(file,'utf8')),manifest); + } + async function cached(asset, expected, directory) { + if (asset.state !== 'uploaded' || asset.size !== expected.size || asset.digest !== `sha256:${expected.sha256}`) fail('release_asset_invalid'); + const file=path.join(directory,expected.name); + if (fs.existsSync(file)) { const stat = fs.lstatSync(file); if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.uid !== process.geteuid()) fail('unsafe_release_storage'); } + if(fs.existsSync(file)&&fs.lstatSync(file).isFile()&&fs.lstatSync(file).size===expected.size + &&await hashFile(file)===expected.sha256)return file; + if(fs.existsSync(file))fs.unlinkSync(file); + const partial=`${file}.partial`; if(fs.existsSync(partial) && !source.supportsResume)fs.unlinkSync(partial); + let last = 0; + await source.download(asset,partial,expected, bytes => { + if (progress && (clock() - last >= 1000 || bytes === expected.size)) { report('download', { asset: expected.name, bytes, totalBytes: expected.size }); last = clock(); } + }); + // Check even injected/alternative transports before committing cached files. + if(fs.statSync(partial).size!==expected.size||await hashFile(partial)!==expected.sha256)fail('release_checksum_failed'); + fs.renameSync(partial,file); return file; + } + async function run() { + const state=privateJson(stateFile,process.geteuid(),true)||{schemaVersion:1,releases:{},retryNonce:null}; + if(state.schemaVersion!==1||!state.releases)fail('unsafe_release_storage'); + const request=retryRequest(); const force=typeof request?.nonce==='string'&&/^[a-f0-9]{32}$/.test(request.nonce)&&request.nonce!==state.retryNonce; + if(force){state.retryNonce=request.nonce;for(const item of Object.values(state.releases))if(item.status==='failed')item.nextAttemptAt=0;save(state);} + let releases; + try{releases=await source.list();} + catch(error){await status({state:'failed',version:null,changelog:[],message:'Unable to check GitHub for updates. Retrying automatically.',retryable:true});return {status:'discovery_failed'};} + const candidates=releases.filter(r=>!r.draft&&!r.prerelease&&typeof r.tag_name==='string'&&VERSION.test(r.tag_name) + &&Number.isSafeInteger(r.id)&&r.id>0&&Number.isFinite(Date.parse(r.published_at))&&Array.isArray(r.assets) + &&(!target || r.tag_name === target.version) + &&r.assets.some(a=>a.name===MANIFEST)) + .sort((a,b)=>compareVersions(b.tag_name,a.tag_name)||Date.parse(b.published_at)-Date.parse(a.published_at)); + for(const release of candidates) { + if (target) await source.verifyCommit(target.version, target.sourceCommit); + const key=String(release.id); + const manifestAssets=release.assets.filter(a=>a.name===MANIFEST); + const manifestAsset=manifestAssets[0]; + const prior=state.releases[key]; + if(prior?.status==='ready') { + // Published release identities are immutable once accepted. + if(prior.fingerprint!==manifestAsset?.digest) { + await status({state:'failed',version:release.tag_name,changelog:[],message:'A published release changed after verification. The prepared copy is preserved.',retryable:false}); + return {status:'release_preparation_failed',code:'immutable_release_conflict'}; + } + else { + // An upgraded watcher can add notes to a release prepared by the old watcher, + // without downloading or installing its Core/DSP packages again. + try { + if(manifestAssets.length!==1||manifestAsset.size<1||manifestAsset.size>256*1024)fail('release_asset_invalid'); + const asset=notesAsset(release,prior);save(state); + if(asset&&!prior.notesReady) { + const directory=path.join(root,`release-${release.id}`);fs.mkdirSync(directory,{recursive:true,mode:0o700}); + const file=await cached(manifestAsset,{name:MANIFEST,size:manifestAsset.size,sha256:manifestAsset.digest.slice(7)},directory); + const manifest=releaseManifest(JSON.parse(fs.readFileSync(file,'utf8'))); + if(manifest.version!==release.tag_name || target && manifest.sourceCommit !== target.sourceCommit)fail('release_commit_mismatch'); + await source.verifyCommit(manifest.version,manifest.sourceCommit); + await publishNotes(await fetchNotes(asset,manifest,directory)); + prior.notesReady=true;save(state); + try{fs.rmSync(directory,{recursive:true,force:true});}catch{} + } + await status({state:'ready',version:release.tag_name,changelog:[],message:null,retryable:false}); + } catch(error) { + await status({state:'failed',version:release.tag_name,changelog:[],message:'Release notes could not be verified.',retryable:error.code!=='immutable_release_conflict'}); + return {status:'release_preparation_failed',code:SAFE_ERRORS.has(error.code)?error.code:'release_preparation_failed'}; + } + } + return {status:'idle'}; // Offer the newest supported release; do not downgrade to older releases. + } + if(prior?.nextAttemptAt>clock())return {status:'backoff'}; + const item=state.releases[key]={...prior,status:'preparing',attempt:(prior?.attempt||0)+1,version:release.tag_name,nextAttemptAt:null};save(state); + let manifest, notes; + progress = { version: release.tag_name, attempt: item.attempt, startedAt: clock(), stages: {}, assets: {} }; + report('manifest'); + async function measured(stage, action) { + report(stage); const start = clock(); + try { return await action(); } finally { progress.stages[stage] = clock() - start; report(stage); } + } + try { + await status({state:'preparing',version:release.tag_name,changelog:[],message:'Downloading and verifying this update…',retryable:false}); + if(manifestAssets.length!==1||!/^sha256:[a-f0-9]{64}$/.test(manifestAsset.digest) + ||manifestAsset.size<1||manifestAsset.size>256*1024)fail('release_asset_invalid'); + if(prior?.fingerprint&&prior.fingerprint!==manifestAsset.digest)fail('immutable_release_conflict'); + item.fingerprint=manifestAsset.digest; save(state); + const presentationAsset=notesAsset(release,item);save(state); + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + if (entry.isDirectory() && /^release-[0-9]+$/.test(entry.name) && entry.name !== `release-${release.id}`) + fs.rmSync(path.join(root, entry.name), { recursive: true }); + } + const directory=path.join(root,`release-${release.id}`);fs.mkdirSync(directory,{recursive:true,mode:0o700}); + const file=await cached(manifestAsset,{name:MANIFEST,size:manifestAsset.size,sha256:manifestAsset.digest.slice(7)},directory); + manifest=releaseManifest(JSON.parse(fs.readFileSync(file,'utf8'))); + if(manifest.version!==release.tag_name || target && manifest.sourceCommit !== target.sourceCommit)fail('release_commit_mismatch'); + await source.verifyCommit(manifest.version,manifest.sourceCommit); + notes=await fetchNotes(presentationAsset,manifest,directory); + await status({state:'preparing',version:manifest.version,changelog:manifest.changelog,...(notes?{notes}:{}),message:'Downloading and verifying this update…',retryable:false}); + const space=fs.statfsSync(root); + const required = manifest.schemaVersion === 2 + ? Object.values(manifest.assets).reduce((sum, asset) => sum + 2 * asset.size + 3 * asset.unpackedSize, 128*1024*1024) + : manifest.assets.runtime.size*3 + 128*1024*1024; + report('preflight', { requiredBytes: required, availableBytes: space.bavail*space.bsize }); + if(space.bavail*space.bsize < required)fail('release_storage_full'); + for(const expected of Object.values(manifest.assets)) { + const matches=release.assets.filter(a=>a.name===expected.name); + if(matches.length!==1)fail('release_asset_invalid'); + const started = clock(); + report('download', { asset: expected.name, bytes: 0, totalBytes: expected.size }); + if (manifest.schemaVersion === 2 && expected === manifest.assets.dependencies) { + const cache = path.join(root, 'dependencies'); fs.mkdirSync(cache, { recursive: true, mode: 0o700 }); + require('./release-delivery-files').trustedDirectory(cache, process.geteuid()); + const dependency = { ...expected, name: expected.sha256 + '.tar.gz' }; + const cachedFile = path.join(cache, dependency.name); + const reused = fs.existsSync(cachedFile) && await hashFile(cachedFile) === expected.sha256; + await cached(matches[0], dependency, cache); + fs.copyFileSync(cachedFile, path.join(directory, expected.name)); + progress.assets[expected.name] = { bytes: expected.size, reused, durationMs: clock() - started }; + } else { + await cached(matches[0],expected,directory); + progress.assets[expected.name] = { bytes: expected.size, durationMs: clock() - started }; + } + } + const prepared=await measured('prepare', () => prepare({manifest,directory,publishedAt:new Date(release.published_at).toISOString()})); + await measured('register', () => publish({release:prepared,runtime:manifest.runtime})); + if(notes){await publishNotes(notes);item.notesReady=true;} + item.status='ready';item.nextAttemptAt=null;item.failureCode=null;save(state); + report('ready', { durationMs: clock() - progress.startedAt, asset: null }); + // Only the newest dependency download is cached; installed releases and + // backup artifacts remain self-contained and never reference this cache. + try { if (manifest.schemaVersion === 2) { + const cache = path.join(root, 'dependencies'); + for (const name of fs.readdirSync(cache)) if (/^[a-f0-9]{64}\.tar\.gz(?:\.partial(?:\.identity)?)?$/.test(name) && name !== manifest.assets.dependencies.sha256 + '.tar.gz') fs.rmSync(path.join(cache, name), { force: true }); + } } catch {} + // Cache cleanup is optional once the immutable release and catalog are durable. + try { fs.rmSync(directory,{recursive:true,force:true}); } catch {} + await status({state:'ready',version:manifest.version,changelog:manifest.changelog,message:null,retryable:false}); + return {status:'release_ready',version:manifest.version}; + } catch(error) { + if(item.status==='ready')throw error; + report('failed', { failedStage: progress.stage, durationMs: clock() - progress.startedAt, code: SAFE_ERRORS.has(error.code) ? error.code : 'release_preparation_failed' }); + item.status='failed';item.failureCode=SAFE_ERRORS.has(error.code)?error.code:'release_preparation_failed'; + item.nextAttemptAt=clock()+Math.min(60*60_000,30_000*2**Math.min(item.attempt-1,7));save(state); + await status({state:'failed',version:release.tag_name,changelog:manifest?.changelog||[],...(notes?{notes}:{}), + message:'This update could not be prepared. Retrying automatically.',retryable:true}); + return {status:'release_preparation_failed',code:item.failureCode}; + } + } + await status({state:'idle',version:null,changelog:[],message:null,retryable:false}); + return {status: target ? 'release_not_found' : 'idle'}; + } + return {run}; +} +module.exports={createReleaseWatcher}; diff --git a/core/core/installations/src/release-dependencies.js b/core/core/installations/src/release-dependencies.js new file mode 100644 index 0000000..472c03b --- /dev/null +++ b/core/core/installations/src/release-dependencies.js @@ -0,0 +1,14 @@ +'use strict'; +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const pinned = require('../runtime-dependencies.json'); +function verify({ node = process.execPath, browserRoot = process.env.DISPATCH_BUILD_BROWSER_ROOT || '/opt/google/chrome', run = spawnSync } = {}) { + const version = executable => { + const result = run(executable, ['--version'], { encoding: 'utf8', timeout: 10_000, env: { PATH: '/usr/bin:/bin' } }); + if (result.error || result.status !== 0) throw Error('release_dependency_unavailable'); + return result.stdout.trim(); + }; + if (version(node) !== `v${pinned.node}` || !new RegExp(`^Google Chrome(?: for Testing)? ${pinned.chrome.replaceAll('.', '\\.')}\\s*$`).test(version(path.join(browserRoot, 'chrome')))) throw Error('release_dependency_version_mismatch'); + return { node: pinned.node, chrome: pinned.chrome }; +} +module.exports = { verify }; diff --git a/core/core/installations/src/release-formats.js b/core/core/installations/src/release-formats.js new file mode 100644 index 0000000..7c34dd9 --- /dev/null +++ b/core/core/installations/src/release-formats.js @@ -0,0 +1,13 @@ +'use strict'; +const contract = require('../release-formats.json'); +function format(name) { + if (!Object.hasOwn(contract.formats, name)) throw Error('invalid_release_format'); + return contract.formats[name]; +} +function componentFiles(name) { + const selected = format(name); + return [...Object.values(selected.assets), contract.manifest, + ...(selected.checksums ? [selected.checksums] : []), + ...(selected.notesSidecar ? [contract.notes] : []), contract.changelog]; +} +module.exports = { ...contract, format, componentFiles }; diff --git a/core/core/installations/src/release-frontend.js b/core/core/installations/src/release-frontend.js new file mode 100644 index 0000000..b5157fa --- /dev/null +++ b/core/core/installations/src/release-frontend.js @@ -0,0 +1,42 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const {spawnSync} = require('node:child_process'); +const {sha} = require('./release-delivery-contract'); +const {removeStage} = require('./release-delivery-install'); + +const ASSETS = ['frontend.js', 'styles.css']; + +// Build only committed source in disposable storage. Never trust a checkout's +// ignored bundles or node_modules as inputs to an immutable release. +function buildFrontend(projectRoot, commit, outputParent, {run = spawnSync} = {}) { + const scratch = fs.mkdtempSync(path.join(outputParent, 'frontend-build-')); + function command(executable, args, options = {}) { + const result = run(executable, args, {cwd: projectRoot, encoding: 'utf8', + timeout: 600_000, maxBuffer: 32 * 1024 * 1024, ...options}); + if (result.error || result.status !== 0) { + if (result.stderr) process.stderr.write(result.stderr.toString().slice(-8192)); + throw Error('release_frontend_build_failed:' + path.basename(executable)); + } + return result.stdout; + } + try { + const archive = command('/usr/bin/git', ['archive', commit, 'dashboard', 'plugins'], {encoding: null}); + command('/usr/bin/tar', ['-xf', '-', '-C', scratch], {input: archive}); + const dashboard = path.join(scratch, 'dashboard'); + command('npm', ['ci', '--no-audit', '--no-fund'], {cwd: dashboard}); + command('npm', ['exec', '--', 'tsc', '--noEmit'], {cwd: dashboard}); + command('npm', ['exec', '--', 'vite', 'build'], {cwd: dashboard}); + return ASSETS.map(name => { + const file = path.join(dashboard, 'public/assets', name); + const stat = fs.lstatSync(file); + if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || !stat.size) throw Error('invalid_frontend_asset'); + const data = fs.readFileSync(file); + return {path: `code/dashboard/public/assets/${name}`, mode: '444', sha256: sha(data), data: data.toString('base64')}; + }); + } finally { + removeStage(scratch); + } +} + +module.exports = {ASSETS, buildFrontend}; diff --git a/core/core/installations/src/release-history-sync.js b/core/core/installations/src/release-history-sync.js new file mode 100644 index 0000000..b024390 --- /dev/null +++ b/core/core/installations/src/release-history-sync.js @@ -0,0 +1,59 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { atomic, privateJson, hashFile } = require('./release-delivery-files'); +const { releaseManifest, VERSION } = require('./release-delivery-contract'); +const { releaseNotes, NAME, LIMIT } = require('./release-notes'); +const { compareVersions } = require('../../../shared/release-version'); + +// Backfill human-facing history only. No installation packages or rollout commands. +function createReleaseHistorySync({ root, source, publish, clock = Date.now }) { + const file = path.join(root, 'history-state.json'); + async function download(asset, name, directory) { + if (!asset || !/^sha256:[a-f0-9]{64}$/.test(asset.digest) || asset.size < 1 || asset.size > LIMIT) throw Error('release_asset_invalid'); + const output = path.join(directory, name); + await source.download(asset, output, { name, size: asset.size, sha256: asset.digest.slice(7) }); + if (fs.statSync(output).size !== asset.size || await hashFile(output) !== asset.digest.slice(7)) throw Error('release_checksum_failed'); + return JSON.parse(fs.readFileSync(output, 'utf8')); + } + async function run() { + const state = privateJson(file, process.geteuid(), true) || { schemaVersion: 1, releases: {} }; + if (state.schemaVersion !== 1 || !state.releases) throw Error('release_history_invalid'); + const releases = (await source.list()).filter(r => !r.draft && !r.prerelease && VERSION.test(r.tag_name) + && Number.isSafeInteger(r.id) && r.id > 0 && Number.isFinite(Date.parse(r.published_at)) && Array.isArray(r.assets)) + .sort((a,b) => compareVersions(b.tag_name,a.tag_name)); + let processed = 0, failed = 0; + for (const release of releases) { + const manifests = release.assets.filter(a => a.name === 'dispatch-release.json'); + if (!manifests.length) continue; // Pre-manifest releases have no verified structured notes. + const sidecars = release.assets.filter(a => a.name === NAME); + const fingerprint = `${manifests[0].digest}:${sidecars[0]?.digest || 'none'}`; + const prior = state.releases[release.id]; + if (prior && prior.fingerprint !== fingerprint) { failed++; continue; } + if (prior?.ready || prior?.retryAt > clock()) continue; + if (processed >= 5) break; // Bound first-run work; later timer ticks continue backfill. + processed++; + const entry = state.releases[release.id] = { ...prior, fingerprint, attempt: (prior?.attempt || 0) + 1 }; + atomic(file, state); + const directory = fs.mkdtempSync(path.join(root, 'history-')); + try { + if (manifests.length !== 1 || sidecars.length > 1) throw Error('release_asset_invalid'); + const manifest = releaseManifest(await download(manifests[0], 'dispatch-release.json', directory)); + if (manifest.version !== release.tag_name) throw Error('release_commit_mismatch'); + await source.verifyCommit(manifest.version, manifest.sourceCommit); + if (manifest.schemaVersion === 2 && sidecars.length) throw Error('release_asset_invalid'); + const notes = manifest.schemaVersion === 2 ? manifest.notes : sidecars.length ? releaseNotes(await download(sidecars[0], NAME, directory), manifest) : null; + await publish({ releaseId: manifest.releaseId, release: { version: manifest.version, sourceCommit: manifest.sourceCommit, + publishedAt: new Date(release.published_at).toISOString(), changelog: manifest.changelog }, notes }); + entry.ready = true; entry.retryAt = null; + } catch { + failed++; entry.retryAt = clock() + Math.min(3600000, 30000 * 2 ** Math.min(entry.attempt - 1, 7)); + } finally { + atomic(file, state); fs.rmSync(directory, { recursive: true, force: true }); + } + } + return { processed, failed }; + } + return { run }; +} +module.exports = { createReleaseHistorySync }; diff --git a/core/core/installations/src/release-history.js b/core/core/installations/src/release-history.js new file mode 100644 index 0000000..6222dc5 --- /dev/null +++ b/core/core/installations/src/release-history.js @@ -0,0 +1,43 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { atomic, privateJson } = require('./release-delivery-files'); +const { authoring } = require('./release-notes'); +const { VERSION } = require('../../../shared/release-version'); +function record(id, release) { + if (typeof id !== 'string' || !/^[a-z][a-z0-9_.-]{2,95}$/.test(id) || !release + || typeof release.version !== 'string' || !VERSION.test(release.version) + || typeof release.sourceCommit !== 'string' || !/^[a-f0-9]{40}$/.test(release.sourceCommit) + || typeof release.publishedAt !== 'string' || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(release.publishedAt) + || !Number.isFinite(Date.parse(release.publishedAt))) throw Error('release_history_invalid'); + return { id, version: release.version, publishedAt: release.publishedAt, sourceCommit: release.sourceCommit, + changelog: authoring(release.changelog).changelog }; +} +function saveReleaseHistory(localRoot, releases) { + const directory = path.join(localRoot, 'config/release-history'); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const stat = fs.lstatSync(directory); + if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== process.geteuid() + || (stat.mode & 0o7777) !== 0o700 || fs.realpathSync(directory) !== directory) throw Error('release_history_invalid'); + for (const [id, release] of Object.entries(releases)) { + const item = record(id, release), file = path.join(directory, `${id}.json`); + const prior = privateJson(file, process.geteuid(), true); + if (prior && JSON.stringify(prior) !== JSON.stringify(item)) throw Error('immutable_release_conflict'); + if (!prior) atomic(file, item); + } +} +function loadReleaseHistory(localRoot) { + const directory = path.join(localRoot, 'config/release-history'); + const releases = {}; + try { + for (const name of fs.readdirSync(directory)) { + if (!/^[a-z][a-z0-9_.-]{2,95}\.json$/.test(name)) continue; + try { + const item = privateJson(path.join(directory, name), process.geteuid()); + if (`${item.id}.json` === name) releases[item.id] = record(item.id, item); + } catch {} // An unreadable historical record does not hide current updates. + } + } catch {} + return releases; +} +module.exports = { saveReleaseHistory, loadReleaseHistory }; diff --git a/core/core/installations/src/release-notes.js b/core/core/installations/src/release-notes.js new file mode 100644 index 0000000..730fe8a --- /dev/null +++ b/core/core/installations/src/release-notes.js @@ -0,0 +1,122 @@ +'use strict'; +// Optional presentation sidecar. The v1 installation manifest remains unchanged. +const fs = require('node:fs'); +const path = require('node:path'); +const { privateJson, atomic } = require('./release-delivery-files'); +const { githubAuthoring, markdown } = require('./github-release-notes'); +const ICONS = new Set(['database', 'users', 'user-plus', 'copy', 'calendar-clock', 'chart-column', + 'shield', 'check-circle', 'trash', 'lock', 'send', 'refresh-cw', 'plus', 'pencil', 'trending-up', 'info']); +const KINDS = ['added', 'changed', 'improved', 'fixed', 'removed']; +const NAME = require('./release-formats').notes; +const LIMIT = 256 * 1024; +const fail = () => { throw Object.assign(new Error('release_notes_invalid'), { code: 'release_notes_invalid' }); }; +function exact(value, keys) { + if (!value || typeof value !== 'object' || Array.isArray(value) + || Object.keys(value).sort().join(',') !== [...keys].sort().join(',')) fail(); +} +function text(value, max, empty = false, multiline = false) { + if (typeof value !== 'string' || value.length > max || (!empty && !value.trim()) + || (multiline ? /[\x00-\x08\x0b-\x1f\x7f]/ : /[\x00-\x1f\x7f]/).test(value)) fail(); +} +function plainChangelog(changelog) { + if (!Array.isArray(changelog) || changelog.length < 1 || changelog.length > 100) fail(); + return changelog.map(({ kind, title, description }) => { + if (!KINDS.includes(kind)) fail(); + text(title, 160); text(description, 600, true); + return { kind, title, description }; + }); +} +function presentation(value) { + exact(value, ['groups', 'changelog', 'afterUpdating']); + if (!Array.isArray(value.groups) || !value.groups.length || value.groups.length > 20 + || !Array.isArray(value.afterUpdating) || value.afterUpdating.length > 10) fail(); + const plain = plainChangelog(value.changelog); + const ids = new Set(); + for (const group of value.groups) { + exact(group, ['id', 'title', 'icon']); + if (typeof group.id !== 'string' || !/^[a-z][a-z0-9-]{0,39}$/.test(group.id) || ids.has(group.id) || !ICONS.has(group.icon)) fail(); + ids.add(group.id); text(group.title, 80); + } + for (const change of value.changelog) { + exact(change, ['kind', 'title', 'description', 'group', 'icon', 'details']); + if (!ids.has(change.group) || !ICONS.has(change.icon)) fail(); + text(change.details, 4000, true, true); + } + if (value.groups.some(group => !value.changelog.some(change => change.group === group.id))) fail(); + for (const action of value.afterUpdating) { + exact(action, ['title', 'description']); text(action.title, 160); text(action.description, 600); + } + // Preparation status carries the legacy copy as well as the rich notes. Keep + // that entire receipt within the existing private-JSON reader's 256 KiB cap. + if (Buffer.byteLength(JSON.stringify(value)) + Buffer.byteLength(JSON.stringify(plain)) > LIMIT - 2048) fail(); + return value; +} +function releaseNotes(value, release) { + exact(value, ['schemaVersion', 'releaseId', 'sourceCommit', 'groups', 'changelog', 'afterUpdating']); + if (value.schemaVersion !== 1 || value.releaseId !== release.releaseId || value.sourceCommit !== release.sourceCommit + || !/^[a-z][a-z0-9_.-]{2,95}$/.test(value.releaseId) || !/^[a-f0-9]{40}$/.test(value.sourceCommit)) fail(); + const notes = presentation({ groups: value.groups, changelog: value.changelog, afterUpdating: value.afterUpdating }); + if (JSON.stringify(plainChangelog(notes.changelog)) !== JSON.stringify(plainChangelog(release.changelog))) fail(); + return value; +} +function authoring(input) { + const { input: content, github } = githubAuthoring(input); + return { ...authoringContent(content), ...(github ? { github } : {}) }; +} +function authoringContent(input) { + if (Array.isArray(input)) { + for (const change of input) exact(change, ['kind', 'title', 'description']); + return { changelog: plainChangelog(input), notes: null }; + } + // Audience and concise popup copy are build inputs only. Strip them before + // producing the v1 notes sidecar so existing release watchers remain compatible. + const curated = input?.changelog?.some(change => Object.hasOwn(change, 'audience') || Object.hasOwn(change, 'popup')); + if (!curated) { + const notes = presentation(input); + return { changelog: plainChangelog(notes.changelog), notes }; + } + exact(input, ['groups', 'changelog', 'afterUpdating']); + if (!Array.isArray(input.afterUpdating)) fail(); + const audience = value => { if (!['platform', 'dsp'].includes(value)) fail(); return value; }; + const changelog = input.changelog.map(change => { + exact(change, ['kind', 'title', 'description', 'group', 'icon', 'details', 'audience', ...(Object.hasOwn(change, 'popup') ? ['popup'] : [])]); + audience(change.audience); + if (change.popup) { + exact(change.popup, ['title', 'description']); text(change.popup.title, 160); text(change.popup.description, 600, true); + } else if (Object.hasOwn(change, 'popup')) fail(); + const { audience: scope, popup, ...original } = change; + return original; + }); + const afterUpdating = input.afterUpdating.map(action => { + exact(action, ['title', 'description', 'audience']); audience(action.audience); + const { audience: scope, ...original } = action; + return original; + }); + const notes = presentation({ groups: input.groups, changelog, afterUpdating }); + const popup = { + changelog: input.changelog.map(change => ({ kind: change.kind, title: change.popup?.title || change.title, + description: change.popup?.description ?? change.description, audience: change.audience })), + afterUpdating: input.afterUpdating.map(({ title, description, audience }) => ({ title, description, audience })), + }; + return { changelog: plainChangelog(notes.changelog), notes, popup }; +} +function saveReleaseNotes(localRoot, notes, release) { + releaseNotes(notes, release); + const directory = path.join(localRoot, 'config/release-notes'); + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + const stat = fs.lstatSync(directory); + if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== process.geteuid() + || (stat.mode & 0o7777) !== 0o700 || fs.realpathSync(directory) !== directory) fail(); + const file = path.join(directory, `${notes.releaseId}.json`); + const prior = privateJson(file, process.geteuid(), true); + if (prior && JSON.stringify(prior) !== JSON.stringify(notes)) fail(); + if (!prior) atomic(file, notes); +} +function loadReleaseNotes(localRoot, releaseId, release) { + try { + if (!/^[a-z][a-z0-9_.-]{2,95}$/.test(releaseId)) return null; + const notes = privateJson(path.join(localRoot, 'config/release-notes', `${releaseId}.json`), process.geteuid(), true); + return notes ? releaseNotes(notes, { ...release, releaseId }) : null; + } catch { return null; } // A presentation failure must not hide valid installation controls. +} +module.exports = { NAME, LIMIT, presentation, releaseNotes, authoring, markdown, loadReleaseNotes, saveReleaseNotes }; diff --git a/core/core/installations/src/release-package.js b/core/core/installations/src/release-package.js new file mode 100644 index 0000000..f2b99ba --- /dev/null +++ b/core/core/installations/src/release-package.js @@ -0,0 +1,45 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { hashFileSync } = require('./release-delivery-files'); +function archive(args) { + const result = spawnSync('/usr/bin/python3', ['-I', path.join(__dirname, "./release-package.py"), ...args], { + encoding: 'utf8', timeout: 300_000, maxBuffer: 4096, env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8' }, + }); + require('./release-build-space').checkArchiveResult(result, 'release_package_invalid'); + return result.stdout; +} +function pack(root, output, kind, commit = '-') { + const { unpackedSize } = JSON.parse(archive(['pack', root, output, kind, commit])); + return { name: path.basename(output), size: fs.statSync(output).size, unpackedSize, sha256: hashFileSync(output) }; +} +function unpack(file, root, kind, commit, sha256, unpackedSize) { archive(['unpack', file, root, kind, commit, sha256, ...(unpackedSize === undefined ? [] : [String(unpackedSize)])]); } +// The v2 runtime identity binds both independently checksummed packages. The +// existing runtime catalog continues to carry a single immutable identity. +function runtimeIdentity(assets) { + return require('./release-delivery-contract').sha(JSON.stringify({ app: assets.app.sha256, dependencies: assets.dependencies.sha256 })); +} +function preparePackages(directory, manifest) { + const { removeStage, seal } = require('./release-delivery-install'); + const stage = path.join(directory, 'packages'); removeStage(stage); fs.mkdirSync(stage, { mode: 0o700 }); + try { + const app = path.join(stage, 'app'), dependencies = path.join(stage, 'dependencies'); + unpack(path.join(directory, manifest.assets.app.name), app, 'app', manifest.sourceCommit, manifest.assets.app.sha256, manifest.assets.app.unpackedSize); + unpack(path.join(directory, manifest.assets.dependencies.name), dependencies, 'dependencies', '-', manifest.assets.dependencies.sha256, manifest.assets.dependencies.unpackedSize); + const runtime = path.join(app, 'runtime'); + fs.chmodSync(runtime, 0o700); + fs.renameSync(path.join(dependencies, 'dependencies'), path.join(runtime, 'dependencies')); + seal(runtime); + require('./native-runtime-artifact').verifyNativeRuntime(runtime, manifest.runtime); + const { collect } = require('./release-delivery-build'); + const { bundle } = require('./release-delivery-contract'); + const read = kind => { + const files = []; + for (const name of fs.readdirSync(path.join(app, kind)).sort()) collect(path.join(app, kind), name, files); + return bundle({ schemaVersion: 1, kind, sourceCommit: manifest.sourceCommit, files }, kind, manifest.sourceCommit); + }; + return { core: read('core'), bridge: read('bridge'), runtime, stage }; + } catch (error) { removeStage(stage); throw error; } +} +module.exports = { pack, unpack, preparePackages, runtimeIdentity }; diff --git a/core/core/installations/src/release-package.py b/core/core/installations/src/release-package.py new file mode 100644 index 0000000..7c5e59f --- /dev/null +++ b/core/core/installations/src/release-package.py @@ -0,0 +1,138 @@ +"""Deterministic release packages with bounded, inventory-checked extraction.""" +import errno +import gzip +import hashlib +import io +import json +import os +import re +import stat +import sys +import tarfile + +MANIFEST = 'package-manifest.json' +MAX_BYTES = 2 * 1024 ** 3 +MAX_FILES = 20000 + + +def require(value): + if not value: + raise ValueError('release_package_invalid') + + +def digest(filename): + result = hashlib.sha256() + with open(filename, 'rb') as source: + while chunk := source.read(1024 * 1024): + result.update(chunk) + return result.hexdigest() + + +def allowed(name, kind): + if kind == 'dependencies': + return name.startswith(('dependencies/node/', 'dependencies/browser/')) + return bool(re.match(r'^(core/(code/(core|host|dashboard|shared|plugins)/|code/bin/dispatch-(dashboard|access-admin)$|host-helper-artifact/)|bridge/bridge-artifact/|runtime/(runtime/|shared/|plugins/|runtime-release-manifest.json$))', name)) + + +def entries(value, kind, commit): + require(set(value) == {'schemaVersion', 'kind', 'sourceCommit', 'files'}) + require(value['schemaVersion'] == 1 and value['kind'] == kind and value['sourceCommit'] == commit) + require(isinstance(value['files'], list) and 1 <= len(value['files']) <= MAX_FILES) + result = {} + total = 0 + for item in value['files']: + require(set(item) == {'path', 'mode', 'size', 'sha256'}) + name = item['path'] + require(isinstance(name, str) and len(name) <= 240 and re.fullmatch(r'[A-Za-z0-9_./+-]+', name)) + require(all(part not in ('', '.', '..') for part in name.split('/')) and allowed(name, kind)) + require(name not in result and item['mode'] in ('444', '555')) + require(type(item['size']) is int and item['size'] >= 0 and re.fullmatch(r'[a-f0-9]{64}', item['sha256'])) + total += item['size'] + require(total <= (MAX_BYTES if kind == 'dependencies' else 128 * 1024 ** 2)) + result[name] = item + for name in result: + parts = name.split('/')[:-1] + while parts: + require('/'.join(parts) not in result) + parts.pop() + return result + + +def pack(root, archive, kind, commit): + require(kind in ('app', 'dependencies') and (re.fullmatch(r'[a-f0-9]{40}', commit) if kind == 'app' else commit == '-')) + files = [] + for directory, dirs, names in os.walk(root, followlinks=False): + for name in dirs: + require(stat.S_ISDIR(os.lstat(os.path.join(directory, name)).st_mode)) + for name in names: + filename = os.path.join(directory, name) + info = os.lstat(filename) + require(stat.S_ISREG(info.st_mode) and info.st_nlink == 1) + files.append({'path': os.path.relpath(filename, root), 'mode': '555' if info.st_mode & 0o111 else '444', + 'size': info.st_size, 'sha256': digest(filename)}) + value = {'schemaVersion': 1, 'kind': kind, 'sourceCommit': commit, 'files': sorted(files, key=lambda item: item['path'])} + entries(value, kind, commit) + total = sum(item['size'] for item in files) + data = (json.dumps(value, separators=(',', ':')) + '\n').encode() + require(len(data) <= 8 * 1024 ** 2) + # Both gzip and tar metadata are independent of the build path and wall clock. + with open(archive, 'xb') as output, gzip.GzipFile(filename='', mode='wb', fileobj=output, mtime=0) as compressed: + with tarfile.open(fileobj=compressed, mode='w|', format=tarfile.USTAR_FORMAT) as target: + member = tarfile.TarInfo(MANIFEST) + member.size, member.mode = len(data), 0o444 + target.addfile(member, io.BytesIO(data)) + for entry in value['files']: + member = tarfile.TarInfo(entry['path']) + member.size, member.mode = entry['size'], int(entry['mode'], 8) + with open(os.path.join(root, entry['path']), 'rb') as source: + target.addfile(member, source) + + return total + +def unpack(archive, root, kind, commit, expected, unpacked_size=None): + require(os.path.isabs(root) and not os.path.lexists(root) and digest(archive) == expected) + os.mkdir(root, 0o700) + with tarfile.open(archive, 'r|gz') as source: + first = source.next() + require(first is not None and first.name == MANIFEST and first.isreg() and first.size <= 8 * 1024 ** 2) + inventory = entries(json.loads(source.extractfile(first).read()), kind, commit) + if unpacked_size is not None: + require(sum(item['size'] for item in inventory.values()) == int(unpacked_size)) + seen = set() + for member in source: + if member is first: + continue + require(member.name in inventory and member.name not in seen and member.isreg()) + entry = inventory[member.name] + require(member.size == entry['size'] and member.mode == int(entry['mode'], 8)) + filename = os.path.join(root, member.name) + os.makedirs(os.path.dirname(filename), mode=0o700, exist_ok=True) + result = hashlib.sha256() + with source.extractfile(member) as incoming, open(filename, 'xb') as outgoing: + while chunk := incoming.read(1024 * 1024): + result.update(chunk) + outgoing.write(chunk) + outgoing.flush() + os.fsync(outgoing.fileno()) + require(result.hexdigest() == entry['sha256']) + os.chmod(filename, int(entry['mode'], 8)) + seen.add(member.name) + require(seen == set(inventory)) + for directory, _, _ in os.walk(root, topdown=False): + os.chmod(directory, 0o555) + + +if __name__ == '__main__': + try: + if sys.argv[1] == 'pack' and len(sys.argv) == 6: + print(json.dumps({'unpackedSize': pack(*sys.argv[2:])})) + elif sys.argv[1] == 'unpack' and len(sys.argv) in (7, 8): + unpack(*sys.argv[2:]) + else: + raise ValueError() + except OSError as error: + print('archive_disk_full' if error.errno in (errno.ENOSPC, errno.EDQUOT) else 'archive_io_failed', file=sys.stderr) + sys.exit(1) + except Exception: + print('release package verification failed', file=sys.stderr) + sys.exit(1) diff --git a/core/core/installations/src/release-ready-notify.js b/core/core/installations/src/release-ready-notify.js new file mode 100644 index 0000000..8a58132 --- /dev/null +++ b/core/core/installations/src/release-ready-notify.js @@ -0,0 +1,7 @@ +'use strict'; +const {atomic} = require('./release-delivery-files'); +function install(localRoot) { + if (!/^\/[A-Za-z0-9_./-]+$/.test(localRoot) || require('node:path').resolve(localRoot) !== localRoot) throw Error('invalid_local_root'); + atomic('/etc/systemd/system/dispatch-release-watch.path', `[Unit]\nDescription=Discover an explicitly published Dispatch release immediately\n\n[Path]\nPathChanged=${localRoot}/run/release-ready\nUnit=dispatch-release-watch.service\n\n[Install]\nWantedBy=multi-user.target\n`, 0o644); +} +module.exports = {install}; diff --git a/core/core/installations/src/release-retention-status.js b/core/core/installations/src/release-retention-status.js new file mode 100644 index 0000000..1daf1b8 --- /dev/null +++ b/core/core/installations/src/release-retention-status.js @@ -0,0 +1,9 @@ +'use strict'; +const path = require('node:path'); +const { RECEIPTS, publicRootJson } = require('./offsite-policy'); +function cleanupReady(rolloutId, releaseId, { root = RECEIPTS, uid = 0 } = {}) { + if (!/^rollout_[a-f0-9]{32}$/.test(rolloutId) || !/^[a-z][a-z0-9_.-]{2,95}$/.test(releaseId)) return false; + const receipt = publicRootJson(path.join(root, `${rolloutId}.cleanup.json`), true, uid); + return receipt?.schemaVersion === 1 && receipt.status === 'completed' && receipt.rolloutId === rolloutId && receipt.releaseId === releaseId; +} +module.exports = { cleanupReady }; diff --git a/core/core/installations/src/release-retention.js b/core/core/installations/src/release-retention.js new file mode 100644 index 0000000..90966b1 --- /dev/null +++ b/core/core/installations/src/release-retention.js @@ -0,0 +1,152 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const { privateJson, atomic, rootParents } = require('./release-delivery-files'); +const { RECEIPTS, publicRootJson, receiptKey } = require('./offsite-policy'); +const { cleanupReady } = require('./release-retention-status'); +const { compareVersions } = require('../../../shared/release-version'); +const BASES = ['dispatch-platform', 'dispatch-runtime', 'dispatch-control', 'dispatch-updater', 'dispatch-release-delivery']; +const fail = () => { throw Error('release_cleanup_unavailable'); }; +function obsoleteReleases(keep) { + const result = []; + for (const base of BASES) { + const parent = `/opt/${base}/releases`; + if (!fs.existsSync(parent)) continue; + rootParents(parent); + for (const name of fs.readdirSync(parent)) { + if (!/^[a-z0-9][a-z0-9_.-]{2,95}$/.test(name)) fail(); + const directory = path.join(parent, name); + if (!keep.has(directory)) { rootParents(directory); result.push(directory); } + } + } + return result; +} +function referencedByProcess(roots) { + for (const pid of fs.readdirSync('/proc').filter(name => /^[0-9]+$/.test(name))) { + try { + const command = fs.readFileSync(`/proc/${pid}/cmdline`).toString().replaceAll('\0', ' '); + const mapped = fs.readFileSync(`/proc/${pid}/maps`, 'utf8'); + const cwd = fs.readlinkSync(`/proc/${pid}/cwd`); + if (roots.some(root => command.includes(root + '/') || mapped.includes(root + '/') || cwd === root || cwd.startsWith(root + '/'))) return true; + } catch (error) { if (!['ENOENT', 'ESRCH', 'EINVAL'].includes(error.code)) throw error; } + } + return false; +} +async function pruneReleases(config) { + if (process.geteuid() !== 0) fail(); + const { account } = require('./host-recovery-bundle'), core = account(config.coreUid), unitRoot = path.join(core.home, '.config/systemd/user'); + const file = path.join(config.localRoot, 'data/access-control/access-control.sqlite3'); + const db = new DatabaseSync(file); + db.exec('PRAGMA foreign_keys=ON; PRAGMA trusted_schema=OFF; PRAGMA busy_timeout=3000'); + let registry; + try { + const rollout = db.prepare('SELECT * FROM platform_rollouts ORDER BY created_at DESC,rowid DESC LIMIT 1').get(); + if (!rollout) return { status: 'idle' }; + if (!/^rollout_[a-f0-9]{32}$/.test(rollout.id)) fail(); + if (cleanupReady(rollout.id, rollout.release_id)) { + fs.rmSync(path.join(config.localRoot, 'backups/platform-core', rollout.id), { recursive: true, force: true }); + return { status: 'idle' }; + } + if (rollout.status !== 'running') return { status: 'idle' }; + const catalog = privateJson(path.join(config.localRoot, 'config/oci-releases.json'), config.coreUid); + const descriptor = catalog.releases?.[rollout.release_id]; + if (descriptor?.backend !== 'native_service_v1') return { status: 'idle' }; + require('./native-deployment').releaseDescriptor(descriptor); + const coreStage = db.prepare('SELECT * FROM platform_rollout_core WHERE rollout_id=?').get(rollout.id); + if (coreStage?.status !== 'succeeded') return { status: 'waiting' }; + const fleet = db.prepare(`SELECT i.*,o.timezone,o.status AS organization_status,s.code AS station_code,m.status AS rollout_status + FROM installations i JOIN organizations o ON o.id=i.organization_id JOIN stations s ON s.organization_id=o.id AND s.is_primary=1 + LEFT JOIN platform_rollout_members m ON m.organization_id=i.organization_id AND m.rollout_id=? + WHERE i.status NOT IN ('decommissioned','decommissioning')`).all(rollout.id); + if (fleet.some(i => i.backend !== 'native_service_v1' || i.release_id !== rollout.release_id || i.rollout_status !== 'updated' + || !['ready', 'suspended', 'pending', 'waiting_for_owner', 'waiting_for_provider_auth'].includes(i.status)) + || db.prepare("SELECT 1 FROM installation_lifecycle_jobs WHERE status IN ('queued','running')").get() + || db.prepare("SELECT 1 FROM installation_provisioning_requests WHERE status IN ('pending','dispatched')").get()) return { status: 'waiting' }; + const { root, config: deployment } = require('./core-recovery-host').rootArtifact(rollout.release_id); + await require('./core-recovery-host').assertHealth(deployment); + const coreRecovery = path.join(config.localRoot, 'backups/platform-core', rollout.id); + const journal = privateJson(path.join(coreRecovery, 'recovery.json'), config.coreUid); + const proof = require('./rollout-backup-proof').rolloutBackupProof(db, rollout.id) + || publicRootJson(path.join(RECEIPTS, receiptKey(path.join(coreRecovery, `attempt-${journal.attempt}`)) + '.json'), true); + const erased = publicRootJson(path.join(RECEIPTS, `${rollout.id}.core-backups-erased.json`), true); + if (journal.phase !== 'promoted' || journal.releaseId !== rollout.release_id + || !(proof?.status === 'verified' && /^[a-f0-9]{64}$/.test(proof.recoveryDigest) + || erased?.status === 'erased' && erased.rolloutId === rollout.id)) fail(); + for (const member of fleet) { + const backup = db.prepare(`SELECT j.backup_id AS id FROM installation_lifecycle_jobs j + JOIN platform_rollout_members m ON m.job_id=j.id WHERE m.rollout_id=? AND m.organization_id=? AND j.operation='upgrade' AND j.backup_id IS NOT NULL`).get(rollout.id, member.organization_id); + if (backup) { + const receipt = privateJson(`/var/lib/dispatch-backup/archives/${backup.id}.json`, 0); + if (receipt.status !== 'verified' || !/^[a-f0-9]{64}$/.test(receipt.recoveryDigest)) fail(); + } + } + const targetCode = path.join(root, 'code'); + for (const name of ['dispatch-dashboard.service', 'dispatch-installation-reconcile.service', 'dispatch-platform-update.service']) { + if (!fs.readFileSync(path.join(unitRoot, name), 'utf8').includes(targetCode + '/')) return { status: 'waiting' }; + } + if (!fs.readFileSync('/etc/systemd/system/dispatch-release-watch.service', 'utf8').includes(targetCode + '/')) return { status: 'waiting' }; + const keep = new Set(['dispatch-platform', 'dispatch-runtime', 'dispatch-control'].map(base => `/opt/${base}/releases/${rollout.release_id}`)); + const platforms = privateJson(path.join(config.localRoot, 'config/platform-releases.json'), config.coreUid); + const current = platforms.releases?.[rollout.release_id]; + for (const [id, release] of Object.entries(platforms.releases || {})) if (current && (compareVersions(release.version, current.version) || release.publishedAt.localeCompare(current.publishedAt)) > 0) { + for (const base of ['dispatch-platform', 'dispatch-runtime', 'dispatch-control']) keep.add(`/opt/${base}/releases/${id}`); + } + // Retained DSPs must still be able to start their exact installed release. + for (const row of db.prepare('SELECT DISTINCT release_id FROM installations').all()) { + for (const base of ['dispatch-platform', 'dispatch-runtime', 'dispatch-control']) keep.add(`/opt/${base}/releases/${row.release_id}`); + } + const obsolete = obsoleteReleases(keep); + if (referencedByProcess(obsolete)) return { status: 'waiting' }; + const host = privateJson('/etc/dispatch/oci-host.json', 0); + if (host.controlReleaseId !== rollout.release_id || fs.realpathSync('/opt/dispatch-control/current') !== `/opt/dispatch-control/releases/${rollout.release_id}`) fail(); + registry = require('./oci-host-account-registry').createOciHostAccountRegistry({ stateRoot: host.stateRoot, identityAvailable: () => false }); + const executor = require('./oci-host-executor').createOciHostExecutor({ registry, stateRoot: host.stateRoot, unitRoot: host.unitRoot, + releaseRoot: host.releaseRoot, centralSocket: host.centralSocket, centralUid: host.centralUid, controllerUid: host.controllerUid }); + // Freeze new Core operations while journals and obsolete artifacts are + // removed. Every installed unit is re-attested against its current plan. + db.exec('BEGIN IMMEDIATE'); + if (db.prepare("SELECT COUNT(*) AS n FROM installations WHERE status NOT IN ('decommissioned','decommissioning')").get().n !== fleet.length + || db.prepare("SELECT 1 FROM installation_lifecycle_jobs WHERE status IN ('queued','running')").get() + || db.prepare("SELECT 1 FROM installation_provisioning_requests WHERE status IN ('pending','dispatched')").get() + || fleet.some(i => { + const current = db.prepare('SELECT revision,release_id,status FROM installations WHERE organization_id=?').get(i.organization_id); + return !current || current.revision !== i.revision || current.release_id !== i.release_id || current.status !== i.status; + })) { db.exec('ROLLBACK'); return { status: 'waiting' }; } + for (const member of fleet) { + const allocation = registry.inspect(member.runtime_key); + if (!allocation) { if (!['pending', 'waiting_for_owner'].includes(member.status)) fail(); continue; } + const manifest = { manifestVersion: 1, revision: member.manifest_revision, + organization: { id: member.organization_id, stationCode: member.station_code, timezone: member.timezone }, + runtime: { key: member.runtime_key, templateId: 'isolated_dsp_v1', releaseId: member.release_id } }; + const authority = { revision: manifest.revision, organization: manifest.organization, runtime: manifest.runtime }; + const plan = require('./native-deployment').createPlan(manifest, authority, descriptor, { + name: allocation.name, uid: allocation.uid, gid: allocation.gid, subuidStart: allocation.subuidStart, + subgidStart: allocation.subgidStart, subidCount: allocation.subidCount }, { + version: 1, backend: 'native_service_v1', channel: descriptor.channel, organizationId: member.organization_id, + runtimeKey: member.runtime_key, manifestRevision: member.manifest_revision, releaseId: member.release_id }); + if (member.status === 'suspended') executor.inspectInactive(plan); else executor.health(plan); + executor.settleCommitted(plan, callback => callback()); + } + for (const directory of obsolete) fs.rmSync(directory, { recursive: true }); + for (const name of fs.readdirSync('/etc/sudoers.d').filter(name => /^dispatch[-_a-z0-9.]*$/.test(name))) { + const file = path.join('/etc/sudoers.d', name), content = fs.readFileSync(file, 'utf8'); + if (obsolete.some(root => content.includes(root + '/'))) fs.unlinkSync(file); + } + for (const name of fs.readdirSync('/etc/systemd/system').filter(name => /^dispatch-backup-enable-[a-z0-9_.-]+\.service$/.test(name))) { + const file = path.join('/etc/systemd/system', name); + if (obsolete.some(root => fs.readFileSync(file, 'utf8').includes(root + '/'))) fs.unlinkSync(file); + } + for (const [name, value] of [['oci-releases.json', catalog], ['platform-releases.json', platforms]]) { + for (const id of Object.keys(value.releases)) if (!keep.has(`/opt/dispatch-platform/releases/${id}`)) delete value.releases[id]; + const target = path.join(config.localRoot, 'config', name); atomic(target, value); fs.chownSync(target, config.coreUid, core.gid); + } + atomic(path.join(RECEIPTS, `${rollout.id}.cleanup.json`), { schemaVersion: 1, status: 'completed', rolloutId: rollout.id, + releaseId: rollout.release_id, completedAt: Date.now() }, 0o644); + db.exec('COMMIT'); + fs.rmSync(coreRecovery, { recursive: true, force: true }); + require('./host-recovery-bundle').command('/usr/bin/systemctl', ['daemon-reload']); + return { status: 'completed', removedReleases: obsolete.length }; + } catch (error) { try { db.exec('ROLLBACK'); } catch {} throw error; } + finally { registry?.close(); db.close(); } +} +module.exports = { pruneReleases, referencedByProcess }; diff --git a/core/core/installations/src/release-reuse-split.js b/core/core/installations/src/release-reuse-split.js new file mode 100644 index 0000000..e16558a --- /dev/null +++ b/core/core/installations/src/release-reuse-split.js @@ -0,0 +1,35 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { pack, unpack, runtimeIdentity } = require('./release-package'); +const { removeStage } = require('./release-delivery-install'); + +// The caller has verified every asset against the exact-commit CI manifest. +// Only the small application archive changes with release-specific popup copy. +function reuse(directory, output, manifest, popup) { + const stage = path.join(output, 'reuse-app'); + const asset = manifest.assets.app; + try { + unpack(path.join(directory, asset.name), stage, 'app', manifest.sourceCommit, asset.sha256, asset.unpackedSize); + const popupFile = path.join(stage, 'core/code/dashboard/release-popup.json'); + fs.chmodSync(path.dirname(popupFile), 0o700); + if (fs.existsSync(popupFile)) fs.unlinkSync(popupFile); + if (popup) { + require('../../accounts/src/release-popup').validatePopup(popup, { + releaseId: popup.releaseId, version: popup.version, sourceCommit: manifest.sourceCommit, + }); + fs.writeFileSync(popupFile, JSON.stringify(popup) + '\n', { flag: 'wx', mode: 0o444 }); + } + const assets = { + app: pack(stage, path.join(output, asset.name), 'app', manifest.sourceCommit), + dependencies: manifest.assets.dependencies, + }; + fs.copyFileSync(path.join(directory, assets.dependencies.name), path.join(output, assets.dependencies.name), fs.constants.COPYFILE_EXCL); + return { + assets, dependencies: manifest.dependencies, + artifact: { artifactSha256: runtimeIdentity(assets), embeddedManifestSha256: manifest.runtime.embeddedManifestSha256 }, + bridgeHash: manifest.runtime.bridgeManifestSha256, + }; + } finally { removeStage(stage); } +} +module.exports = { reuse }; diff --git a/core/core/installations/src/retire-oci-credentials.js b/core/core/installations/src/retire-oci-credentials.js new file mode 100644 index 0000000..3476f93 --- /dev/null +++ b/core/core/installations/src/retire-oci-credentials.js @@ -0,0 +1,27 @@ +'use strict'; + +// Repeated after reconciliation so a crash between durable retirement and file +// deletion cannot leave a usable registration credential behind. +function retireOciCredentials({ store, credentialPort }) { + const rows = store.db.prepare(`SELECT i.runtime_key,a.token_hash,i.organization_id,i.backend FROM installations i + LEFT JOIN runtime_agent_authorities a ON i.organization_id=a.organization_id AND i.runtime_key=a.runtime_key + WHERE i.backend IN ('oci_container_v1','native_service_v1') AND i.status='decommissioned' + AND (a.status='revoked' OR (a.runtime_key IS NULL AND i.backend='native_service_v1' AND EXISTS + (SELECT 1 FROM installation_lifecycle_jobs j WHERE j.organization_id=i.organization_id AND j.operation='destroy' AND j.status='succeeded')))`).all(); + let removed = 0; + for (const row of rows) { + if (credentialPort.revoke(row.runtime_key, row.token_hash)) removed += 1; + else { + // Missing is the successful replay case; a changed credential is a conflict. + try { credentialPort.read(row.runtime_key); } + catch (error) { if (error?.code !== 'ENOENT') throw error; + if (row.backend === 'native_service_v1' && store.db.prepare("SELECT 1 FROM installation_lifecycle_jobs WHERE organization_id=? AND operation='destroy' AND status='succeeded'").get(row.organization_id)) store.eraseOrganization(row.organization_id); + continue; + } + throw Object.assign(new Error('runtime_agent_authority_conflict'), { code: 'runtime_agent_authority_conflict' }); + } + if (row.backend === 'native_service_v1' && store.db.prepare("SELECT 1 FROM installation_lifecycle_jobs WHERE organization_id=? AND operation='destroy' AND status='succeeded'").get(row.organization_id)) store.eraseOrganization(row.organization_id); + } + return removed; +} +module.exports = { retireOciCredentials }; diff --git a/core/core/installations/src/retired-dsp-metadata.js b/core/core/installations/src/retired-dsp-metadata.js new file mode 100644 index 0000000..0c308a9 --- /dev/null +++ b/core/core/installations/src/retired-dsp-metadata.js @@ -0,0 +1,83 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const { spawnSync } = require('node:child_process'); +const { publicRootJson, RECEIPTS } = require('./offsite-policy'); +const { privateJson, atomic } = require('./release-delivery-files'); +const { HOST_TENANT_ROOT, HOST_BRIDGE_ROOT, opaqueRuntimeSuffix, hostAccountName } = require('../../runtime-host-identity'); +function eraseProvisionerRows(db, organizationId) { + db.exec('PRAGMA secure_delete=ON; BEGIN IMMEDIATE; PRAGMA defer_foreign_keys=ON'); + try { + const jobs = db.prepare('SELECT id FROM jobs WHERE organization_id=?').all(organizationId); + for (const { name } of db.prepare("SELECT name FROM sqlite_schema WHERE type='table' AND name NOT LIKE 'sqlite_%'").all()) { + if (!/^[a-z_]+$/.test(name)) throw Error('retirement_failed'); + const columns = db.prepare(`PRAGMA table_info(${name})`).all().map(c => c.name); + if (columns.includes('job_id')) for (const job of jobs) db.prepare(`DELETE FROM ${name} WHERE job_id=?`).run(job.id); + if (columns.includes('organization_id')) db.prepare(`DELETE FROM ${name} WHERE organization_id=?`).run(organizationId); + } + if (db.prepare('PRAGMA foreign_key_check').all().length) throw Error('retirement_failed'); + db.exec('COMMIT; PRAGMA wal_checkpoint(TRUNCATE)'); + } catch (error) { try { db.exec('ROLLBACK'); } catch {} throw error; } +} +function purgeRetiredMetadata(config) { + if (process.geteuid() !== 0) throw Error('retirement_requires_root'); + const access = new DatabaseSync(path.join(config.localRoot, 'data/access-control/access-control.sqlite3'), { readOnly: true }); + let removed = 0; + try { + for (const name of fs.readdirSync(RECEIPTS).filter(name => /^deleted-life_[a-f0-9]{32}\.json$/.test(name))) { + const proof = publicRootJson(path.join(RECEIPTS, name)); + if (proof.status !== 'destroyed' || !/^[a-z][a-z0-9_-]{2,95}$/.test(proof.organizationId)) throw Error('retirement_failed'); + if (access.prepare('SELECT 1 FROM organizations WHERE id=?').get(proof.organizationId) + || access.prepare('SELECT 1 FROM installations WHERE runtime_key=?').get(proof.runtimeKey)) continue; + const suffix = opaqueRuntimeSuffix(proof.runtimeKey); + if ([path.join(HOST_TENANT_ROOT, suffix), path.join(HOST_BRIDGE_ROOT, suffix), + `/etc/systemd/system/dispatch-dsp-${suffix}.service`, `/etc/systemd/system/dispatch-runtime-agent-bridge-${suffix}.service`].some(p => fs.existsSync(p)) + || spawnSync('/usr/bin/getent', ['passwd', hostAccountName(proof.runtimeKey)]).status !== 2) throw Error('retirement_failed'); + const provisionerFile = path.join(config.localRoot, 'state/provisioner/provisioner.sqlite3'); + if (fs.existsSync(provisionerFile)) { + const db = new DatabaseSync(provisionerFile); + try { db.exec('PRAGMA foreign_keys=ON; PRAGMA busy_timeout=3000'); eraseProvisionerRows(db, proof.organizationId); } finally { db.close(); } + } + const host = privateJson('/etc/dispatch/oci-host.json', 0); + for (const [file, statements] of [[path.join(host.authorityRoot, 'authority.sqlite3'), + ['DELETE FROM actions WHERE runtime_key=?', 'DELETE FROM leases WHERE runtime_key=?']], + [path.join(host.stateRoot, 'oci-host.sqlite3'), ["DELETE FROM allocations WHERE runtime_key=? AND status='retired'"]]]) { + const db = new DatabaseSync(file); + try { db.exec('PRAGMA foreign_keys=ON; PRAGMA secure_delete=ON; BEGIN IMMEDIATE'); + for (const sql of statements) db.prepare(sql).run(proof.runtimeKey); + db.exec('COMMIT; PRAGMA wal_checkpoint(TRUNCATE)'); + } finally { db.close(); } + } + for (const file of [`journals/${suffix}.json`, `journals/${suffix}.settled.json`, + `candidates/dispatch-dsp-${suffix}.service`, `candidates/dispatch-runtime-agent-bridge-${suffix}.service`]) fs.rmSync(path.join(host.stateRoot, file), { force: true }); + const scheduled = path.join(config.localRoot, 'backups/scheduled-core'); + if (fs.existsSync(scheduled)) { + if (fs.realpathSync(scheduled) !== scheduled) throw Error('retirement_failed'); + for (const row of access.prepare("SELECT id FROM platform_backup_records WHERE kind='core' AND deleted_at IS NOT NULL").all()) { + if (!/^breq_[a-f0-9]{32}$/.test(row.id)) throw Error('retirement_failed'); + for (const name of [row.id, `.creating-${row.id}`]) fs.rmSync(path.join(scheduled, name), { recursive: true, force: true }); + } + } + const archives = '/var/lib/dispatch-backup/archives'; + if (fs.existsSync(archives)) for (const file of fs.readdirSync(archives).filter(name => /^(backup|breq)_[a-f0-9]{32}\.json$/.test(name))) { + const receipt = privateJson(path.join(archives, file), 0); + if (receipt.organizationId === proof.organizationId || receipt.kind === 'core' && receipt.status === 'destroyed') { + if (receipt.status !== 'destroyed') throw Error('retirement_failed'); + if (receipt.kind === 'core') fs.rmSync(path.join(config.localRoot, 'backups/scheduled-core', receipt.id), { recursive: true, force: true }); + fs.unlinkSync(path.join(archives, file)); + } + } + const coreBackups = path.join(config.localRoot, 'backups/platform-core'); + if (fs.existsSync(coreBackups)) for (const rollout of fs.readdirSync(coreBackups).filter(name => /^rollout_[a-f0-9]{32}$/.test(name))) { + const root = path.join(coreBackups, rollout); + const journal = privateJson(path.join(root, 'recovery.json'), config.coreUid, true); + if (!journal || !['promoted', 'recovered'].includes(journal.phase)) throw Error('retirement_failed'); + for (const attempt of fs.readdirSync(root).filter(name => /^attempt-[1-9][0-9]*$/.test(name))) fs.rmSync(path.join(root, attempt), { recursive: true }); + atomic(path.join(RECEIPTS, `${rollout}.core-backups-erased.json`), { schemaVersion: 1, status: 'erased', rolloutId: rollout, erasedAt: Date.now() }, 0o644); + } + fs.unlinkSync(path.join(RECEIPTS, name)); removed++; + } + } finally { access.close(); } + return removed; +} +module.exports = { purgeRetiredMetadata, eraseProvisionerRows }; diff --git a/core/core/installations/src/rollout-backup-proof.js b/core/core/installations/src/rollout-backup-proof.js new file mode 100644 index 0000000..049e866 --- /dev/null +++ b/core/core/installations/src/rollout-backup-proof.js @@ -0,0 +1,24 @@ +'use strict'; +const crypto = require('node:crypto'); +const { privateJson } = require('./release-delivery-files'); +function rolloutBackupProof(db, rolloutId, read = id => privateJson(`/var/lib/dispatch-backup/archives/${id}.json`, 0)) { + if (!db.prepare("SELECT 1 FROM sqlite_schema WHERE name='platform_rollout_backups'").get()) return null; + const progress = require('../../accounts/src/rollout-backups').rolloutBackupProgress(db, rolloutId); + if (!progress) return null; + const fail = () => { throw Error('release_cleanup_unavailable'); }; + if (progress.status !== 'completed') fail(); + let core; + for (const member of progress.members) { + if (!/^(backup|breq)_[a-f0-9]{32}$/.test(member.backupId)) fail(); + const row = db.prepare('SELECT * FROM platform_backup_records WHERE id=? AND deleted_at IS NULL').get(member.backupId); + const proof = read(member.backupId); + if (!row || row.organization_id !== member.organizationId || proof.status !== 'verified' + || proof.id !== row.id || proof.organizationId !== row.organization_id + || proof.metadataDigest !== crypto.createHash('sha256').update(row.metadata_json).digest('hex') + || !/^[a-f0-9]{64}$/.test(proof.recoveryDigest)) fail(); + if (!member.organizationId) core = proof; + } + if (!core) fail(); + return core; +} +module.exports = { rolloutBackupProof }; diff --git a/core/core/installations/src/runtime-agent-credential.js b/core/core/installations/src/runtime-agent-credential.js new file mode 100644 index 0000000..9cf2c00 --- /dev/null +++ b/core/core/installations/src/runtime-agent-credential.js @@ -0,0 +1,165 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { INSTALLATION_IDENTIFIER_RE } = require('../../../shared/contracts/src'); +const { PROJECT_ROOT } = require('../../../shared/paths/runtime-paths'); +const { registrationToken, readPrivateRegistrationToken } = require('../../agents/src'); + +const PRIVATE_DIRECTORY_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; + +function fail(code = 'runtime_boundary_violation') { + throw Object.assign(new Error(code), { code }); +} + +function contains(root, candidate) { + const relative = path.relative(root, candidate); + return relative === '' || (relative !== '..' && !relative.startsWith(`..${path.sep}`) + && !path.isAbsolute(relative)); +} + +function directoryIdentity(target, expectedDevice = null) { + let info; + try { info = fs.lstatSync(target); } catch { fail(); } + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== process.geteuid() + || (info.mode & 0o7777) !== PRIVATE_DIRECTORY_MODE + || expectedDevice !== null && info.dev !== expectedDevice || fs.realpathSync(target) !== target) fail(); + return Object.freeze({ dev: info.dev, ino: info.ino }); +} + +function syncDirectory(target) { + let handle; + try { + handle = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_DIRECTORY); + fs.fsyncSync(handle); + } catch { fail(); } + finally { if (handle !== undefined) try { fs.closeSync(handle); } catch {} } +} + +function ensureDirectory(target, parent, expectedDevice) { + const parentBefore = directoryIdentity(parent, expectedDevice); + let changed = false; + try { + fs.mkdirSync(target, { mode: PRIVATE_DIRECTORY_MODE }); + syncDirectory(parent); + changed = true; + } catch (error) { + if (error?.code !== 'EEXIST') fail(); + } + directoryIdentity(target, parentBefore.dev); + const parentAfter = directoryIdentity(parent, expectedDevice); + if (parentBefore.dev !== parentAfter.dev || parentBefore.ino !== parentAfter.ino) fail(); + return changed; +} + +function writeToken(file, token, parentDevice) { + const temporary = path.join(path.dirname(file), `.registration-token.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`); + try { + fs.writeFileSync(temporary, `${registrationToken(token)}\n`, { mode: PRIVATE_FILE_MODE, flag: 'wx' }); + const handle = fs.openSync(temporary, 'r'); + try { fs.fsyncSync(handle); } finally { fs.closeSync(handle); } + const info = fs.lstatSync(temporary); + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() || info.nlink !== 1 + || info.dev !== parentDevice || (info.mode & 0o7777) !== PRIVATE_FILE_MODE) fail(); + fs.renameSync(temporary, file); + fs.chmodSync(file, PRIVATE_FILE_MODE); + syncDirectory(path.dirname(file)); + return readPrivateRegistrationToken(file); + } finally { + try { fs.rmSync(temporary, { force: true }); } catch {} + } +} + +function removeOrphanTokens(agentRoot, expectedDevice) { + let changed = false; + for (const name of fs.readdirSync(agentRoot)) { + if (name === 'registration-token') continue; + if (!/^\.registration-token\.[1-9][0-9]*\.[a-f0-9]{16}\.tmp$/.test(name)) fail(); + const target = path.join(agentRoot, name); + const info = fs.lstatSync(target); + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() || info.nlink !== 1 + || info.dev !== expectedDevice || (info.mode & 0o7777) !== PRIVATE_FILE_MODE + || fs.realpathSync(target) !== target) fail(); + fs.unlinkSync(target); + changed = true; + } + if (changed) syncDirectory(agentRoot); + return changed; +} + +function createRuntimeAgentCredentialManager({ installationsRoot } = {}) { + if (typeof installationsRoot !== 'string' || !path.isAbsolute(installationsRoot) + || path.resolve(installationsRoot) !== installationsRoot || contains(PROJECT_ROOT, installationsRoot) + || contains(installationsRoot, PROJECT_ROOT)) fail(); + const rootIdentity = directoryIdentity(installationsRoot); + + function paths(runtimeKey) { + if (typeof runtimeKey !== 'string' || !INSTALLATION_IDENTIFIER_RE.test(runtimeKey) || runtimeKey === 'local') fail(); + const installationRoot = path.join(installationsRoot, runtimeKey); + if (path.dirname(installationRoot) !== installationsRoot) fail(); + const secretsRoot = path.join(installationRoot, 'secrets'); + const agentRoot = path.join(secretsRoot, 'runtime-agent'); + return Object.freeze({ + installationRoot, + secretsRoot, + agentRoot, + tokenFile: path.join(agentRoot, 'registration-token'), + }); + } + + function assertRoot() { + const current = directoryIdentity(installationsRoot); + if (current.dev !== rootIdentity.dev || current.ino !== rootIdentity.ino) fail(); + } + + function issue(runtimeKey, { rotate = false } = {}) { + if (typeof rotate !== 'boolean') fail(); + assertRoot(); + const selected = paths(runtimeKey); + let changed = ensureDirectory(selected.installationRoot, installationsRoot, rootIdentity.dev); + const installationIdentity = directoryIdentity(selected.installationRoot, rootIdentity.dev); + changed = ensureDirectory(selected.secretsRoot, selected.installationRoot, installationIdentity.dev) || changed; + changed = ensureDirectory(selected.agentRoot, selected.secretsRoot, installationIdentity.dev) || changed; + const agentIdentity = directoryIdentity(selected.agentRoot, installationIdentity.dev); + changed = removeOrphanTokens(selected.agentRoot, agentIdentity.dev) || changed; + let token; + let tokenChanged = false; + if (!rotate && fs.existsSync(selected.tokenFile)) token = readPrivateRegistrationToken(selected.tokenFile); + else { + token = writeToken(selected.tokenFile, crypto.randomBytes(32).toString('base64url'), agentIdentity.dev); + changed = true; + tokenChanged = true; + } + assertRoot(); + return Object.freeze({ + runtimeKey, + tokenHash: crypto.createHash('sha256').update(token).digest('hex'), + changed, + tokenChanged, + }); + } + + function revoke(runtimeKey, expectedTokenHash = null) { + if (expectedTokenHash !== null && !/^[a-f0-9]{64}$/.test(expectedTokenHash)) fail(); + assertRoot(); + const selected = paths(runtimeKey); + if (!fs.existsSync(selected.agentRoot)) return false; + const installationIdentity = directoryIdentity(selected.installationRoot, rootIdentity.dev); + directoryIdentity(selected.secretsRoot, installationIdentity.dev); + const agentIdentity = directoryIdentity(selected.agentRoot, installationIdentity.dev); + const changed = removeOrphanTokens(selected.agentRoot, agentIdentity.dev); + if (!fs.existsSync(selected.tokenFile)) return changed; + const token = readPrivateRegistrationToken(selected.tokenFile); + const tokenHash = crypto.createHash('sha256').update(token).digest('hex'); + if (expectedTokenHash !== null && tokenHash !== expectedTokenHash) return changed; + fs.unlinkSync(selected.tokenFile); + syncDirectory(selected.agentRoot); + return true; + } + + return Object.freeze({ issue, revoke, paths }); +} + +module.exports = { createRuntimeAgentCredentialManager }; diff --git a/core/core/installations/src/services.js b/core/core/installations/src/services.js new file mode 100644 index 0000000..b748a58 --- /dev/null +++ b/core/core/installations/src/services.js @@ -0,0 +1,1084 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); +const { + PROJECT_ROOT, + resolveManagedInstallationRuntimePaths, + managedInstallationRuntimeEnvironment, +} = require('../../../shared/paths/runtime-paths'); +const { serverInstallationManifest } = require('../../../shared/contracts/src'); +const { trustedCommandPath, resolveRootExecutable } = require('../../../shared/trusted-command-path'); + +const INSTALLATION_BASE_SERVICE_PLAN_VERSION = 2; +const INSTALLATION_SERVICE_PLAN_VERSION = 3; +const INSTALLATION_SERVICE_COUNT = 3; +const INSTALLATION_AGENT_SERVICE_COUNT = 4; +const INSTALLATION_SERVICE_COUNTS = Object.freeze([INSTALLATION_SERVICE_COUNT, INSTALLATION_AGENT_SERVICE_COUNT]); + +function validServicePlanShape(version, count) { + return version === INSTALLATION_BASE_SERVICE_PLAN_VERSION && count === INSTALLATION_SERVICE_COUNT + || version === INSTALLATION_SERVICE_PLAN_VERSION && count === INSTALLATION_AGENT_SERVICE_COUNT; +} +const PRIVATE_DIRECTORY_MODE = 0o700; +const PRIVATE_FILE_MODE = 0o600; +const MAX_UNIT_BYTES = 32 * 1024; +const MAX_JOURNAL_BYTES = 96 * 1024; +const MAX_SOURCE_EXECUTABLE_BYTES = 4 * 1024 * 1024; +const MAX_UNIX_SOCKET_PATH_BYTES = 107; +const RESERVED_SYSTEMD_VALUE = /[\0\r\n%"'\\]/; +const ENVIRONMENT_NAME_RE = /^[A-Z][A-Z0-9_]{1,63}$/; +const ISSUED_SERVICE_PLANS = new WeakSet(); + +function fail(code = 'service_installation_failed') { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, allowed, required, code = 'runtime_boundary_violation') { + if (!plain(value)) fail(code); + const keys = Object.keys(value); + if (keys.some(key => !allowed.includes(key)) || required.some(key => !Object.hasOwn(value, key))) { + fail(code); + } +} + +function absolute(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || RESERVED_SYSTEMD_VALUE.test(value)) fail('runtime_boundary_violation'); + return value; +} + +function contains(root, candidate) { + const relative = path.relative(root, candidate); + return relative === '' || (relative !== '..' && !relative.startsWith(`..${path.sep}`) + && !path.isAbsolute(relative)); +} + +function lstatMaybe(target) { + try { return fs.lstatSync(target); } + catch (error) { + if (error?.code === 'ENOENT') return null; + fail(); + } +} + +function directoryIdentity(target, { exactPrivate = false, expectedDevice = null, trustedCode = false } = {}) { + const selected = absolute(target); + const info = lstatMaybe(selected); + if (!info || !info.isDirectory() || info.isSymbolicLink() || (info.uid !== process.geteuid() && !(trustedCode && info.uid === 0)) + || (exactPrivate ? (info.mode & 0o7777) !== PRIVATE_DIRECTORY_MODE : (info.mode & 0o7022) !== 0) + || expectedDevice !== null && info.dev !== expectedDevice) fail('runtime_boundary_violation'); + let canonical; + try { canonical = fs.realpathSync(selected); } catch { fail('runtime_boundary_violation'); } + if (canonical !== selected) fail('runtime_boundary_violation'); + return Object.freeze({ dev: info.dev, ino: info.ino }); +} + +function sameIdentity(left, right) { + return Boolean(left && right && left.dev === right.dev && left.ino === right.ino); +} + +function regularFile(target, expectedDevice, { executable = false } = {}) { + const selected = absolute(target); + const info = lstatMaybe(selected); + if (!info || !info.isFile() || info.isSymbolicLink() || info.nlink !== 1 + || !new Set([0, process.geteuid()]).has(info.uid) || (info.mode & 0o022) !== 0 + || executable && (info.mode & 0o111) === 0 + || expectedDevice !== null && info.dev !== expectedDevice) fail('runtime_boundary_violation'); + let canonical; + try { canonical = fs.realpathSync(selected); } catch { fail('runtime_boundary_violation'); } + if (canonical !== selected) fail('runtime_boundary_violation'); + return info; +} + +function sourceIdentity(target) { + const info = regularFile(target, null, { executable: true }); + if (info.size < 1 || info.size > MAX_SOURCE_EXECUTABLE_BYTES) fail('runtime_boundary_violation'); + let content; + try { content = fs.readFileSync(target); } catch { fail('runtime_boundary_violation'); } + return Object.freeze({ dev: info.dev, ino: info.ino, size: info.size, sha256: digest(content) }); +} + +function assertTrustedProjectPath(projectRoot, target) { + const selected = absolute(target); + if (!contains(projectRoot, selected) || selected === projectRoot) fail('runtime_boundary_violation'); + let current = path.dirname(selected); + for (;;) { + directoryIdentity(current, { trustedCode: true }); + if (current === projectRoot) break; + const parent = path.dirname(current); + if (parent === current || !contains(projectRoot, parent)) fail('runtime_boundary_violation'); + current = parent; + } +} + +function privateFile(target, expectedDevice, maximumBytes) { + const info = regularFile(target, expectedDevice); + if ((info.mode & 0o7777) !== PRIVATE_FILE_MODE || info.size < 1 || info.size > maximumBytes) fail(); + return info; +} + +function candidateFile(target, expectedDevice) { + return privateFile(target, expectedDevice, MAX_UNIT_BYTES); +} + +function canonicalProjectRoot(value) { + const selected = absolute(value); + const identity = directoryIdentity(selected, { trustedCode: true }); + if (identity.dev === undefined) fail('runtime_boundary_violation'); + return selected; +} + +function systemdEscape(value) { + if (typeof value !== 'string' || RESERVED_SYSTEMD_VALUE.test(value)) fail('runtime_boundary_violation'); + let escaped = ''; + for (const character of value) { + if (/^[A-Za-z0-9_./:@+-]$/.test(character)) escaped += character; + else for (const byte of Buffer.from(character)) escaped += `\\x${byte.toString(16).padStart(2, '0')}`; + } + return escaped; +} + +function cleanExecStart(environmentLauncher, environment, executable) { + const assignments = Object.entries(environment) + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, value]) => { + if (!ENVIRONMENT_NAME_RE.test(key)) fail('runtime_boundary_violation'); + return `${key}=${value}`; + }); + return [environmentLauncher, '--ignore-environment', ...assignments, executable] + .map(systemdEscape).join(' '); +} + +function digest(content) { + return crypto.createHash('sha256').update(content, 'utf8').digest('hex'); +} + +function serviceJournalPath(unitRoot, runtimeKey) { + return path.join(unitRoot, `.dispatch-service-${digest(runtimeKey).slice(0, 32)}.json`); +} + +function receipt(selected, status, changed = false) { + return Object.freeze({ + servicePlanVersion: selected.servicePlanVersion, + status, + serviceCount: selected.units.length, + changed, + }); +} + +function directMutation(operation) { + return operation(); +} + +function isInstallationServicePlan(value) { + return plain(value) && ISSUED_SERVICE_PLANS.has(value); +} + +function syncDirectory(target) { + let handle; + try { + handle = fs.openSync(target, fs.constants.O_RDONLY | fs.constants.O_DIRECTORY); + fs.fsyncSync(handle); + } catch { fail(); } + finally { if (handle !== undefined) try { fs.closeSync(handle); } catch {} } +} + +function writePrivateFile(target, content, expectedDevice, mutate, maximumBytes = MAX_UNIT_BYTES) { + if (Buffer.byteLength(content, 'utf8') < 1 || Buffer.byteLength(content, 'utf8') > maximumBytes) fail(); + const existing = lstatMaybe(target); + if (existing) { + privateFile(target, expectedDevice, maximumBytes); + let current; + try { current = fs.readFileSync(target, 'utf8'); } catch { fail(); } + if (current === content) return false; + } + const temporary = path.join(path.dirname(target), `.${path.basename(target)}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`); + try { + return mutate(() => { + try { + fs.writeFileSync(temporary, content, { mode: PRIVATE_FILE_MODE, flag: 'wx' }); + const handle = fs.openSync(temporary, 'r'); + try { fs.fsyncSync(handle); } finally { fs.closeSync(handle); } + fs.renameSync(temporary, target); + fs.chmodSync(target, PRIVATE_FILE_MODE); + syncDirectory(path.dirname(target)); + privateFile(target, expectedDevice, maximumBytes); + return true; + } catch (error) { + if (error?.code === 'installation_operation_in_progress') throw error; + fail(); + } + }) === true; + } finally { + try { fs.rmSync(temporary, { force: true }); } catch {} + } +} + +function authUnit({ runtimeKey, projectRoot, commandPath, environment, environmentLauncher }) { + const workingDirectory = path.join(projectRoot, 'runtime', 'auth-broker'); + const executable = path.join(workingDirectory, 'bin', 'dispatch-auth-broker'); + const healthCommand = path.join(workingDirectory, 'bin', 'dispatch-auth-brokerctl'); + const name = `dispatch-runtime-${runtimeKey}-auth-broker.service`; + const serviceEnvironment = Object.freeze({ + ...environment, + PATH: commandPath, + LANG: 'C.UTF-8', + LC_ALL: 'C.UTF-8', + TZ: 'UTC', + NODE_NO_WARNINGS: '1', + }); + const lines = [ + '[Unit]', + `Description=Dispatch managed Auth Broker (${runtimeKey})`, + 'After=network-online.target', + 'Wants=network-online.target', + 'StartLimitIntervalSec=60', + 'StartLimitBurst=5', + '', + '[Service]', + 'Type=simple', + 'UnsetEnvironment=DISPATCH_LOCAL_ROOT DISPATCH_ACCESS_CONTROL_DATABASE_ROOT NODE_OPTIONS LD_PRELOAD LD_LIBRARY_PATH', + `WorkingDirectory=${systemdEscape(workingDirectory)}`, + `ExecStart=${cleanExecStart(environmentLauncher, serviceEnvironment, executable)}`, + 'Restart=always', + 'RestartSec=2', + 'KillMode=control-group', + 'TimeoutStartSec=30', + 'TimeoutStopSec=30', + 'UMask=0077', + 'NoNewPrivileges=true', + 'RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6', + 'RestrictSUIDSGID=true', + 'LockPersonality=true', + 'SystemCallArchitectures=native', + '', + '[Install]', + 'WantedBy=default.target', + '', + ]; + return { + id: 'auth_broker', + name, + environmentLauncher, + executable, + expectedProcessArgument: executable, + socketPath: serviceEnvironment.DISPATCH_AUTH_SOCKET, + healthCommand, + healthArguments: Object.freeze(['health']), + workingDirectory, + environment: serviceEnvironment, + content: `${lines.join('\n')}\n`, + }; +} + +function collectionUnit({ runtimeKey, projectRoot, commandPath, environment, authName, environmentLauncher }) { + const workingDirectory = path.join(projectRoot, 'runtime', 'collection-manager'); + const executable = path.join(workingDirectory, 'bin', 'dispatch-collection-manager'); + const expectedProcessArgument = executable; + const healthCommand = path.join(workingDirectory, 'bin', 'dispatch-collectionctl'); + const name = `dispatch-runtime-${runtimeKey}-collection-manager.service`; + const serviceEnvironment = Object.freeze({ + ...environment, + DISPATCH_MANAGED_RUNTIME: '1', + PATH: commandPath, + LANG: 'C.UTF-8', + LC_ALL: 'C.UTF-8', + TZ: 'UTC', + NODE_NO_WARNINGS: '1', + }); + const lines = [ + '[Unit]', + `Description=Dispatch managed Collection Manager (${runtimeKey})`, + `After=local-fs.target ${authName}`, + `Wants=${authName}`, + 'StartLimitIntervalSec=60', + 'StartLimitBurst=5', + '', + '[Service]', + 'Type=simple', + 'UnsetEnvironment=DISPATCH_LOCAL_ROOT DISPATCH_ACCESS_CONTROL_DATABASE_ROOT NODE_OPTIONS LD_PRELOAD LD_LIBRARY_PATH', + `WorkingDirectory=${systemdEscape(workingDirectory)}`, + `ExecStart=${cleanExecStart(environmentLauncher, serviceEnvironment, executable)}`, + 'Restart=always', + 'RestartSec=3', + 'KillMode=control-group', + 'TimeoutStartSec=30', + 'TimeoutStopSec=30', + 'UMask=0077', + 'NoNewPrivileges=true', + 'RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6', + 'RestrictSUIDSGID=true', + 'LockPersonality=true', + 'RestrictNamespaces=true', + 'SystemCallArchitectures=native', + '', + '[Install]', + 'WantedBy=default.target', + '', + ]; + return { + id: 'collection_manager', + name, + environmentLauncher, + executable, + expectedProcessArgument, + socketPath: null, + healthCommand, + healthArguments: Object.freeze(['status']), + workingDirectory, + environment: serviceEnvironment, + content: `${lines.join('\n')}\n`, + }; +} + +function gatewayUnit({ + runtimeKey, + projectRoot, + commandPath, + environment, + authName, + collectionName, + environmentLauncher, +}) { + const workingDirectory = path.join(projectRoot, 'runtime', 'gateway'); + const executable = path.join(workingDirectory, 'bin', 'dispatch-runtime-gateway'); + const healthCommand = path.join(workingDirectory, 'bin', 'dispatch-runtime-gatewayctl'); + const name = `dispatch-runtime-${runtimeKey}-gateway.service`; + const serviceEnvironment = Object.freeze({ + ...environment, + PATH: commandPath, + LANG: 'C.UTF-8', + LC_ALL: 'C.UTF-8', + TZ: 'UTC', + NODE_NO_WARNINGS: '1', + }); + const lines = [ + '[Unit]', + `Description=Dispatch managed Runtime Gateway (${runtimeKey})`, + `After=local-fs.target ${authName} ${collectionName}`, + `Wants=${authName} ${collectionName}`, + 'StartLimitIntervalSec=60', + 'StartLimitBurst=5', + '', + '[Service]', + 'Type=simple', + 'UnsetEnvironment=DISPATCH_LOCAL_ROOT DISPATCH_ACCESS_CONTROL_DATABASE_ROOT NODE_OPTIONS LD_PRELOAD LD_LIBRARY_PATH', + `WorkingDirectory=${systemdEscape(workingDirectory)}`, + `ExecStart=${cleanExecStart(environmentLauncher, serviceEnvironment, executable)}`, + 'Restart=always', + 'RestartSec=2', + 'KillMode=control-group', + 'TimeoutStartSec=30', + 'TimeoutStopSec=30', + 'UMask=0077', + 'NoNewPrivileges=true', + 'RestrictAddressFamilies=AF_UNIX', + 'RestrictSUIDSGID=true', + 'LockPersonality=true', + 'RestrictNamespaces=true', + 'SystemCallArchitectures=native', + '', + '[Install]', + 'WantedBy=default.target', + '', + ]; + return { + id: 'runtime_gateway', + name, + environmentLauncher, + executable, + expectedProcessArgument: executable, + socketPath: serviceEnvironment.DISPATCH_RUNTIME_GATEWAY_SOCKET, + healthCommand, + healthArguments: Object.freeze(['health']), + workingDirectory, + environment: serviceEnvironment, + content: `${lines.join('\n')}\n`, + }; +} + +function runtimeAgentUnit({ + runtimeKey, + projectRoot, + commandPath, + environment, + gatewayName, + gatewaySocket, + hubSocket, + tokenFile, + statusSocket, + environmentLauncher, +}) { + const workingDirectory = path.join(projectRoot, 'runtime', 'agent'); + const executable = path.join(workingDirectory, 'bin', 'dispatch-runtime-agent'); + const healthCommand = path.join(workingDirectory, 'bin', 'dispatch-runtime-agentctl'); + const name = `dispatch-runtime-${runtimeKey}-agent.service`; + const serviceEnvironment = Object.freeze({ + ...environment, + DISPATCH_RUNTIME_KEY: runtimeKey, + DISPATCH_RUNTIME_GATEWAY_SOCKET: gatewaySocket, + DISPATCH_RUNTIME_AGENT_HUB_SOCKET: hubSocket, + DISPATCH_RUNTIME_AGENT_TOKEN_FILE: tokenFile, + DISPATCH_RUNTIME_AGENT_STATUS_SOCKET: statusSocket, + PATH: commandPath, + LANG: 'C.UTF-8', + LC_ALL: 'C.UTF-8', + TZ: 'UTC', + NODE_NO_WARNINGS: '1', + }); + const lines = [ + '[Unit]', + `Description=Dispatch managed Runtime Agent (${runtimeKey})`, + `After=local-fs.target ${gatewayName}`, + `Wants=${gatewayName}`, + 'StartLimitIntervalSec=60', + 'StartLimitBurst=5', + '', + '[Service]', + 'Type=simple', + 'UnsetEnvironment=DISPATCH_LOCAL_ROOT DISPATCH_ACCESS_CONTROL_DATABASE_ROOT NODE_OPTIONS LD_PRELOAD LD_LIBRARY_PATH', + `WorkingDirectory=${systemdEscape(workingDirectory)}`, + `ExecStart=${cleanExecStart(environmentLauncher, serviceEnvironment, executable)}`, + 'Restart=always', + 'RestartSec=2', + 'KillMode=control-group', + 'TimeoutStartSec=30', + 'TimeoutStopSec=30', + 'UMask=0077', + 'NoNewPrivileges=true', + 'RestrictAddressFamilies=AF_UNIX', + 'RestrictSUIDSGID=true', + 'LockPersonality=true', + 'RestrictNamespaces=true', + 'SystemCallArchitectures=native', + '', + '[Install]', + 'WantedBy=default.target', + '', + ]; + return { + id: 'runtime_agent', + name, + environmentLauncher, + executable, + expectedProcessArgument: executable, + socketPath: statusSocket, + healthCommand, + healthArguments: Object.freeze(['health']), + workingDirectory, + environment: serviceEnvironment, + content: `${lines.join('\n')}\n`, + }; +} + +function createInstallationServiceManager(options) { + exact(options, ['unitRoot', 'projectRoot', 'commandPath', 'systemdAnalyze', 'runtimeAgentHubSocket'], ['unitRoot']); + const unitRoot = absolute(options.unitRoot); + const projectRoot = canonicalProjectRoot(options.projectRoot === undefined ? PROJECT_ROOT : options.projectRoot); + if (contains(projectRoot, unitRoot) || contains(unitRoot, projectRoot)) fail('runtime_boundary_violation'); + const unitRootIdentity = directoryIdentity(unitRoot, { exactPrivate: true }); + let commandPath; + try { commandPath = trustedCommandPath(options.commandPath === undefined ? process.env.PATH : options.commandPath); } + catch { fail('runtime_boundary_violation'); } + if (RESERVED_SYSTEMD_VALUE.test(commandPath)) fail('runtime_boundary_violation'); + const systemdAnalyze = resolveRootExecutable(options.systemdAnalyze, ['systemd-analyze']); + if (!systemdAnalyze) fail('runtime_boundary_violation'); + const environmentLauncher = resolveRootExecutable(undefined, ['env']); + if (!environmentLauncher) fail('runtime_boundary_violation'); + const runtimeAgentHubSocket = options.runtimeAgentHubSocket === undefined + ? null : absolute(options.runtimeAgentHubSocket); + if (runtimeAgentHubSocket !== null + && path.basename(runtimeAgentHubSocket) !== 'runtime-agent-hub.sock') fail('runtime_boundary_violation'); + const issuedPlans = new WeakSet(); + + function assertUnitRoot() { + const current = directoryIdentity(unitRoot, { exactPrivate: true }); + if (!sameIdentity(unitRootIdentity, current)) fail('runtime_boundary_violation'); + } + + function plan(manifestValue, authorityValue, layout) { + assertUnitRoot(); + const manifest = serverInstallationManifest(manifestValue, authorityValue); + if (!plain(layout) || layout.runtimeKey !== manifest.runtime.key || layout.projectRoot !== projectRoot) { + fail('runtime_identity_mismatch'); + } + const managed = resolveManagedInstallationRuntimePaths(layout); + if (contains(unitRoot, managed.installationRoot) || contains(managed.installationRoot, unitRoot)) { + fail('runtime_boundary_violation'); + } + const gatewaySocket = path.join(managed.runtimeRoot, 'runtime-gateway.sock'); + if ([managed.auth.socket, gatewaySocket] + .some(socket => Buffer.byteLength(socket, 'utf8') > MAX_UNIX_SOCKET_PATH_BYTES)) { + fail('runtime_boundary_violation'); + } + const fullEnvironment = managedInstallationRuntimeEnvironment(layout); + const authEnvironment = Object.freeze({ + DISPATCH_PROJECT_ROOT: fullEnvironment.DISPATCH_PROJECT_ROOT, + DISPATCH_RUNTIME_ROOT: fullEnvironment.DISPATCH_RUNTIME_ROOT, + DISPATCH_AUTH_DATABASE_ROOT: fullEnvironment.DISPATCH_AUTH_DATABASE_ROOT, + DISPATCH_AUTH_SECRET_ROOT: fullEnvironment.DISPATCH_AUTH_SECRET_ROOT, + DISPATCH_AUTH_STATE_ROOT: fullEnvironment.DISPATCH_AUTH_STATE_ROOT, + DISPATCH_AUTH_SOCKET: fullEnvironment.DISPATCH_AUTH_SOCKET, + }); + const candidateRoot = path.join(managed.configRoot, 'systemd'); + absolute(candidateRoot); + const auth = authUnit({ + runtimeKey: manifest.runtime.key, + projectRoot, + commandPath, + environment: authEnvironment, + environmentLauncher, + }); + const collection = collectionUnit({ + runtimeKey: manifest.runtime.key, + projectRoot, + commandPath, + environment: fullEnvironment, + authName: auth.name, + environmentLauncher, + }); + const gateway = gatewayUnit({ + runtimeKey: manifest.runtime.key, + projectRoot, + commandPath, + environment: Object.freeze({ + ...fullEnvironment, + DISPATCH_RUNTIME_KEY: manifest.runtime.key, + DISPATCH_RUNTIME_GATEWAY_SOCKET: gatewaySocket, + }), + authName: auth.name, + collectionName: collection.name, + environmentLauncher, + }); + const unitDefinitions = [auth, collection, gateway]; + if (runtimeAgentHubSocket !== null) { + if (contains(projectRoot, runtimeAgentHubSocket) + || contains(managed.installationRoot, runtimeAgentHubSocket) + || contains(path.dirname(runtimeAgentHubSocket), managed.installationRoot) + || [runtimeAgentHubSocket, managed.runtimeAgent.statusSocket] + .some(socket => Buffer.byteLength(socket, 'utf8') > MAX_UNIX_SOCKET_PATH_BYTES)) { + fail('runtime_boundary_violation'); + } + unitDefinitions.push(runtimeAgentUnit({ + runtimeKey: manifest.runtime.key, + projectRoot, + commandPath, + environment: fullEnvironment, + gatewayName: gateway.name, + gatewaySocket, + hubSocket: runtimeAgentHubSocket, + tokenFile: managed.runtimeAgent.registrationToken, + statusSocket: managed.runtimeAgent.statusSocket, + environmentLauncher, + })); + } + const units = unitDefinitions.map(unit => { + if (!/^[A-Za-z0-9_.@-]{1,220}\.service$/.test(unit.name)) fail('runtime_boundary_violation'); + assertTrustedProjectPath(projectRoot, unit.executable); + assertTrustedProjectPath(projectRoot, unit.healthCommand); + assertTrustedProjectPath(projectRoot, unit.workingDirectory); + const executableIdentity = sourceIdentity(unit.executable); + const healthCommandIdentity = sourceIdentity(unit.healthCommand); + const workingDirectoryIdentity = directoryIdentity(unit.workingDirectory); + return Object.freeze({ + ...unit, + executableIdentity, + healthCommandIdentity, + workingDirectoryIdentity, + candidate: path.join(candidateRoot, unit.name), + installed: path.join(unitRoot, unit.name), + sha256: digest(unit.content), + }); + }); + const selectedPlan = Object.freeze({ + servicePlanVersion: runtimeAgentHubSocket === null + ? INSTALLATION_BASE_SERVICE_PLAN_VERSION : INSTALLATION_SERVICE_PLAN_VERSION, + runtimeKey: manifest.runtime.key, + candidateRoot, + journal: serviceJournalPath(unitRoot, manifest.runtime.key), + unitRoot, + units: Object.freeze(units), + }); + issuedPlans.add(selectedPlan); + ISSUED_SERVICE_PLANS.add(selectedPlan); + return selectedPlan; + } + + function ensureCandidateRoot(selected, mutate) { + const configRoot = path.dirname(selected.candidateRoot); + const configIdentity = directoryIdentity(configRoot, { exactPrivate: true }); + if (!lstatMaybe(selected.candidateRoot)) { + mutate(() => { + if (!lstatMaybe(selected.candidateRoot)) { + fs.mkdirSync(selected.candidateRoot, { mode: PRIVATE_DIRECTORY_MODE }); + syncDirectory(configRoot); + } + }); + } + directoryIdentity(selected.candidateRoot, { exactPrivate: true, expectedDevice: configIdentity.dev }); + const after = directoryIdentity(configRoot, { exactPrivate: true }); + if (!sameIdentity(configIdentity, after)) fail(); + } + + function validatePlan(selected) { + if (!plain(selected) || !issuedPlans.has(selected) + || !validServicePlanShape(selected.servicePlanVersion, selected.units?.length) + || selected.unitRoot !== unitRoot || !Array.isArray(selected.units) + || !INSTALLATION_SERVICE_COUNTS.includes(selected.units.length) + || selected.units.length !== (runtimeAgentHubSocket === null + ? INSTALLATION_SERVICE_COUNT : INSTALLATION_AGENT_SERVICE_COUNT) + || selected.journal !== serviceJournalPath(unitRoot, selected.runtimeKey)) { + fail('runtime_boundary_violation'); + } + assertUnitRoot(); + for (const unit of selected.units) { + const currentLauncher = resolveRootExecutable(unit.environmentLauncher, []); + if (currentLauncher !== environmentLauncher) fail('runtime_boundary_violation'); + assertTrustedProjectPath(projectRoot, unit.executable); + assertTrustedProjectPath(projectRoot, unit.healthCommand); + assertTrustedProjectPath(projectRoot, unit.workingDirectory); + const executable = sourceIdentity(unit.executable); + const health = sourceIdentity(unit.healthCommand); + const working = directoryIdentity(unit.workingDirectory); + if (!sameIdentity(executable, unit.executableIdentity) + || executable.size !== unit.executableIdentity.size + || executable.sha256 !== unit.executableIdentity.sha256 + || !sameIdentity(health, unit.healthCommandIdentity) + || health.size !== unit.healthCommandIdentity.size + || health.sha256 !== unit.healthCommandIdentity.sha256 + || !sameIdentity(working, unit.workingDirectoryIdentity)) { + fail('runtime_boundary_violation'); + } + } + return selected; + } + + function render(selectedValue, mutationCapability = directMutation) { + const selected = validatePlan(selectedValue); + if (typeof mutationCapability !== 'function') fail('runtime_boundary_violation'); + ensureCandidateRoot(selected, mutationCapability); + let entries; + try { entries = fs.readdirSync(selected.candidateRoot); } catch { fail(); } + const expected = new Set(selected.units.map(unit => unit.name)); + if (entries.some(entry => !expected.has(entry))) fail(); + const identity = directoryIdentity(selected.candidateRoot, { exactPrivate: true }); + let changed = false; + for (const unit of selected.units) { + assertUnitRoot(); + changed = writePrivateFile(unit.candidate, unit.content, identity.dev, mutationCapability) || changed; + } + inspect(selected); + return receipt(selected, 'rendered', changed); + } + + function inspect(selectedValue) { + const selected = validatePlan(selectedValue); + const identity = directoryIdentity(selected.candidateRoot, { exactPrivate: true }); + const expected = new Set(selected.units.map(unit => unit.name)); + let entries; + try { entries = fs.readdirSync(selected.candidateRoot); } catch { fail(); } + if (entries.length !== expected.size || entries.some(entry => !expected.has(entry))) fail(); + for (const unit of selected.units) { + candidateFile(unit.candidate, identity.dev); + let content; + try { content = fs.readFileSync(unit.candidate, 'utf8'); } catch { fail(); } + if (content !== unit.content || digest(content) !== unit.sha256) fail(); + } + assertUnitRoot(); + return receipt(selected, 'verified'); + } + + function validate(selectedValue) { + const selected = validatePlan(selectedValue); + inspect(selected); + const result = spawnSync(systemdAnalyze, ['verify', ...selected.units.map(unit => unit.candidate)], { + cwd: projectRoot, + env: { PATH: commandPath, LANG: 'C.UTF-8', LC_ALL: 'C.UTF-8' }, + encoding: 'utf8', + timeout: 30_000, + maxBuffer: 64 * 1024, + }); + if (result.error || result.status !== 0) fail(); + inspect(selected); + return receipt(selected, 'validated'); + } + + function readPrivateContent(target, expectedDevice, maximumBytes) { + privateFile(target, expectedDevice, maximumBytes); + let raw; + try { raw = fs.readFileSync(target); } catch { fail(); } + const content = raw.toString('utf8'); + if (!raw.equals(Buffer.from(content, 'utf8')) || content.includes('\0')) fail(); + return content; + } + + function checkedSupervisorState(selected, value) { + if (!Array.isArray(value) || value.length !== selected.units.length) fail('runtime_boundary_violation'); + return Object.freeze(selected.units.map((unit, index) => { + const state = value[index]; + exact( + state, + ['id', 'name', 'enabled', 'active', 'enableMode'], + ['id', 'name', 'enabled', 'active', 'enableMode'], + ); + if (state.id !== unit.id || state.name !== unit.name + || typeof state.enabled !== 'boolean' || typeof state.active !== 'boolean' + || !['none', 'runtime', 'persistent'].includes(state.enableMode) + || state.enabled !== (state.enableMode !== 'none')) { + fail('runtime_boundary_violation'); + } + return Object.freeze({ ...state }); + })); + } + + function checkedJournal(selected, value) { + exact( + value, + ['version', 'servicePlanVersion', 'runtimeKey', 'phase', 'units'], + ['version', 'servicePlanVersion', 'runtimeKey', 'phase', 'units'], + ); + if (value.version !== 1 || value.servicePlanVersion !== selected.servicePlanVersion + || value.runtimeKey !== selected.runtimeKey + || !['installing', 'installed', 'verified', 'restored'].includes(value.phase) + || !Array.isArray(value.units) || value.units.length !== selected.units.length) { + fail('runtime_boundary_violation'); + } + const units = selected.units.map((unit, index) => { + const entry = value.units[index]; + exact( + entry, + ['id', 'name', 'candidateSha256', 'previous'], + ['id', 'name', 'candidateSha256', 'previous'], + ); + exact( + entry.previous, + ['present', 'content', 'sha256', 'enabled', 'active', 'enableMode'], + ['present', 'content', 'sha256', 'enabled', 'active', 'enableMode'], + ); + if (entry.id !== unit.id || entry.name !== unit.name || entry.candidateSha256 !== unit.sha256 + || typeof entry.previous.present !== 'boolean' || typeof entry.previous.enabled !== 'boolean' + || typeof entry.previous.active !== 'boolean' + || !['none', 'runtime', 'persistent'].includes(entry.previous.enableMode) + || entry.previous.enabled !== (entry.previous.enableMode !== 'none')) { + fail('runtime_boundary_violation'); + } + if (entry.previous.present) { + if (typeof entry.previous.content !== 'string' || typeof entry.previous.sha256 !== 'string' + || digest(entry.previous.content) !== entry.previous.sha256 + || Buffer.byteLength(entry.previous.content, 'utf8') < 1 + || Buffer.byteLength(entry.previous.content, 'utf8') > MAX_UNIT_BYTES) fail('runtime_boundary_violation'); + } else if (entry.previous.content !== null || entry.previous.sha256 !== null + || entry.previous.enabled || entry.previous.active || entry.previous.enableMode !== 'none') { + fail('runtime_boundary_violation'); + } + return Object.freeze({ + id: entry.id, + name: entry.name, + candidateSha256: entry.candidateSha256, + previous: Object.freeze({ ...entry.previous }), + }); + }); + return Object.freeze({ ...value, units: Object.freeze(units) }); + } + + function loadJournal(selectedValue, { required = false } = {}) { + const selected = validatePlan(selectedValue); + const identity = directoryIdentity(unitRoot, { exactPrivate: true }); + if (!lstatMaybe(selected.journal)) { + if (required) fail(); + return null; + } + const raw = readPrivateContent(selected.journal, identity.dev, MAX_JOURNAL_BYTES); + if (!raw.endsWith('\n') || raw.includes('\r')) fail(); + let parsed; + try { parsed = JSON.parse(raw.slice(0, -1)); } catch { fail(); } + return checkedJournal(selected, parsed); + } + + function writeJournal(selected, journal, mutationCapability) { + const checked = checkedJournal(selected, journal); + const identity = directoryIdentity(unitRoot, { exactPrivate: true }); + writePrivateFile( + selected.journal, + `${JSON.stringify(checked)}\n`, + identity.dev, + mutationCapability, + MAX_JOURNAL_BYTES, + ); + return loadJournal(selected, { required: true }); + } + + function captureJournal(selected, supervisorState, mutationCapability) { + const existing = loadJournal(selected); + if (existing) return existing; + const states = checkedSupervisorState(selected, supervisorState); + const identity = directoryIdentity(unitRoot, { exactPrivate: true }); + const units = selected.units.map((unit, index) => { + const present = Boolean(lstatMaybe(unit.installed)); + let content = null; + let sha256 = null; + if (present) { + content = readPrivateContent(unit.installed, identity.dev, MAX_UNIT_BYTES); + sha256 = digest(content); + } else if (states[index].enabled || states[index].active) { + fail('runtime_boundary_violation'); + } + return { + id: unit.id, + name: unit.name, + candidateSha256: unit.sha256, + previous: { + present, + content, + sha256, + enabled: states[index].enabled, + active: states[index].active, + enableMode: states[index].enableMode, + }, + }; + }); + return writeJournal(selected, { + version: 1, + servicePlanVersion: selected.servicePlanVersion, + runtimeKey: selected.runtimeKey, + phase: 'installing', + units, + }, mutationCapability); + } + + function inspectInstalled(selectedValue) { + const selected = validatePlan(selectedValue); + const identity = directoryIdentity(unitRoot, { exactPrivate: true }); + for (const unit of selected.units) { + const content = readPrivateContent(unit.installed, identity.dev, MAX_UNIT_BYTES); + if (content !== unit.content || digest(content) !== unit.sha256) fail(); + } + return receipt(selected, 'installed'); + } + + function install(selectedValue, supervisorState, mutationCapability = directMutation) { + const selected = validatePlan(selectedValue); + if (typeof mutationCapability !== 'function') fail('runtime_boundary_violation'); + inspect(selected); + const journal = captureJournal(selected, supervisorState, mutationCapability); + if (['verified', 'restored'].includes(journal.phase)) fail(); + const identity = directoryIdentity(unitRoot, { exactPrivate: true }); + let changed = false; + for (const unit of selected.units) { + changed = writePrivateFile(unit.installed, unit.content, identity.dev, mutationCapability) || changed; + } + writeJournal(selected, { ...journal, phase: 'installed' }, mutationCapability); + inspectInstalled(selected); + return receipt(selected, 'installed', changed); + } + + function rollbackState(selectedValue) { + const selected = validatePlan(selectedValue); + const journal = loadJournal(selected); + if (!journal) return null; + return Object.freeze(journal.units.map(entry => Object.freeze({ + id: entry.id, + name: entry.name, + enabled: entry.previous.enabled, + active: entry.previous.active, + enableMode: entry.previous.enableMode, + }))); + } + + function restoreFiles(selectedValue, mutationCapability = directMutation) { + const selected = validatePlan(selectedValue); + if (typeof mutationCapability !== 'function') fail('runtime_boundary_violation'); + const journal = loadJournal(selected, { required: true }); + const identity = directoryIdentity(unitRoot, { exactPrivate: true }); + let changed = false; + for (let index = selected.units.length - 1; index >= 0; index -= 1) { + const unit = selected.units[index]; + const previous = journal.units[index].previous; + const exists = Boolean(lstatMaybe(unit.installed)); + let current = null; + if (exists) current = readPrivateContent(unit.installed, identity.dev, MAX_UNIT_BYTES); + if (previous.present) { + if (current === previous.content) continue; + if (current !== unit.content) fail(); + changed = writePrivateFile( + unit.installed, + previous.content, + identity.dev, + mutationCapability, + ) || changed; + } else { + if (!exists) continue; + if (current !== unit.content) fail(); + mutationCapability(() => { + const verified = readPrivateContent(unit.installed, identity.dev, MAX_UNIT_BYTES); + if (verified !== unit.content) fail(); + fs.unlinkSync(unit.installed); + syncDirectory(selected.unitRoot); + }); + changed = true; + } + } + writeJournal(selected, { ...journal, phase: 'restored' }, mutationCapability); + return receipt(selected, 'restored', changed); + } + + function verifyRestored(selected, journal) { + const identity = directoryIdentity(unitRoot, { exactPrivate: true }); + for (let index = 0; index < selected.units.length; index += 1) { + const unit = selected.units[index]; + const previous = journal.units[index].previous; + if (!previous.present) { + if (lstatMaybe(unit.installed)) fail(); + continue; + } + const current = readPrivateContent(unit.installed, identity.dev, MAX_UNIT_BYTES); + if (current !== previous.content || digest(current) !== previous.sha256) fail(); + } + } + + function removeJournal(selected, mutationCapability) { + const identity = directoryIdentity(unitRoot, { exactPrivate: true }); + mutationCapability(() => { + privateFile(selected.journal, identity.dev, MAX_JOURNAL_BYTES); + fs.unlinkSync(selected.journal); + syncDirectory(unitRoot); + }); + if (lstatMaybe(selected.journal)) fail(); + } + + function finishRollback(selectedValue, mutationCapability = directMutation) { + const selected = validatePlan(selectedValue); + const journal = loadJournal(selected, { required: true }); + if (journal.phase !== 'restored') fail(); + verifyRestored(selected, journal); + removeJournal(selected, mutationCapability); + return receipt(selected, 'rolled_back', true); + } + + function markVerified(selectedValue, mutationCapability = directMutation) { + const selected = validatePlan(selectedValue); + if (typeof mutationCapability !== 'function') fail('runtime_boundary_violation'); + inspectInstalled(selected); + const journal = loadJournal(selected, { required: true }); + if (journal.phase === 'verified') return receipt(selected, 'verified'); + if (journal.phase !== 'installed') fail(); + writeJournal(selected, { ...journal, phase: 'verified' }, mutationCapability); + return receipt(selected, 'verified', true); + } + + function commit(selectedValue, mutationCapability = directMutation) { + const selected = validatePlan(selectedValue); + const journal = loadJournal(selected, { required: true }); + if (journal.phase !== 'verified') fail(); + inspectInstalled(selected); + removeJournal(selected, mutationCapability); + return receipt(selected, 'committed', true); + } + + function finalizeSettled(selectedValue, mutationCapability = directMutation) { + const selected = validatePlan(selectedValue); + if (typeof mutationCapability !== 'function') fail('runtime_boundary_violation'); + const journal = loadJournal(selected); + if (!journal) return receipt(selected, 'settled'); + if (journal.phase === 'verified') return commit(selected, mutationCapability); + if (journal.phase === 'restored') return finishRollback(selected, mutationCapability); + fail(); + } + + function removeCandidate(selectedValue, mutationCapability = directMutation) { + const selected = validatePlan(selectedValue); + if (typeof mutationCapability !== 'function') fail('runtime_boundary_violation'); + const journal = loadJournal(selected); + if (journal && journal.phase !== 'restored') fail(); + if (!lstatMaybe(selected.candidateRoot)) return receipt(selected, 'candidate_removed'); + const candidateIdentity = directoryIdentity(selected.candidateRoot, { exactPrivate: true }); + const expected = new Set(selected.units.map(unit => unit.name)); + let entries; + try { entries = fs.readdirSync(selected.candidateRoot); } catch { fail(); } + if (entries.some(entry => !expected.has(entry))) fail(); + for (const unit of [...selected.units].reverse()) { + if (!lstatMaybe(unit.candidate)) continue; + mutationCapability(() => { + const content = readPrivateContent(unit.candidate, candidateIdentity.dev, MAX_UNIT_BYTES); + if (content !== unit.content) fail(); + fs.unlinkSync(unit.candidate); + syncDirectory(selected.candidateRoot); + }); + } + const configRoot = path.dirname(selected.candidateRoot); + mutationCapability(() => { + const current = directoryIdentity(selected.candidateRoot, { + exactPrivate: true, + expectedDevice: candidateIdentity.dev, + }); + if (!sameIdentity(candidateIdentity, current) || fs.readdirSync(selected.candidateRoot).length !== 0) fail(); + fs.rmdirSync(selected.candidateRoot); + syncDirectory(configRoot); + }); + if (lstatMaybe(selected.candidateRoot)) fail(); + return receipt(selected, 'candidate_removed', true); + } + + function removeInstalled(selectedValue, mutationCapability = directMutation) { + const selected = validatePlan(selectedValue); + if (typeof mutationCapability !== 'function' || loadJournal(selected)) fail('runtime_boundary_violation'); + const identity = directoryIdentity(unitRoot, { exactPrivate: true }); + let changed = false; + for (const unit of [...selected.units].reverse()) { + if (!lstatMaybe(unit.installed)) continue; + mutationCapability(() => { + const content = readPrivateContent(unit.installed, identity.dev, MAX_UNIT_BYTES); + if (content !== unit.content || digest(content) !== unit.sha256) fail(); + fs.unlinkSync(unit.installed); + syncDirectory(unitRoot); + }); + changed = true; + } + for (const unit of selected.units) if (lstatMaybe(unit.installed)) fail(); + return receipt(selected, 'removed', changed); + } + + function inspectAbsent(selectedValue) { + const selected = validatePlan(selectedValue); + if (loadJournal(selected) || selected.units.some(unit => lstatMaybe(unit.installed))) fail(); + return receipt(selected, 'absent'); + } + + return Object.freeze({ + plan, + render, + inspect, + validate, + install, + inspectInstalled, + rollbackState, + restoreFiles, + finishRollback, + markVerified, + commit, + finalizeSettled, + removeCandidate, + removeInstalled, + inspectAbsent, + }); +} + +module.exports = { + INSTALLATION_BASE_SERVICE_PLAN_VERSION, + INSTALLATION_SERVICE_PLAN_VERSION, + INSTALLATION_SERVICE_COUNT, + INSTALLATION_AGENT_SERVICE_COUNT, + INSTALLATION_SERVICE_COUNTS, + validServicePlanShape, + MAX_UNIT_BYTES, + isInstallationServicePlan, + createInstallationServiceManager, +}; diff --git a/core/core/installations/src/system-backup-manifests.js b/core/core/installations/src/system-backup-manifests.js new file mode 100644 index 0000000..6a60ca6 --- /dev/null +++ b/core/core/installations/src/system-backup-manifests.js @@ -0,0 +1,98 @@ +'use strict'; +const fs = require('node:fs'), + path = require('node:path'); +const { atomic } = require('./release-delivery-files'); +// A full-system recovery point is a small encrypted manifest referencing +// independent repositories. It never embeds a second copy of component data. +async function syncSystemManifests({ + config, + db, + runFactory, + workRoot, + record, + storage, + excludedOrganizations = new Set(), + clock = Date.now, +}) { + const sets = {}; + const digest = (value) => + require('node:crypto').createHash('sha256').update(JSON.stringify(value)).digest('hex'); + for (const set of db.prepare('SELECT * FROM backup_sets').all()) { + if (!/^breq_[a-f0-9]{32}$/.test(set.id)) throw Error('backup_set_invalid'); + const members = JSON.parse(set.members_json); + if (members.some((m) => excludedOrganizations.has(m.organizationId))) continue; + if (!['deleting', 'deleted'].includes(set.status) && (!members.length || members.some((m) => !m.backupId))) continue; + const components = members.filter(m => m.backupId).map((member) => { + const row = db + .prepare('SELECT * FROM platform_backup_records WHERE id=?') + .get(member.backupId); + return { + id: member.backupId, + organizationId: member.organizationId, + kind: member.organizationId ? 'dsp' : 'core', + retentionDays: row?.retention_days ?? null, + deleted: !row || !!row.deleted_at, + }; + }); + if ( + set.status === 'verified' && + components.some((c) => { + const proof = record?.(c.id); + return ( + !proof || + proof.status !== 'verified' || + proof.organizationId !== c.organizationId || + proof.kind !== c.kind + ); + }) + ) + continue; + const manifest = { + schemaVersion: 1, + id: set.id, + kind: 'system', + createdAt: set.created_at, + status: set.status, + systemSchedule: JSON.parse( + db.prepare('SELECT settings_json FROM backup_set_settings WHERE set_id=?').get(set.id) + ?.settings_json || 'null', + ), + components, + }; + const encoded = JSON.stringify(manifest), + file = path.join(workRoot, `set-${set.id}.json`); + if (fs.existsSync(file) && fs.readFileSync(file, 'utf8').trim() === encoded) { + sets[set.id] = { + status: ['deleting', 'deleted'].includes(set.status) ? 'deleted' : 'verified', + setDigest: digest(set), + }; + continue; + } + const run = runFactory({ + ...config.environment, + RESTIC_REPOSITORY: `s3:https://${config.accountId}.r2.cloudflarestorage.com/${config.bucket}/sets/${set.id}`, + }); + if (['deleting', 'deleted'].includes(set.status)) { + await storage.removeSet(set.id); + atomic(file, manifest); + sets[set.id] = { status: 'deleted', setDigest: digest(set) }; + continue; + } + const work = fs.mkdtempSync(path.join(workRoot, 'system-manifest-')); + try { + atomic(path.join(work, 'system.json'), manifest); + try { + run(['cat', 'config']); + } catch { + run(['init', '--repository-version', '2']); + } + run(['backup', '--host', 'dispatch', '--', 'system.json'], work); + atomic(file, manifest); + sets[set.id] = { status: 'verified', setDigest: digest(set), verifiedAt: clock() }; + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } + } + return sets; +} +module.exports = { syncSystemManifests }; diff --git a/core/core/installations/src/systemd-user.js b/core/core/installations/src/systemd-user.js new file mode 100644 index 0000000..077bd3b --- /dev/null +++ b/core/core/installations/src/systemd-user.js @@ -0,0 +1,525 @@ +'use strict'; + +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); +const { resolveRootExecutable, trustedCommandPath } = require('../../../shared/trusted-command-path'); +const { + isInstallationServicePlan, + INSTALLATION_SERVICE_COUNTS, + validServicePlanShape, +} = require('./services'); + +const DEFAULT_WAIT_TIMEOUT_MS = 30_000; +const DEFAULT_POLL_MS = 50; +const MAX_COMMAND_OUTPUT_BYTES = 64 * 1024; + +function fail(code = 'service_installation_failed') { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, allowed, required, code = 'runtime_boundary_violation') { + if (!plain(value)) fail(code); + const keys = Object.keys(value); + if (keys.some(key => !allowed.includes(key)) || required.some(key => !Object.hasOwn(value, key))) { + fail(code); + } +} + +function positiveInteger(value, minimum, maximum) { + if (!Number.isSafeInteger(value) || value < minimum || value > maximum) fail('runtime_boundary_violation'); + return value; +} + +function absoluteExecutable(value, names) { + const resolved = resolveRootExecutable(value, names); + if (!resolved) fail('runtime_boundary_violation'); + return resolved; +} + +function sleep(milliseconds) { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds); +} + +function parseProperties(output) { + if (typeof output !== 'string' || Buffer.byteLength(output, 'utf8') > MAX_COMMAND_OUTPUT_BYTES) fail(); + const selected = {}; + for (const line of output.trimEnd().split('\n')) { + const separator = line.indexOf('='); + if (separator < 1) fail(); + const key = line.slice(0, separator); + if (Object.hasOwn(selected, key)) fail(); + selected[key] = line.slice(separator + 1); + } + return selected; +} + +function parseJsonLine(output) { + if (typeof output !== 'string' || Buffer.byteLength(output, 'utf8') > MAX_COMMAND_OUTPUT_BYTES + || !output.endsWith('\n') || output.includes('\r') || output.slice(0, -1).includes('\n')) fail(); + try { + const value = JSON.parse(output.slice(0, -1)); + if (!plain(value)) fail(); + return value; + } catch (error) { + if (error?.code) throw error; + fail(); + } +} + +function receipt(plan, status, changed = false) { + return Object.freeze({ + servicePlanVersion: plan.servicePlanVersion, + status, + serviceCount: plan.units.length, + changed, + }); +} + +function createSystemdUserSupervisor(options = {}) { + exact( + options, + ['systemctl', 'commandPath', 'runtimeOnly', 'waitTimeoutMs', 'pollMs', 'clock'], + [], + ); + const systemctl = absoluteExecutable(options.systemctl, ['systemctl']); + if (typeof process.geteuid !== 'function') fail('runtime_boundary_violation'); + const runtimeDirectory = `/run/user/${process.geteuid()}`; + let runtimeInfo; + try { runtimeInfo = fs.lstatSync(runtimeDirectory); } catch { fail('runtime_boundary_violation'); } + if (!runtimeInfo.isDirectory() || runtimeInfo.isSymbolicLink() || runtimeInfo.uid !== process.geteuid() + || (runtimeInfo.mode & 0o7777) !== 0o700 || fs.realpathSync(runtimeDirectory) !== runtimeDirectory) { + fail('runtime_boundary_violation'); + } + const busPath = path.join(runtimeDirectory, 'bus'); + let commandPath; + try { commandPath = trustedCommandPath(options.commandPath === undefined ? process.env.PATH : options.commandPath); } + catch { fail('runtime_boundary_violation'); } + const runtimeOnly = options.runtimeOnly === undefined ? true : options.runtimeOnly; + if (typeof runtimeOnly !== 'boolean') fail('runtime_boundary_violation'); + const waitTimeoutMs = positiveInteger( + options.waitTimeoutMs === undefined ? DEFAULT_WAIT_TIMEOUT_MS : options.waitTimeoutMs, + 100, + 120_000, + ); + const pollMs = positiveInteger(options.pollMs === undefined ? DEFAULT_POLL_MS : options.pollMs, 10, 1_000); + const clock = options.clock === undefined ? Date.now : options.clock; + if (typeof clock !== 'function') fail('runtime_boundary_violation'); + + function execute(args, { accepted = [0], timeout = 30_000 } = {}) { + if (!Array.isArray(args) || args.some(argument => typeof argument !== 'string' + || /[\0\r\n]/.test(argument))) fail('runtime_boundary_violation'); + const result = spawnSync(systemctl, ['--user', '--no-pager', ...args], { + env: { + PATH: commandPath, + LANG: 'C.UTF-8', + LC_ALL: 'C.UTF-8', + XDG_RUNTIME_DIR: runtimeDirectory, + DBUS_SESSION_BUS_ADDRESS: `unix:path=${busPath}`, + }, + encoding: 'utf8', + timeout, + maxBuffer: MAX_COMMAND_OUTPUT_BYTES, + }); + if (result.error || !accepted.includes(result.status) + || Buffer.byteLength(result.stdout || '', 'utf8') > MAX_COMMAND_OUTPUT_BYTES + || Buffer.byteLength(result.stderr || '', 'utf8') > MAX_COMMAND_OUTPUT_BYTES) fail(); + return result; + } + + function units(plan) { + if (!isInstallationServicePlan(plan) + || !validServicePlanShape(plan.servicePlanVersion, plan.units?.length) || !Array.isArray(plan.units) + || !INSTALLATION_SERVICE_COUNTS.includes(plan.units.length)) fail('runtime_boundary_violation'); + for (const unit of plan.units) { + if (!plain(unit) || !['auth_broker', 'collection_manager', 'runtime_gateway', 'runtime_agent'].includes(unit.id) + || typeof unit.name !== 'string' || !/^[A-Za-z0-9_.@-]{1,220}\.service$/.test(unit.name) + || typeof unit.installed !== 'string' || path.basename(unit.installed) !== unit.name + || !path.isAbsolute(unit.installed) || typeof unit.expectedProcessArgument !== 'string' + || !path.isAbsolute(unit.expectedProcessArgument) || typeof unit.healthCommand !== 'string' + || !path.isAbsolute(unit.healthCommand) || !Array.isArray(unit.healthArguments) + || unit.socketPath !== null && (typeof unit.socketPath !== 'string' || !path.isAbsolute(unit.socketPath)) + || !plain(unit.environment)) fail('runtime_boundary_violation'); + } + if (new Set(plan.units.map(unit => unit.id)).size !== plan.units.length + || new Set(plan.units.map(unit => unit.name)).size !== plan.units.length) { + fail('runtime_boundary_violation'); + } + return plan.units; + } + + function state(unit) { + const result = execute([ + 'show', + unit.name, + '--property=Id,LoadState,ActiveState,SubState,MainPID,NRestarts,FragmentPath,ExecStart,Restart,KillMode,UMask,UnitFileState,ControlGroup', + ], { accepted: [0, 1] }); + if (result.status !== 0) return Object.freeze({ loaded: false, active: false, enabled: false }); + const properties = parseProperties(result.stdout); + if (properties.Id !== unit.name) fail(); + return Object.freeze({ + loaded: properties.LoadState === 'loaded', + active: properties.ActiveState === 'active' && properties.SubState === 'running', + activeState: properties.ActiveState, + enabled: ['enabled', 'enabled-runtime', 'linked', 'linked-runtime'].includes(properties.UnitFileState), + pid: Number(properties.MainPID), + restarts: Number(properties.NRestarts), + fragment: properties.FragmentPath, + execStart: properties.ExecStart, + restart: properties.Restart, + killMode: properties.KillMode, + umask: properties.UMask, + unitFileState: properties.UnitFileState, + controlGroup: properties.ControlGroup, + }); + } + + function snapshot(plan) { + return Object.freeze(units(plan).map(unit => { + const current = state(unit); + if (current.loaded && current.fragment !== unit.installed) fail('runtime_boundary_violation'); + const enableMode = ['enabled-runtime', 'linked-runtime'].includes(current.unitFileState) + ? 'runtime' + : (['enabled', 'linked'].includes(current.unitFileState) ? 'persistent' : 'none'); + return Object.freeze({ + id: unit.id, + name: unit.name, + enabled: current.enabled === true, + active: current.loaded === true && !['inactive', 'failed'].includes(current.activeState), + enableMode, + }); + })); + } + + function mutateCall(mutationCapability, callback) { + if (typeof mutationCapability !== 'function') fail('runtime_boundary_violation'); + return mutationCapability(callback); + } + + function reload(plan, mutationCapability) { + units(plan); + mutateCall(mutationCapability, () => execute(['daemon-reload'])); + return receipt(plan, 'reloaded', true); + } + + function enable(plan, mutationCapability) { + const selected = units(plan); + for (const unit of selected) { + const current = state(unit); + if (!current.loaded || current.fragment !== unit.installed) fail('runtime_boundary_violation'); + const args = ['enable', '--no-reload']; + if (runtimeOnly) args.push('--runtime'); + args.push(unit.name); + mutateCall(mutationCapability, () => execute(args)); + } + reload(plan, mutationCapability); + return receipt(plan, 'enabled', true); + } + + function disable(plan, mutationCapability) { + const selected = [...units(plan)].reverse(); + for (const unit of selected) { + const current = state(unit); + if (current.loaded && current.fragment !== unit.installed) fail('runtime_boundary_violation'); + if (!current.loaded) continue; + const args = ['disable', '--no-reload']; + if (runtimeOnly) args.push('--runtime'); + args.push(unit.name); + mutateCall(mutationCapability, () => execute(args, { accepted: [0, 1] })); + } + reload(plan, mutationCapability); + return receipt(plan, 'disabled', true); + } + + function waitFor(unit, predicate) { + const deadline = clock() + waitTimeoutMs; + for (;;) { + const current = state(unit); + if (predicate(current)) return current; + if (clock() >= deadline) fail('runtime_health_failed'); + sleep(pollMs); + } + } + + function start(plan, mutationCapability) { + const selected = units(plan); + for (const unit of selected) { + const current = state(unit); + if (!current.loaded || current.fragment !== unit.installed) fail('runtime_boundary_violation'); + mutateCall(mutationCapability, () => execute(['start', '--no-block', unit.name])); + waitFor(unit, current => { + if (!current.active || !Number.isSafeInteger(current.pid) || current.pid < 1) return false; + try { return processArguments(current.pid).includes(unit.expectedProcessArgument); } catch { return false; } + }); + } + return receipt(plan, 'started', true); + } + + function stop(plan, mutationCapability) { + const selected = [...units(plan)].reverse(); + for (const unit of selected) { + const current = state(unit); + if (current.loaded && current.fragment !== unit.installed) fail('runtime_boundary_violation'); + if (!current.loaded) continue; + mutateCall(mutationCapability, () => execute(['stop', '--no-block', unit.name], { accepted: [0, 1] })); + waitFor(unit, current => ['inactive', 'failed'].includes(current.activeState)); + } + return receipt(plan, 'stopped', true); + } + + function resetFailed(plan, mutationCapability) { + for (const unit of [...units(plan)].reverse()) { + const current = state(unit); + if (current.loaded && current.fragment !== unit.installed) fail('runtime_boundary_violation'); + if (!current.loaded) continue; + mutateCall(mutationCapability, () => execute( + ['reset-failed', unit.name], + { accepted: [0, 1] }, + )); + } + return receipt(plan, 'failure_state_reset', true); + } + + function restoreState(plan, snapshotValue, mutationCapability) { + const selected = units(plan); + if (!Array.isArray(snapshotValue) || snapshotValue.length !== selected.length) { + fail('runtime_boundary_violation'); + } + const snapshot = selected.map((unit, index) => { + const value = snapshotValue[index]; + exact( + value, + ['id', 'name', 'enabled', 'active', 'enableMode'], + ['id', 'name', 'enabled', 'active', 'enableMode'], + ); + if (value.id !== unit.id || value.name !== unit.name + || typeof value.enabled !== 'boolean' || typeof value.active !== 'boolean' + || !['none', 'runtime', 'persistent'].includes(value.enableMode) + || value.enabled !== (value.enableMode !== 'none')) { + fail('runtime_boundary_violation'); + } + return value; + }); + for (let index = 0; index < selected.length; index += 1) { + if (!snapshot[index].enabled) continue; + const current = state(selected[index]); + if (!current.loaded || current.fragment !== selected[index].installed) fail('runtime_boundary_violation'); + const args = ['enable', '--no-reload']; + if (snapshot[index].enableMode === 'runtime') args.push('--runtime'); + args.push(selected[index].name); + mutateCall(mutationCapability, () => execute(args)); + } + reload(plan, mutationCapability); + for (let index = 0; index < selected.length; index += 1) { + if (!snapshot[index].active) continue; + const current = state(selected[index]); + if (!current.loaded || current.fragment !== selected[index].installed) fail('runtime_boundary_violation'); + mutateCall(mutationCapability, () => execute(['start', '--no-block', selected[index].name])); + waitFor(selected[index], current => current.active === true); + } + return receipt(plan, 'state_restored', true); + } + + function processEnvironment(pid) { + if (!Number.isSafeInteger(pid) || pid < 1) fail('runtime_health_failed'); + let raw; + try { raw = fs.readFileSync(`/proc/${pid}/environ`); } catch { fail('runtime_health_failed'); } + if (raw.length > MAX_COMMAND_OUTPUT_BYTES) fail('runtime_health_failed'); + const result = {}; + for (const entry of raw.toString('utf8').split('\0').filter(Boolean)) { + const separator = entry.indexOf('='); + if (separator < 1) fail('runtime_health_failed'); + result[entry.slice(0, separator)] = entry.slice(separator + 1); + } + return result; + } + + function processArguments(pid) { + let raw; + try { raw = fs.readFileSync(`/proc/${pid}/cmdline`); } catch { fail('runtime_health_failed'); } + if (raw.length < 2 || raw.length > MAX_COMMAND_OUTPUT_BYTES) fail('runtime_health_failed'); + return raw.toString('utf8').split('\0').filter(Boolean); + } + + function verifyUnixSocketOwner(socketPath, pid) { + let info; + try { info = fs.lstatSync(socketPath); } catch { fail('runtime_health_failed'); } + if (!info.isSocket() || info.isSymbolicLink() || info.uid !== process.geteuid() + || (info.mode & 0o7777) !== 0o600 || fs.realpathSync(socketPath) !== socketPath) { + fail('runtime_health_failed'); + } + let table; + try { table = fs.readFileSync('/proc/net/unix', 'utf8'); } catch { fail('runtime_health_failed'); } + let inode = null; + for (const line of table.split('\n').slice(1)) { + const fields = line.trim().split(/\s+/); + if (fields.length >= 8 && fields.slice(7).join(' ') === socketPath && /^\d+$/.test(fields[6])) { + inode = fields[6]; + break; + } + } + if (!inode) fail('runtime_health_failed'); + let descriptors; + try { descriptors = fs.readdirSync(`/proc/${pid}/fd`); } catch { fail('runtime_health_failed'); } + const expected = `socket:[${inode}]`; + const owned = descriptors.some(descriptor => { + try { return fs.readlinkSync(`/proc/${pid}/fd/${descriptor}`) === expected; } catch { return false; } + }); + if (!owned) fail('runtime_health_failed'); + } + + function inspect(plan) { + for (const unit of units(plan)) { + const current = state(unit); + if (!current.loaded || !current.active || !current.enabled + || current.fragment !== unit.installed || current.restart !== 'always' + || current.killMode !== 'control-group' || current.umask !== '0077' + || !Number.isSafeInteger(current.pid) || current.pid < 1 + || !Number.isSafeInteger(current.restarts) || current.restarts < 0 + || typeof current.execStart !== 'string' + || !current.execStart.includes(`path=${unit.environmentLauncher}`) + || !current.execStart.includes(unit.executable)) { + fail('runtime_health_failed'); + } + let processInfo; + try { processInfo = fs.statSync(`/proc/${current.pid}`); } catch { fail('runtime_health_failed'); } + if (processInfo.uid !== process.geteuid()) fail('runtime_health_failed'); + if (typeof current.controlGroup !== 'string' || !current.controlGroup.startsWith('/')) { + fail('runtime_health_failed'); + } + let cgroups; + try { cgroups = fs.readFileSync(`/proc/${current.pid}/cgroup`, 'utf8'); } catch { fail('runtime_health_failed'); } + if (!cgroups.split('\n').some(line => { + const separator = line.indexOf('::'); + return separator >= 0 && line.slice(separator + 2) === current.controlGroup; + })) { + fail('runtime_health_failed'); + } + const argumentsList = processArguments(current.pid); + if (!argumentsList.includes(unit.expectedProcessArgument)) fail('runtime_health_failed'); + const environment = processEnvironment(current.pid); + for (const [key, value] of Object.entries(unit.environment)) { + if (environment[key] !== value) fail('runtime_health_failed'); + } + if (JSON.stringify(Object.keys(environment).sort()) + !== JSON.stringify(Object.keys(unit.environment).sort())) fail('runtime_health_failed'); + if (Object.hasOwn(environment, 'DISPATCH_LOCAL_ROOT') + || Object.keys(environment).some(key => key.startsWith('DISPATCH_ACCESS_CONTROL')) + || ['NODE_OPTIONS', 'LD_PRELOAD', 'LD_LIBRARY_PATH'].some(key => Object.hasOwn(environment, key))) { + fail('runtime_health_failed'); + } + } + return receipt(plan, 'active'); + } + + function health(plan) { + inspect(plan); + for (const unit of units(plan)) { + const deadline = clock() + waitTimeoutMs; + for (;;) { + const result = spawnSync(unit.healthCommand, unit.healthArguments, { + cwd: unit.workingDirectory, + env: unit.environment, + encoding: 'utf8', + timeout: 15_000, + maxBuffer: MAX_COMMAND_OUTPUT_BYTES, + }); + let response = null; + try { + if (!result.error && result.signal === null) response = parseJsonLine(result.stdout); + } catch {} + const statusReady = response?.ok === true && ['ready', 'verified'].includes(response.status); + const managerReady = unit.id !== 'collection_manager' + || (plain(response?.data) && plain(response.data.manager) && response.data.manager.running === true); + if (result.status === 0 && statusReady && managerReady) break; + const startupState = [ + 'broker_unavailable', + 'collection_manager_not_initialized', + 'runtime_gateway_unavailable', + 'runtime_agent_unavailable', + ].includes(response?.status) + || (unit.id === 'collection_manager' && response?.ok === true && managerReady === false); + if (!startupState || clock() >= deadline) fail('runtime_health_failed'); + sleep(pollMs); + } + if (unit.socketPath !== null) verifyUnixSocketOwner(unit.socketPath, state(unit).pid); + } + inspect(plan); + return receipt(plan, 'healthy'); + } + + function restartEvidence(plan, serviceId, mutationCapability) { + const unit = units(plan).find(candidate => candidate.id === serviceId); + if (!unit) fail('runtime_boundary_violation'); + const before = state(unit); + if (!before.active || before.fragment !== unit.installed + || !Number.isSafeInteger(before.pid) || before.pid < 1) fail('runtime_health_failed'); + mutateCall(mutationCapability, () => execute([ + 'kill', '--kill-whom=main', '--signal=SIGTERM', unit.name, + ])); + const after = waitFor(unit, current => { + if (!current.active || current.pid === before.pid || current.restarts <= before.restarts) return false; + try { return processArguments(current.pid).includes(unit.expectedProcessArgument); } catch { return false; } + }); + if (!after.active) fail('runtime_health_failed'); + health(plan); + return receipt(plan, 'restart_verified', true); + } + + function boundedRestartEvidence(plan, serviceId, mutationCapability) { + const selected = units(plan); + const unit = selected.find(value => value.id === serviceId); + if (!unit) fail('runtime_boundary_violation'); + inspect(plan); + const initial = state(unit); + let current = initial; + for (let termination = 0; termination < 8 && current.activeState !== 'failed'; termination += 1) { + const priorPid = current.pid; + if (current.fragment !== unit.installed) fail('runtime_health_failed'); + mutateCall(mutationCapability, () => execute([ + 'kill', '--kill-whom=main', '--signal=SIGTERM', unit.name, + ])); + const deadline = clock() + waitTimeoutMs; + for (;;) { + current = state(unit); + if (current.activeState === 'failed') break; + if (current.active && current.pid !== priorPid) { + try { + if (processArguments(current.pid).includes(unit.expectedProcessArgument)) break; + } catch {} + } + if (clock() >= deadline) fail('runtime_health_failed'); + sleep(pollMs); + } + } + if (current.activeState !== 'failed' || current.active + || current.restarts < initial.restarts + || current.restarts - initial.restarts > 5) fail('runtime_health_failed'); + return receipt(plan, 'restart_bounded', true); + } + + return Object.freeze({ + snapshot, + reload, + enable, + disable, + start, + stop, + resetFailed, + restoreState, + inspect, + health, + restartEvidence, + boundedRestartEvidence, + }); +} + +module.exports = { + DEFAULT_WAIT_TIMEOUT_MS, + createSystemdUserSupervisor, +}; diff --git a/core/core/installations/src/worker-notify.js b/core/core/installations/src/worker-notify.js new file mode 100644 index 0000000..d0e19d1 --- /dev/null +++ b/core/core/installations/src/worker-notify.js @@ -0,0 +1,30 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +// An unprivileged worker can signal only this fixed root-side exporter through +// its path unit. The file contains no command, path, credentials or job payload. +function exportReady(localRoot = process.env.DISPATCH_LOCAL_ROOT) { + if (!localRoot || !path.isAbsolute(localRoot)) return; + try { + const root = path.join(localRoot, 'run'); + const s = fs.lstatSync(root); + if (!s.isDirectory() || s.uid !== process.geteuid() || s.mode & 0o077 || fs.realpathSync(root) !== root) return; + const fd = fs.openSync(path.join(root, 'backup-ready'), fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_NOFOLLOW, 0o600); + try { + const stat = fs.fstatSync(fd); + if (!stat.isFile() || stat.nlink !== 1 || stat.uid !== process.geteuid()) return; + fs.ftruncateSync(fd, 0); + fs.writeSync(fd, String(Date.now())); fs.fsyncSync(fd); + } finally { fs.closeSync(fd); } + } catch {} // Fallback exporter timer recovers missed notifications. +} +function userWorkers(config) { + if (process.geteuid() !== 0) return; + try { + const account = require('./host-recovery-bundle').account(config.coreUid); + require('node:child_process').spawnSync('/usr/sbin/runuser', ['--user', account.name, '--', '/usr/bin/env', + `XDG_RUNTIME_DIR=/run/user/${account.uid}`, `DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/${account.uid}/bus`, + '/usr/bin/systemctl', '--user', '--no-block', 'start', 'dispatch-installation-reconcile.service', 'dispatch-platform-update.service'], + { timeout: 5000, stdio: 'ignore' }); + } catch {} +} +module.exports = { exportReady, userWorkers }; diff --git a/core/core/installations/tests/activation.test.js b/core/core/installations/tests/activation.test.js new file mode 100644 index 0000000..f74039d --- /dev/null +++ b/core/core/installations/tests/activation.test.js @@ -0,0 +1,218 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const test = require('node:test'); +const { serverInstallationActivation } = require('../../../shared/contracts/src'); +const { runManagedPaycomActivation } = require('../../../compatibility/provisioner/src/activation.js'); + +function manifest() { + return { + manifestVersion: 1, + revision: 1, + organization: { id: 'org_activation_fixture', stationCode: 'TST1', timezone: 'America/Chicago' }, + runtime: { key: 'fixture_activation', templateId: 'isolated_dsp_v1', releaseId: 'dispatch_fixture_1' }, + }; +} + +function manifestAuthority(value) { + return { + revision: value.revision, + organization: { ...value.organization }, + runtime: { ...value.runtime }, + }; +} + +function activationAuthority({ state = 'waiting_for_provider_auth' } = {}) { + const selected = manifest(); + const authority = manifestAuthority(selected); + const calls = []; + let revision = 1; + let currentState = state; + let job = state === 'verifying' ? { + id: 'job_activation_001', operation: 'resume', status: 'running', + installationState: 'verifying', revision: 2, replayed: false, failure: null, + } : null; + if (job) revision = 2; + const context = () => ({ + manifest: selected, + manifestAuthority: authority, + installation: { state: currentState, revision, currentJobId: job?.id || null }, + job, + owner: { active: true }, + }); + return { + calls, + inspect: async () => context(), + heartbeat: async () => context(), + begin: async evidence => { + calls.push(['begin', evidence.status]); + assert.equal(currentState, 'waiting_for_provider_auth'); + currentState = 'verifying'; + revision += 1; + job = { + id: 'job_activation_001', operation: 'resume', status: 'running', + installationState: 'verifying', revision, replayed: false, failure: null, + }; + return context(); + }, + commit: async (activation, activationAuthorityValue) => { + calls.push(['commit', activation.readiness.gates.first_publication]); + serverInstallationActivation(activation, activationAuthorityValue); + assert.equal(currentState, 'verifying'); + currentState = 'ready'; + revision += 1; + job = { ...job, status: 'succeeded', installationState: 'ready', revision }; + return { state: currentState, revision }; + }, + fail: async (jobId, failure) => { + calls.push(['fail', jobId, failure.code]); + currentState = 'failed'; + revision += 1; + }, + current: () => ({ state: currentState, revision, job }), + }; +} + +function activationRuntime(overrides = {}) { + const calls = []; + let definitionDigest = null; + let requestDigest = null; + const runtime = { + calls, + verifyInfrastructure: async selected => { + calls.push(['infrastructure', selected.runtime.key]); + return { + runtimeKey: selected.runtime.key, + runtime_layout: true, + service_supervision: true, + auth_broker: true, + collection_manager: true, + runtime_gateway: true, + }; + }, + configure: async definition => { + calls.push(['configure', definition.digest]); + definitionDigest = definition.digest; + return { digest: definition.digest, collectors: 1, sources: 1, plans: 15, syncs: 1 }; + }, + testProvider: async profileId => { + calls.push(['provider', profileId]); + return { profileId, provider: 'paycom', status: 'authenticated', testedAt: '2026-09-02T21:30:00.000Z' }; + }, + publishFirst: async (request, options) => { + calls.push(['publication', request.scope, options.idempotencyKey]); + requestDigest = crypto.createHash('sha256').update(JSON.stringify(request)).digest('hex'); + return { + batchId: 'batch_activation_001', preparationRunId: 'run_periods', status: 'succeeded', runCount: 5, + succeededRuns: 5, failedRuns: 0, cancelledRuns: 0, + }; + }, + verifyPublication: async batchId => { + calls.push(['audit', batchId]); + return { + definitionDigest, + requestDigest, + previewDigest: 'a'.repeat(64), + batchId, + preparationRunId: 'run_periods', + target: '2026-09-05', + runs: [ + { id: 'run_roster', taskId: 'roster', plan: 'paycom-period-roster', method: 'roster.period' }, + { id: 'run_timecards', taskId: 'timecards', plan: 'paycom-period-timecards-from-roster', method: 'timecards.from-published-roster' }, + { id: 'run_timecards_audit', taskId: 'timecards-audit', plan: 'paycom-period-timecards-audit', method: 'timecards.audit' }, + { id: 'run_links', taskId: 'links', plan: 'paycom-period-resource-links', method: 'resource-links.period' }, + { id: 'run_links_audit', taskId: 'links-audit', plan: 'paycom-period-resource-links-audit', method: 'resource-links.audit' }, + ], + publications: { + payPeriods: { id: 'pub_periods', runId: 'run_periods', originRunId: 'run_periods', contentSha256: '1'.repeat(64), batchBound: false }, + roster: { id: 'pub_roster', runId: 'run_roster', originRunId: 'run_roster', contentSha256: '2'.repeat(64), batchBound: true }, + timecards: { id: 'pub_timecards', runId: 'run_timecards', originRunId: 'run_timecards', contentSha256: '3'.repeat(64), batchBound: true }, + resourceLinks: { id: 'pub_links', runId: 'run_links', originRunId: 'run_links', contentSha256: '4'.repeat(64), batchBound: true }, + }, + capturedAt: '2026-09-02T21:30:00.000Z', + }; + }, + }; + return Object.assign(runtime, overrides); +} + +test('managed activation reaches ready only after all authority, runtime, auth, and publication gates pass', async () => { + const authority = activationAuthority(); + const runtime = activationRuntime(); + const result = await runManagedPaycomActivation({ authority, runtime }); + assert.deepEqual(result, { + ok: true, status: 'ready', state: 'ready', revision: 3, manifestRevision: 1, gates: 9, + }); + assert.deepEqual(authority.calls, [['begin', 'authenticated'], ['commit', 'passed']]); + assert.equal(runtime.calls.filter(call => call[0] === 'infrastructure').length, 3); + assert.equal(runtime.calls.filter(call => call[0] === 'provider').length, 1); + assert.deepEqual(runtime.calls.find(call => call[0] === 'publication'), + ['publication', 'full', 'activation:job_activation_001']); + const callsAfterCommit = runtime.calls.length; + const replay = await runManagedPaycomActivation({ authority, runtime }); + assert.deepEqual(replay, result); + assert.equal(runtime.calls.length, callsAfterCommit); + assert.deepEqual(authority.calls, [['begin', 'authenticated'], ['commit', 'passed']]); +}); + +test('managed activation resumes the same verifying job and fails closed on publication errors', async () => { + const resumedAuthority = activationAuthority({ state: 'verifying' }); + const resumedRuntime = activationRuntime(); + const resumed = await runManagedPaycomActivation({ authority: resumedAuthority, runtime: resumedRuntime }); + assert.equal(resumed.ok, true); + assert.equal(resumedAuthority.calls.some(call => call[0] === 'begin'), false); + assert.equal(resumedRuntime.calls.some(call => call[0] === 'provider'), false); + assert.deepEqual(resumedRuntime.calls.find(call => call[0] === 'publication'), + ['publication', 'full', 'activation:job_activation_001']); + + const failedAuthority = activationAuthority(); + const failedRuntime = activationRuntime({ + publishFirst: async () => ({ + batchId: 'batch_activation_001', preparationRunId: 'run_periods', status: 'failed', runCount: 5, + succeededRuns: 4, failedRuns: 1, cancelledRuns: 0, + }), + }); + const failed = await runManagedPaycomActivation({ authority: failedAuthority, runtime: failedRuntime }); + assert.equal(failed.ok, false); + assert.equal(failed.status, 'first_publication_failed'); + assert.deepEqual(failedAuthority.calls, [ + ['begin', 'authenticated'], + ['fail', 'job_activation_001', 'first_publication_failed'], + ]); + assert.equal(failedAuthority.current().state, 'failed'); +}); + +test('provider authentication failure cannot begin verification or manufacture readiness', async () => { + const authority = activationAuthority(); + const runtime = activationRuntime({ + testProvider: async profileId => ({ + profileId, provider: 'paycom', status: 'captcha_required', testedAt: '2026-09-02T21:30:00.000Z', + }), + }); + const result = await runManagedPaycomActivation({ authority, runtime }); + assert.equal(result.ok, false); + assert.equal(result.status, 'provider_auth_required'); + assert.deepEqual(authority.calls, []); + assert.equal(authority.current().state, 'waiting_for_provider_auth'); +}); + +test('Core activates a remote DSP without local provider definitions and rejects inconsistent receipts', async () => { + const { runManagedPaycomActivation: runRemote } = require('../src/activation'); + for (const scenario of ['valid', 'invalid_digest', 'changed_digest']) { + const authority = activationAuthority(); + const runtime = activationRuntime(); + const configure = runtime.configure; + runtime.configure = async definition => { + assert.equal(definition, null); + return configure({ digest: scenario === 'invalid_digest' ? 'invalid' : 'd'.repeat(64) }); + }; + if (scenario === 'changed_digest') { + const audit = runtime.verifyPublication; + runtime.verifyPublication = async batchId => ({ ...await audit(batchId), definitionDigest: 'e'.repeat(64) }); + } + const result = await runRemote({ authority, runtime, projectRoot: '/a/core/without/provider/files' }); + assert.equal(result.ok, scenario === 'valid', scenario); + assert.equal(authority.current().state, scenario === 'valid' ? 'ready' : 'failed'); + } +}); diff --git a/core/core/installations/tests/backup-archives.test.js b/core/core/installations/tests/backup-archives.test.js new file mode 100644 index 0000000..0753d97 --- /dev/null +++ b/core/core/installations/tests/backup-archives.test.js @@ -0,0 +1,207 @@ +'use strict'; +const test = require('node:test'), + assert = require('node:assert/strict'), + fs = require('node:fs'), + os = require('node:os'), + path = require('node:path'), + crypto = require('node:crypto'); +const { DatabaseSync } = require('node:sqlite'); +const { createBackupArchives } = require('../src/backup-archives'); +const { createRestic } = require('../src/offsite-backup'); +const { atomic, hashFileSync } = require('../src/release-delivery-files'); +test( + 'archive encrypts DSP metadata with its snapshot, independently restores all files and cannot change accepted retention', + { skip: !fs.existsSync('/usr/bin/restic') }, + (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-archive-test-')); + fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const source = path.join(root, 'source'), + workRoot = path.join(root, 'work'), + receiptRoot = path.join(root, 'receipts'); + for (const p of [source, workRoot, receiptRoot]) fs.mkdirSync(p, { mode: 0o700 }); + const file = path.join(source, 'access-control-before.sqlite3'), + db = new DatabaseSync(file); + db.exec("CREATE TABLE data(value TEXT); INSERT INTO data VALUES('company record')"); + db.close(); + fs.chmodSync(file, 0o600); + atomic(path.join(source, 'manifest.json'), { + version: 1, + kind: 'core', + sha256: hashFileSync(file), + size: fs.statSync(file).size, + }); + const password = path.join(root, 'password'); + fs.writeFileSync(password, crypto.randomBytes(32).toString('hex'), { mode: 0o600 }); + const calls = [], + runFactory = (env) => { + const run = createRestic({ + ...env, + RESTIC_REPOSITORY: path.join(root, 'repository'), + RESTIC_PASSWORD_FILE: password, + }); + return (args, cwd) => { + calls.push(args); + return run(args, cwd); + }; + }; + const archives = createBackupArchives( + { + accountId: 'a'.repeat(32), + bucket: 'dispatch-test', + environment: { PATH: '/usr/bin:/bin' }, + }, + { + workRoot, + receiptRoot, + ownerUid: process.geteuid(), + runFactory, + storage: {}, + pathResolver: () => ({ source, uid: process.geteuid() }), + clock: () => 1000000000, + recoveryCapture: ({ destination }) => ({ organizationIds: [], ...require('../src/recovery-capsule').capture(destination, + [{ source, target: '/home/fixture/local/data' }], { kind: 'core', platform: 'ubuntu-24.04-amd64', + localRoot: '/home/fixture/local', accounts: [{ name: 'fixture', uid: process.geteuid(), gid: process.getgid(), home: '/home/fixture' }], + services: [], installations: [] }, new Set([process.geteuid()])) }), + }, + ); + const row = { + id: `breq_${'a'.repeat(32)}`, + kind: 'core', + organization_id: null, + metadata_json: JSON.stringify({ privateConfiguration: 'secret configuration' }), + retention_days: 30, + created_at: 1000, + }; + return (async () => { + const result = await archives.exportRecord(row); + assert.equal(result.status, 'verified'); + assert.match(result.recoveryDigest, /^[a-f0-9]{64}$/); + assert.equal(result.expiresAt, 1000000000 + 30 * 86400000); + assert.equal( + calls.some((a) => a[0] === '--no-lock' && a[1] === 'restore' || a[1] === 'check'), + false, + ); + const before = calls.length; + assert.deepEqual(await archives.exportRecord({ ...row, retention_days: 7 }), result); + assert.equal(calls.length, before); + await assert.rejects(() => archives.exportRecord({ ...row, metadata_json: '{}' })); + const restored = path.join(root, 'independent'); + createRestic({ + PATH: '/usr/bin:/bin', + RESTIC_REPOSITORY: path.join(root, 'repository'), + RESTIC_PASSWORD_FILE: password, + })(['restore', result.snapshotId, '--target', restored, '--verify']); + assert.equal( + JSON.parse(fs.readFileSync(path.join(restored, 'bundle/dsp.json'))).metadata + .privateConfiguration, + 'secret configuration', + ); + const check = new DatabaseSync( + path.join(restored, 'bundle/snapshot/access-control-before.sqlite3'), + { readOnly: true }, + ); + assert.equal(check.prepare('SELECT value FROM data').get().value, 'company record'); + check.close(); + const walk = (p) => + fs + .readdirSync(p, { withFileTypes: true }) + .flatMap((e) => (e.isDirectory() ? walk(path.join(p, e.name)) : [path.join(p, e.name)])); + for (const f of walk(path.join(root, 'repository'))) + assert.equal(fs.readFileSync(f).includes(Buffer.from('secret configuration')), false); + // The restored Core database does not yet know the backup used to restore + // it. Rebuild its catalog from the encrypted archive, without local data. + const localRoot = path.join(root, 'restored-core'); + fs.mkdirSync(path.join(localRoot, 'data/access-control'), { recursive: true }); + const catalogDb = new DatabaseSync(path.join(localRoot, 'data/access-control/access-control.sqlite3')); + catalogDb.exec('CREATE TABLE organizations(id TEXT PRIMARY KEY); CREATE TABLE platform_backup_records(id TEXT PRIMARY KEY,organization_id TEXT,kind TEXT,metadata_json TEXT,retention_days INTEGER,created_at INTEGER,expires_at INTEGER,deleted_at INTEGER)'); + t.after(() => catalogDb.close()); + atomic(path.join(workRoot, 'rediscover.json'), { schemaVersion: 1 }); + const recovered = new Map(); + await require('../src/recover-archive-catalog').recoverArchiveCatalog({ + config: { localRoot }, storage: { listArchives: async () => [{ id: row.id, retentionDays: 30 }] }, + runFor: () => runFactory({ PATH: '/usr/bin:/bin' }), workRoot, ownerUid: process.geteuid(), + record: id => recovered.get(id), save: (id, value) => recovered.set(id, value), clock: Date.now, + }); + assert.equal(fs.existsSync(path.join(workRoot, 'rediscover.json')), false); + assert.equal(recovered.get(row.id).digest, result.digest); + assert.equal(recovered.get(row.id).bundleDigest, result.bundleDigest); + assert.equal(catalogDb.prepare('SELECT metadata_json FROM platform_backup_records WHERE id=?').get(row.id).metadata_json, row.metadata_json); + atomic(path.join(workRoot, 'rediscover.json'), { schemaVersion: 1 }); + recovered.clear(); + await assert.rejects(require('../src/recover-archive-catalog').recoverArchiveCatalog({ + config: { localRoot }, storage: { listArchives: async () => [{ id: row.id, retentionDays: 30 }] }, + runFor: () => (args, cwd) => { + const value = runFactory({ PATH: '/usr/bin:/bin' })(args, cwd); + if (args[0] === 'restore') atomic(path.join(args[args.indexOf('--target') + 1], 'bundle/recovery-proof.json'), { sha256: '0'.repeat(64) }); + return value; + }, workRoot, ownerUid: process.geteuid(), record: id => recovered.get(id), + save: (id, value) => recovered.set(id, value), clock: Date.now, + }), /recovery_capsule_invalid/); + assert.equal(fs.existsSync(path.join(workRoot, 'rediscover.json')), true); + assert.equal(recovered.size, 0); + })(); + }, +); + +test('removed DSP backups are neither exported nor expired, and normal retention resumes after restoration', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-retained-archives-')); + fs.chmodSync(root, 0o700); + const data = path.join(root, 'data/access-control'); fs.mkdirSync(data, { recursive: true }); + const dbFile = path.join(data, 'access-control.sqlite3'); + const db = new DatabaseSync(dbFile); fs.chmodSync(dbFile, 0o600); + t.after(() => { db.close(); fs.rmSync(root, { recursive: true, force: true }); }); + db.exec(`CREATE TABLE installations(organization_id TEXT,runtime_key TEXT,current_job_id TEXT); + CREATE TABLE dsp_removals(organization_id TEXT); + CREATE TABLE platform_backup_records(id TEXT,organization_id TEXT,kind TEXT,metadata_json TEXT,retention_days INTEGER,created_at INTEGER,expires_at INTEGER,deleted_at INTEGER); + CREATE TABLE installation_lifecycle_jobs(id TEXT,organization_id TEXT,operation TEXT,authority_scope TEXT,status TEXT,backup_id TEXT,safety_backup_id TEXT,stage_receipts_json TEXT); + CREATE TABLE installation_backups(id TEXT,organization_id TEXT); + CREATE TABLE platform_backup_requests(status TEXT,kind TEXT,input_json TEXT); + INSERT INTO installations VALUES('org_retained','runtime_retained',NULL); + INSERT INTO dsp_removals VALUES('org_retained');`); + const id = 'backup_' + 'c'.repeat(32), pending = 'backup_' + 'd'.repeat(32); + for (const selected of [id, pending]) db.prepare('INSERT INTO platform_backup_records VALUES(?,?,?,?,?,?,?,NULL)').run(selected, 'org_retained', 'dsp', '{}', 7, 1, 2); + const workRoot = path.join(root, 'work'), receiptRoot = path.join(root, 'receipts'); + fs.mkdirSync(workRoot, { mode: 0o700 }); fs.mkdirSync(receiptRoot, { mode: 0o700 }); + const expired = [], exported = []; + const archives = createBackupArchives({ localRoot: root, coreUid: process.geteuid(), environment: {} }, { + ownerUid: process.geteuid(), workRoot, receiptRoot, clock: () => 100, + storage: { ensureLocks: async () => {}, removeExpired: async rec => expired.push(rec.id), usage:async()=>({archives:expired.includes(id)?{}:{[id]:300},sets:{},legacyBytes:0}) }, + runFactory: () => () => { throw Error('must not transfer retained backup'); }, + pathResolver: (config, row) => { exported.push(row.id); return { source: path.join(root, 'absent'), uid: process.geteuid() }; }, + }); + const proofFile = path.join(workRoot, 'archives', id + '.json'); + atomic(proofFile, { id, metadataDigest: crypto.createHash('sha256').update('{}').digest('hex'), status: 'verified', expiresAt: 2, localPruned: true }); + let result = await archives.scan(); + assert.equal(result.failed, 0); + assert.deepEqual(result.heldRuntimes, ['runtime_retained']); + assert.deepEqual(expired, []); + assert.ok(!exported.includes(pending)); + let catalog = JSON.parse(fs.readFileSync(path.join(receiptRoot, 'catalog.json'))); + assert.equal(catalog.backups[id].expiresAt, null); + assert.equal(catalog.backups[id].retained, true); + assert.equal(catalog.usage.status,'ready');assert.equal(catalog.usage.dsps[0].bytes,300); + assert.equal(JSON.parse(fs.readFileSync(proofFile)).status, 'verified'); + db.exec('DELETE FROM dsp_removals'); + result = await archives.scan(); + assert.equal(result.failed, 0); + assert.deepEqual(expired, [id]); + assert.ok(exported.includes(pending)); + catalog=JSON.parse(fs.readFileSync(path.join(receiptRoot,'catalog.json')));assert.equal(catalog.usage.bytes,0); +}); + +test('live queue discovery excludes removed, deleting and explicitly deleted snapshots', t => { + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-discovery-'));t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + const file=path.join(root,'db.sqlite3'),db=new DatabaseSync(file); + db.exec(`CREATE TABLE platform_backup_records(id TEXT,organization_id TEXT,kind TEXT,deleted_at INTEGER,created_at INTEGER); + CREATE TABLE dsp_removals(organization_id TEXT); + CREATE TABLE installation_lifecycle_jobs(organization_id TEXT,operation TEXT,status TEXT); + CREATE TABLE backup_deletions(backup_id TEXT,status TEXT); + INSERT INTO platform_backup_records VALUES('core',NULL,'core',NULL,1),('live','org_live','dsp',NULL,1),('removed','org_removed','dsp',NULL,1),('destroy','org_destroy','dsp',NULL,1),('delete','org_live','dsp',NULL,1); + INSERT INTO dsp_removals VALUES('org_removed'); + INSERT INTO installation_lifecycle_jobs VALUES('org_destroy','destroy','queued'); + INSERT INTO backup_deletions VALUES('delete','queued');`); + db.close(); + const rows=require('../src/backup-archives').pendingExports(file,()=>null); + assert.deepEqual(rows.map(row=>row.id),['live']); +}); diff --git a/core/core/installations/tests/backup-readiness.test.js b/core/core/installations/tests/backup-readiness.test.js new file mode 100644 index 0000000..c95bcb4 --- /dev/null +++ b/core/core/installations/tests/backup-readiness.test.js @@ -0,0 +1,54 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const { inspectInstallation, readReadiness, localIdentity } = require('../src/backup-readiness'); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'backup-readiness-')); + fs.chmodSync(root, 0o700); + for (const name of ['data', 'state', 'config', 'secrets/auth-broker', 'backups']) fs.mkdirSync(path.join(root, name), { recursive: true, mode: 0o700 }); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return root; +} +test('readiness finds both rollout blockers without reading or changing profile contents', t => { + const root = fixture(t), cache = path.join(root, 'state/cache'); + fs.mkdirSync(cache, { mode: 0o755 }); fs.chmodSync(cache, 0o755); + fs.writeFileSync(path.join(root, 'data/private'), 'SECRET', { mode: 0o600 }); + const outside = path.join(root, 'outside'); fs.writeFileSync(outside, 'keep'); + fs.symlinkSync(outside, path.join(root, 'state/runtime-link')); + const before = fs.readFileSync(path.join(root, 'data/private')); + t.mock.method(fs, 'readFileSync', () => { throw Error('must not read payloads'); }); + const result = inspectInstallation(root, process.geteuid()); + assert.equal(result.status, 'attention'); + assert.deepEqual(result.issues, ['backup_directory_permissions', 'state_symlink']); + assert.equal(fs.statSync(cache).mode & 0o777, 0o755); + assert(fs.lstatSync(path.join(root, 'state/runtime-link')).isSymbolicLink()); + assert(!JSON.stringify(result).includes('SECRET')); + t.mock.restoreAll(); assert.deepEqual(fs.readFileSync(path.join(root, 'data/private')), before); +}); +test('readiness reports hardlinks, unsafe files, missing roots and bounded scans', t => { + const root = fixture(t), file = path.join(root, 'data/file'); + fs.writeFileSync(file, 'fixture', { mode: 0o600 }); fs.chmodSync(file, 0o644); + fs.linkSync(file, path.join(root, 'data/linked')); + const result = inspectInstallation(root, process.geteuid()); + assert(result.issues.includes('backup_file_permissions')); assert(result.issues.includes('backup_hardlink')); + assert(inspectInstallation(root, process.geteuid(), { maxEntries: 1 }).issues.includes('inspection_limit')); + fs.rmSync(path.join(root, 'state'), { recursive: true }); + assert(inspectInstallation(root, process.geteuid()).issues.includes('tree_changed_or_missing')); +}); +test('readiness refuses links outside the installation and detects insufficient space', t => { + const root = fixture(t); + fs.rmSync(path.join(root, 'state'), { recursive: true }); fs.symlinkSync('/etc', path.join(root, 'state')); + assert.deepEqual(inspectInstallation(root, process.geteuid()).issues, ['state_symlink']); + t.mock.method(fs, 'statfsSync', () => ({ bavail: 0, bsize: 4096 })); + assert(inspectInstallation(root, process.geteuid()).issues.includes('backup_insufficient_space')); +}); +test('receipt is bound to the local root, trusted owner and freshness', t => { + const root = fixture(t), file = path.join(root, 'receipt.json'); + const value = { schemaVersion: 1, localRootHash: localIdentity(root), checkedAt: 1000, status: 'ready', members: [] }; + fs.writeFileSync(file, JSON.stringify(value), { mode: 0o644 }); + const options = { file, uid: process.geteuid(), now: () => 2000 }; + assert.equal(readReadiness(root, options).status, 'ready'); + assert.equal(readReadiness(root, { ...options, now: () => 200000 }).status, 'stale'); + assert.throws(() => readReadiness('/another-host', options), /invalid/); + fs.chmodSync(file, 0o666); assert.throws(() => readReadiness(root, options)); +}); diff --git a/core/core/installations/tests/backup-recovery.test.js b/core/core/installations/tests/backup-recovery.test.js new file mode 100644 index 0000000..a392f35 --- /dev/null +++ b/core/core/installations/tests/backup-recovery.test.js @@ -0,0 +1,75 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const test = require('node:test'); +const { createInstallationBackupManager } = require('../src/backups'); + +for (const full of [false, true]) for (const boundary of [...Array.from({length: full ? 8 : 4}, (_, i) => i + 1), 'committed']) { + test(`restore format ${full ? 2 : 1} recovers after process death at durable boundary ${boundary}`, t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-restore-crash-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const layout = { installationRoot: root, directories: { + dataRoot: path.join(root, 'data'), stateRoot: path.join(root, 'state'), backupsRoot: path.join(root, 'backups'), + } }; + if (full) Object.assign(layout.directories, {configRoot:path.join(root,'config'),authSecretsRoot:path.join(root,'secrets/auth-broker')}); + const labels = full ? ['data','state','config','secrets/auth-broker'] : ['data','state']; + for (const directory of Object.values(layout.directories)) fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + for (const label of labels) fs.writeFileSync(path.join(root, label, 'value'), `backup-${label}`, { mode: 0o600 }); + const manager = createInstallationBackupManager({ layout, full }); + const spec = { id: 'backup_crash', purpose: 'manual', manifestRevision: 1, releaseId: 'release_crash', status: 'reserved' }; + const captured = manager.snapshot(spec, callback => callback()); + const source = { ...spec, status: 'available', treeDigest: captured.treeDigest, + fileCount: captured.fileCount, totalBytes: captured.totalBytes }; + for (const label of labels) fs.writeFileSync(path.join(root, label, 'value'), `prior-${label}`); + const input = path.join(root, 'input.json'); + fs.writeFileSync(input, JSON.stringify({ layout, source, boundary, full }), { mode: 0o600 }); + const child = spawnSync(process.execPath, ['--no-warnings', '-e', ` + const fs = require('node:fs'); + const { layout, source, boundary, full } = JSON.parse(fs.readFileSync(process.argv[1])); + const originalRename = fs.renameSync, originalSync = fs.fsyncSync; + let renames = 0; + fs.renameSync = (...args) => { + const value = originalRename(...args); + if (++renames === boundary) process.kill(process.pid, 'SIGKILL'); + return value; + }; + fs.fsyncSync = fd => { + const value = originalSync(fd); + if (boundary === 'committed' && fs.readlinkSync('/proc/self/fd/' + fd).endsWith('/committed.json')) { + process.kill(process.pid, 'SIGKILL'); + } + return value; + }; + require(process.argv[2]).createInstallationBackupManager({ layout, full }) + .restore(source, 'operation_crash', callback => callback()); + `, input, path.resolve(__dirname, "../src/backups.js")], { encoding: 'utf8', timeout: 15_000 }); + assert.equal(child.signal, 'SIGKILL', child.stderr); + assert.equal(manager.restore(source, 'operation_crash', callback => callback()).status, 'restored'); + for (const label of labels) assert.equal(fs.readFileSync(path.join(root, label, 'value'), 'utf8'), `backup-${label}`); + assert.equal(fs.readdirSync(layout.directories.backupsRoot).some(name => name.startsWith('.restore-')), false); + assert.equal(manager.inspectRestored(source).status, 'verified'); + }); +} + +test('complete DSP snapshot restores configuration and credential encryption keys with data, while keeping the live registration token',t=>{ + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-full-restore-'));fs.chmodSync(root,0o700);t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + const directories={dataRoot:path.join(root,'data'),stateRoot:path.join(root,'state'),configRoot:path.join(root,'config'),authSecretsRoot:path.join(root,'secrets/auth-broker'),backupsRoot:path.join(root,'backups')}; + fs.mkdirSync(path.join(root,'secrets'),{mode:0o700}); + for(const dir of Object.values(directories))fs.mkdirSync(dir,{mode:0o700}); + const liveToken=path.join(root,'secrets/registration-token');fs.writeFileSync(liveToken,'current-registration-token',{mode:0o600}); + for(const dir of Object.values(directories).filter(d=>d!==directories.backupsRoot))fs.writeFileSync(path.join(dir,'record'),'original',{mode:0o600}); + const manager=createInstallationBackupManager({layout:{installationRoot:root,directories},full:true}); + const spec={id:'backup_complete',purpose:'manual',manifestRevision:1,releaseId:'release_full',status:'reserved'}; + const captured=manager.snapshot(spec,cb=>cb()),source={...spec,status:'available',treeDigest:captured.treeDigest,fileCount:captured.fileCount,totalBytes:captured.totalBytes}; + const snapshot=path.join(directories.backupsRoot,spec.id),manifest=JSON.parse(fs.readFileSync(path.join(snapshot,'manifest.json'))); + assert.equal(manifest.version,2);assert.equal(manifest.entries.some(e=>e.path==='auth-secrets/record'),true); + assert.equal(require('../src/offsite-backup').verifySnapshot(snapshot,process.geteuid()).digest,captured.treeDigest); + for(const dir of Object.values(directories).filter(d=>d!==directories.backupsRoot))fs.writeFileSync(path.join(dir,'record'),'changed',{mode:0o600}); + manager.restore(source,'restore_complete',cb=>cb()); + assert.equal(manager.inspectRestored(source).status,'verified'); + for(const dir of Object.values(directories).filter(d=>d!==directories.backupsRoot))assert.equal(fs.readFileSync(path.join(dir,'record'),'utf8'),'original'); + assert.equal(fs.readFileSync(liveToken,'utf8'),'current-registration-token'); +}); diff --git a/core/core/installations/tests/backup-scratch.test.js b/core/core/installations/tests/backup-scratch.test.js new file mode 100644 index 0000000..76da9d9 --- /dev/null +++ b/core/core/installations/tests/backup-scratch.test.js @@ -0,0 +1,33 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const { cleanupBackupScratch } = require('../src/backup-scratch'); +test('worker restart erases interrupted transfer history and preserves the archive catalog', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-scratch-test-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + fs.mkdirSync(path.join(root, 'archives'), { mode: 0o700 }); + for (const prefix of ['archive-transfer-', 'archive-restore-', 'rediscover-', 'transfer-', 'canary-']) { + const dir = fs.mkdtempSync(path.join(root, prefix)); + fs.writeFileSync(path.join(dir, 'tenant-secret'), 'synthetic history'); + } + cleanupBackupScratch(root, process.geteuid()); + assert.deepEqual(fs.readdirSync(root), ['archives']); + fs.symlinkSync(path.join(root, 'archives'), path.join(root, 'rediscover-abc123')); + assert.throws(() => cleanupBackupScratch(root, process.geteuid()), /unsafe_backup_scratch/); + assert.equal(fs.existsSync(path.join(root, 'archives')), true); +}); + +test('worker restart removes private abandoned imports without following links', t => { + const { cleanupRestoreStaging } = require('../src/backup-scratch'); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-import-test-')); + fs.chmodSync(root, 0o711); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const work = fs.mkdtempSync(path.join(root, 'import-')); + fs.writeFileSync(path.join(work, 'tenant-secret'), 'synthetic history'); + cleanupRestoreStaging(root, process.geteuid()); + assert.deepEqual(fs.readdirSync(root), []); + fs.mkdirSync(path.join(root, 'unrelated'), { mode: 0o700 }); + fs.symlinkSync(path.join(root, 'unrelated'), path.join(root, 'import-AbC123')); + assert.throws(() => cleanupRestoreStaging(root, process.geteuid()), /unsafe_backup_scratch/); + assert.equal(fs.existsSync(path.join(root, 'unrelated')), true); +}); diff --git a/core/core/installations/tests/backup-storage-usage.test.js b/core/core/installations/tests/backup-storage-usage.test.js new file mode 100644 index 0000000..4459675 --- /dev/null +++ b/core/core/installations/tests/backup-storage-usage.test.js @@ -0,0 +1,127 @@ +"use strict"; +const test = require("node:test"), + assert = require("node:assert/strict"), + fs = require("node:fs"), + os = require("node:os"), + path = require("node:path"); +const { + summarize, + measureStorageUsage, +} = require("../src/backup-storage-usage"); +test("storage counts each physical archive once, including retained DSPs and partial deletion, while sets reuse component totals", () => { + const records = [ + { id: "core", kind: "core" }, + { id: "one", kind: "dsp", organization_id: "org_one" }, + { + id: "removed", + kind: "dsp", + organization_id: "org_removed", + deleted_at: 123, + }, + ]; + const sets = [ + { + id: "set1", + members_json: JSON.stringify([ + { backupId: "core" }, + { backupId: "one" }, + { backupId: "one" }, + ]), + }, + { + id: "set2", + members_json: JSON.stringify([ + { backupId: "core" }, + { backupId: "removed" }, + ]), + }, + ]; + const result = summarize( + { + archives: { core: 100, one: 200, removed: 300, unassigned: 50 }, + sets: { set1: 10, set2: 20 }, + legacyBytes: 40, + }, + records, + sets, + ); + assert.equal(result.bytes, 720); + assert.equal(result.backupCount, 4); + assert.deepEqual(result.core, { bytes: 100, backupCount: 1 }); + assert.equal( + result.dsps.find((s) => s.organizationId === "org_removed").bytes, + 300, + ); + assert.deepEqual(result.other, { bytes: 50, backupCount: 1 }); + assert.equal(result.sets[0].bytes, 310); + assert.equal(result.sets[0].backupCount, 2); + assert.equal(result.sets[1].bytes, 420); + const remaining = summarize( + { archives: { core: 100, one: 200 }, sets: { set1: 10 }, legacyBytes: 0 }, + records, + sets, + ); + assert.equal(remaining.bytes, 310); + assert.equal( + remaining.dsps.some((s) => s.organizationId === "org_removed"), + false, + ); +}); +test("storage totals include archives beyond the dashboard history limit", () => { + const records = Array.from({ length: 1205 }, (_, i) => ({ + id: "id_" + i, + kind: "dsp", + organization_id: "org_one", + })); + const result = summarize( + { + archives: Object.fromEntries(records.map((r) => [r.id, 10])), + sets: {}, + legacyBytes: 0, + }, + records, + [], + ); + assert.equal(result.bytes, 12050); + assert.equal(result.dsps[0].backupCount, 1205); +}); +test("usage caches read-only scans, refreshes on changes, and preserves explicitly stale measurements on failure", async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "dispatch-usage-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + let calls = 0, + now = 1000, + broken = false; + const options = { + config: { accountId: "a", bucket: "fixture" }, + workRoot: root, + ownerUid: process.geteuid(), + clock: () => now, + records: [], + sets: [], + version: 1, + storage: { + usage: async () => { + calls++; + if (broken) throw Error("offline"); + return { archives: {}, sets: {}, legacyBytes: 42 }; + }, + }, + }; + assert.equal((await measureStorageUsage(options)).bytes, 42); + await measureStorageUsage(options); + assert.equal(calls, 1); + options.version = 2; + await measureStorageUsage(options); + assert.equal(calls, 2); + now += 300001; + broken = true; + const stale = await measureStorageUsage(options); + assert.equal(stale.status, "stale"); + assert.equal(stale.bytes, 42); + assert.equal(stale.checkedAt, 1000); + options.config.bucket = "another-bucket"; + assert.deepEqual(await measureStorageUsage(options), { + status: "unavailable", + checkedAt: null, + }); +}); diff --git a/core/core/installations/tests/core-backup-files.test.js b/core/core/installations/tests/core-backup-files.test.js new file mode 100644 index 0000000..faefa8b --- /dev/null +++ b/core/core/installations/tests/core-backup-files.test.js @@ -0,0 +1,36 @@ +'use strict'; +const test=require('node:test'),assert=require('node:assert/strict'),fs=require('node:fs'),path=require('node:path'),os=require('node:os'); +const files=require('../src/core-backup-files'); +test('Core configuration and tunnel credentials restore without touching DSP registration secrets',t=>{ + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-core-files-'));t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + const local=path.join(root,'local'),source=path.join(root,'source'); + for(const dir of ['config','secrets/cloudflared','secrets/oci-runtime-agents'])fs.mkdirSync(path.join(local,dir),{recursive:true,mode:0o700}); + fs.writeFileSync(path.join(local,'config/dashboard.env'),'PORT=4100\n');fs.writeFileSync(path.join(local,'secrets/cloudflared/tunnel.json'),'{"fixture":"old-core-secret"}\n');fs.writeFileSync(path.join(local,'secrets/oci-runtime-agents/runtime_dsp.token'),'DSP-secret'); + files.capture(local,source);assert.equal(fs.existsSync(path.join(source,'secrets/oci-runtime-agents')),false); + fs.writeFileSync(path.join(local,'config/dashboard.env'),'PORT=4200\n');fs.writeFileSync(path.join(local,'secrets/cloudflared/tunnel.json'),'{"fixture":"changed"}');files.restore(local,source); + assert.equal(fs.readFileSync(path.join(local,'config/dashboard.env'),'utf8'),'PORT=4100\n');assert.equal(fs.readFileSync(path.join(local,'secrets/cloudflared/tunnel.json'),'utf8'),'{"fixture":"old-core-secret"}\n');assert.equal(fs.readFileSync(path.join(local,'secrets/oci-runtime-agents/runtime_dsp.token'),'utf8'),'DSP-secret'); +}); +test('Core restore removes newer Core files and recovery uses the selected snapshot contents',t=>{ + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-core-files-'));t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + const local=path.join(root,'local'),snapshot=path.join(root,'snapshot'),source=path.join(snapshot,'core-files'); + fs.mkdirSync(path.join(local,'config'),{recursive:true});fs.mkdirSync(path.join(local,'secrets/cloudflared'),{recursive:true}); + fs.writeFileSync(path.join(local,'config/dashboard.env'),'saved');files.capture(local,source); + fs.writeFileSync(path.join(local,'config/dashboard.env'),'newer');fs.writeFileSync(path.join(local,'config/provisioning.env'),'newer');fs.writeFileSync(path.join(local,'secrets/cloudflared/new.json'),'newer'); + const roots=files.recoveryFileRoots(local,snapshot); + assert.equal(roots.length,1);assert.equal(roots[0].target,path.join(local,'config/dashboard.env'));assert.equal(fs.readFileSync(roots[0].source,'utf8'),'saved'); + files.restore(local,source);assert.equal(fs.readFileSync(path.join(local,'config/dashboard.env'),'utf8'),'saved');assert.equal(fs.existsSync(path.join(local,'config/provisioning.env')),false);assert.equal(fs.existsSync(path.join(local,'secrets/cloudflared/new.json')),false); +}); + +test('Turnstile secret is restored with its private directory and removed when absent from the selected backup',t=>{ + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-turnstile-backup-'));t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + const local=path.join(root,'local'),source=path.join(root,'source'),empty=path.join(root,'empty'); + const secret=path.join(local,'secrets/turnstile/secret-key'); + fs.mkdirSync(path.dirname(secret),{recursive:true,mode:0o700}); + fs.writeFileSync(secret,'fixture-turnstile-secret',{mode:0o600}); + files.capture(local,source);fs.rmSync(path.dirname(secret),{recursive:true}); + files.restore(local,source); + assert.equal(fs.readFileSync(secret,'utf8'),'fixture-turnstile-secret'); + assert.equal(fs.statSync(secret).mode & 0o777,0o600); + assert.equal(fs.statSync(path.dirname(secret)).mode & 0o777,0o700); + files.restore(local,empty);assert.equal(fs.existsSync(secret),false); +}); diff --git a/core/core/installations/tests/core-recovery-host.test.js b/core/core/installations/tests/core-recovery-host.test.js new file mode 100644 index 0000000..3e055c9 --- /dev/null +++ b/core/core/installations/tests/core-recovery-host.test.js @@ -0,0 +1,130 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { AccessStore } = require('../../accounts/src/store'); +const { SCHEMA_VERSION } = require('../../accounts/src/schema'); +const { createCoreRecovery } = require('../src/core-recovery'); +const { createHostRecovery } = require('../src/core-recovery-host'); +const { atomic } = require('../src/release-delivery-files'); +function setup(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-host-recovery-')); fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const localRoot = path.join(root, 'local'), unitRoot = path.join(root, 'units'); + for (const dir of [localRoot, unitRoot, path.join(localRoot, 'config'), path.join(localRoot, 'config/systemd'), path.join(localRoot, 'config/systemd/user')]) fs.mkdirSync(dir, { mode: 0o700 }); + fs.mkdirSync(path.join(localRoot, 'data'), { mode: 0o700 }); + const dbRoot = path.join(localRoot, 'data/access-control'); + const store = new AccessStore({ databaseRoot: dbRoot, database: path.join(dbRoot, 'access-control.sqlite3') }); + t.after(() => store.close()); store.db.exec("CREATE TABLE business(value TEXT); INSERT INTO business VALUES('prior work')"); + const config = { localRoot, unitRoot, releaseId: 'dispatch_candidate', version: '9.0.0', sourceCommit: 'a'.repeat(40), port: 4310, publicOrigin: 'https://dispatch.example.test' }; + const previous = { ...config, releaseId: 'dispatch_previous', version: '8.0.0', sourceCommit: 'b'.repeat(40) }; + const artifacts = {}; + const descriptor = c => ({ version: 2, backend: 'oci_container_v1', releaseId: c.releaseId, channel: 'production', + image: 'ghcr.io/example-organization/dispatch-runtime@sha256:' + 'a'.repeat(64), imageDigest: 'sha256:' + 'a'.repeat(64), imageId: 'c'.repeat(64), + sourceCommit: c.sourceCommit, platform: 'linux/amd64', runtimeAgentProtocol: 1, runtimeGatewayProtocol: 1, + embeddedManifestSha256: 'd'.repeat(64), imageArchiveSha256: 'e'.repeat(64), bridgeManifestSha256: 'f'.repeat(64) }); + atomic(path.join(localRoot, 'config/oci-releases.json'), { schemaVersion: 1, releases: { [config.releaseId]: descriptor(config), [previous.releaseId]: descriptor(previous) } }); + for (const c of [config, previous]) { + const dir = path.join(root, c.releaseId); artifacts[c.releaseId] = { root: dir, config: c }; + fs.mkdirSync(path.join(dir, 'code/core/accounts/src'), { recursive: true, mode: 0o700 }); + fs.mkdirSync(path.join(dir, 'units'), { mode: 0o700 }); + fs.writeFileSync(path.join(dir, 'code/core/accounts/src/schema.js'), 'exports.SCHEMA_VERSION=' + SCHEMA_VERSION); + fs.writeFileSync(path.join(dir, 'code/core/accounts/src/store.js'), 'module.exports=require(' + JSON.stringify(require.resolve('../../accounts/src/store')) + ')'); + for (const name of ['dispatch-dashboard.service', 'dispatch-installation-reconcile.service']) { + const content = '[Service]\nExecStart=' + dir + '/code/fixture\n'; + fs.writeFileSync(path.join(dir, 'units', name), content); + if (c === previous) atomic(path.join(unitRoot, name), content); + } + } + let current = previous, running = true, broken = false; + const commands = [], sleeps = []; + const healthCheck = async (expected, nonce) => { + assert.equal(running, true); assert.equal(expected.releaseId, current.releaseId); + if (nonce) assert.equal(JSON.parse(fs.readFileSync(path.join(localRoot, 'config/core-maintenance.json'))).nonce, nonce); + if (broken && current === config) throw Error('candidate unhealthy'); + }; + const adapter = createHostRecovery(config, artifacts[config.releaseId].root, { + offsitePolicy: { assertOffsiteReady() {}, async waitForOffsiteBackup() {} }, + artifact: id => artifacts[id], sleep: async ms => sleeps.push(ms), + health: { read: async () => current, wait: healthCheck, assert: healthCheck }, + switchHost: (dir, check) => { if (!check) current = Object.values(artifacts).find(a => a.root === dir).config; }, + command: args => { + commands.push(args); + if (args[0] === 'show') return args[1].endsWith('.timer') ? 'active' : 'inactive'; + if (args[0] === 'is-active') return running ? 'active' : 'inactive'; + if (args[1] === 'dispatch-dashboard.service') { + running = args[0] === 'start'; + if (running && current === config) store.db.exec("UPDATE business SET value='candidate work'"); + } + return ''; + }, + }); + const context = { rolloutId: 'rollout_' + 'a'.repeat(32), releaseId: config.releaseId, attempt: 1 }; + return { store, config, previous, artifacts, adapter, commands, sleeps, context, localRoot, + recovery: () => createCoreRecovery({ localRoot, context, adapter }), breakHealth: () => broken = true, + state: () => ({ current: current.releaseId, running }) }; +} +test('production recovery adapter snapshots, arms the independent timer, restores units and SQLite after failed verification', async t => { + const f = setup(t); + await f.recovery().apply(); + assert.equal(f.state().current, 'dispatch_candidate'); + assert.match(fs.readFileSync(path.join(f.config.unitRoot, 'dispatch-core-recovery.service'), 'utf8'), /dispatch-core-recover watch/); + assert.equal(f.commands.some(a => a.join(' ') === 'enable --now dispatch-core-recovery.timer'), true); + f.breakHealth(); await assert.rejects(f.recovery().verify()); + assert.deepEqual(f.state(), { current: 'dispatch_previous', running: true }); + assert.equal(f.store.db.prepare('SELECT value FROM business').get().value, 'prior work'); + assert.match(fs.readFileSync(path.join(f.config.unitRoot, 'dispatch-dashboard.service'), 'utf8'), /dispatch_previous\/code/); + assert.equal(fs.existsSync(path.join(f.localRoot, 'config/core-maintenance.json')), false); + // Recovery leaves the watchdog armed until the CLI durably pauses the rollout. + assert.equal(f.commands.some(a => a.join(' ') === 'disable --now dispatch-core-recovery.timer'), false); +}); +test('production verification observes health before promotion and disarms the watchdog after opening traffic', async t => { + const f = setup(t); await f.recovery().apply(); await f.recovery().verify(); + assert.deepEqual(f.sleeps, [5000, 5000, 5000]); + assert.equal(f.recovery().view().phase, 'promoted'); + assert.equal(f.commands.some(a => a.join(' ') === 'disable --now dispatch-core-recovery.timer'), true); +}); +test('preflight refuses a schema transition without stopping the previous Core', async t => { + const f = setup(t); + fs.writeFileSync(path.join(f.artifacts.dispatch_candidate.root, 'code/core/accounts/src/schema.js'), 'exports.SCHEMA_VERSION=9999'); + await assert.rejects(f.recovery().apply(), { code: 'core_schema_transition_requires_review' }); + assert.equal(f.commands.some(a => ['stop', 'start', 'enable'].includes(a[0])), false); + assert.deepEqual(f.state(), { current: 'dispatch_previous', running: true }); +}); + +test('first native update preflight refuses a legacy fleet before stopping old Core', async t => { + const f = setup(t); + const file = path.join(f.localRoot, 'config/oci-releases.json'); + const catalog = JSON.parse(fs.readFileSync(file)); + catalog.releases[f.config.releaseId] = { version: 1, backend: 'native_service_v1', releaseId: f.config.releaseId, + channel: 'production', sourceCommit: f.config.sourceCommit, platform: 'linux/amd64', runtimeAgentProtocol: 1, + runtimeGatewayProtocol: 1, artifactSha256: 'a'.repeat(64), embeddedManifestSha256: 'b'.repeat(64), bridgeManifestSha256: 'c'.repeat(64) }; + atomic(file, catalog); + f.store.db.exec(`INSERT INTO organizations(id,name,abbreviation,timezone,status,created_at,updated_at) + VALUES('legacy','Legacy','LEG','UTC','active',1,1); + INSERT INTO installations(organization_id,runtime_key,status,backend,revision,manifest_revision,created_at,updated_at) + VALUES('legacy','runtime_legacy','ready','oci_container_v1',1,1,1,1);`); + await assert.rejects(f.recovery().apply(), { code: 'native_migration_required' }); + assert.deepEqual(f.commands, []); + assert.equal(fs.existsSync(path.join(f.localRoot, 'config/core-maintenance.json')), false); + assert.deepEqual(f.state(), { current: 'dispatch_previous', running: true }); + f.store.db.prepare("UPDATE installations SET status='decommissioned' WHERE organization_id='legacy'").run(); + await f.recovery().apply(); + await f.recovery().verify(); + assert.deepEqual(f.state(), { current: 'dispatch_candidate', running: true }); +}); + +test('shared rollout backup avoids a second offsite snapshot while local migration rollback remains recoverable', async t => { + const f = setup(t), db = f.store.db; + db.prepare("INSERT INTO platform_rollout_backups VALUES(?, 'breq_shared')").run(f.context.rolloutId); + db.prepare("INSERT INTO backup_sets VALUES('breq_shared',1,?,'verified')").run(JSON.stringify([{ organizationId: null, requestId: 'breq_core' }])); + db.prepare("INSERT INTO platform_backup_requests VALUES('breq_core',NULL,'core','completed','completed',NULL,'{}',NULL,'shared:core',1,1,NULL)").run(); + await f.recovery().apply(); + const manifest = JSON.parse(fs.readFileSync(path.join(f.localRoot, 'backups/platform-core', f.context.rolloutId, 'attempt-1/manifest.json'))); + assert.equal(manifest.localOnly, true); + f.breakHealth(); await assert.rejects(f.recovery().verify()); + assert.equal(f.store.db.prepare('SELECT value FROM business').get().value, 'prior work'); + assert.equal(f.recovery().view().phase, 'recovered'); +}); diff --git a/core/core/installations/tests/core-recovery.test.js b/core/core/installations/tests/core-recovery.test.js new file mode 100644 index 0000000..2f87731 --- /dev/null +++ b/core/core/installations/tests/core-recovery.test.js @@ -0,0 +1,132 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const test = require('node:test'); +const { spawnSync } = require('node:child_process'); +const { AccessStore } = require('../../accounts/src/store'); +const { createCoreRecovery } = require('../src/core-recovery'); +const { snapshotDatabase, restoreDatabase } = require('../src/core-recovery-host'); +const { atomic } = require('../src/release-delivery-files'); +const context = { rolloutId: 'rollout_' + 'a'.repeat(32), releaseId: 'dispatch_9.0.0', attempt: 1 }; +function fixture(t, suppliedRoot) { + const localRoot = suppliedRoot || fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-core-recovery-')); + fs.chmodSync(localRoot, 0o700); + const dbRoot = path.join(localRoot, 'access'), database = path.join(dbRoot, 'access-control.sqlite3'); + const store = new AccessStore({ databaseRoot: dbRoot, database }); + store.db.exec("CREATE TABLE IF NOT EXISTS business_data(id TEXT PRIMARY KEY,value TEXT) STRICT; INSERT OR IGNORE INTO business_data VALUES('important','previous data');"); + const calls = []; let broken = null, crash = null; + const service = path.join(localRoot, 'service.json'), gate = path.join(localRoot, 'maintenance'); + if (!fs.existsSync(service)) atomic(service, { version: 'old', running: true }); + const read = () => JSON.parse(fs.readFileSync(service)); + const step = async (name, action) => { + calls.push(name); + if (broken === name) throw Error(name); + const result = await action(); + if (crash === name) process.kill(process.pid, 'SIGKILL'); + return result; + }; + const adapter = { + preflight: () => step('preflight', () => ({ version: '8.0.0', timerWasActive: true })), + enterMaintenance: () => step('enterMaintenance', () => atomic(gate, 'closed')), + drain: () => step('drain', () => {}), + stopCandidate: () => step('stopCandidate', () => atomic(service, { ...read(), running: false })), + snapshot: dir => step('snapshot', () => snapshotDatabase(database, dir)), + verifyOffsite: () => step('verifyOffsite', () => {}), + installCandidate: () => step('installCandidate', () => atomic(service, { version: 'new', running: false })), + startCandidate: () => step('startCandidate', () => { + store.db.exec("UPDATE business_data SET value='candidate change'; CREATE TABLE IF NOT EXISTS migration_fixture(id TEXT) STRICT;"); + atomic(service, { version: 'new', running: true }); + }), + verifyCandidate: () => step('verifyCandidate', () => assert.equal(fs.existsSync(gate), true)), + restore: (dir, receipt) => step('restore', () => restoreDatabase(database, dir, receipt)), + restoreServices: () => step('restoreServices', () => atomic(service, { version: 'old', running: false })), + startPrior: () => step('startPrior', () => atomic(service, { version: 'old', running: true })), + verifyPrior: () => step('verifyPrior', () => assert.deepEqual(read(), { version: 'old', running: true })), + releaseMaintenance: () => step('releaseMaintenance', () => fs.rmSync(gate, { force: true })), + restoreScheduling: () => step('restoreScheduling', () => {}), + verifyPromoted: () => step('verifyPromoted', () => assert.equal(read().version, 'new')), + }; + const recovery = (attempt = 1) => createCoreRecovery({ localRoot, context: { ...context, attempt }, adapter }); + if (t) t.after(() => { store.close(); fs.rmSync(localRoot, { recursive: true, force: true }); }); + return { localRoot, recovery, store, adapter, calls, read, gate, breakAt: name => broken = name, crashAt: name => crash = name }; +} +if (process.env.DISPATCH_RECOVERY_CRASH_FIXTURE) { + const f = fixture(null, process.env.DISPATCH_RECOVERY_CRASH_FIXTURE); + f.crashAt(process.env.DISPATCH_RECOVERY_CRASH_AT); + const r = f.recovery(); + (async () => { if (process.env.DISPATCH_RECOVERY_CRASH_MODE === 'verify') await r.verify(); else if (process.env.DISPATCH_RECOVERY_CRASH_MODE === 'recover') await r.recover(); else await r.apply(); })().catch(() => process.exit(1)); +} else { + for (const stage of ['drain', 'snapshot', 'verifyOffsite', 'installCandidate', 'startCandidate', 'verifyCandidate']) { + test(`Core failure at ${stage} restores the prior service and preserves business data`, async t => { + const f = fixture(t); f.breakAt(stage); const recovery = f.recovery(); + await assert.rejects(async () => { await recovery.apply(); await recovery.verify(); }); + assert.equal(f.recovery().view().phase, 'recovered'); + assert.deepEqual(f.read(), { version: 'old', running: true }); + assert.equal(f.store.db.prepare('SELECT value FROM business_data').get().value, 'previous data'); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM sqlite_master WHERE name='migration_fixture'").get().n, 0); + assert.equal(fs.existsSync(f.gate), false); + }); + } + test('failed preflight leaves the working service untouched', async t => { + const f = fixture(t); f.breakAt('preflight'); await assert.rejects(f.recovery().apply()); + assert.deepEqual(f.calls, ['preflight']); assert.equal(f.recovery().view(), null); assert.equal(f.read().running, true); + }); + test('a verified promotion cannot roll back over newly accepted writes', async t => { + const f = fixture(t); await f.recovery().apply(); await f.recovery().verify(); + f.store.db.exec("UPDATE business_data SET value='new customer work'"); + await assert.rejects(f.recovery().recover(), { code: 'core_already_promoted' }); + await f.recovery(2).apply(); await f.recovery(2).verify(); + assert.equal(f.store.db.prepare('SELECT value FROM business_data').get().value, 'new customer work'); + assert.equal(f.calls.includes('restore'), false); + }); + test('a recovery retry after the old service starts never restores the database again', async t => { + const f = fixture(t); await f.recovery().apply(); f.breakAt('verifyPrior'); + await assert.rejects(f.recovery().recover()); assert.equal(f.recovery().view().phase, 'restored'); + f.store.db.exec("UPDATE business_data SET value='work after recovery'"); + f.breakAt(null); await f.recovery().recover(); + assert.equal(f.calls.filter(x => x === 'restore').length, 1); + assert.equal(f.store.db.prepare('SELECT value FROM business_data').get().value, 'work after recovery'); + }); + test('corrupt backup prevents a destructive restore and leaves traffic closed', async t => { + const f = fixture(t); await f.recovery().apply(); + const file = path.join(f.localRoot, 'backups/platform-core', context.rolloutId, 'attempt-1/access-control-before.sqlite3'); + fs.appendFileSync(file, 'corrupt'); f.breakAt('verifyCandidate'); + await assert.rejects(f.recovery().verify(), { code: 'core_recovery_required' }); + assert.equal(f.read().running, false); assert.equal(fs.existsSync(f.gate), true); + }); + test('a resumed failed rollout gets a fresh backup rather than overwriting the previous attempt', async t => { + const f = fixture(t); await f.recovery().apply(); f.breakAt('verifyCandidate'); await assert.rejects(f.recovery().verify()); + f.store.db.exec("UPDATE business_data SET value='work between attempts'"); + f.breakAt(null); await f.recovery(2).apply(); f.breakAt('verifyCandidate'); await assert.rejects(f.recovery(2).verify()); + assert.equal(f.store.db.prepare('SELECT value FROM business_data').get().value, 'work between attempts'); + assert.equal(fs.existsSync(path.join(f.localRoot, 'backups/platform-core', context.rolloutId, 'attempt-1/access-control-before.sqlite3')), true); + }); + for (const stage of ['enterMaintenance', 'snapshot', 'installCandidate', 'startCandidate']) { + test(`SIGKILL after ${stage} resumes as recovery with no data loss`, async t => { + const f = fixture(t); + const child = spawnSync(process.execPath, ['--no-warnings', __filename], { env: { ...process.env, + DISPATCH_RECOVERY_CRASH_FIXTURE: f.localRoot, DISPATCH_RECOVERY_CRASH_AT: stage }, timeout: 15_000 }); + assert.equal(child.signal, 'SIGKILL', child.stderr.toString()); + await assert.rejects(f.recovery(2).apply(), { code: 'core_interrupted_update_recovered' }); + assert.equal(f.store.db.prepare('SELECT value FROM business_data').get().value, 'previous data'); + assert.deepEqual(f.read(), { version: 'old', running: true }); + }); + } +} + +if (!process.env.DISPATCH_RECOVERY_CRASH_FIXTURE) { + for (const [mode, stage] of [['verify', 'releaseMaintenance'], ['recover', 'startPrior']]) { + test(`SIGKILL during ${mode} after ${stage} cannot rewind work accepted after restart`, async t => { + const f = fixture(t); await f.recovery().apply(); + const child = spawnSync(process.execPath, ['--no-warnings', __filename], { env: { ...process.env, + DISPATCH_RECOVERY_CRASH_FIXTURE: f.localRoot, DISPATCH_RECOVERY_CRASH_AT: stage, + DISPATCH_RECOVERY_CRASH_MODE: mode }, timeout: 15_000 }); + assert.equal(child.signal, 'SIGKILL', child.stderr.toString()); + f.store.db.exec("UPDATE business_data SET value='work accepted after restart'"); + if (mode === 'verify') await f.recovery().verify(); else await f.recovery().recover(); + assert.equal(f.store.db.prepare('SELECT value FROM business_data').get().value, 'work accepted after restart'); + }); + } +} diff --git a/core/core/installations/tests/create-bridge-artifact.test.js b/core/core/installations/tests/create-bridge-artifact.test.js new file mode 100644 index 0000000..c8755bb --- /dev/null +++ b/core/core/installations/tests/create-bridge-artifact.test.js @@ -0,0 +1,81 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const test = require('node:test'); + +test('bridge release artifact is a deterministic read-only allowlist with a content manifest', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-bridge-release-')); + const target = path.join(root, 'bridge-artifact'); + t.after(() => { + if (fs.existsSync(target)) { + const makeWritable = directory => { + fs.chmodSync(directory, 0o700); + for (const name of fs.readdirSync(directory)) { + const child = path.join(directory, name); + if (fs.lstatSync(child).isDirectory()) makeWritable(child); + } + }; + makeWritable(target); + } + fs.rmSync(root, { recursive: true, force: true }); + }); + const result = spawnSync('/usr/bin/node', [ + '--no-warnings', path.resolve(__dirname, "../src/create-bridge-artifact.js"), target, + ], { encoding: 'utf8', timeout: 30_000 }); + assert.equal(result.status, 0, result.stderr); + const receipt = JSON.parse(result.stdout.trim()); + const loaded = spawnSync(process.execPath, ['--no-warnings', '-e', + `require(${JSON.stringify(path.join(target, 'core/agent-bridge/src/bridge.js'))})`], { encoding: 'utf8' }); + assert.equal(loaded.status, 0, loaded.stderr); + + const manifestFile = path.join(target, 'manifest.json'); + const manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf8')); + assert.equal(receipt.manifestSha256, + crypto.createHash('sha256').update(fs.readFileSync(manifestFile)).digest('hex')); + assert.equal(receipt.files, manifest.files.length); + assert.equal(fs.lstatSync(target).mode & 0o7777, 0o555); + for (const entry of manifest.files) { + const file = path.join(target, entry.path); + assert.equal(fs.lstatSync(file).mode & 0o7777, 0o444); + assert.equal(crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'), entry.sha256); + } +}); + +test('privileged host-helper artifact contains only its explicit immutable dependency closure', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-host-helper-release-')); + const target = path.join(root, 'host-helper-artifact'); + t.after(() => { + if (fs.existsSync(target)) { + const makeWritable = directory => { + fs.chmodSync(directory, 0o700); + for (const name of fs.readdirSync(directory)) { + const child = path.join(directory, name); + if (fs.lstatSync(child).isDirectory()) makeWritable(child); + } + }; + makeWritable(target); + } + fs.rmSync(root, { recursive: true, force: true }); + }); + const result = spawnSync('/usr/bin/node', [ + '--no-warnings', path.resolve(__dirname, "../src/create-host-helper-artifact.js"), target, + ], { encoding: 'utf8', timeout: 30_000 }); + assert.equal(result.status, 0, result.stderr); + const receipt = JSON.parse(result.stdout.trim()); + const loaded = spawnSync(process.execPath, ['--no-warnings', '-e', + `require(${JSON.stringify(path.join(target, 'core/installations/src/oci-host-helper.js'))}); require(${JSON.stringify(path.join(target, 'core/installations/src/oci-host-issuer.js'))});`], { encoding: 'utf8' }); + assert.equal(loaded.status, 0, loaded.stderr); + + const manifest = JSON.parse(fs.readFileSync(path.join(target, 'manifest.json'), 'utf8')); + assert.equal(receipt.files, manifest.files.length); + assert.equal(manifest.files.some(file => file.path.includes('access-control')), false); + assert.equal(manifest.files.some(file => file.path.includes('interfaces/')), false); + for (const entry of manifest.files) { + assert.equal((fs.lstatSync(path.join(target, entry.path)).mode & 0o7777).toString(8), entry.mode); + } +}); diff --git a/core/core/installations/tests/diagnostics.test.js b/core/core/installations/tests/diagnostics.test.js new file mode 100644 index 0000000..0f343cf --- /dev/null +++ b/core/core/installations/tests/diagnostics.test.js @@ -0,0 +1,136 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const os = require('node:os'); +const { DatabaseSync } = require('node:sqlite'); +const { AccessStore, AccessControlService } = require('../../accounts/src'); +const { SCHEMA_VERSION } = require('../../accounts/src/schema'); +const { createDiagnosticsWorker } = require('../src/diagnostics-worker'); +const { createDiagnosticsSeed } = require('dispatch-dsp/runtime/supervisor/src/diagnostics-seed.js'); +const { success } = require('../../../shared/contracts/src/result'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { PaycomStore } = require('dispatch-dsp/plugins/paycom/backend/src/store.js'); + +async function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-diagnostics-')); fs.chmodSync(root, 0o700); + const paths = { databaseRoot: root + '/access', database: root + '/access/access-control.sqlite3' }; + const store = new AccessStore(paths); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const access = new AccessControlService(store, { installationOperatorEnabled: true, installationBackend: 'native_service_v1' }); + const invitation = access.createPlatformBootstrap({ email: 'platform@example.test' }); + const owner = await access.acceptNewUser({ token: invitation.token, firstName: 'Platform', lastName: 'Owner', + password: 'test platform password', confirmPassword: 'test platform password' }); + return { root, store, access, owner, paths }; +} + +test('schema 11 upgrade adds diagnostics without changing existing users or organizations', async t => { + const f = await fixture(t); + const users = f.store.db.prepare('SELECT * FROM users').all(); + f.store.db.exec('DROP TABLE diagnostic_dsps; PRAGMA user_version=11'); + f.store.close(); + const upgraded = new AccessStore(f.paths); + try { + assert.equal(upgraded.db.prepare('PRAGMA user_version').get().user_version, SCHEMA_VERSION); + assert.deepEqual(upgraded.db.prepare('SELECT * FROM users').all(), users); + assert.equal(upgraded.db.prepare('SELECT count(*) AS n FROM diagnostic_dsps').get().n, 0); + } finally { upgraded.close(); } +}); + +test('diagnostic creation is owner-only, atomic, idempotent and read-only when polling', async t => { + const f = await fixture(t); + assert.throws(() => f.access.platformDiagnostics({ ...f.owner.session, platformPermissions: [] }, {}), /platform_forbidden/); + assert.throws(() => f.access.platformDiagnostics({ ...f.owner.session, user: { ...f.owner.session.user, platformRole: null } }), /platform_forbidden/); + assert.throws(() => f.access.platformDiagnostics(f.owner.session, { idempotencyKey: 'test:diagnostics', organizationId: 'existing' }), /invalid_input/); + const first = f.access.platformDiagnostics(f.owner.session, { idempotencyKey: 'test:diagnostics' }); + const again = f.access.platformDiagnostics(f.owner.session, { idempotencyKey: 'test:diagnostics' }); + assert.deepEqual(first, again); + assert.equal(first.dsps.length, 1); + assert.match(first.dsps[0].name, /^TEST DSP /); + assert.equal(f.store.organizations().length, 1); + const organizationId = f.store.organizations()[0].id; + assert.equal(f.store.activeOwnerCount(organizationId), 1); + assert.equal(f.store.invitations(organizationId)[0].status, 'accepted'); + assert.equal(f.store.db.prepare('SELECT count(*) AS n FROM installation_provisioning_requests').get().n, 1); + const writer = new DatabaseSync(f.paths.database); + try { + writer.exec('BEGIN IMMEDIATE'); + assert.equal(f.access.platformDiagnostics(f.owner.session).dsps.length, 1); + } finally { writer.exec('ROLLBACK'); writer.close(); } + f.store.db.prepare("UPDATE installations SET status='decommissioned' WHERE organization_id=?").run(organizationId); + const authority = require('../../accounts/src/installation-lifecycle').createAccessInstallationLifecycleAuthority({ + store: f.store, organizationId, authorityScope: 'platform_removal', actorUserId: f.owner.session.user.id, destructionEnabled: true, + }); + const job = authority.request({ operation: 'destroy', expectedRevision: f.store.installationControl(organizationId).revision, idempotencyKey: 'test:diagnostic:delete' }); + f.store.db.prepare("UPDATE installation_lifecycle_jobs SET status='succeeded',result_json='{}',finished_at=? WHERE id=?").run(Date.now(), job.id); + f.store.db.prepare("UPDATE installations SET status='decommissioned' WHERE organization_id=?").run(organizationId); + f.store.eraseOrganization(organizationId); + assert.equal(f.access.platformDiagnostics(f.owner.session).dsps.length, 0); + assert.equal(f.store.userById(f.owner.session.user.id).platform_role, 'owner'); +}); + +test('private diagnostic command accepts only a DSP identity and is excluded from SDK capabilities', async () => { + const { validateGatewayRequest } = require('../../../shared/gateway/protocol'); + const request = { protocolVersion: 1, runtimeKey: 'runtime_' + 'a'.repeat(32), action: 'diagnostics.seed', input: { requestId: 'org_' + 'a'.repeat(32) } }; + assert.deepEqual(validateGatewayRequest(request), request); + for (const input of [{}, { requestId: '../existing' }, { ...request.input, command: 'sh' }, { ...request.input, data: {} }]) { + assert.throws(() => validateGatewayRequest({ ...request, input }), /invalid_request/); + } + const client = require('../../../shared/gateway/client').createRuntimeGatewayDispatchClient({ runtimeKey: request.runtimeKey, socketPath: '/tmp/unused.sock' }); + assert.equal(client.capabilities().data.actions.includes('diagnostics.seed'), false); +}); + +test('diagnostics worker records a failed setup and never processes an ordinary DSP', async t => { + const f = await fixture(t); + f.access.platformDiagnostics(f.owner.session, { idempotencyKey: 'test:diagnostics:failure' }); + const ordinary = f.access.createOrganization(f.owner.session, { ownerEmail: 'ordinary@example.test', idempotencyKey: 'test:ordinary:dsp' }); + f.store.db.prepare("UPDATE installations SET status='waiting_for_provider_auth'").run(); + const seen = []; + const worker = createDiagnosticsWorker({ store: f.store, invoke: async () => { throw Error('unused'); }, activate: async options => { + seen.push(options.organizationId); throw Error('test seed failure'); + } }); + assert.equal((await worker.runPending('worker_diagnostics_failure')).failed, 1); + assert.equal(seen.includes(ordinary.organization.id), false); + assert.equal(f.access.platformDiagnostics(f.owner.session).dsps[0].status, 'failed'); +}); + +test('diagnostic worker publishes real synthetic data, activates once and leaves collection stopped', async t => { + const f = await fixture(t); + f.access.platformDiagnostics(f.owner.session, { idempotencyKey: 'test:diagnostics:seed' }); + const organizationId = f.store.organizations()[0].id; + f.store.db.prepare("UPDATE installations SET status='waiting_for_provider_auth',release_id='dispatch_diagnostics_1' WHERE organization_id=?").run(organizationId); + require('../../accounts/src/organization-profile').applyOrganizationProfiles(f.store); + const config = { runtimeKey: `runtime_${organizationId.slice(4)}`, layout: { directories: { stateRoot: f.root + '/state' } }, paths: { + projectRoot: path.resolve(__dirname, "../../.."), + paycom: { database: f.root + '/paycom/paycom.sqlite3', stagingRoot: f.root + '/staging' }, + collection: { databaseRoot: f.root + '/collection', database: f.root + '/collection/collection-manager.sqlite3', stateRoot: f.root + '/collection-state' }, + } }; + for (const directory of ['state', 'paycom', 'staging', 'collection', 'collection-state']) fs.mkdirSync(f.root + '/' + directory, { mode: 0o700 }); + const seed = createDiagnosticsSeed(config); + assert.equal(seed({ requestId: 'org_' + '0'.repeat(32) }).ok, false); + let calls = 0; + const worker = createDiagnosticsWorker({ store: f.store, invoke: async (runtimeKey, action, input) => { + assert.equal(runtimeKey, config.runtimeKey); + if (action === 'health') return success('ready', {}); + assert.equal(action, 'diagnostics.seed'); calls++; + return seed(input); + } }); + const result = await worker.runPending('worker_diagnostics_test'); + assert.equal(result.completed, 1); + assert.equal(f.store.installationControl(organizationId).status, 'ready'); + assert.equal(f.store.installationControl(organizationId).releaseId, 'dispatch_diagnostics_1'); + assert.equal(f.store.organization(organizationId).status, 'active'); + assert.equal((await worker.runPending('worker_diagnostics_replay')).processed, 0); + assert.equal(calls, 1); + const replay = seed({ requestId: organizationId }); + assert.equal(replay.ok, true); + const paycom = new PaycomStore(config.paths.paycom.database); + try { assert.equal(paycom.db.prepare('SELECT count(*) AS n FROM roster_employees').get().n, 2); } finally { paycom.close(); } + const collection = new CollectionStore(config.paths.collection); + try { + assert.equal(collection.sync('paycom-main-workforce').desiredState, 'stopped'); + } finally { collection.close(); } + fs.rmSync(f.root + '/state/diagnostics/seed.json'); + assert.equal(seed({ requestId: organizationId }).status, 'installation_operation_not_allowed'); +}); diff --git a/core/core/installations/tests/directory-backup-files.test.js b/core/core/installations/tests/directory-backup-files.test.js new file mode 100644 index 0000000..e7df82e --- /dev/null +++ b/core/core/installations/tests/directory-backup-files.test.js @@ -0,0 +1,39 @@ +'use strict'; +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const assert = require('node:assert/strict'), test = require('node:test'); +const { scan, clone, clear, copyContents } = require('../../../host/storage/backup-files'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'directory-backup-')); + const source = path.join(root, 'source'); fs.mkdirSync(source, { mode: 0o700 }); + fs.mkdirSync(path.join(source, 'empty'), { mode: 0o700 }); + fs.mkdirSync(path.join(source, 'nested'), { mode: 0o700 }); + fs.writeFileSync(path.join(source, 'nested/data'), 'synthetic preserved data', { mode: 0o600 }); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return { root, source }; +} + +test('private backup copies preserve files and empty directories, and detect changed payloads', t => { + const { root, source } = fixture(t), target = path.join(root, 'backup'); + const receipt = clone(source, target); + assert.equal(receipt.treeDigest, scan(source).treeDigest); + assert.equal(fs.statSync(path.join(target, 'nested/data')).mode & 0o777, 0o600); + clear(source); assert.deepEqual(fs.readdirSync(source), []); + copyContents(target, source); assert.equal(scan(source).treeDigest, receipt.treeDigest); + fs.writeFileSync(path.join(target, 'nested/data'), 'changed'); + assert.notEqual(scan(target).treeDigest, receipt.treeDigest); +}); + +test('backup scans and deletion reject links and shared files before deleting original content', t => { + const { root, source } = fixture(t), original = path.join(source, 'nested/data'); + const outside = path.join(root, 'outside'); fs.writeFileSync(outside, 'outside data', { mode: 0o600 }); + const link = path.join(source, 'unsafe'); fs.symlinkSync(outside, link); + assert.throws(() => scan(source), { code: 'directory_backup_unsafe' }); + assert.throws(() => clear(source), { code: 'directory_backup_unsafe' }); + assert.equal(fs.readFileSync(original, 'utf8'), 'synthetic preserved data'); + fs.unlinkSync(link); fs.linkSync(outside, link); + assert.throws(() => scan(source), { code: 'directory_backup_unsafe' }); + fs.unlinkSync(link); fs.chmodSync(original, 0o644); + assert.throws(() => clone(source, path.join(root, 'backup')), { code: 'directory_backup_unsafe' }); + assert.equal(fs.readFileSync(outside, 'utf8'), 'outside data'); +}); diff --git a/core/core/installations/tests/directory-dsp.test.js b/core/core/installations/tests/directory-dsp.test.js new file mode 100644 index 0000000..1a62fe8 --- /dev/null +++ b/core/core/installations/tests/directory-dsp.test.js @@ -0,0 +1,74 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { platformPaths, loadPlatformPaths } = require('../../../shared/paths/platform-paths'); +const { createDsp, inspectDsp } = require('../../../host/storage/storage'); +const { sandboxArguments } = require('./support/directory-sandbox'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'directory-dsp-test-')); + for (const name of ['live', 'local', 'dsps', 'dev', 'worktrees']) fs.mkdirSync(path.join(root, name), { mode: 0o700 }); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return platformPaths(root); +} + +test('creating a DSP preserves existing tenants and rejects traversal and duplicate identities', t => { + const paths = fixture(t), first = createDsp(paths); + assert.match(first.id, /^dsp_[a-f0-9]{32}$/); + const marker = path.join(first.root, 'data/marker'); + fs.writeFileSync(marker, 'preserve'); + assert.throws(() => createDsp(paths, { id: first.id }), { code: 'EEXIST' }); + for (const id of ['../escape', 'dsp_/../escape', 'DSP_' + 'a'.repeat(32), 'local']) { + assert.throws(() => createDsp(paths, { id })); + } + const second = createDsp(paths); + assert.notEqual(first.root, second.root); + assert.equal(fs.readFileSync(marker, 'utf8'), 'preserve'); + assert.equal(fs.statSync(path.join(first.root, 'secrets')).mode & 0o777, 0o700); +}); + +test('incomplete provisioning and symlinked DSP roots or storage fail closed', t => { + const paths = fixture(t), dsp = createDsp(paths); + fs.writeFileSync(path.join(dsp.root, '.provisioning'), '1'); + assert.throws(() => inspectDsp(paths, dsp.id)); + fs.unlinkSync(path.join(dsp.root, '.provisioning')); + fs.renameSync(path.join(dsp.root, 'data'), path.join(dsp.root, 'data-original')); + fs.symlinkSync(paths.local, path.join(dsp.root, 'data')); + assert.throws(() => inspectDsp(paths, dsp.id)); + const id = 'dsp_' + 'f'.repeat(32); + fs.symlinkSync(paths.local, path.join(paths.dsps, id)); + assert.throws(() => createDsp(paths, { id })); + assert.throws(() => inspectDsp(paths, id)); +}); + +test('private deployment config rejects public-tree locations and unsafe permissions', t => { + const paths = fixture(t); + const config = JSON.stringify({ version: 1, platformRoot: paths.platformRoot }); + const file = path.join(paths.local, 'platform.json'); + fs.writeFileSync(file, config, { mode: 0o600 }); + assert.equal(loadPlatformPaths(file).dsps, paths.dsps); + fs.chmodSync(file, 0o644); + assert.throws(() => loadPlatformPaths(file)); + const publicFile = path.join(paths.live, 'platform.json'); + fs.writeFileSync(publicFile, config, { mode: 0o600 }); + assert.throws(() => loadPlatformPaths(publicFile)); +}); + +test('sandbox accepts only local tools and source-contained scripts', t => { + const paths = fixture(t), dsp = createDsp(paths); + const toolsRoot = path.join(paths.local, 'tools'); + fs.mkdirSync(toolsRoot, { mode: 0o700 }); + fs.writeFileSync(path.join(paths.live, 'fixture.js'), ''); + const args = sandboxArguments(paths, dsp.id, { toolsRoot, script: 'fixture.js' }); + assert.ok(args.includes('--unshare-all')); + assert.equal(args.includes(dsp.root), false, 'the parent containing host control files must never be mounted'); + assert.ok(args.includes(path.join(dsp.root, '.storage-view'))); + assert.throws(() => sandboxArguments(paths, dsp.id, { toolsRoot, script: '../outside.js' })); + fs.symlinkSync('/usr/bin/true', path.join(paths.live, 'escape.js')); + assert.throws(() => sandboxArguments(paths, dsp.id, { toolsRoot, script: 'escape.js' })); + assert.throws(() => sandboxArguments(paths, dsp.id, { toolsRoot: paths.dev, script: 'fixture.js' })); +}); diff --git a/core/core/installations/tests/directory-erasure.test.js b/core/core/installations/tests/directory-erasure.test.js new file mode 100644 index 0000000..1fa1717 --- /dev/null +++ b/core/core/installations/tests/directory-erasure.test.js @@ -0,0 +1,63 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'); +const { eraseRuntime } = require('../../../host/storage/erase-runtime'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'erase-directory-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const dsps = path.join(root, 'dsps'); fs.mkdirSync(dsps, { mode: 0o700 }); + const id = 'dsp_' + 'a'.repeat(32), selected = path.join(dsps, id); fs.mkdirSync(selected, { mode: 0o700 }); + const info = fs.statSync(selected), job = { runtimeKey: id, rootInode: info.ino, rootDevice: info.dev }; + let stopped = 0, unmounted = 0; + const volumes = { stopped: async () => { stopped++; }, unmount: async () => { unmounted++; } }; + return { root, paths: { dsps }, selected, job, volumes, counts: () => [stopped, unmounted] }; +} + +test('filesystem erasure verifies identity, stops and unmounts before removing only the DSP tree', async t => { + const c = fixture(t), sibling = path.join(c.paths.dsps, 'sibling'); fs.mkdirSync(sibling, { mode: 0o700 }); + fs.writeFileSync(path.join(sibling, 'keep'), 'synthetic neighbor'); + fs.symlinkSync(sibling, path.join(c.selected, 'link-outside')); + let calls = 0; + await eraseRuntime(c.paths, c.job, c.volumes, 42, async (args, options) => { + calls++; assert.deepEqual(c.counts(), [1, 1]); assert.equal(options.lockFd, 42); + assert.deepEqual(args, ['/usr/bin/rm', '-rf', '--one-file-system', '--', c.selected]); + fs.rmSync(args.at(-1), { recursive: true }); + }); + assert.equal(fs.readFileSync(path.join(sibling, 'keep'), 'utf8'), 'synthetic neighbor'); + assert.equal(fs.existsSync(c.selected), false); + await eraseRuntime(c.paths, c.job, c.volumes, 42, () => { throw Error('must not run'); }); + assert.equal(calls, 1); +}); + +test('filesystem erasure rejects replaced roots, live workers and unexpected mounts', async t => { + const c = fixture(t), never = () => { throw Error('must not erase'); }; + await assert.rejects(eraseRuntime(c.paths, { ...c.job, rootInode: c.job.rootInode + 1 }, c.volumes, 42, never), /identity_changed/); + await assert.rejects(eraseRuntime(c.paths, c.job, { ...c.volumes, stopped: async () => { throw Error('runtime_active'); } }, 42, never), /runtime_active/); + const read = fs.readFileSync; + t.mock.method(fs, 'readFileSync', (file, ...args) => file === '/proc/self/mountinfo' + ? `1 2 0:1 / ${c.selected}/unexpected rw - tmpfs tmpfs rw\n` : read(file, ...args)); + await assert.rejects(eraseRuntime(c.paths, c.job, c.volumes, 42, never), /deletion_mounted/); + assert.equal(fs.existsSync(c.selected), true); +}); + +test('dedicated backup erasure resumes after interruption has removed its manifest', t => { + const c = fixture(t), root = path.join(c.root, 'backups'); fs.mkdirSync(root, { mode: 0o700 }); + const name = 'mbk_' + 'b'.repeat(32), snapshot = path.join(root, name); + fs.mkdirSync(snapshot, { mode: 0o700 }); + fs.writeFileSync(path.join(snapshot, 'manifest.json'), '{}', { mode: 0o600 }); + const job = { ...c.job, id: 'c'.repeat(64) }; + const backups = { root, jobs: () => [], manifest: () => ({ scope: 'dsp', roots: [], dsps: [{ id: job.runtimeKey }] }) }; + const erase = require('../../../host/storage/erase-backups').eraseBackups, rename = fs.renameSync; + const renamed = t.mock.method(fs, 'renameSync', (from, to) => { + rename(from, to); + if (from === snapshot) { + fs.unlinkSync(path.join(to, 'manifest.json')); fs.unlinkSync(path.join(to, '.erasing.json')); + throw Error('simulated_interruption'); + } + }); + assert.throws(() => erase(backups, job), /simulated_interruption/); + assert.equal(fs.existsSync(snapshot), false); assert.equal(fs.readdirSync(root).length, 1); + renamed.mock.restore(); erase(backups, job); + assert.deepEqual(fs.readdirSync(root), []); +}); diff --git a/core/core/installations/tests/directory-lifecycle.test.js b/core/core/installations/tests/directory-lifecycle.test.js new file mode 100644 index 0000000..5964c8b --- /dev/null +++ b/core/core/installations/tests/directory-lifecycle.test.js @@ -0,0 +1,109 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); +const { once } = require('node:events'); +const test = require('node:test'); +const { platformPaths } = require('../../../shared/paths/platform-paths'); +const { ensureDsp, inspectDsp } = require('../../../host/storage/storage'); +const { acquireLock } = require('../../../host/controller/operations'); +const { DirectoryJournal } = require('../../../host/controller/journal'); +const { DirectoryManager } = require('../../../host/controller/manager'); +const { request } = require('../../../host/controller/controller'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'directory-lifecycle-')); + for (const name of ['live', 'local', 'dsps', 'dev', 'worktrees']) fs.mkdirSync(path.join(root, name), { mode: 0o700 }); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return platformPaths(root); +} + +function manager(paths) { + const active = new Set(), starts = [], stops = []; + const host = { prepare: async () => {}, start: async id => { starts.push(id); active.add(id); }, + stop: async id => { stops.push(id); active.delete(id); } }; + const hub = { connected: id => active.has(id), invoke: async () => ({ ok: true }) }; + const value = new DirectoryManager({ paths, host, hub }); + value.bridge = async () => {}; + return { value, host, active, starts, stops }; +} + +test('operation lock excludes another process and is released after a controller crash', async t => { + const paths = fixture(t); + const code = `const { acquireLock } = require(process.argv[1]); acquireLock({local:process.argv[2]}); process.stdout.write('ready'); setInterval(()=>{},1000);`; + const child = spawn(process.execPath, ['-e', code, require.resolve('../../../host/controller/operations'), paths.local], { stdio: ['ignore', 'pipe', 'pipe'] }); + t.after(() => child.kill('SIGKILL')); + await once(child.stdout, 'data'); + assert.throws(() => acquireLock(paths), { code: 'directory_operation_busy' }); + const exited = once(child, 'exit'); child.kill('SIGKILL'); await exited; + const fd = acquireLock(paths); fs.closeSync(fd); +}); + +test('interrupted storage resumes only its bound reservation and never follows storage symlinks', t => { + const paths = fixture(t), journal = new DirectoryJournal(paths); + const job = journal.request('create', 'storage_retry'); + const root = path.join(paths.dsps, job.dspId); + fs.mkdirSync(root, { mode: 0o700 }); + fs.writeFileSync(path.join(root, '.provisioning'), JSON.stringify({ version: 1, id: job.dspId, creationId: job.creationId }), { mode: 0o600 }); + fs.mkdirSync(path.join(root, 'config'), { mode: 0o700 }); + assert.throws(() => ensureDsp(paths, job.dspId, 'create_' + 'f'.repeat(32))); + fs.symlinkSync(paths.local, path.join(root, 'data')); + assert.throws(() => ensureDsp(paths, job.dspId, job.creationId)); + assert.deepEqual(fs.readdirSync(paths.local).sort(), ['state']); + fs.unlinkSync(path.join(root, 'data')); + assert.equal(ensureDsp(paths, job.dspId, job.creationId).creationId, job.creationId); + assert.equal(ensureDsp(paths, job.dspId, job.creationId).id, job.dspId); +}); + +test('failed creation retains identity and credentials; completed replay has no host effects', async t => { + const paths = fixture(t), m = manager(paths); + const start = m.host.start; + m.host.start = async () => { throw new Error('injected interruption'); }; + await assert.rejects(m.value.apply('create', 'create_retry'), { code: 'directory_operation_failed' }); + const [record] = m.value.journal.all(); + const token = fs.readFileSync(path.join(paths.dsps, record.id, 'secrets/runtime-agent/registration-token')); + const next = manager(paths); + next.host.start = start; + next.value.hub = m.value.hub; + const result = await next.value.apply('create', 'create_retry'); + assert.equal(result.dspId, record.id); + assert.deepEqual(fs.readFileSync(path.join(paths.dsps, record.id, 'secrets/runtime-agent/registration-token')), token); + assert.deepEqual(await next.value.apply('create', 'create_retry'), result); + assert.equal(m.starts.length, 1); + assert.equal(fs.readdirSync(paths.dsps).length, 1); + await assert.rejects(next.value.apply('stop', 'create_retry', record.id), { code: 'directory_request_conflict' }); +}); + +test('stop supersedes an older failed start and recovery preserves desired states and sibling data', async t => { + const paths = fixture(t), m = manager(paths); + const first = await m.value.apply('create', 'create_first'); + const second = await m.value.apply('create', 'create_second'); + const keep = path.join(paths.dsps, first.dspId, 'data/retained'); fs.writeFileSync(keep, 'synthetic'); + const start = m.host.start; + m.host.start = async () => { throw new Error('injected interruption'); }; + await assert.rejects(m.value.apply('restart', 'failed_restart', first.dspId)); + await m.value.apply('stop', 'stop_after_failure', first.dspId); + m.host.start = start; + await assert.rejects(m.value.apply('restart', 'failed_restart', first.dspId), { code: 'directory_request_superseded' }); + await m.value.recover(); + assert.equal(m.active.has(first.dspId), false); + assert.equal(m.active.has(second.dspId), true); + await m.value.apply('start', 'resume_first', first.dspId); + await m.value.apply('retire', 'retire_first', first.dspId); + assert.equal(m.value.journal.authorityCatalog().resolve(first.dspId), null); + assert.equal(m.active.has(second.dspId), true); + assert.equal(fs.readFileSync(keep, 'utf8'), 'synthetic'); + assert.equal(inspectDsp(paths, first.dspId).id, first.dspId); + await assert.rejects(m.value.apply('start', 'restart_retired', first.dspId), { code: 'directory_dsp_retired' }); +}); + +test('controller accepts only explicit lifecycle fields', () => { + assert.equal(request('{"action":"list"}').action, 'list'); + for (const value of [{ action: 'create', requestId: 'request_key', command: '/bin/true' }, + { action: 'start', requestId: 'request_key' }, { action: 'remove', dspId: 'invalid' }, []]) { + assert.throws(() => request(JSON.stringify(value)), { code: 'directory_request_invalid' }); + } +}); diff --git a/core/core/installations/tests/directory-manual-backups.test.js b/core/core/installations/tests/directory-manual-backups.test.js new file mode 100644 index 0000000..1816c88 --- /dev/null +++ b/core/core/installations/tests/directory-manual-backups.test.js @@ -0,0 +1,159 @@ +'use strict'; +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const test = require('node:test'), assert = require('node:assert/strict'); +const { platformPaths, loadPlatformPaths } = require('../../../shared/paths/platform-paths'); +const { AccessStore } = require('../../accounts/src/store'); +const { administerOwner } = require('../../accounts/src/owner-admin'); +const { AccessControlService } = require('../../accounts/src/service'); +const { snapshotCore, restoreCore } = require('../../../host/storage/backup-core'); +const { ManualBackups, interruptedRestore } = require('../../../host/storage/manual-backups'); +const { main } = require('../../../host/storage/manual-backup-cli'); +const { privateDirectory, acquireLock } = require('../../../host/controller/operations'); +const files = require('../../../host/storage/backup-files'); +const { ensureDsp } = require('../../../host/storage/storage'); +const { atomic } = require('../src/release-delivery-files'); +const crypto = require('node:crypto'); + +async function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'manual-backup-')); + for (const name of ['live','local','dsps','dev','worktrees']) fs.mkdirSync(path.join(root, name), { mode: 0o700 }); + const paths = platformPaths(root); + const databaseRoot = privateDirectory(path.join(paths.local, 'state/access-control')); + const store = new AccessStore({ databaseRoot, database: path.join(databaseRoot, 'access-control.sqlite3') }); + privateDirectory(path.join(paths.local, 'config')); + fs.writeFileSync(path.join(paths.local, 'config/platform.json'), JSON.stringify({ version: 1, platformRoot: root }), { mode: 0o600 }); + const credentials = { email: 'backup-owner@example.test', password: 'synthetic backup password' }; + await administerOwner(store, 'owner-create', { ...credentials, firstName: 'Backup', lastName: 'Fixture', confirmPassword: credentials.password }); + const errors = [], backups = new ManualBackups({ paths, store, onError: error => errors.push(error) }); + const owner = store.userByEmail(credentials.email); + const intent = (action, requestId, backupId = null) => ({ action, scope: 'platform', organizationId: null, + expectedRevision: null, backupId, requestId, confirmRestore: action === 'restore' }); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + return { paths, store, owner, backups, credentials, intent, errors }; +} + +test('Core snapshots restore account data, revoke sessions, and reject schema changes before overwriting', async t => { + const f = await fixture(t), access = new AccessControlService(f.store); + await access.signIn(f.credentials); + const file = path.join(f.paths.local, 'snapshot.sqlite3'); + await snapshotCore(f.store.paths.database, file); + const saved = new DatabaseSync(file); + assert.equal(saved.prepare('SELECT count(*) n FROM sessions').get().n, 0); saved.close(); + f.store.db.prepare("UPDATE users SET first_name='Changed' WHERE id=?").run(f.owner.id); + restoreCore(f.store, file); + assert.equal(f.store.userById(f.owner.id).first_name, 'Backup'); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM sessions').get().n, 0); + assert.equal((await access.signIn(f.credentials)).session.user.platformRole, 'owner'); + const altered = new DatabaseSync(file); altered.exec('CREATE TABLE unexpected(value TEXT)'); altered.close(); + f.store.db.prepare("UPDATE users SET first_name='Retained' WHERE id=?").run(f.owner.id); + assert.throws(() => restoreCore(f.store, file), { code: 'directory_backup_schema_changed' }); + assert.equal(f.store.userById(f.owner.id).first_name, 'Retained'); +}); + +test('manual platform backup requires an owner and a stopped controller, and detects altered backup data', async t => { + const f = await fixture(t); + assert.throws(() => f.backups.request('usr_missing', f.intent('backup','manual:fixture:denied')), { code: 'platform_forbidden' }); + const id = f.backups.request(f.owner.id, f.intent('backup','manual:fixture:first')); + const lock = acquireLock(f.paths, 'controller'); + try { await assert.rejects(f.backups.resume(id), { code: 'directory_operation_busy' }); } + finally { fs.closeSync(lock); } + const completed = await f.backups.resume(id); + assert.equal(completed.status, 'complete', f.errors[0]?.stack); + const manifest = f.backups.inspect(completed.backupId); + assert.equal(manifest.scope, 'platform'); + assert.equal(f.backups.request(f.owner.id, f.intent('backup','manual:fixture:first')), id); + fs.appendFileSync(path.join(f.backups.root, completed.backupId, 'payload/platform-config/platform.json'), ' '); + assert.throws(() => f.backups.inspect(completed.backupId), { code: 'directory_backup_changed' }); +}); + +test('interrupted restore keeps bootstrap configuration and resumes the same owner request', async t => { + const f = await fixture(t); + fs.writeFileSync(path.join(f.paths.local, 'config/fixture.json'), '{"value":"saved"}', { mode: 0o600 }); + const backup = await f.backups.resume(f.backups.request(f.owner.id, f.intent('backup','manual:fixture:before'))); + assert.equal(backup.status, 'complete', f.errors[0]?.stack); + fs.writeFileSync(path.join(f.paths.local, 'config/fixture.json'), '{"value":"new"}'); + f.store.db.prepare("UPDATE users SET first_name='Changed' WHERE id=?").run(f.owner.id); + const id = f.backups.request(f.owner.id, f.intent('restore','manual:fixture:restore',backup.backupId)); + const copy = files.copyContents; + files.copyContents = (source, target, options) => { + if (target === path.join(f.paths.local, 'config')) throw new Error('injected interruption after deletion'); + return copy(source, target, options); + }; + try { assert.equal((await f.backups.resume(id)).status, 'failed'); } + finally { files.copyContents = copy; } + assert.equal(interruptedRestore(f.paths), true); + assert.equal(loadPlatformPaths(path.join(f.paths.local, 'config/platform.json')).platformRoot, f.paths.platformRoot); + assert.equal((await f.backups.resume(id)).status, 'complete', f.errors.at(-1)?.stack); + assert.equal(interruptedRestore(f.paths), false); + assert.equal(f.store.userById(f.owner.id).first_name, 'Backup'); + assert.equal(JSON.parse(fs.readFileSync(path.join(f.paths.local, 'config/fixture.json'))).value, 'saved'); +}); + +test('offline CLI rejects wrong credentials without creating an operation and returns no private identity', async t => { + const f = await fixture(t), output = []; + const options = { paths: f.paths, write: value => output.push(value), read: async () => ({ ...f.credentials, + password: 'wrong synthetic password', scope: 'platform', requestId: 'manual:fixture:cli' }) }; + assert.equal(await main(['backup'], options), 1); + assert.equal(f.backups.jobs().length, 0); + options.read = async () => ({ ...f.credentials, scope: 'platform', requestId: 'manual:fixture:cli' }); + assert.equal(await main(['backup'], options), 0); + assert.equal(output.join('').includes(f.credentials.email), false); + assert.equal(output.join('').includes(f.paths.platformRoot), false); + assert.equal(f.backups.view().operations.length, 1); +}); + +test('DSP restore preserves siblings and repairs token authority, while rejecting a running DSP', async t => { + const f = await fixture(t); + const dsps = ['a','b'].map(letter => { + const id = 'dsp_' + letter.repeat(32), creationId = 'create_' + letter.repeat(32), organizationId = 'org_' + letter.repeat(32); + const dsp = ensureDsp(f.paths, id, creationId), token = crypto.randomBytes(32).toString('base64url'); + const record = { version: 1, id, creationId, latestRequest: letter.repeat(64), desiredState: 'stopped', + tokenHash: crypto.createHash('sha256').update(token).digest('hex') }; + f.backups.journal.saveRecord(record); + f.store.createOrganization({ id: organizationId, name: 'Synthetic DSP', abbreviation: 'SYN', timezone: 'UTC', status: 'suspended', createdBy: f.owner.id, timestamp: 1 }); + f.store.createInstallation(organizationId, id, 'suspended', 1, 'dispatch_current_1', 'directory_service_v1'); + atomic(path.join(dsp.root, 'config/installation.json'), { version: 1, organizationId, runtimeKey: id }); + atomic(path.join(dsp.root, 'secrets/runtime-agent/registration-token'), token + '\n'); + fs.writeFileSync(path.join(dsp.root, 'data/fixture'), 'saved ' + letter, { mode: 0o600 }); + return { ...dsp, record, organizationId, token }; + }); + // Native service isolation and stopped-unit checks have separate acceptance + // coverage; this fixture exercises data and authority on ordinary directories. + f.backups.stopped = async records => assert.equal(records.every(row => row.desiredState === 'stopped'), true); + const [first, second] = dsps; + const packageRoot = privateDirectory(path.join(f.paths.dev, 'package')); + privateDirectory(path.join(packageRoot, 'backend')); + const plugin = { ...require('../../../tests/fixtures/paycom-plugin.json'), frontend: null, dashboard: null, published: null, runtime: 'backend/index.js' }; + atomic(path.join(packageRoot, 'dispatch-plugin.json'), plugin); + fs.writeFileSync(path.join(packageRoot, 'backend/index.js'), 'module.exports = {};', { mode: 0o600 }); + const { digest } = require('../../../tooling/build-plugin-package').sealPackage(packageRoot); + const installer = require('../../../host/plugins/install'); + const staged = installer.stagePackage({ dspRoot: first.root, packageRoot, expectedDigest: digest }); + installer.activatePackage({ dspRoot: first.root, staged, revision: 1 }); + require('../../accounts/tests/plugin-fixture').enableFixturePlugin(f.store, first.organizationId); + const intent = action => ({ ...f.intent(action, 'manual:fixture:dsp:' + action), scope: 'dsp', organizationId: first.organizationId, expectedRevision: 1 }); + f.backups.journal.saveRecord({ ...first.record, desiredState: 'running' }); + assert.throws(() => f.backups.request(f.owner.id, intent('backup')), { code: 'backup_requires_suspended_dsps' }); + f.backups.journal.saveRecord(first.record); + const backup = await f.backups.resume(f.backups.request(f.owner.id, intent('backup'))); + assert.equal(backup.status, 'complete', f.errors[0]?.stack); + f.store.db.prepare("UPDATE dsp_plugins SET desired_state='disabled',applied_state='disabled',revision=2,applied_revision=2 WHERE organization_id=?").run(first.organizationId); + fs.writeFileSync(path.join(first.root, 'data/fixture'), 'new a'); + fs.writeFileSync(path.join(second.root, 'data/fixture'), 'new b'); + const id = f.backups.request(f.owner.id, { ...intent('restore'), backupId: backup.backupId }); + const pluginBackup = require('../../../host/plugins/backup-state'), restorePluginState = pluginBackup.restore; + pluginBackup.restore = (...args) => { restorePluginState(...args); throw new Error('synthetic interruption after plugin authority restore'); }; + try { assert.equal((await f.backups.resume(id)).status, 'failed'); } + finally { pluginBackup.restore = restorePluginState; } + assert.equal(interruptedRestore(f.paths), true); + assert.equal((await f.backups.resume(id)).status, 'complete', f.errors.at(-1)?.stack); + assert.equal(fs.readFileSync(path.join(first.root, 'data/fixture'), 'utf8'), 'saved a'); + assert.equal(fs.readFileSync(path.join(second.root, 'data/fixture'), 'utf8'), 'new b'); + assert.equal(f.store.runtimeAgentAuthority(first.id).token_hash, first.record.tokenHash); + assert.equal(f.backups.journal.record(first.id).desiredState, 'stopped'); + const restored = f.store.db.prepare('SELECT * FROM dsp_plugins WHERE organization_id=?').get(first.organizationId); + assert.equal(restored.desired_state, 'enabled'); assert.equal(restored.revision, 3); assert.equal(restored.applied_revision, 3); + assert.equal(installer.installedPackage({ dspRoot: first.root, pluginId: 'paycom', revision: 3 }).receipt.digest, digest); + assert.equal(f.store.db.prepare('SELECT 1 FROM dsp_plugins WHERE organization_id=?').get(second.organizationId), undefined); +}); diff --git a/core/core/installations/tests/directory-network.test.js b/core/core/installations/tests/directory-network.test.js new file mode 100644 index 0000000..6b9c492 --- /dev/null +++ b/core/core/installations/tests/directory-network.test.js @@ -0,0 +1,115 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const net = require('node:net'); +const { once } = require('node:events'); +const { networkPolicy, publicAddress, loadNetworkPolicy } = require('../../../host/networking/network-policy'); +const { DirectoryEgress } = require('../../../host/networking/egress'); +const { createEgressRelay } = require('dispatch-dsp/runtime/supervisor/src/egress-relay.js'); + +async function fixture(t, options = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'eg-')); + fs.mkdirSync(path.join(root, '.control'), { mode: 0o700 }); + const events = [], addresses = [], clients = []; + const origin = net.createServer(socket => { clients.push(socket); socket.pipe(socket); }); + origin.listen(0, '127.0.0.1'); await once(origin, 'listening'); + const egress = new DirectoryEgress({ dspRoot: root, policy: networkPolicy({ version: 1, hosts: ['provider.example'] }), + resolve: async () => ['93.184.216.34'], onEvent: event => events.push(event), + connect: input => { addresses.push(input); return net.createConnection({ host: '127.0.0.1', port: origin.address().port }); }, ...options }); + await egress.start(); + const relay = createEgressRelay({ socketPath: path.join(root, '.control/egress.sock'), port: 0 }); + await relay.start(); + t.after(async () => { + await relay.close(); await egress.close(); + for (const client of clients) client.destroy(); + await new Promise(resolve => origin.close(resolve)); + fs.rmSync(root, { recursive: true, force: true }); + }); + return { root, egress, relay, events, addresses, + async request(raw) { + const socket = net.createConnection({ host: '127.0.0.1', port: relay.server.address().port }); + socket.on('error', () => {}); + await once(socket, 'connect'); + socket.write(raw); + return socket; + }, + }; +} + +test('policy admits exact provider names and bounded subdomains, with all special addresses excluded', () => { + const policy = networkPolicy({ version: 1, hosts: ['provider.example', '*.assets.example'] }); + for (const host of ['provider.example', 'img.assets.example']) assert.equal(policy.allows(host), true); + for (const host of ['provider.example.evil.test', 'evilprovider.example', 'assets.example', '127.0.0.1', '[::1]', + 'provider.example.', 'provider.example:443', 'PROVIDER.EXAMPLE']) assert.equal(policy.allows(host), false); + for (const ip of ['0.1.2.3', '10.1.2.3', '100.64.0.1', '100.127.255.255', '127.0.0.1', '169.254.169.254', + '172.16.0.1', '172.31.0.1', '192.168.1.1', '192.0.0.9', '192.0.2.1', '192.88.99.1', '198.18.0.1', + '198.51.100.1', '203.0.113.1', '224.0.0.1', '255.255.255.255', '::1', '::ffff:8.8.8.8', '2130706433']) { + assert.equal(publicAddress(ip), false, ip); + } + for (const ip of ['8.8.8.8', '93.184.216.34', '100.128.0.1', '172.32.0.1']) assert.equal(publicAddress(ip), true, ip); + for (const hosts of [['*'], ['*.com'], ['127.0.0.1'], ['provider.example/path'], ['provider.example', 'provider.example']]) { + assert.throws(() => networkPolicy({ version: 1, hosts })); + } +}); + +test('real Unix relay tunnels only to a pinned approved address and keeps application bytes private', async t => { + const f = await fixture(t); + const socket = await f.request('CONNECT provider.example:443 HTTP/1.1\r\nHost: provider.example:443\r\n\r\n'); + assert.match(String((await once(socket, 'data'))[0]), /^HTTP\/1.1 200/); + socket.write('synthetic private bytes'); + assert.equal(String((await once(socket, 'data'))[0]), 'synthetic private bytes'); + assert.deepEqual(f.addresses, [{ host: '93.184.216.34', port: 443, family: 4 }]); + assert.deepEqual(f.events, [{ status: 'connected', host: 'provider.example' }]); + socket.destroy(); +}); + +test('IP literals, HTTP forwarding, alternate ports and unapproved names fail before DNS or connection', async t => { + let resolutions = 0; + const f = await fixture(t, { resolve: async () => { resolutions++; return ['8.8.8.8']; } }); + for (const target of ['127.0.0.1:443', '[::1]:443', '2130706433:443', 'provider.example:80', + 'provider.example.evil.test:443', 'user@provider.example:443']) { + const socket = await f.request(`CONNECT ${target} HTTP/1.1\r\nHost: ${target}\r\n\r\n`); + assert.match(String((await once(socket, 'data'))[0]), /^HTTP\/1.1 403/); socket.destroy(); + } + const socket = await f.request('GET http://provider.example/ HTTP/1.1\r\nHost: provider.example\r\n\r\n'); + assert.match(String((await once(socket, 'data'))[0]), /^HTTP\/1.1 405/); socket.destroy(); + assert.equal(resolutions, 0); assert.deepEqual(f.addresses, []); +}); + +test('DNS rebinding, mixed public/private answers and revocation during resolution cannot reach a host socket', async t => { + let answer = ['127.0.0.1'], permitted = true; + const f = await fixture(t, { permitted: () => permitted, resolve: async () => answer }); + for (const values of [['127.0.0.1'], ['8.8.8.8', '169.254.169.254'], ['::1'], []]) { + answer = values; + const socket = await f.request('CONNECT provider.example:443 HTTP/1.1\r\n\r\n'); + assert.match(String((await once(socket, 'data'))[0]), /^HTTP\/1.1 403/); socket.destroy(); + } + f.egress.resolve = async () => { permitted = false; return ['8.8.8.8']; }; + const socket = await f.request('CONNECT provider.example:443 HTTP/1.1\r\n\r\n'); + await once(socket, 'close'); assert.deepEqual(f.addresses, []); +}); + +test('revoking a running DSP closes existing tunnels', async t => { + let permitted = true; + const f = await fixture(t, { permitted: () => permitted }); + const socket = await f.request('CONNECT provider.example:443 HTTP/1.1\r\n\r\n'); + await once(socket, 'data'); + permitted = false; + await once(socket, 'close'); + assert.equal(f.egress.sockets.size, 0); +}); + +test('private policy loads outside source and refuses permissive file modes or links', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'policy-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + fs.mkdirSync(path.join(root, 'config'), { mode: 0o700 }); + const paths = { local: root }, file = path.join(root, 'config/directory-network.json'); + assert.equal(loadNetworkPolicy(paths).allows('www.amazon.com'), true); + fs.writeFileSync(file, JSON.stringify({ version: 1, hosts: [] }), { mode: 0o600 }); + assert.equal(loadNetworkPolicy(paths).allows('www.amazon.com'), false); + fs.chmodSync(file, 0o644); assert.throws(() => loadNetworkPolicy(paths)); +}); diff --git a/core/core/installations/tests/directory-owner-import.test.js b/core/core/installations/tests/directory-owner-import.test.js new file mode 100644 index 0000000..586962b --- /dev/null +++ b/core/core/installations/tests/directory-owner-import.test.js @@ -0,0 +1,41 @@ +'use strict'; +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const test = require('node:test'), assert = require('node:assert/strict'); +const { AccessStore } = require('../../accounts/src/store'); +const { AccessControlService } = require('../../accounts/src/service'); +const { administerOwner } = require('../../accounts/src/owner-admin'); +const { importInitialOwner } = require('../../../host/controller/import-owner'); + +async function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'directory-import-')); + const sourcePaths = { databaseRoot: path.join(root, 'source'), database: path.join(root, 'source/access-control.sqlite3') }; + const source = new AccessStore(sourcePaths); + const target = new AccessStore({ databaseRoot: path.join(root, 'target'), database: path.join(root, 'target/access-control.sqlite3') }); + const credentials = { email: 'migration@example.test', password: 'synthetic import password' }; + await administerOwner(source, 'owner-create', { ...credentials, firstName: 'Migration', lastName: 'Fixture', confirmPassword: credentials.password }); + t.after(() => { source.close(); target.close(); fs.rmSync(root, { recursive: true, force: true }); }); + return { root, source, target, file: sourcePaths.database, credentials }; +} + +test('owner-only migration preserves login, leaves source unchanged and never reverts a later password on replay', async t => { + const f = await fixture(t), sourceOwner = f.source.userByEmail(f.credentials.email); + const before = JSON.stringify(sourceOwner); + assert.equal(importInitialOwner(f.target, f.file).status, 'owner_imported'); + const access = new AccessControlService(f.target); + const session = await access.signIn(f.credentials); + assert.equal(session.session.user.platformRole, 'owner'); + assert.equal(f.target.db.prepare('SELECT count(*) n FROM memberships').get().n, 0); + await administerOwner(f.target, 'owner-recover', { email: f.credentials.email, newEmail: '', password: 'new synthetic import password', confirmPassword: 'new synthetic import password' }); + assert.equal(importInitialOwner(f.target, f.file).status, 'owner_already_imported'); + await assert.rejects(access.signIn(f.credentials)); + assert.equal(JSON.stringify(f.source.userByEmail(f.credentials.email)), before); +}); + +test('owner import refuses a source with DSPs and rejects linked databases', async t => { + const f = await fixture(t); + const linked = path.join(f.root, 'linked.sqlite3'); fs.symlinkSync(f.file, linked); + assert.throws(() => importInitialOwner(f.target, linked), { code: 'directory_import_unsafe' }); + f.source.db.prepare("INSERT INTO organizations VALUES('org_fixture','Synthetic','SYN','UTC','active',NULL,1,1)").run(); + assert.throws(() => importInitialOwner(f.target, f.file), { code: 'directory_import_mapping_required' }); + assert.equal(f.target.db.prepare('SELECT count(*) n FROM users').get().n, 0); +}); diff --git a/core/core/installations/tests/directory-tools.test.js b/core/core/installations/tests/directory-tools.test.js new file mode 100644 index 0000000..7bd3c47 --- /dev/null +++ b/core/core/installations/tests/directory-tools.test.js @@ -0,0 +1,46 @@ +'use strict'; +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { installTools } = require('../../../host/services/tools'); + +function fixture(t) { + // Source trust requires a non-writable ancestor chain; use the operator's + // private home rather than a shared system temporary directory. + const root = fs.mkdtempSync(path.join(process.env.TMPDIR || os.homedir(), '.tools-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const paths = { local: path.join(root, 'local'), dsps: path.join(root, 'dsps') }; + fs.mkdirSync(paths.local, { mode: 0o700 }); fs.mkdirSync(paths.dsps, { mode: 0o700 }); + const source = name => { + const file = path.join(root, name); fs.writeFileSync(file, 'synthetic tool ' + name, { mode: 0o755 }); return file; + }; + return { root, paths, inputs: { nodeSource: source('node'), tiniSource: source('tini') } }; +} + +test('private tool installation replays unchanged and stages changed versions without replacing active binaries', t => { + const f = fixture(t), installed = installTools(f.paths, f.inputs); + const target = path.join(installed, 'node'); + assert.deepEqual(fs.readFileSync(target), fs.readFileSync(f.inputs.nodeSource)); + assert.equal(fs.statSync(target).nlink, 1); assert.equal(fs.statSync(target).mode & 0o777, 0o755); + const inode = fs.statSync(target).ino; + assert.equal(installTools(f.paths, f.inputs), installed); assert.equal(fs.statSync(target).ino, inode); + fs.writeFileSync(f.inputs.nodeSource, 'new synthetic version'); + const next = installTools(f.paths, f.inputs); + assert.notEqual(next, installed); + assert.equal(fs.readFileSync(path.join(next, 'node'), 'utf8'), 'new synthetic version'); + assert.equal(fs.readFileSync(target, 'utf8'), 'synthetic tool node'); + fs.writeFileSync(path.join(next, 'node'), 'modified installed binary'); + assert.throws(() => installTools(f.paths, f.inputs), { code: 'directory_tool_conflict' }); +}); + +test('tool installation rejects source links, unsafe permissions and DSP-owned tool sources', t => { + const f = fixture(t); + const link = path.join(f.root, 'linked'); fs.symlinkSync(f.inputs.nodeSource, link); + assert.throws(() => installTools(f.paths, { ...f.inputs, nodeSource: link })); + fs.chmodSync(f.inputs.nodeSource, 0o777); assert.throws(() => installTools(f.paths, f.inputs)); + fs.chmodSync(f.inputs.nodeSource, 0o755); + const file = path.join(f.paths.dsps, 'untrusted'); fs.writeFileSync(file, 'synthetic', { mode: 0o755 }); + assert.throws(() => installTools(f.paths, { ...f.inputs, nodeSource: file })); +}); diff --git a/core/core/installations/tests/directory-volume.test.js b/core/core/installations/tests/directory-volume.test.js new file mode 100644 index 0000000..0170601 --- /dev/null +++ b/core/core/installations/tests/directory-volume.test.js @@ -0,0 +1,59 @@ +'use strict'; +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const assert = require('node:assert/strict'), test = require('node:test'); +const { platformPaths } = require('../../../shared/paths/platform-paths'); +const { ensureDsp, inspectDsp } = require('../../../host/storage/storage'); +const { DirectoryVolumes, volumePolicy, pristine } = require('../../../host/storage/volume'); +const { mounts, volumeState } = require('../../../host/storage/volume-state'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'directory-volume-')); + for (const name of ['live', 'local', 'dsps', 'dev', 'worktrees']) fs.mkdirSync(path.join(root, name), { mode: 0o700 }); + const paths = platformPaths(root), id = 'dsp_' + 'e'.repeat(32), creationId = 'create_' + 'a'.repeat(32); + const dsp = ensureDsp(paths, id, creationId); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return { paths, dsp, creationId }; +} + +test('volume metadata prevents runtime access to underlying directories after an unmount', t => { + const { paths, dsp, creationId } = fixture(t); + const metadata = { version: 1, id: dsp.id, uuid: '01234567-89ab-cdef-0123-456789abcdef', bytes: 64 * 1024 ** 2, phase: 'ready' }; + const file = path.join(dsp.root, '.volume.json'); + fs.writeFileSync(file, JSON.stringify(metadata), { mode: 0o600 }); + assert.throws(() => inspectDsp(paths, dsp.id), { code: 'directory_volume_unmounted' }); + assert.equal(ensureDsp(paths, dsp.id, creationId).creationId, creationId); + fs.writeFileSync(file, JSON.stringify({ ...metadata, bytes: -1 })); + assert.throws(() => volumeState(dsp.root), { code: 'directory_volume_unsafe' }); +}); + +test('populated or linked DSP storage is never silently converted', async t => { + const { paths, dsp } = fixture(t); + assert.equal(pristine(dsp.root), true); + const file = path.join(dsp.root, 'data/retained'); + fs.writeFileSync(file, 'synthetic retained data'); + assert.equal(pristine(dsp.root), false); + assert.deepEqual(await new DirectoryVolumes(paths).ensure(dsp), { limited: false }); + assert.equal(fs.existsSync(path.join(dsp.root, '.volume.json')), false); + fs.unlinkSync(file); fs.symlinkSync(paths.local, file); + assert.equal(pristine(dsp.root), false); +}); + +test('storage capacity policy is private, closed and bounded', t => { + const { paths } = fixture(t); + fs.mkdirSync(path.join(paths.local, 'config'), { mode: 0o700 }); + const file = path.join(paths.local, 'config/directory-storage.json'); + assert.equal(volumePolicy(paths).bytes, 4 * 1024 ** 3); + fs.writeFileSync(file, JSON.stringify({ version: 1, dspGiB: 8, reserveGiB: 16 }), { mode: 0o600 }); + assert.equal(volumePolicy(paths).bytes, 8 * 1024 ** 3); + fs.chmodSync(file, 0o644); assert.throws(() => volumePolicy(paths)); fs.chmodSync(file, 0o600); + fs.writeFileSync(file, JSON.stringify({ version: 1, dspGiB: 8, reserveGiB: 0 })); + assert.throws(() => volumePolicy(paths), { code: 'directory_volume_policy_invalid' }); +}); + +test('mount parsing preserves device identity and decodes mount paths', () => { + const parsed = mounts('41 20 7:3 /data /private/data rw,nosuid,nodev,noexec shared:4 - ext4 /dev/loop3 rw\n' + + '42 20 7:4 / /private/with\\040space rw - ext4 /dev/loop4 rw\n'); + assert.equal(parsed.get('/private/data').root, '/data'); + assert.equal(parsed.get('/private/data').device, '7:3'); + assert.equal(parsed.get('/private/with space').source, '/dev/loop4'); +}); diff --git a/core/core/installations/tests/dsp-backup-deletion.test.js b/core/core/installations/tests/dsp-backup-deletion.test.js new file mode 100644 index 0000000..24309a8 --- /dev/null +++ b/core/core/installations/tests/dsp-backup-deletion.test.js @@ -0,0 +1,113 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const { purgeDspBackups } = require('../src/dsp-backup-deletion'); +const { receiptKey } = require('../src/offsite-policy'); +const { HOST_TENANT_ROOT, opaqueRuntimeSuffix } = require('../../runtime-host-identity'); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-dsp-purge-')); + fs.mkdirSync(path.join(root, 'archives')); t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const job = { id: 'life_'+'a'.repeat(32), organization_id: 'org_deleted', runtime_key: 'runtime_deleted', authority_scope: 'platform_removal', + stage_receipts_json: JSON.stringify({__request: JSON.stringify({operation:'destroy'})}) }; + const backupId = 'backup_'+'b'.repeat(32); + const tag = receiptKey(path.join(HOST_TENANT_ROOT, opaqueRuntimeSuffix(job.runtime_key), 'runtime', job.runtime_key, 'backups', backupId)); + let snapshots = [{ id:'c'.repeat(64), tags:[tag], hostname:'dispatch' }, {id:'d'.repeat(64), tags:['other-dsp'], hostname:'dispatch'}]; + const deleted=[], calls=[]; + const options = { config:{prefix:'dispatch'}, jobs:[job], backups:[{id:backupId,organization_id:job.organization_id}], + records:[{id:backupId,kind:'dsp',organization_id:job.organization_id,retention_days:null}, + {id:'backup_'+'e'.repeat(32),kind:'dsp',organization_id:'org_other',retention_days:null}], + storage:{withDeletionAccess:async (_p,call)=>{calls.push('unlock');await call();calls.push('relock')},removePermanent:async r=>deleted.push(r)}, + run:args=>{if(args[0]==='prune')assert.deepEqual(args,['prune','--max-unused','0']);calls.push(args[0]);if(args[0]==='snapshots')return [snapshots];if(args[0]==='forget')snapshots=snapshots.filter(s=>!args.includes(s.id));return []}, + workRoot:root,receiptRoot:root,ownerUid:process.geteuid() }; + return { root, options, job, deleted, calls, snapshots:()=>snapshots }; +} +test('purge removes only the target DSP across archive tiers and legacy snapshots, then publishes proof', async t=>{ + const f=fixture(t), result=await purgeDspBackups(f.options); + assert.equal(result.failed,0);assert.equal(f.deleted.length,5); + assert.ok(f.deleted.every(row=>row.id===f.options.backups[0].id)); + assert.deepEqual(f.snapshots().map(s=>s.id),['d'.repeat(64)]); + const proof=JSON.parse(fs.readFileSync(path.join(f.root,`deleted-${f.job.id}.json`))); + assert.equal(proof.organizationId,f.job.organization_id);assert.equal(proof.status,'destroyed'); + assert.equal(f.calls.at(-1),'relock'); + const count=f.calls.length;assert.equal((await purgeDspBackups(f.options)).failed,0);assert.equal(f.calls.length,count); +}); +test('partial deletion publishes no success and retries prune after legacy forget already succeeded', async t=>{ + const f=fixture(t), run=f.options.run; + f.options.run=args=>{if(args[0]==='prune')throw Error('storage unavailable');return run(args)}; + assert.equal((await purgeDspBackups(f.options)).failed,1); + assert.equal(fs.existsSync(path.join(f.root,`deleted-${f.job.id}.json`)),false); + f.options.run=run; + assert.equal((await purgeDspBackups(f.options)).failed,0); + assert.ok(f.calls.includes('prune'));assert.deepEqual(f.snapshots().map(s=>s.id),['d'.repeat(64)]); +}); +test('invalid deletion authority and malformed snapshot listings cannot delete objects',async t=>{ + const f=fixture(t);f.job.authority_scope='tenant'; + assert.equal((await purgeDspBackups(f.options)).failed,1);assert.deepEqual(f.deleted,[]); + f.job.authority_scope='platform_removal';f.options.run=()=>[{unexpected:true}]; + assert.equal((await purgeDspBackups(f.options)).failed,1);assert.deepEqual(f.deleted,[]); +}); + +test('real encrypted legacy repository retains and restores a peer DSP after target data is pruned', + {skip:!fs.existsSync('/usr/bin/restic')}, async t=>{ + const f=fixture(t), {createRestic}=require('../src/offsite-backup'); + const run=createRestic({PATH:'/usr/bin:/bin',RESTIC_REPOSITORY:path.join(f.root,'repository'),RESTIC_PASSWORD:'synthetic deletion fixture password'}); + run(['init','--repository-version','2']); + const targetPath=path.join(f.root,'target'), peerPath=path.join(f.root,'peer'); + fs.mkdirSync(targetPath);fs.mkdirSync(peerPath); + fs.writeFileSync(path.join(targetPath,'data.txt'),'permanently deleted DSP data'); + fs.writeFileSync(path.join(peerPath,'data.txt'),'retained peer DSP data'); + const targetTag=f.snapshots()[0].tags[0]; + run(['backup','--host','dispatch','--tag',targetTag,'--','data.txt'],targetPath); + const peer=run(['backup','--host','dispatch','--tag','other-dsp','--','data.txt'],peerPath).find(s=>s?.message_type==='summary').snapshot_id; + f.options.run=run; + assert.equal((await purgeDspBackups(f.options)).failed,0); + assert.deepEqual(run(['snapshots']).flat().map(s=>s.id),[peer]); + const restored=path.join(f.root,'restored');run(['restore',peer,'--target',restored,'--verify']); + assert.equal(fs.readFileSync(path.join(restored,'data.txt'),'utf8'),'retained peer DSP data'); +}); + + +test('DSP deletion removes all shared Core archives while preserving peer individual archives', async t => { + const f = fixture(t), id = 'breq_' + 'f'.repeat(32); + f.options.config.localRoot = path.join(f.root, 'local'); + f.options.records.push({ id, kind: 'core', organization_id: null, retention_days: 30 }); + const result = await purgeDspBackups(f.options); + assert.equal(result.failed, 0); + assert.equal(f.deleted.filter(row => row.id === id).length, 5); + assert.equal(f.deleted.some(row => row.id === 'backup_' + 'e'.repeat(32)), false); +}); + +test('complete Core inventories preserve unrelated archives and legacy snapshots', async t => { + const f = fixture(t), id = 'breq_' + '1'.repeat(32), tag = '2'.repeat(64); + f.options.config.localRoot = path.join(f.root, 'local'); + f.options.records.push({ id, kind: 'core', organization_id: null, retention_days: null }); + fs.writeFileSync(path.join(f.root, 'archives', id + '.json'), JSON.stringify({ + id, kind: 'core', organizationId: null, organizationInventoryVersion: 1, organizationIds: ['org_other'], + }), { mode: 0o600 }); + fs.writeFileSync(path.join(f.root, tag + '.json'), JSON.stringify({ + organizationInventoryVersion: 1, organizationIds: [], + }), { mode: 0o600 }); + f.snapshots().push({ id: '3'.repeat(64), tags: [tag], hostname: 'dispatch' }); + assert.equal((await purgeDspBackups(f.options)).failed, 0); + assert.equal(f.deleted.some(row => row.id === id), false); + assert.ok(f.snapshots().some(row => row.id === '3'.repeat(64))); + assert.ok(fs.existsSync(path.join(f.root, tag + '.json'))); +}); + +test('missing, partial and malformed inventories cannot exclude Core archives from deletion', () => { + const { mayContainOrganization } = require('../src/dsp-backup-deletion'); + for (const proof of [null, { organizationIds: [] }, + { organizationInventoryVersion: 1, organizationIds: [null] }, + { organizationInventoryVersion: 1, organizationIds: ['org_deleted'] }]) + assert.equal(mayContainOrganization(proof, 'org_deleted'), true); + assert.equal(mayContainOrganization({ organizationInventoryVersion: 1, organizationIds: [] }, 'org_deleted'), false); +}); + +test('permanent DSP deletion erases referencing system manifests while retaining isolated Core archives',async t=>{ + const f=fixture(t),core='breq_'+'8'.repeat(32),set='breq_'+'9'.repeat(32),removed=[]; + f.options.records.push({id:core,kind:'core',organization_id:null,retention_days:null}); + fs.writeFileSync(path.join(f.root,'archives',core+'.json'),JSON.stringify({id:core,kind:'core',organizationId:null,organizationInventoryVersion:1,organizationIds:[]}),{mode:0o600}); + f.options.sets=[{id:set,members_json:JSON.stringify([{organizationId:f.job.organization_id,backupId:'backup_'+'b'.repeat(32)},{organizationId:null,backupId:core}])}]; + f.options.storage.removeSet=async id=>removed.push(id); + assert.equal((await purgeDspBackups(f.options)).failed,0);assert.deepEqual(removed,[set]);assert.equal(f.deleted.some(r=>r.id===core),false); +}); diff --git a/core/core/installations/tests/fixture-paycom-collector.js b/core/core/installations/tests/fixture-paycom-collector.js new file mode 100644 index 0000000..eda403f --- /dev/null +++ b/core/core/installations/tests/fixture-paycom-collector.js @@ -0,0 +1,172 @@ +#!/usr/bin/env node +'use strict'; + +const path = require('node:path'); +const projectRoot = process.env.DISPATCH_PROJECT_ROOT; +if (typeof projectRoot !== 'string' || !path.isAbsolute(projectRoot) || path.resolve(projectRoot) !== projectRoot) { + process.exitCode = 1; + return; +} +const plugin = relative => path.join(projectRoot, 'plugins', 'paycom', 'backend', relative); +const { + PaycomStore, + stageCandidate, + cleanupStage, +} = require(plugin('src/store')); +const { DATABASE, STAGING_ROOT } = require(plugin('src/paths')); +const { + periodContaining, + previousPeriod, + nextPeriod, + periodFromEnd, +} = require(plugin('src/timecard-period')); +const { + TIMECARD_SUMMARY, + ROUTE_VERSION, + linkRows, +} = require(plugin('src/resource-links')); +const { timecardRecord, rosterRow } = require(plugin('tests/helpers')); + +const TODAY = '2026-09-02'; +const TARGET = '2026-09-05'; +const COLLECTED_AT = '2026-09-02T21:30:00.000Z'; + +function publish(store, request, candidate) { + const stage = stageCandidate(STAGING_ROOT, candidate); + let result; + try { result = store.publish(stage); } + finally { cleanupStage(stage, STAGING_ROOT); } + return { + ok: true, + status: result.disposition === 'no_change' ? 'no_change' : 'published', + data: { + method: request.method, + target: candidate.target, + publicationId: result.publicationId, + disposition: result.disposition, + rowCount: result.rowCount, + contentSha256: result.contentSha256, + }, + }; +} + +function execute(request) { + if (request.method === 'collection.resolve-targets') { + return { + ok: true, + status: 'succeeded', + data: { + targetType: 'pay-period', + targets: [{ key: TARGET, start: '2026-08-23', end: TARGET, values: { periodEnd: TARGET } }], + }, + }; + } + const store = new PaycomStore(DATABASE); + try { + if (request.method === 'pay-periods.discover') { + const current = periodContaining(TODAY); + const rows = [ + { ...previousPeriod(current), relation: 'previous' }, + { ...current, relation: 'current' }, + { ...nextPeriod(current), relation: 'next' }, + ].map(({ start, end, key, relation }) => ({ start, end, key, relation })); + return publish(store, request, { + kind: 'pay_periods', + target: TODAY, + runId: request.runId, + attempt: request.attempt, + collectedAt: COLLECTED_AT, + metadata: { timezone: request.source.config.timezone, basis: 'fixture' }, + rows, + }); + } + const period = periodFromEnd(request.input.periodEnd || TARGET); + if (request.method === 'roster.period') { + return publish(store, request, { + kind: 'roster', + target: period.end, + runId: request.runId, + attempt: request.attempt, + collectedAt: COLLECTED_AT, + metadata: { sourceSha256: 'a'.repeat(64) }, + rows: [rosterRow('A001', 'Fixture One'), rosterRow('A002', 'Fixture Two')], + }); + } + if (request.method === 'timecards.from-published-roster') { + const roster = store.activeRoster(period.end); + const rows = roster.employees.filter(employee => employee.isActive).map((employee, index) => ({ + employeeCode: employee.employeeCode, + employeeName: employee.employeeName, + record: timecardRecord(employee.employeeCode, period.end), + sourceSha256: String(index + 1).repeat(64), + })); + return publish(store, request, { + kind: 'timecards', + target: period.end, + periodKey: period.key, + runId: request.runId, + attempt: request.attempt, + collectedAt: COLLECTED_AT, + metadata: { + periodStart: period.start, + periodEnd: period.end, + mode: 'published_roster', + rosterPublicationId: roster.publication.id, + rosterContentSha256: roster.publication.content_sha256, + }, + rows, + }); + } + if (request.method === 'timecards.audit') { + const audit = store.auditTimecards(period.end); + if (!audit.verified) return { ok: false, status: 'failed', error: { code: audit.code } }; + return { ok: true, status: 'succeeded', data: { method: request.method, audit } }; + } + if (request.method === 'resource-links.period') { + const roster = store.activeRoster(period.end); + const employees = roster.employees.filter(employee => employee.isActive); + const receipt = publish(store, request, { + kind: 'resource_links', + target: period.end, + periodKey: period.key, + runId: request.runId, + attempt: request.attempt, + collectedAt: COLLECTED_AT, + metadata: { + resourceType: TIMECARD_SUMMARY, + periodStart: period.start, + periodEnd: period.end, + rosterPublicationId: roster.publication.id, + rosterContentSha256: roster.publication.content_sha256, + routeVersion: ROUTE_VERSION, + }, + rows: linkRows(TIMECARD_SUMMARY, employees, period), + }); + const audit = store.auditResourceLinks(TIMECARD_SUMMARY, period.end); + if (!audit.verified) throw new Error(audit.code); + receipt.data.audit = { verified: true, activeEmployees: employees.length, links: audit.rowCount }; + return receipt; + } + if (request.method === 'resource-links.audit') { + const audit = store.auditResourceLinks(TIMECARD_SUMMARY, period.end); + if (!audit.verified) return { ok: false, status: 'failed', error: { code: audit.code } }; + return { ok: true, status: 'succeeded', data: { method: request.method, audit } }; + } + throw new Error('invalid_request'); + } finally { + store.close(); + } +} + +const chunks = []; +process.stdin.on('data', chunk => chunks.push(chunk)); +process.stdin.on('end', () => { + try { + const request = JSON.parse(Buffer.concat(chunks).toString('utf8')); + process.stdout.write(`${JSON.stringify(execute(request))}\n`); + } catch (error) { + const code = /^[a-z][a-z0-9_]{0,63}$/.test(error?.code || error?.message) + ? error.code || error.message : 'fixture_failed'; + process.stdout.write(`${JSON.stringify({ ok: false, status: 'failed', error: { code } })}\n`); + } +}); diff --git a/core/core/installations/tests/fixtures/directory-network-probe.js b/core/core/installations/tests/fixtures/directory-network-probe.js new file mode 100644 index 0000000..129544a --- /dev/null +++ b/core/core/installations/tests/fixtures/directory-network-probe.js @@ -0,0 +1,61 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const path = require('node:path'); +const fs = require('node:fs'); +const net = require('node:net'); +const { createEgressRelay } = require('dispatch-dsp/runtime/supervisor/src/egress-relay.js'); +const { ChromeBrowserRuntime } = require('dispatch-dsp/runtime/auth-broker/src/browser-runtime.js'); +const { CdpConnection, createTarget } = require('dispatch-dsp/runtime/auth-broker/src/cdp.js'); +const { configuration, assertMountBoundary } = require('dispatch-dsp/runtime/supervisor/src/supervisor.js'); +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); + +async function main() { + assertMountBoundary(configuration()); + assert.deepEqual(fs.readdirSync('/sys/class/net'), ['lo']); + const relay = createEgressRelay(); await relay.start(); + let browser, connection; + const results = []; + const options = { stateRoot: path.join(path.dirname(process.env.DISPATCH_DATA_ROOT), 'browser/network-verification'), + socketRoot: process.env.DISPATCH_RUNTIME_ROOT }; + try { + browser = await new ChromeBrowserRuntime(options).launch({ provider: 'paycom', profile: 'synthetic-network' }); + const target = await createTarget(browser.endpoint, 'about:blank'); + connection = await CdpConnection.connect(target.webSocketDebuggerUrl); + await connection.command('Network.enable'); + for (const url of ['https://www.paycomonline.net/v4/cl/web.php', 'https://logistics.amazon.com/operations/execution']) { + const navigated = await connection.command('Page.navigate', { url }); + assert.equal(navigated.errorText, undefined, JSON.stringify(navigated)); + let view; + for (let attempt = 0; attempt < 200; attempt++) { + view = await connection.evaluate(`({url:location.origin,ready:document.readyState,inputs:document.querySelectorAll('input').length,secure:window.isSecureContext})`); + if (view.ready === 'complete' && view.inputs > 0) break; + await delay(100); + } + assert.ok(view.secure && view.inputs > 0, JSON.stringify(view)); + assert.match(view.url, /^https:\/\/(www\.paycomonline\.net|www\.amazon\.com|logistics\.amazon\.com)$/); + results.push({ origin: view.url, secureContext: view.secure, loginInputs: view.inputs }); + } + await connection.command('Network.setCookie', { name: 'dispatch_rebuild_probe', value: 'synthetic', + url: 'https://www.paycomonline.net', secure: true, httpOnly: true }); + connection.close(); connection = null; await browser.close(); browser = null; + browser = await new ChromeBrowserRuntime(options).launch({ provider: 'paycom', profile: 'synthetic-network' }); + const next = await createTarget(browser.endpoint, 'about:blank'); + connection = await CdpConnection.connect(next.webSocketDebuggerUrl); + const cookies = await connection.command('Network.getCookies', { urls: ['https://www.paycomonline.net'] }); + assert.ok(cookies.cookies.some(cookie => cookie.name === 'dispatch_rebuild_probe' && cookie.value === 'synthetic')); + await connection.command('Network.deleteCookies', { name: 'dispatch_rebuild_probe', url: 'https://www.paycomonline.net' }); + // Direct public connections still have no route from this network namespace. + await new Promise((resolve, reject) => { + const socket = net.createConnection({ host: '8.8.8.8', port: 443 }); + socket.once('connect', () => { socket.destroy(); reject(new Error('direct_network_visible')); }); + socket.once('error', () => { socket.destroy(); resolve(); }); + socket.setTimeout(1000, () => { socket.destroy(); resolve(); }); + }); + process.stdout.write(JSON.stringify({ ok: true, providers: results, sessionRetained: true, directNetworkBlocked: true }) + '\n'); + } finally { connection?.close(); await browser?.close(); await relay.close(); } +} +main().catch(error => { + fs.writeFileSync(path.join(process.env.DISPATCH_LOGS_ROOT, 'network-verification-error.log'), error.stack + '\n', { mode: 0o600 }); + process.stderr.write(error.stack + '\n'); process.exitCode = 1; +}); diff --git a/core/core/installations/tests/fixtures/directory-sandbox-probe.js b/core/core/installations/tests/fixtures/directory-sandbox-probe.js new file mode 100644 index 0000000..1e709e3 --- /dev/null +++ b/core/core/installations/tests/fixtures/directory-sandbox-probe.js @@ -0,0 +1,45 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { CollectionManager } = require('dispatch-dsp/runtime/collection-manager/src/manager.js'); +const { defaultPaths } = require('dispatch-runtime-kit/collection-manager/src/paths'); + +async function main() { + process.umask(0o077); + const [siblingId, hostRoot, hostMarker, hostPid] = process.argv.slice(2); + const ownRoot = path.dirname(process.env.DISPATCH_DATA_ROOT); + const blocked = [hostRoot, `/var/lib/dispatch/${siblingId}`, '/etc/shadow', '/run/docker.sock']; + for (const selected of blocked) assert.equal(fs.existsSync(selected), false, 'unexpected host path'); + for (const name of ['.control', '.service-root', '.code-view']) { + assert.equal(fs.existsSync(path.join(ownRoot, name)), false, 'host control path visible'); + } + const processes = fs.readdirSync('/proc').filter(name => /^\d+$/.test(name)); + for (const pid of processes) { + let cmdline; + try { cmdline = fs.readFileSync(`/proc/${pid}/cmdline`, 'utf8'); } catch { continue; } + assert.equal(cmdline.split('\0')[0] === hostMarker, false, 'host process visible'); + } + // The marker uses a distinct host process, not a numeric PID assumption: PID + // values can be reused inside a private PID namespace. + assert.notEqual(fs.readlinkSync('/proc/self/ns/pid'), hostPid); + assert.throws(() => fs.writeFileSync('/opt/dispatch/.sandbox-write-test', 'forbidden')); + assert.throws(() => fs.writeFileSync('/usr/.sandbox-write-test', 'forbidden')); + assert.equal(process.env.DISPATCH_PRIVATE_TEST_VALUE, undefined); + assert.equal(fs.readFileSync('/proc/self/status', 'utf8').match(/^CapEff:\s*(\w+)/m)[1], '0000000000000000'); + + const marker = path.join(ownRoot, 'data/directory-runtime-marker'); + fs.writeFileSync(marker, 'synthetic runtime'); + const store = new CollectionStore(defaultPaths()); + const manager = new CollectionManager(store); + try { + await manager.start(); + assert.ok(fs.statSync(defaultPaths().database).isFile()); + process.stdout.write(JSON.stringify({ ok: true, id: process.env.DISPATCH_RUNTIME_KEY, + collectionManager: 'started', isolation: 'passed', visibleProcesses: processes.length }) + '\n'); + } finally { await manager.stop(); store.close(); } +} + +main().catch(error => { process.stderr.write(`${error.stack}\n`); process.exitCode = 1; }); diff --git a/core/core/installations/tests/fixtures/directory-service-probe.js b/core/core/installations/tests/fixtures/directory-service-probe.js new file mode 100644 index 0000000..d0f8141 --- /dev/null +++ b/core/core/installations/tests/fixtures/directory-service-probe.js @@ -0,0 +1,103 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const http = require('node:http'); +const crypto = require('node:crypto'); +const { spawn } = require('node:child_process'); +const { configuration, assertMountBoundary } = require('dispatch-dsp/runtime/supervisor/src/supervisor.js'); +const { ChromeBrowserRuntime } = require('dispatch-dsp/runtime/auth-broker/src/browser-runtime.js'); +const { CdpConnection, createTarget } = require('dispatch-dsp/runtime/auth-broker/src/cdp.js'); + +async function main() { + process.umask(0o077); + const [hostRoot, sibling, hostPidNamespace] = process.argv.slice(2); + const config = configuration(); + const root = path.dirname(process.env.DISPATCH_DATA_ROOT); + assert.ok(process.pid > 1, 'namespace init must own subprocess reaping'); + try { assertMountBoundary(config); } + catch (error) { + fs.writeFileSync(path.join(root, 'logs/isolation-mounts.log'), fs.readFileSync('/proc/self/mountinfo'), { mode: 0o600 }); + throw error; + } + assert.ok(process.geteuid() > 0); + assert.notEqual(fs.readlinkSync('/proc/self/ns/pid'), hostPidNamespace); + for (const file of [hostRoot, `/var/lib/dispatch/${sibling}`, '/run/docker.sock', '/etc/shadow']) { + assert.equal(fs.existsSync(file), false, 'host or sibling path visible'); + } + for (const name of ['.service-root', '.code-view', '.control']) { + assert.throws(() => fs.readdirSync(path.join(root, name))); + assert.throws(() => fs.renameSync(path.join(root, name), path.join(root, name + '-moved'))); + } + assert.throws(() => fs.writeFileSync('/opt/dispatch/.write-probe', 'forbidden')); + assert.equal(fs.statSync(process.env.DISPATCH_CHROME_EXECUTABLE).uid, 0); + assert.equal(fs.statSync('/usr/bin/setpriv').uid, 0); + const status = fs.readFileSync('/proc/self/status', 'utf8'); + assert.match(status, /^CapEff:\s*0000000000000000$/m); + assert.match(status, /^NoNewPrivs:\s*1$/m); + const hostInterfaces = fs.readdirSync('/sys/class/net'); + assert.deepEqual(hostInterfaces, ['lo']); + + let seeded = false, seedCount = 0, browser, connection; + const sessionValue = crypto.randomBytes(12).toString('hex'); + const server = http.createServer((request, response) => { + if (request.url === '/seed' && !seeded) { + seeded = true; seedCount++; + response.setHeader('Set-Cookie', `directory_fixture_session=${sessionValue}; HttpOnly; Path=/`); + } + response.end('Synthetic directory session'); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const origin = `http://127.0.0.1:${server.address().port}`; + const diagnosticFile = path.join(root, 'logs/browser-verification.log'); + fs.writeFileSync(diagnosticFile, '', { mode: 0o600 }); + const options = { stateRoot: path.join(root, 'browser/service-verification'), socketRoot: process.env.DISPATCH_RUNTIME_ROOT, + // This acceptance-only origin is inside the guest, without an external proxy. + directoryNetwork: false, + spawnImpl: (executable, args, selected) => { + const stdio = [...selected.stdio]; stdio[2] = 'pipe'; + const child = spawn(executable, args, { ...selected, stdio }); + child.stderr.on('data', data => fs.appendFileSync(diagnosticFile, data)); + child.on('error', error => fs.appendFileSync(diagnosticFile, `${error.code}\n`)); + return child; + }, + }; + const cookies = async () => (await connection.command('Network.getCookies', { urls: [origin] })).cookies; + try { + browser = await new ChromeBrowserRuntime(options).launch({ provider: 'paycom', profile: 'synthetic-directory-session' }); + let target = await createTarget(browser.endpoint, `${origin}/seed`); + connection = await CdpConnection.connect(target.webSocketDebuggerUrl); + for (let i = 0; i < 100 && !(await cookies()).some(cookie => cookie.value === sessionValue); i++) await new Promise(resolve => setTimeout(resolve, 50)); + assert.ok((await cookies()).some(cookie => cookie.name === 'directory_fixture_session' && cookie.value === sessionValue)); + // Inspect Chrome's own sandbox report; no unsafe launch flags are added. + const sandboxTarget = await createTarget(browser.endpoint, 'chrome://sandbox'); + const sandboxConnection = await CdpConnection.connect(sandboxTarget.webSocketDebuggerUrl); + let sandboxText = ''; + try { + for (let i = 0; i < 100; i++) { + sandboxText = (await sandboxConnection.command('Runtime.evaluate', { expression: 'document.body.innerText', returnByValue: true })).result.value || ''; + if (sandboxText.includes('Seccomp')) break; + await new Promise(resolve => setTimeout(resolve, 50)); + } + assert.match(sandboxText, /Seccomp[^\n]*\s+Yes/i); + assert.match(sandboxText, /PID namespaces?[^\n]*\s+Yes/i); + } finally { sandboxConnection.close(); } + connection.close(); connection = null; + await browser.close(); browser = null; + const restarted = new ChromeBrowserRuntime(options); + await restarted.reconcile(); + browser = await restarted.launch({ provider: 'paycom', profile: 'synthetic-directory-session' }); + target = await createTarget(browser.endpoint, 'about:blank'); + connection = await CdpConnection.connect(target.webSocketDebuggerUrl); + assert.ok((await cookies()).some(cookie => cookie.name === 'directory_fixture_session' && cookie.value === sessionValue)); + assert.equal(seedCount, 1); + process.stdout.write(JSON.stringify({ ok: true, id: process.env.DISPATCH_RUNTIME_KEY, + browser: 'launched', sandbox: 'enabled', sessionCookie: 'retained-after-restart', isolation: 'passed', + pidNamespace: fs.readlinkSync('/proc/self/ns/pid') }) + '\n'); + } finally { + connection?.close(); await browser?.close(); + server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); + } +} +main().catch(error => { process.stderr.write(`${error.stack}\n`); process.exitCode = 1; }); diff --git a/core/core/installations/tests/fixtures/directory-volume-probe.js b/core/core/installations/tests/fixtures/directory-volume-probe.js new file mode 100644 index 0000000..26608a7 --- /dev/null +++ b/core/core/installations/tests/fixtures/directory-volume-probe.js @@ -0,0 +1,17 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'), assert = require('node:assert/strict'); +const root = process.env.DISPATCH_DATA_ROOT; +const size = fs.statfsSync(root), cap = size.blocks * size.bsize; +assert.ok(cap < 80 * 1024 ** 2, 'acceptance requires a small isolated volume'); +const file = path.join(root, 'quota-fill'), fd = fs.openSync(file, 'w', 0o600); +let limited = false, written = 0; +try { + const block = Buffer.alloc(1024 ** 2, 65); + while (written < cap + block.length) written += fs.writeSync(fd, block); +} catch (error) { if (error.code !== 'ENOSPC') throw error; limited = true; } +finally { fs.closeSync(fd); fs.unlinkSync(file); } +assert.equal(limited, true); +fs.writeFileSync(path.join(root, 'quota-retained'), 'synthetic quota recovery', { mode: 0o600 }); +const retained = fs.openSync(path.join(root, 'quota-retained'), 'r'); fs.fsyncSync(retained); fs.closeSync(retained); +assert.equal(fs.statSync(root).dev, fs.statSync(path.dirname(root)).dev); +process.stdout.write(JSON.stringify({ ok: true, limited, writtenBytes: written, capacityBytes: cap, storageIdentity: true }) + '\n'); diff --git a/core/core/installations/tests/github-release-notes.test.js b/core/core/installations/tests/github-release-notes.test.js new file mode 100644 index 0000000..bd43a41 --- /dev/null +++ b/core/core/installations/tests/github-release-notes.test.js @@ -0,0 +1,140 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fixture = require('../examples/github-changelog.json'); +const { authoring, markdown, releaseNotes } = require('../src/release-notes'); +const input = () => structuredClone(fixture); +const render = source => { + const { changelog, notes, github } = authoring(source); + return markdown('0.0.1', changelog, notes, github); +}; + +test('GitHub uses inline author and PR links under change types without a duplicate release heading', () => { + const body = render(input()); + assert.deepEqual([...body.matchAll(/^## (.+)$/gm)].map(match => match[1]), ['New', 'Improved', 'Fixed', 'Maintenance']); + assert.ok(body.includes(fixture.github.summary.replaceAll('.', '\\.'))); + assert.ok(body.includes('- **Capacity\\-aware shift templates** — Preview conflicts while building shifts\\. by [@example\\-author](https://github.com/example-author) · [PR #101]')); + assert.doesNotMatch(body, /^# Dispatch/m); + assert.ok(body.startsWith(fixture.github.summary.replaceAll('.', '\\.'))); + assert.match(body, /## New[\s\S]*#101[\s\S]*#103[\s\S]*## Improved[\s\S]*#102[\s\S]*#105[\s\S]*## Fixed[\s\S]*#104[\s\S]*#106[\s\S]*## Maintenance[\s\S]*#107/); + for (let number = 101; number <= 107; number++) { + assert.equal(body.split(`[PR #${number}](https://github.com/example-organization/dispatch-platform/pull/${number})`).length, 2); + } + assert.ok(body.endsWith('[Full changelog](https://github.com/example-organization/dispatch-platform/compare/0.0.0...0.0.1)\n')); + assert.doesNotMatch(body, /^## (Scheduling|Workforce|Dashboard|Highlights|Changed|Added)$/m); +}); + +test('GitHub-only metadata leaves the complete Updates, installation and popup data unchanged', () => { + const source = input(), original = structuredClone(source); + const withoutGithub = input(); + delete withoutGithub.github; + for (const entry of withoutGithub.changelog) delete entry.github; + const { github, ...actual } = authoring(source); + assert.deepEqual(actual, authoring(withoutGithub)); + assert.deepEqual(source, original); + assert.equal(actual.notes.groups.length, 4); + assert.equal(actual.notes.changelog.at(-1).kind, 'changed'); + assert.equal(actual.popup.changelog.at(-1).audience, 'platform'); + assert.equal(actual.popup.changelog[0].title, source.changelog[0].popup.title); + const release = { releaseId: 'dispatch_0.0.1', sourceCommit: 'a'.repeat(40), changelog: actual.changelog }; + assert.ok(releaseNotes({ schemaVersion: 1, releaseId: release.releaseId, sourceCommit: release.sourceCommit, ...actual.notes }, release)); + assert.equal(github.changes.at(-1).maintenance, true); +}); + +test('changed maps to Improved, removed remains visible, and empty sections are omitted', () => { + const source = input(); + source.changelog.at(-1).github.maintenance = false; + source.changelog[1].kind = 'removed'; + const body = render(source); + assert.deepEqual([...body.matchAll(/^## (.+)$/gm)].map(match => match[1]), ['New', 'Improved', 'Fixed', 'Removed']); + assert.match(body, /## Improved[\s\S]*#107[\s\S]*## Fixed/); + const legacy = authoring([{ ...authoring(source).changelog[0], kind: 'fixed' }]); + assert.deepEqual([...markdown('0.0.1', legacy.changelog, legacy.notes).matchAll(/^## (.+)$/gm)].map(match => match[1]), ['Fixed']); +}); + +test('required actions stay prominent and complete details use a disclosure under their entry', () => { + const source = input(); + source.afterUpdating = [{ title: 'Action', description: 'Required action.', audience: 'dsp' }]; + source.changelog[0].details = 'First paragraph.\n\nSecond paragraph.'; + const body = render(source); + assert.ok(body.indexOf('## After updating') < body.indexOf('## New')); + assert.ok(body.includes('- **Action** — Required action\\.')); + assert.ok(body.includes('
\n Details\n\n First paragraph\\.\n \n Second paragraph\\.\n\n
')); +}); + +test('old rich inputs, entry-only metadata, and first releases work without fabricated comparison links', () => { + const source = input(); + delete source.github; + assert.doesNotMatch(render(source), /Full changelog/); + assert.match(render(source), /\/pull\/101/); + for (const entry of source.changelog) delete entry.github; + assert.equal(authoring(source).github, undefined); + assert.match(render(source), /## New/); + assert.doesNotMatch(render(source), /Full changelog|\/pull\//); + for (const entry of source.changelog) { delete entry.audience; delete entry.popup; } + source.github = { summary: fixture.github.summary }; + assert.equal(authoring(source).popup, undefined); + assert.doesNotMatch(render(source), /Full changelog/); +}); + +test('plain-text copy cannot create headings, raw HTML, or authored Markdown links', () => { + const source = input(); + source.github.summary = ' [link](https://example.test)'; + source.changelog[0].title = '**text**'; + source.changelog[0].details = '\n## Injected'; + const body = render(source); + assert.ok(body.includes('<img src=x> \\[link\\]\\(https://example\\.test\\)')); + assert.ok(body.includes('\\*\\*text\\*\\*')); + assert.ok(body.includes('</details>\n \\#\\# Injected')); + assert.doesNotMatch(body, / { + const source = input(); + source.github.previousTag = '0.0.7+hotfix.1'; + assert.match(render(source), /compare\/0\.0\.7%2Bhotfix\.1\.\.\.0\.0\.1/); + source.github.previousTag = 'releases/0.0.0'; + assert.match(render(source), /compare\/releases%2F0\.0\.0\.\.\.0\.0\.1/); +}); + +test('each PR retains its own author inline, including bots and legacy references', () => { + const source = input(); + source.changelog[0].github.pullRequests = [ + { number: 101, author: 'first-author' }, + { number: 108, author: 'dependabot[bot]' }, + 109, + ]; + const body = render(source); + const line = body.split('\n').find(line => line.includes('PR #101')); + assert.ok(line.includes('by [@first\\-author](https://github.com/first-author) · [PR #101](https://github.com/example-organization/dispatch-platform/pull/101); by [@dependabot\\[bot\\]](https://github.com/dependabot%5Bbot%5D) · [PR #108](https://github.com/example-organization/dispatch-platform/pull/108); [PR #109](https://github.com/example-organization/dispatch-platform/pull/109)')); + assert.ok(body.indexOf('PR #101') < body.indexOf('
')); + assert.doesNotMatch(line, /github-actions|by undefined/); +}); + +for (const [name, mutate] of [ + ['null metadata', n => n.github = null], + ['unsupported field', n => n.github.url = 'https://example.test'], + ['empty summary', n => n.github.summary = ''], + ['multiline summary', n => n.github.summary = 'one\ntwo'], + ['oversized summary', n => n.github.summary = 'x'.repeat(601)], + ['invalid tag', n => n.github.previousTag = 'tag?query'], + ['null entry metadata', n => n.changelog[0].github = null], + ['unsupported entry field', n => n.changelog[0].github.summary = 'text'], + ['nonboolean maintenance', n => n.changelog[0].github.maintenance = 'true'], + ['duplicate PR', n => n.changelog[0].github.pullRequests = [101, 101]], + ['duplicate attributed PR', n => n.changelog[0].github.pullRequests = [{ number: 101, author: 'one' }, { number: 101, author: 'two' }]], + ['duplicate mixed PR', n => n.changelog[0].github.pullRequests = [101, { number: 101, author: 'one' }]], + ['missing author', n => n.changelog[0].github.pullRequests = [{ number: 101 }]], + ['empty author', n => n.changelog[0].github.pullRequests = [{ number: 101, author: '' }]], + ['author URL', n => n.changelog[0].github.pullRequests = [{ number: 101, author: 'https://example.test' }]], + ['author markup', n => n.changelog[0].github.pullRequests = [{ number: 101, author: 'name](https://example.test)' }]], + ['null reference', n => n.changelog[0].github.pullRequests = [null]], + ['unsupported reference field', n => n.changelog[0].github.pullRequests = [{ number: 101, author: 'one', url: 'https://example.test' }]], + ['invalid PR', n => n.changelog[0].github.pullRequests = [0]], + ['noninteger PR', n => n.changelog[0].github.pullRequests = [1.5]], + ['string PR', n => n.changelog[0].github.pullRequests = ['101']], + ['too many PRs', n => n.changelog[0].github.pullRequests = Array.from({ length: 21 }, (_, i) => i + 1)], +]) test(`GitHub authoring rejects ${name}`, () => { + const source = input(); mutate(source); + assert.throws(() => authoring(source), { code: 'release_notes_invalid' }); +}); diff --git a/core/core/installations/tests/helpers/full-host-recovery-vm.js b/core/core/installations/tests/helpers/full-host-recovery-vm.js new file mode 100644 index 0000000..a2fff9c --- /dev/null +++ b/core/core/installations/tests/helpers/full-host-recovery-vm.js @@ -0,0 +1,119 @@ +'use strict'; +// Destructive disposable-VM drill. Refuses the development/production host. +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'), crypto = require('node:crypto'); +const { execFileSync } = require('node:child_process'); +const root = path.resolve(__dirname, "../../../.."); +const recovery = require('../../src/host-recovery-bundle'); +const { AccessStore, AccessControlService } = require('../../../accounts/src'); +const { MANAGED_INSTALLATION_DIRECTORY_FIELDS } = require('../../../../shared/paths/runtime-paths'); +const { createOciFixtureDeploymentPlan, renderOciSystemUnit, renderOciBridgeSystemUnit, hostAccountName } = require('../../src/oci-deployment'); +const call = (cmd, args) => execFileSync(cmd, args, { encoding: 'utf8', timeout: 120000, stdio: ['ignore', 'pipe', 'pipe'] }).trim(); +const localRoot = '/home/dispatchfixture/local', uid = 1001, unitRoot = '/home/dispatchfixture/.config/systemd/user'; +const config = { coreUid: uid, localRoot }, selected = { scope: 'user', account: { name: 'dispatchfixture', uid } }; +const userctl = args => recovery.systemctl(selected, args); +const releaseId = 'dispatch_native_fixture', coreRoot = `/opt/dispatch-platform/releases/${releaseId}/core-artifact`; +const runtimeRoot = `/opt/dispatch-runtime/releases/${releaseId}`; +async function main() { + if (process.geteuid() !== 0 || os.hostname() !== 'dispatch-recovery-test' || root !== '/work' + || fs.existsSync('/etc/dispatch') || fs.existsSync(coreRoot)) throw Error('disposable_vm_required'); + call('/usr/sbin/groupadd', ['--gid', String(uid), 'dispatchfixture']); + call('/usr/sbin/useradd', ['--uid', String(uid), '--gid', String(uid), '--create-home', 'dispatchfixture']); + for (const child of ['data', 'state', 'config', 'secrets', 'run', 'installations']) fs.mkdirSync(path.join(localRoot, child), { recursive: true, mode: 0o700 }); + fs.mkdirSync(unitRoot, { recursive: true, mode: 0o700 }); + fs.mkdirSync(coreRoot, { recursive: true }); + fs.cpSync(root, path.join(coreRoot, 'code'), { recursive: true }); + const immutable = directory => { + for (const item of fs.readdirSync(directory, { withFileTypes: true })) { + const file = path.join(directory, item.name); + if (item.isDirectory()) immutable(file); + else fs.chmodSync(file, fs.statSync(file).mode & 0o111 ? 0o555 : 0o444); + } + fs.chmodSync(directory, 0o555); + }; + immutable(path.join(coreRoot, 'code')); + const deployment = { releaseId, version: 'fixture', sourceCommit: 'a'.repeat(40), localRoot, unitRoot, + port: 4310, publicOrigin: 'https://dispatch.example.test' }; + require('../../src/core-artifact-layout').finishCoreArtifact(coreRoot, deployment); + fs.copyFileSync(path.join(coreRoot, 'units/dispatch-dashboard.service'), path.join(unitRoot, 'dispatch-dashboard.service')); + fs.chmodSync(path.join(unitRoot, 'dispatch-dashboard.service'), 0o600); + fs.writeFileSync(path.join(localRoot, 'config/provisioning.env'), `DISPATCH_INSTALLATIONS_ROOT=${localRoot}/installations\nDISPATCH_RUNTIME_AGENT_CONTROL_SOCKET=${localRoot}/run/runtime-agent-control.sock\n`, { mode: 0o600 }); + fs.mkdirSync('/etc/dispatch', { mode: 0o755 }); + fs.writeFileSync('/etc/dispatch/fixture-secret', 'synthetic host secret', { mode: 0o600 }); + fs.writeFileSync(path.join(localRoot, 'secrets/fixture-secret'), 'synthetic core secret', { mode: 0o600 }); + const release = JSON.parse(fs.readFileSync('/root/package/descriptor.json')); + fs.mkdirSync(runtimeRoot, { recursive: true }); + const nativeArtifact = require('../../src/native-runtime-artifact'); + if (fs.existsSync(path.join(runtimeRoot, 'runtime-artifact'))) nativeArtifact.verifyNativeRuntime(path.join(runtimeRoot, 'runtime-artifact'), release); + else nativeArtifact.unpackNativeRuntime('/root/package/runtime.tar.gz', path.join(runtimeRoot, 'runtime-artifact'), release); + if (!fs.existsSync(path.join(runtimeRoot, 'bridge-artifact'))) require('../../src/create-bridge-artifact').main([path.join(runtimeRoot, 'bridge-artifact')]); + require('../../src/release-delivery-install').installBrowserSandboxProfile(); + const store = new AccessStore({ databaseRoot: path.join(localRoot, 'data/access-control'), database: path.join(localRoot, 'data/access-control/access-control.sqlite3') }); + const access = new AccessControlService(store); + const invitation = access.createPlatformBootstrap({ email: 'recovery@example.test' }); + const owner = await access.acceptNewUser({ token: invitation.token, firstName: 'Recovery', lastName: 'Fixture', + password: 'disposable recovery fixture password', confirmPassword: 'disposable recovery fixture password' }); + for (const directory of ['/var/lib/dispatch', '/var/lib/dispatch/tenants']) { fs.mkdirSync(directory, { recursive: true, mode: 0o755 }); fs.chmodSync(directory, 0o755); } + const plans = []; + for (let i = 0; i < 2; i++) { + const id = `org_recovery_${i}`, key = `runtime_recovery_${i}`, accountUid = 20501 + i, name = hostAccountName(key); + const manifest = { manifestVersion: 1, revision: 1, organization: { id, stationCode: 'DXX1', timezone: 'UTC' }, + runtime: { key, templateId: 'isolated_dsp_v1', releaseId } }; + const authority = { revision: 1, organization: manifest.organization, runtime: manifest.runtime }; + const plan = createOciFixtureDeploymentPlan(manifest, authority, release, + { name, uid: accountUid, gid: accountUid, subuidStart: 300000 + i * 65536, subgidStart: 300000 + i * 65536, subidCount: 65536 }, + { version: 1, backend: 'native_service_v1', channel: 'fixture', organizationId: id, runtimeKey: key, manifestRevision: 1, releaseId }); + plans.push(plan); + call('/usr/sbin/groupadd', ['--gid', String(accountUid), name]); + call('/usr/sbin/useradd', ['--uid', String(accountUid), '--gid', String(accountUid), '--no-create-home', '--home-dir', plan.host.accountHome, name]); + for (const directory of [plan.host.accountHome, plan.host.installationRoot, ...Object.values(MANAGED_INSTALLATION_DIRECTORY_FIELDS).map(p => path.join(plan.host.installationRoot, p))]) fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + fs.mkdirSync(plan.host.bridgeRoot, { recursive: true, mode: 0o711 }); fs.chmodSync(plan.host.bridgeRoot, 0o711); + const token = crypto.randomBytes(32).toString('base64url'); + fs.writeFileSync(path.join(plan.host.installationRoot, 'secrets/runtime-agent/registration-token'), token + '\n', { mode: 0o600 }); + fs.writeFileSync(path.join(plan.host.installationRoot, 'data/fixture-record'), `DSP ${i} private records`, { mode: 0o600 }); + call('/usr/bin/chown', ['-R', `${accountUid}:${accountUid}`, plan.host.tenantRoot]); + store.createOrganization({ id, name: `Recovery DSP ${i}`, abbreviation: null, timezone: 'UTC', status: i ? 'suspended' : 'setup_required', createdBy: null, timestamp: Date.now() }); + store.insertStation(id, 'DXX1', true, Date.now()); + store.createInstallation(id, key, i ? 'suspended' : 'waiting_for_provider_auth', Date.now(), releaseId, 'native_service_v1'); + store.recordRuntimeAgentAuthority({ organizationId: id, runtimeKey: key, tokenHash: crypto.createHash('sha256').update(token).digest('hex'), timestamp: Date.now() }); + fs.writeFileSync(plan.host.unitPath, renderOciSystemUnit(plan)); + fs.writeFileSync(plan.host.bridgeUnitPath, renderOciBridgeSystemUnit(plan, { bridgeExecutable: path.join(runtimeRoot, 'bridge-artifact/core/agent-bridge/src/service-cli.js'), + centralSocket: path.join(localRoot, 'run/runtime-agent-hub.sock'), centralUid: uid, controllerUid: 0 })); + } + store.close(); + call('/usr/bin/chown', ['-R', `${uid}:${uid}`, '/home/dispatchfixture']); + call('/usr/bin/loginctl', ['enable-linger', 'dispatchfixture']); + call('/usr/bin/systemctl', ['start', `user@${uid}.service`]); userctl(['daemon-reload']); userctl(['enable', '--now', 'dispatch-dashboard.service']); + call('/usr/bin/systemctl', ['daemon-reload']); + call('/usr/bin/systemctl', ['enable', '--now', plans[0].identity.bridgeUnitName, plans[0].identity.unitName]); + // Deliberately enabled but stopped: restore must also correct this old suspension state. + call('/usr/bin/systemctl', ['enable', plans[1].identity.bridgeUnitName, plans[1].identity.unitName]); + const health = async () => { + for (let attempt = 0; attempt < 40; attempt++) { + try { + const script = `require('/work/core/agents/src/control').runtimeAgentControlInvoke('${localRoot}/run/runtime-agent-control.sock','${plans[0].runtimeKey}','health',{}).then(r=>{if(!r.ok)process.exitCode=1}).catch(()=>process.exitCode=1)`; + call('/usr/sbin/runuser', ['--user', 'dispatchfixture', '--', '/usr/bin/node', '--no-warnings', '-e', script]); return; + } catch { await new Promise(resolve => setTimeout(resolve, 500)); } + } + throw Error('runtime_not_healthy'); + }; + await health(); + const directory = '/root/full-recovery'; + const proof = recovery.captureHostRecovery({ config, destination: directory }); + const manifest = JSON.parse(fs.readFileSync(path.join(directory, 'recovery.json'))); + for (const service of manifest.metadata.services) recovery.systemctl(service, ['disable', '--now', service.name]); + call('/usr/bin/loginctl', ['disable-linger', 'dispatchfixture']); call('/usr/bin/systemctl', ['stop', `user@${uid}.service`]); + for (const account of manifest.metadata.accounts) { call('/usr/sbin/userdel', [account.name]); try { call('/usr/sbin/groupdel', [account.name]); } catch {} } + for (const target of manifest.roots) fs.rmSync(target, { recursive: true, force: true }); + fs.rmSync('/home/dispatchfixture', { recursive: true, force: true }); + const restored = await recovery.restoreHostRecovery({ directory, digest: proof.sha256 }); + await health(); + const after = new (require('node:sqlite').DatabaseSync)(path.join(localRoot, 'data/access-control/access-control.sqlite3'), { readOnly: true }); + if (!after.prepare('SELECT id FROM users WHERE id=?').get(owner.session.user.id) || after.prepare('SELECT count(*) AS n FROM organizations').get().n !== 2) throw Error('restored_database_mismatch'); + after.close(); + for (let i = 0; i < 2; i++) if (fs.readFileSync(path.join(plans[i].host.installationRoot, 'data/fixture-record'), 'utf8') !== `DSP ${i} private records`) throw Error('restored_dsp_mismatch'); + if (recovery.command('/usr/bin/systemctl', ['show', plans[1].identity.unitName, '--property=UnitFileState', '--value']) !== 'disabled') throw Error('suspended_dsp_enabled'); + if (fs.readFileSync('/etc/dispatch/fixture-secret', 'utf8') !== 'synthetic host secret') throw Error('restored_secret_mismatch'); + fs.rmSync(directory, { recursive: true }); + console.log(JSON.stringify({ ...restored, accountsRecreated: true, actualDashboard: true, actualDspHealth: true, suspendedDisabled: true })); +} +main().catch(error => { console.error(error.stack); process.exitCode = 1; }); diff --git a/core/core/installations/tests/helpers/job-crash-worker.js b/core/core/installations/tests/helpers/job-crash-worker.js new file mode 100644 index 0000000..6d448e2 --- /dev/null +++ b/core/core/installations/tests/helpers/job-crash-worker.js @@ -0,0 +1,26 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { createInstallationLayoutManager } = require('../../src'); +const { createInstallationJobStore } = require('../../src/job-store'); + +const options = JSON.parse(process.argv[2]); +const store = createInstallationJobStore({ stateRoot: options.stateRoot }); +const layout = createInstallationLayoutManager({ installationsRoot: options.installationsRoot }); +const claim = store.claimNext('worker_interrupted', 1_020, 100); +assert.ok(claim); +assert.equal(store.claimNext('worker_competing', 1_021, 100), null); +const work = store.work(claim, 1_030); +const receipt = layout.materialize( + work.manifest, + work.authority, + mutation => store.mutateClaim(claim, 1_040, mutation), +); +store.completeStage( + claim, + work.stage, + receipt, + 1_040, +); +process.stdout.write(JSON.stringify(claim)); +process.exit(86); diff --git a/core/core/installations/tests/helpers/job-request-worker.js b/core/core/installations/tests/helpers/job-request-worker.js new file mode 100644 index 0000000..245da70 --- /dev/null +++ b/core/core/installations/tests/helpers/job-request-worker.js @@ -0,0 +1,24 @@ +'use strict'; + +const fs = require('node:fs'); +const { createInstallationJobStore } = require('../../src/job-store'); + +const options = JSON.parse(process.argv[2]); +const store = createInstallationJobStore({ stateRoot: options.stateRoot }); +try { + const result = store.request( + options.manifest, + options.authority, + options.operation, + { + scope: 'operator_fixture', + permission: 'platform.installations.manage', + operatorEnabled: true, + }, + options.jobId, + 1_010, + ); + fs.writeSync(1, JSON.stringify(result)); +} finally { + store.close(); +} diff --git a/core/core/installations/tests/helpers/release-retention-vm.js b/core/core/installations/tests/helpers/release-retention-vm.js new file mode 100644 index 0000000..0df0e6a --- /dev/null +++ b/core/core/installations/tests/helpers/release-retention-vm.js @@ -0,0 +1,56 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'); +const assert = require('node:assert/strict'); +const { spawn } = require('node:child_process'); +const { DatabaseSync } = require('node:sqlite'); +const { atomic } = require('../../src/release-delivery-files'); +const { pruneReleases } = require('../../src/release-retention'); +const { receiptKey, RECEIPTS } = require('../../src/offsite-policy'); +async function main() { + assert.equal(os.hostname(), 'dispatch-recovery-test'); assert.equal(process.geteuid(), 0); + const localRoot = '/home/dispatchfixture/local', releaseId = 'dispatch_native_fixture', rolloutId = 'rollout_' + '1'.repeat(32); + const code = `/opt/dispatch-platform/releases/${releaseId}/core-artifact/code`; + const unitRoot = '/home/dispatchfixture/.config/systemd/user'; + const store = { db: new DatabaseSync(path.join(localRoot, 'data/access-control/access-control.sqlite3')), close() { this.db.close(); } }; + // The full recovery drill already verifies running and suspended DSPs. This + // test isolates the root fleet-completion cleanup with an empty active fleet. + store.db.prepare("UPDATE installations SET status='decommissioned'").run(); + const user = store.db.prepare("SELECT id FROM users WHERE platform_role='owner'").get(); + store.db.prepare("INSERT INTO platform_rollouts VALUES(?,?,?,?,'running',?,?)").run(rolloutId, releaseId, user.id, 'fixture:cleanup:123456', Date.now(), Date.now()); + store.db.prepare("INSERT INTO platform_rollout_core VALUES(?,'succeeded',?,1,NULL,?)").run(rolloutId, '{}', Date.now()); + store.close(); + const userFile = (file, value) => { atomic(file, value); fs.chownSync(file, 1001, 1001); }; + const descriptor = JSON.parse(fs.readFileSync('/root/package/descriptor.json')); + userFile(path.join(localRoot, 'config/oci-releases.json'), { schemaVersion: 1, releases: { [releaseId]: descriptor } }); + userFile(path.join(localRoot, 'config/platform-releases.json'), { schemaVersion: 1, releases: { [releaseId]: { publishedAt: '2026-09-06T00:00:00.000Z' }, dispatch_obsolete: { publishedAt: '2026-09-05T00:00:00.000Z' } } }); + for (const name of ['dispatch-installation-reconcile.service', 'dispatch-platform-update.service']) userFile(path.join(unitRoot, name), `[Service]\nExecStart=/usr/bin/node ${code}/fixture-unused.js\n`); + atomic('/etc/systemd/system/dispatch-release-watch.service', `[Service]\nExecStart=/usr/bin/node ${code}/fixture-unused.js\n`, 0o644); + const stateRoot = '/var/lib/dispatch-host/state'; fs.mkdirSync(stateRoot, { recursive: true, mode: 0o700 }); fs.chmodSync(stateRoot, 0o700); + fs.mkdirSync(`/opt/dispatch-control/releases/${releaseId}`, { recursive: true, mode: 0o755 }); + fs.symlinkSync(`/opt/dispatch-control/releases/${releaseId}`, '/opt/dispatch-control/current'); + atomic('/etc/dispatch/oci-host.json', { stateRoot, authorityRoot: '/var/lib/dispatch-host/authority', unitRoot: '/etc/systemd/system', + releaseRoot: '/opt/dispatch-runtime/releases', centralSocket: path.join(localRoot, 'run/runtime-agent-hub.sock'), centralUid: 1001, controllerUid: 0, controlReleaseId: releaseId }); + const obsolete = []; + for (const base of ['dispatch-platform', 'dispatch-runtime', 'dispatch-control', 'dispatch-updater', 'dispatch-release-delivery']) { + const directory = `/opt/${base}/releases/dispatch_obsolete`; fs.mkdirSync(directory, { recursive: true, mode: 0o755 }); + fs.writeFileSync(path.join(directory, 'old-code'), 'obsolete fixture code'); obsolete.push(directory); + } + const recoveryRoot = path.join(localRoot, 'backups/platform-core', rolloutId); + fs.mkdirSync(path.join(recoveryRoot, 'attempt-1'), { recursive: true, mode: 0o700 }); + userFile(path.join(recoveryRoot, 'recovery.json'), { phase: 'promoted', releaseId, attempt: 1 }); + fs.mkdirSync(RECEIPTS, { recursive: true, mode: 0o755 }); + await assert.rejects(pruneReleases({ localRoot, coreUid: 1001 }), /release_cleanup_unavailable/); + assert.ok(obsolete.every(file => fs.existsSync(file))); + atomic(path.join(RECEIPTS, receiptKey(path.join(recoveryRoot, 'attempt-1')) + '.json'), { status: 'verified', recoveryDigest: 'a'.repeat(64) }, 0o644); + const reader = spawn('/usr/bin/sleep', ['120'], { cwd: obsolete[0], stdio: 'ignore' }); + await new Promise((resolve, reject) => { reader.once('spawn', resolve); reader.once('error', reject); }); + try { assert.equal((await pruneReleases({ localRoot, coreUid: 1001 })).status, 'waiting'); } + finally { reader.kill(); await new Promise(resolve => reader.once('exit', resolve)); } + const result = await pruneReleases({ localRoot, coreUid: 1001 }); + assert.equal(result.status, 'completed'); assert.equal(result.removedReleases, 5); + assert.ok(obsolete.every(file => !fs.existsSync(file))); + assert.ok(fs.existsSync(code)); assert.equal(fs.existsSync(recoveryRoot), false); + assert.equal((await pruneReleases({ localRoot, coreUid: 1001 })).status, 'idle'); + console.log(JSON.stringify({ status: 'retention_verified', missingProofBlocked: true, runningOldProcessBlocked: true, obsoleteRemoved: 5, currentPreserved: true })); +} +main().catch(error => { console.error(error.stack); process.exitCode = 1; }); diff --git a/core/core/installations/tests/helpers/restored-boot-vm.js b/core/core/installations/tests/helpers/restored-boot-vm.js new file mode 100644 index 0000000..07235f5 --- /dev/null +++ b/core/core/installations/tests/helpers/restored-boot-vm.js @@ -0,0 +1,31 @@ +'use strict'; +const fs = require('node:fs'), os = require('node:os'); +const { execFileSync } = require('node:child_process'); +const assert = require('node:assert/strict'); +const { createOciFixtureDeploymentPlan, renderOciSystemUnit, renderOciBridgeSystemUnit, hostAccountName } = require('../../src/oci-deployment'); +async function main() { + assert.equal(os.hostname(), 'dispatch-recovery-test'); assert.equal(process.geteuid(), 0); + const release = JSON.parse(fs.readFileSync('/root/package/descriptor.json')); + for (let i = 0; i < 2; i++) { + const id = `org_recovery_${i}`, key = `runtime_recovery_${i}`, uid = 20501 + i; + const manifest = { manifestVersion: 1, revision: 1, organization: { id, stationCode: 'DXX1', timezone: 'UTC' }, runtime: { key, templateId: 'isolated_dsp_v1', releaseId: release.releaseId } }; + const plan = createOciFixtureDeploymentPlan(manifest, { revision: 1, organization: manifest.organization, runtime: manifest.runtime }, release, + { name: hostAccountName(key), uid, gid: uid, subuidStart: 300000 + i * 65536, subgidStart: 300000 + i * 65536, subidCount: 65536 }, + { version: 1, backend: 'native_service_v1', channel: 'fixture', organizationId: id, runtimeKey: key, manifestRevision: 1, releaseId: release.releaseId }); + if (process.argv[2] === 'prepare') { + fs.writeFileSync(plan.host.unitPath, renderOciSystemUnit(plan)); + fs.writeFileSync(plan.host.bridgeUnitPath, renderOciBridgeSystemUnit(plan, { bridgeExecutable: `/opt/dispatch-runtime/releases/${release.releaseId}/bridge-artifact/core/agent-bridge/src/service-cli.js`, + centralSocket: '/home/dispatchfixture/local/run/runtime-agent-hub.sock', centralUid: 1001, controllerUid: 0 })); + } else { + const state = execFileSync('/usr/bin/systemctl', ['show', plan.identity.unitName, '--property=UnitFileState', '--value'], { encoding: 'utf8' }).trim(); + assert.equal(state, i ? 'disabled' : 'enabled'); + const active = execFileSync('/usr/bin/systemctl', ['show', plan.identity.unitName, '--property=ActiveState', '--value'], { encoding: 'utf8' }).trim(); + assert.equal(active, i ? 'inactive' : 'active'); + } + } + if (process.argv[2] === 'prepare') { execFileSync('/usr/bin/systemctl', ['daemon-reload']); return; } + const script = `require('/work/core/agents/src/control').runtimeAgentControlInvoke('/home/dispatchfixture/local/run/runtime-agent-control.sock','runtime_recovery_0','health',{}).then(r=>{if(!r.ok)process.exitCode=1}).catch(()=>process.exitCode=1)`; + execFileSync('/usr/sbin/runuser', ['--user', 'dispatchfixture', '--', '/usr/bin/node', '--no-warnings', '-e', script]); + console.log(JSON.stringify({ status: 'reboot_verified', activeDspHealthy: true, suspendedDspDisabled: true })); +} +main().catch(error => { console.error(error.stack); process.exitCode = 1; }); diff --git a/core/core/installations/tests/helpers/rollout-backups.js b/core/core/installations/tests/helpers/rollout-backups.js new file mode 100644 index 0000000..6c04f0c --- /dev/null +++ b/core/core/installations/tests/helpers/rollout-backups.js @@ -0,0 +1,22 @@ +'use strict'; +// Simulate only the snapshot executor/upload boundary for coordinator tests. +// Requests, lifecycle authority, identity, categories and persistence stay real. +const { createAccessInstallationLifecycleAuthority } = require('../../../accounts/src/installation-lifecycle'); +function completeRolloutBackups(store) { + for (const request of store.db.prepare("SELECT * FROM platform_backup_requests WHERE json_extract(input_json,'$.category')='pre_update' AND status!='completed'").all()) { + store.db.prepare("UPDATE platform_backup_requests SET status='running' WHERE id=?").run(request.id); + if (request.organization_id) { + const control = store.installationControl(request.organization_id); + const job = createAccessInstallationLifecycleAuthority({ store, organizationId: request.organization_id, authorityScope: 'platform_backups' }) + .request({ operation: 'backup', expectedRevision: control.revision, idempotencyKey: `${request.id}:backing_up` }); + const saved = store.lifecycleJob(job.id); + store.db.prepare("UPDATE installation_lifecycle_jobs SET status='succeeded',finished_at=?,result_json='{}' WHERE id=?").run(Date.now(), job.id); + store.db.prepare('UPDATE installations SET status=? WHERE organization_id=?').run(control.status, request.organization_id); + store.db.prepare("UPDATE installation_backups SET status='available',tree_digest=?,file_count=1,total_bytes=10,completed_at=? WHERE id=?") + .run('a'.repeat(64), Date.now(), saved.backup_id); + store.db.prepare('UPDATE platform_backup_requests SET job_id=? WHERE id=?').run(job.id, request.id); + } + store.db.prepare("UPDATE platform_backup_requests SET status='completed',phase='completed' WHERE id=?").run(request.id); + } +} +module.exports = { completeRolloutBackups }; diff --git a/core/core/installations/tests/immutable-artifact.test.js b/core/core/installations/tests/immutable-artifact.test.js new file mode 100644 index 0000000..b54be52 --- /dev/null +++ b/core/core/installations/tests/immutable-artifact.test.js @@ -0,0 +1,28 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { createImmutableArtifact } = require('../src/immutable-artifact'); + +test('artifact failures remove partial output and preserve existing destinations', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-artifact-failure-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const projectRoot = path.join(root, 'source'); + fs.mkdirSync(projectRoot, { mode: 0o755 }); + fs.writeFileSync(path.join(projectRoot, 'valid.js'), 'module.exports = {};\n', { mode: 0o644 }); + fs.symlinkSync('valid.js', path.join(projectRoot, 'linked.js')); + fs.writeFileSync(path.join(projectRoot, 'writable.js'), 'unsafe'); + fs.chmodSync(path.join(projectRoot, 'writable.js'), 0o666); + const target = path.join(root, 'artifact'); + for (const rejected of ['missing.js', 'linked.js', 'writable.js', './valid.js']) { + assert.throws(() => createImmutableArtifact({ projectRoot, target, sourceFiles: ['valid.js', rejected] })); + assert.equal(fs.existsSync(target), false, `${rejected} left partial output`); + } + fs.mkdirSync(target, { mode: 0o755 }); + fs.writeFileSync(path.join(target, 'keep'), 'existing artifact'); + assert.throws(() => createImmutableArtifact({ projectRoot, target, sourceFiles: ['valid.js'] })); + assert.equal(fs.readFileSync(path.join(target, 'keep'), 'utf8'), 'existing artifact'); +}); diff --git a/core/core/installations/tests/install-command.test.js b/core/core/installations/tests/install-command.test.js new file mode 100644 index 0000000..7329bef --- /dev/null +++ b/core/core/installations/tests/install-command.test.js @@ -0,0 +1,22 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const { parse, preflight } = require('../src/install-command'); +const config = { uid: 1001, gid: 1001, localRoot: '/srv/dispatch/local', unitRoot: '/srv/dispatch/units' }; +test('installation commands require an exact version and commit and separate setup inputs', () => { + const args = ['prepare', '--version', '1.2.3', '--commit', 'a'.repeat(40)]; + assert.equal(parse(args).version, '1.2.3'); + for (const invalid of [['update'], [...args, '--version', '1.2.4'], [...args, '--force', 'true'], [...args, '--config', '/tmp/config'], ['setup', '--config', '../config', '--core', '/tmp/core']]) assert.throws(() => parse(invalid)); + assert.equal(parse(['setup', '--config', '/root/config.json', '--core', '/root/core.json']).action, 'setup'); +}); +test('preflight reports every missing prerequisite before any installation action', () => { + const result = preflight(config, { exists: () => false, run: () => ({ status: 2, stdout: '' }), platform: 'linux', arch: 'x64' }); + assert.equal(result.ok, false); assert.ok(result.missing.includes('configured_service_account_required')); + assert.ok(result.missing.includes('missing:config/provisioning.env')); + assert.ok(result.missing.includes('release_delivery_setup_required')); +}); +test('dependency pins reject an unexpected runtime version before packaging', () => { + const { verify } = require('../src/release-dependencies'); + const pins = require('../runtime-dependencies.json'); + assert.deepEqual(verify({ run: file => ({ status: 0, stdout: file.endsWith('chrome') ? `Google Chrome for Testing ${pins.chrome}` : `v${pins.node}` }) }), { node: pins.node, chrome: pins.chrome }); + assert.throws(() => verify({ run: () => ({ status: 0, stdout: 'v0.0.0' }) }), /release_dependency_version_mismatch/); +}); diff --git a/core/core/installations/tests/install-layout.test.js b/core/core/installations/tests/install-layout.test.js new file mode 100644 index 0000000..8b33bce --- /dev/null +++ b/core/core/installations/tests/install-layout.test.js @@ -0,0 +1,20 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'), { spawnSync } = require('node:child_process'); +test('initial layout creates owned private catalogs and preserves configuration on retry', t => { + if (process.geteuid() === 0 || spawnSync('/usr/bin/sudo', ['-n', '/usr/bin/true']).status !== 0) return t.skip('requires unprivileged account with sudo'); + const root = fs.mkdtempSync(path.join(os.homedir(), '.dispatch-install-layout-')); fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const config = { uid: process.geteuid(), gid: process.getegid(), localRoot: path.join(root, 'local'), unitRoot: path.join(root, 'units') }; + const script = 'require(process.argv[1]).initializeLayout(JSON.parse(process.argv[2]))'; + const run = () => spawnSync('/usr/bin/sudo', ['-n', process.execPath, '-e', script, path.resolve(__dirname, "../src/install-layout.js"), JSON.stringify(config)], { encoding: 'utf8' }); + let result = run(); assert.equal(result.status, 0, result.stderr); + const file = path.join(config.localRoot, 'config/platform-releases.json'); + assert.equal(fs.statSync(file).uid, config.uid); assert.equal(fs.statSync(file).mode & 0o777, 0o600); + const value = { schemaVersion: 1, releases: { fixture: {} } }; fs.writeFileSync(file, JSON.stringify(value)); + const env = path.join(config.localRoot, 'config/provisioning.env'); fs.appendFileSync(env, 'FIXTURE=preserved\n'); + result = run(); assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(fs.readFileSync(file)), value); assert.match(fs.readFileSync(env, 'utf8'), /FIXTURE=preserved/); + fs.unlinkSync(file); fs.symlinkSync('/etc/passwd', file); + assert.notEqual(run().status, 0); fs.unlinkSync(file); +}); diff --git a/core/core/installations/tests/jobs.test.js b/core/core/installations/tests/jobs.test.js new file mode 100644 index 0000000..6e3dc9e --- /dev/null +++ b/core/core/installations/tests/jobs.test.js @@ -0,0 +1,903 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { spawn } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const test = require('node:test'); +const { + INSTALLATION_LAYOUT_TEMPLATE, + PRIVATE_DIRECTORY_MODE, + createInstallationLayoutManager, + createDurableInstallationProvisioner, +} = require('../src'); +const { + INSTALLATION_JOB_SCHEMA_VERSION, + INSTALLATION_JOB_STAGES, + INSTALLATION_SERVICE_PIPELINE_ID, + INSTALLATION_JOB_MAX_ATTEMPTS, + PROVISIONER_DATABASE_NAME, + PRIVATE_FILE_MODE, + createInstallationJobStore, +} = require('../src/job-store'); + +const MANAGE = Object.freeze({ + scope: 'operator_fixture', + permission: 'platform.installations.manage', + operatorEnabled: true, +}); +const READ = Object.freeze({ scope: 'operator_fixture', permission: 'platform.installations.read' }); +const FIXTURE_REGISTRATION = Object.freeze({ + fixture: true, + installationState: 'pending', + retainedData: false, +}); +const LIVE_REGISTRATION = Object.freeze({ + source: 'access_control', + installationState: 'pending', + organizationStatus: 'pending_owner', + retainedData: false, +}); + +function manifest(id = 'alpha') { + return { + manifestVersion: 1, + revision: 1, + organization: { id: `org_${id}`, stationCode: 'TST1', timezone: 'America/Los_Angeles' }, + runtime: { + key: `fixture_${id}`, + templateId: INSTALLATION_LAYOUT_TEMPLATE, + releaseId: 'dispatch_fixture_1', + }, + }; +} + +function liveManifest(id = 'live') { + const selected = manifest(id); + selected.runtime.key = `runtime_${id}`; + return selected; +} + +function authority(value) { + return { + revision: value.revision, + organization: { ...value.organization }, + runtime: { ...value.runtime }, + }; +} + +function provisionRequest(key, expectedRevision) { + return { operation: 'provision', idempotencyKey: key, expectedRevision }; +} + +function retryRequest(key, expectedRevision) { + return { operation: 'retry', idempotencyKey: key, expectedRevision }; +} + +function cancelRequest(key, expectedRevision) { + return { operation: 'cancel', idempotencyKey: key, expectedRevision }; +} + +function isCode(code) { + return error => error?.code === code && error.message === code; +} + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-jobs-test-')); + fs.chmodSync(root, PRIVATE_DIRECTORY_MODE); + const stateRoot = path.join(root, 'control'); + const installationsRoot = path.join(root, 'installations'); + fs.mkdirSync(stateRoot, { mode: PRIVATE_DIRECTORY_MODE }); + fs.mkdirSync(installationsRoot, { mode: PRIVATE_DIRECTORY_MODE }); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return { root, stateRoot, installationsRoot }; +} + +function newStore(stateRoot) { + return createInstallationJobStore({ stateRoot }); +} + +function requestInChild(options) { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [ + '--no-warnings', + path.join(__dirname, "./helpers/job-request-worker.js"), + JSON.stringify(options), + ], { stdio: ['ignore', 'pipe', 'pipe'] }); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', chunk => { stdout += chunk; }); + child.stderr.on('data', chunk => { stderr += chunk; }); + child.once('error', reject); + child.once('close', status => { + if (status !== 0) return reject(new Error(`request worker failed: ${stderr.slice(0, 1024)}`)); + try { return resolve(JSON.parse(stdout)); } + catch { return reject(new Error('request worker returned invalid output')); } + }); + }); +} + +test('durable job storage is external, private, fixed, and restart-safe', t => { + const { root, stateRoot, installationsRoot } = fixture(t); + const selected = manifest(); + let store = newStore(stateRoot); + assert.deepEqual(store.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION, 1_000), { + state: 'pending', revision: 1, generation: 0, + }); + assert.throws(() => store.registerFixture( + selected, + authority(selected), + { ...FIXTURE_REGISTRATION, fixture: false }, + 1_001, + ), isCode('runtime_boundary_violation')); + const managed = manifest('managed_registration'); + managed.runtime.key = 'runtime_managed_registration'; + assert.throws(() => store.registerFixture( + managed, + authority(managed), + FIXTURE_REGISTRATION, + 1_002, + ), isCode('runtime_boundary_violation')); + assert.deepEqual(store.health(), { + ok: true, + status: 'ready', + schemaVersion: INSTALLATION_JOB_SCHEMA_VERSION, + databaseIntegrity: 'ok', + installations: 1, + jobs: { queued: 0, running: 0, succeeded: 0, failed: 0, cancelled: 0 }, + }); + const database = path.join(stateRoot, PROVISIONER_DATABASE_NAME); + const info = fs.lstatSync(database); + assert.equal(info.isFile(), true); + assert.equal(info.isSymbolicLink(), false); + assert.equal(info.nlink, 1); + assert.equal(info.mode & 0o7777, PRIVATE_FILE_MODE); + assert.equal(fs.realpathSync(database), database); + store.close(); + + store = newStore(stateRoot); + assert.deepEqual(store.installation(selected, authority(selected), READ), { + state: 'pending', revision: 1, generation: 0, + }); + store.close(); + + const unsafeRoot = path.join(root, 'unsafe'); + fs.mkdirSync(unsafeRoot, { mode: 0o755 }); + assert.throws(() => newStore(unsafeRoot), isCode('runtime_boundary_violation')); + for (const projectRoot of ['', null, false, 0]) { + assert.throws(() => createInstallationJobStore({ stateRoot, projectRoot }), + isCode('runtime_boundary_violation')); + } + const expanded = new DatabaseSync(database); + expanded.exec('CREATE TABLE unexpected_state(value TEXT) STRICT'); + expanded.close(); + assert.throws(() => newStore(stateRoot), isCode('runtime_boundary_violation')); + + const driftRoot = path.join(root, 'schema-drift'); + fs.mkdirSync(driftRoot, { mode: PRIVATE_DIRECTORY_MODE }); + const driftStore = newStore(driftRoot); + driftStore.close(); + const driftDatabase = path.join(driftRoot, PROVISIONER_DATABASE_NAME); + const drifted = new DatabaseSync(driftDatabase); + drifted.exec(`PRAGMA writable_schema=ON; + UPDATE sqlite_schema SET sql=replace(sql,'CHECK(revision>=1)','') WHERE name='installations'; + PRAGMA writable_schema=OFF;`); + drifted.close(); + assert.throws(() => newStore(driftRoot), isCode('runtime_boundary_violation')); + + const futureVersionRoot = path.join(root, 'future-version'); + fs.mkdirSync(futureVersionRoot, { mode: PRIVATE_DIRECTORY_MODE }); + const currentVersionStore = newStore(futureVersionRoot); + currentVersionStore.close(); + const futureVersion = new DatabaseSync(path.join(futureVersionRoot, PROVISIONER_DATABASE_NAME)); + futureVersion.exec(`PRAGMA user_version=${INSTALLATION_JOB_SCHEMA_VERSION + 1}`); + futureVersion.close(); + assert.throws(() => newStore(futureVersionRoot), isCode('runtime_boundary_violation')); + + const versionOneRoot = path.join(root, 'version-one'); + fs.mkdirSync(versionOneRoot, { mode: PRIVATE_DIRECTORY_MODE }); + const versionTwoStore = newStore(versionOneRoot); + const legacyRuntime = manifest('migration'); + versionTwoStore.registerFixture( + legacyRuntime, authority(legacyRuntime), FIXTURE_REGISTRATION, 100, + ); + versionTwoStore.request( + legacyRuntime, + authority(legacyRuntime), + { operation: 'provision', idempotencyKey: 'migration_request_key', expectedRevision: 1 }, + MANAGE, + 'job_migration_fixture', + 101, + ); + const legacyClaim = versionTwoStore.claimNext('worker_migration_one', 102, 100); + versionTwoStore.completeStage( + legacyClaim, + INSTALLATION_JOB_STAGES[0], + { layoutVersion: 1, status: 'verified', directoryCount: 15, changed: true }, + 103, + ); + versionTwoStore.close(); + const versionOneDatabase = new DatabaseSync(path.join(versionOneRoot, PROVISIONER_DATABASE_NAME)); + versionOneDatabase.exec('DROP TABLE live_job_authorizations; DROP TABLE job_compensations; PRAGMA user_version=1;'); + versionOneDatabase.close(); + const migrated = createInstallationJobStore({ + stateRoot: versionOneRoot, + pipelineId: INSTALLATION_SERVICE_PIPELINE_ID, + }); + assert.equal(migrated.health().schemaVersion, INSTALLATION_JOB_SCHEMA_VERSION); + const resumedClaim = migrated.claimNext('worker_migration_two', 203, 100); + const resumed = migrated.work(resumedClaim, 204); + assert.equal(resumed.pipelineId, 'installation_layout_v1'); + assert.equal(resumed.stage, INSTALLATION_JOB_STAGES[1]); + migrated.completeStage( + resumedClaim, + INSTALLATION_JOB_STAGES[1], + { layoutVersion: 1, status: 'verified', directoryCount: 15, changed: false }, + 205, + ); + assert.equal(migrated.finishSucceeded(resumedClaim, 206).status, 'succeeded'); + assert.deepEqual( + migrated.progress(legacyRuntime, authority(legacyRuntime), READ, 'job_migration_fixture'), + { completedStages: 2, totalStages: 2, attempts: 2 }, + ); + migrated.close(); + + const emptyRoot = path.join(root, 'empty-database'); + fs.mkdirSync(emptyRoot, { mode: PRIVATE_DIRECTORY_MODE }); + fs.writeFileSync(path.join(emptyRoot, PROVISIONER_DATABASE_NAME), '', { mode: PRIVATE_FILE_MODE }); + const recovered = newStore(emptyRoot); + assert.equal(recovered.health().status, 'ready'); + recovered.close(); + + const dirtyRoot = path.join(root, 'unexpected-entry'); + fs.mkdirSync(dirtyRoot, { mode: PRIVATE_DIRECTORY_MODE }); + fs.writeFileSync(path.join(dirtyRoot, 'unexpected'), 'fixture', { mode: PRIVATE_FILE_MODE }); + assert.throws(() => newStore(dirtyRoot), isCode('runtime_boundary_violation')); + assert.equal(fs.existsSync(path.join(dirtyRoot, PROVISIONER_DATABASE_NAME)), false); + + const corruptRoot = path.join(root, 'corrupt-database'); + fs.mkdirSync(corruptRoot, { mode: PRIVATE_DIRECTORY_MODE }); + fs.writeFileSync( + path.join(corruptRoot, PROVISIONER_DATABASE_NAME), + 'not a sqlite database', + { mode: PRIVATE_FILE_MODE }, + ); + assert.throws(() => createDurableInstallationProvisioner({ + stateRoot: corruptRoot, + installationsRoot, + }), error => error?.code === 'installation_operation_failed' + && error.message === 'installation_operation_failed' + && !JSON.stringify(error).includes('SQLITE')); +}); + +test('live jobs remain unrunnable until the Access Control acknowledgement is durably authorized', t => { + const { stateRoot } = fixture(t); + const store = newStore(stateRoot); + const selected = liveManifest('authorization'); + assert.deepEqual(store.registerLive(selected, authority(selected), LIVE_REGISTRATION, 300), { + state: 'pending', revision: 1, generation: 0, + }); + const job = store.request( + selected, + authority(selected), + provisionRequest('live:provision:authorization', 1), + MANAGE, + 'job_live_authorization', + 301, + ); + assert.equal(job.status, 'queued'); + assert.equal(store.claimNext('worker_before_ack', 302, 100), null); + assert.throws(() => store.authorizeLive( + selected, + authority(selected), + MANAGE, + job.id, + { + source: 'access_control', + installationState: 'provisioning', + currentJobId: 'job_other_authorization', + organizationStatus: 'pending_owner', + }, + 303, + ), isCode('runtime_boundary_violation')); + const authorized = store.authorizeLive( + selected, + authority(selected), + MANAGE, + job.id, + { + source: 'access_control', + installationState: 'provisioning', + currentJobId: job.id, + organizationStatus: 'pending_owner', + }, + 304, + ); + assert.equal(authorized.replayed, false); + assert.equal(store.authorizeLive( + selected, + authority(selected), + MANAGE, + job.id, + { + source: 'access_control', + installationState: 'provisioning', + currentJobId: job.id, + organizationStatus: 'pending_owner', + }, + 305, + ).replayed, true); + const claim = store.claimNext('worker_after_ack', 306, 100); + assert.equal(claim.jobId, job.id); + assert.equal(store.work(claim, 307).fixture, false); + store.close(); +}); + +test('competing request processes converge on one idempotent job', async t => { + const { stateRoot } = fixture(t); + const selected = manifest('concurrent'); + let store = newStore(stateRoot); + store.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION, 1_000); + store.close(); + const shared = { + stateRoot, + manifest: selected, + authority: authority(selected), + operation: provisionRequest('fixture:provision:concurrent', 1), + }; + const results = await Promise.all([ + requestInChild({ ...shared, jobId: 'job_concurrent_a' }), + requestInChild({ ...shared, jobId: 'job_concurrent_b' }), + ]); + assert.equal(results[0].id, results[1].id); + assert.deepEqual(results.map(result => result.replayed).sort(), [false, true]); + store = newStore(stateRoot); + assert.equal(store.health().jobs.queued, 1); + store.close(); +}); + +test('the provisioner boundary discards unexpected internal errors', t => { + const { stateRoot, installationsRoot } = fixture(t); + const selected = manifest('public_failure'); + const provisioner = createDurableInstallationProvisioner({ + stateRoot, + installationsRoot, + idFactory: () => { throw new Error(`internal ${stateRoot}`); }, + }); + t.after(() => provisioner.close()); + provisioner.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION); + assert.throws(() => provisioner.request( + selected, + authority(selected), + provisionRequest('fixture:provision:public-failure', 1), + MANAGE, + ), error => error?.code === 'installation_operation_failed' + && error.message === 'installation_operation_failed' + && !JSON.stringify(error).includes(stateRoot)); +}); + +test('target-bound requests are authorized, revisioned, and durably idempotent', t => { + const { stateRoot } = fixture(t); + const selected = manifest('idempotent'); + const store = newStore(stateRoot); + t.after(() => store.close()); + store.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION, 1_000); + + const firstRequest = provisionRequest('fixture:provision:idempotent', 1); + const first = store.request(selected, authority(selected), firstRequest, MANAGE, 'job_idempotent', 1_010); + assert.deepEqual(first, { + id: 'job_idempotent', + operation: 'provision', + status: 'queued', + installationState: 'provisioning', + revision: 2, + replayed: false, + failure: null, + }); + const replayed = store.request(selected, authority(selected), firstRequest, MANAGE, 'job_unused', 1_020); + assert.equal(replayed.id, first.id); + assert.equal(replayed.replayed, true); + assert.throws(() => store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:idempotent', 2), + MANAGE, + 'job_conflict', + 1_030, + ), isCode('idempotency_conflict')); + assert.throws(() => store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:stale', 1), + MANAGE, + 'job_stale', + 1_040, + ), isCode('installation_revision_conflict')); + assert.throws(() => store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:active', 2), + MANAGE, + 'job_active', + 1_050, + ), isCode('installation_operation_in_progress')); + assert.throws(() => store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:denied', 2), + { ...MANAGE, operatorEnabled: false }, + 'job_denied', + 1_060, + ), isCode('installation_operation_not_allowed')); + assert.throws(() => store.current(selected, authority(selected), MANAGE), + isCode('runtime_boundary_violation')); + assert.throws(() => store.progress(selected, authority(selected), MANAGE, first.id), + isCode('runtime_boundary_violation')); + assert.equal(JSON.stringify(first).includes(stateRoot), false); + assert.equal(JSON.stringify(first).includes(selected.runtime.key), false); + assert.deepEqual(Object.keys(first).sort(), [ + 'failure', 'id', 'installationState', 'operation', 'replayed', 'revision', 'status', + ]); +}); + +test('expired claims resume checkpoints and stale fences cannot mutate', t => { + const { stateRoot, installationsRoot } = fixture(t); + const selected = manifest('resume'); + const manager = createInstallationLayoutManager({ installationsRoot }); + let store = newStore(stateRoot); + store.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION, 1_000); + store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:resume', 1), + MANAGE, + 'job_resume', + 1_010, + ); + + const firstClaim = store.claimNext('worker_first', 1_020, 100); + assert.equal(store.claimNext('worker_other', 1_030, 100), null); + const firstWork = store.work(firstClaim, 1_040); + assert.equal(firstWork.stage, INSTALLATION_JOB_STAGES[0]); + const materialized = manager.materialize( + firstWork.manifest, + firstWork.authority, + mutation => store.mutateClaim(firstClaim, 1_045, mutation), + ); + assert.throws(() => store.completeStage( + firstClaim, + firstWork.stage, + { ...materialized, path: stateRoot }, + 1_049, + ), isCode('runtime_layout_failed')); + store.completeStage(firstClaim, firstWork.stage, materialized, 1_050); + assert.deepEqual(store.progress(selected, authority(selected), READ, 'job_resume'), { completedStages: 1, totalStages: 2, attempts: 1 }); + store.close(); + + store = newStore(stateRoot); + t.after(() => store.close()); + const resumedClaim = store.claimNext('worker_resumed', 1_120, 100); + assert.equal(resumedClaim.fence > firstClaim.fence, true); + assert.throws(() => store.work(firstClaim, 1_121), isCode('installation_operation_in_progress')); + const resumed = store.work(resumedClaim, 1_122); + assert.equal(resumed.stage, INSTALLATION_JOB_STAGES[1]); + store.completeStage( + resumedClaim, + resumed.stage, + manager.inspect(resumed.manifest, resumed.authority), + 1_123, + ); + const completed = store.finishSucceeded(resumedClaim, 1_124); + assert.equal(completed.status, 'succeeded'); + assert.equal(completed.installationState, 'provisioning'); + assert.deepEqual(store.progress(selected, authority(selected), READ, completed.id), { completedStages: 2, totalStages: 2, attempts: 2 }); + assert.equal(manager.inspect(selected, authority(selected)).status, 'verified'); +}); + +test('reclaim fences every durable filesystem mutation', t => { + const { stateRoot, installationsRoot } = fixture(t); + const selected = manifest('filesystem_fence'); + const manager = createInstallationLayoutManager({ installationsRoot }); + const store = newStore(stateRoot); + t.after(() => store.close()); + store.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION, 900); + store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:filesystem-fence', 1), + MANAGE, + 'job_filesystem_fence', + 950, + ); + const staleClaim = store.claimNext('worker_stale_filesystem', 1_000, 100); + const staleWork = store.work(staleClaim, 1_010); + const replacementClaim = store.claimNext('worker_current_filesystem', 1_100, 100); + const layout = manager.derive(selected, authority(selected)); + assert.equal(fs.existsSync(layout.installationRoot), false); + assert.throws(() => manager.materialize( + staleWork.manifest, + staleWork.authority, + mutation => store.mutateClaim(staleClaim, 1_101, mutation), + ), isCode('installation_operation_in_progress')); + assert.equal(fs.existsSync(layout.installationRoot), false); + + const currentWork = store.work(replacementClaim, 1_102); + const receipt = manager.materialize( + currentWork.manifest, + currentWork.authority, + mutation => store.mutateClaim(replacementClaim, 1_103, mutation), + ); + assert.equal(receipt.status, 'verified'); +}); + +test('queued and running cancellation converge without stale-worker completion', t => { + const { stateRoot } = fixture(t); + const selected = manifest('cancel'); + const store = newStore(stateRoot); + t.after(() => store.close()); + store.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION, 1_000); + + store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:cancel-queued', 1), + MANAGE, + 'job_cancel_queued', + 1_010, + ); + const queuedCancellation = cancelRequest('fixture:cancel:queued', 2); + const queued = store.request( + selected, authority(selected), queuedCancellation, MANAGE, 'job_unused', 1_020, + ); + assert.equal(queued.status, 'cancelled'); + assert.equal(queued.installationState, 'pending'); + assert.equal(queued.revision, 3); + assert.equal(store.request( + selected, authority(selected), queuedCancellation, MANAGE, 'job_unused_2', 1_030, + ).replayed, true); + + store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:cancel-running', 3), + MANAGE, + 'job_cancel_running', + 1_040, + ); + const claim = store.claimNext('worker_cancel', 1_050, 100); + const running = store.request( + selected, + authority(selected), + cancelRequest('fixture:cancel:running', 4), + MANAGE, + 'job_unused_3', + 1_060, + ); + assert.equal(running.status, 'running'); + assert.equal(store.work(claim, 1_070).cancelRequested, true); + const cancelled = store.finishCancelled(claim, 1_080); + assert.equal(cancelled.status, 'cancelled'); + assert.equal(cancelled.installationState, 'pending'); + assert.equal(cancelled.revision, 5); + assert.throws(() => store.finishSucceeded(claim, 1_081), isCode('installation_operation_in_progress')); +}); + +test('requested cancellation wins when an expired job reaches its attempt bound', t => { + const { stateRoot } = fixture(t); + const selected = manifest('cancel_attempt_bound'); + const store = newStore(stateRoot); + t.after(() => store.close()); + store.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION, 900); + store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:cancel-attempt-bound', 1), + MANAGE, + 'job_cancel_attempt_bound', + 950, + ); + store.claimNext('worker_cancel_bound_0', 1_000, 100); + for (let index = 1; index < INSTALLATION_JOB_MAX_ATTEMPTS; index += 1) { + store.claimNext(`worker_cancel_bound_${index}`, 1_000 + index * 100, 100); + } + store.request( + selected, + authority(selected), + cancelRequest('fixture:cancel:attempt-bound', 2), + MANAGE, + 'job_cancel_attempt_request', + 1_750, + ); + const terminal = store.claimNext('worker_cancel_terminal', 1_800, 100); + assert.equal(terminal.terminalJob.status, 'cancelled'); + assert.equal(terminal.terminalJob.installationState, 'pending'); +}); + +test('recoverable failures retry with a new fenced generation', t => { + const { stateRoot, installationsRoot } = fixture(t); + const selected = manifest('retry'); + const manager = createInstallationLayoutManager({ installationsRoot }); + const store = newStore(stateRoot); + t.after(() => store.close()); + store.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION, 1_000); + store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:failure', 1), + MANAGE, + 'job_failure', + 1_010, + ); + const failedClaim = store.claimNext('worker_failure', 1_020, 100); + const failed = store.finishFailed(failedClaim, { code: 'runtime_layout_failed', path: '/private' }, 1_030); + assert.deepEqual(failed.failure, { + code: 'runtime_layout_failed', category: 'infrastructure', recoverable: true, + }); + assert.equal(failed.installationState, 'failed'); + assert.equal(JSON.stringify(failed).includes('/private'), false); + + const retried = store.request( + selected, + authority(selected), + retryRequest('fixture:retry:failure', 3), + MANAGE, + 'job_retry', + 1_040, + ); + assert.equal(retried.operation, 'retry'); + assert.equal(retried.installationState, 'provisioning'); + assert.equal(retried.revision, 4); + const retryClaim = store.claimNext('worker_retry', 1_050, 100); + assert.equal(retryClaim.generation > failedClaim.generation, true); + assert.throws(() => store.completeStage( + failedClaim, + INSTALLATION_JOB_STAGES[0], + { layoutVersion: 1, status: 'verified', directoryCount: 15, changed: false }, + 1_051, + ), isCode('installation_operation_in_progress')); + for (;;) { + const work = store.work(retryClaim, 1_060); + if (work.stage === null) break; + const receipt = work.stage === INSTALLATION_JOB_STAGES[0] + ? manager.materialize( + work.manifest, + work.authority, + mutation => store.mutateClaim(retryClaim, 1_060, mutation), + ) + : manager.inspect(work.manifest, work.authority); + store.completeStage(retryClaim, work.stage, receipt, 1_061); + } + assert.equal(store.finishSucceeded(retryClaim, 1_062).status, 'succeeded'); +}); + +test('a cancelled retry retains its durable failed source', t => { + const { stateRoot } = fixture(t); + const selected = manifest('cancelled_retry'); + const store = newStore(stateRoot); + t.after(() => store.close()); + store.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION, 1_000); + store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:cancelled-retry', 1), + MANAGE, + 'job_cancelled_retry_failure', + 1_010, + ); + const failedClaim = store.claimNext('worker_cancelled_retry_failure', 1_020, 100); + store.finishFailed(failedClaim, 'runtime_layout_failed', 1_030); + store.request( + selected, + authority(selected), + retryRequest('fixture:retry:cancelled-once', 3), + MANAGE, + 'job_cancelled_retry_first', + 1_040, + ); + const cancelled = store.request( + selected, + authority(selected), + cancelRequest('fixture:cancel:cancelled-retry', 4), + MANAGE, + 'job_cancelled_retry_cancel', + 1_050, + ); + assert.equal(cancelled.status, 'cancelled'); + assert.equal(cancelled.installationState, 'failed'); + const retriedAgain = store.request( + selected, + authority(selected), + retryRequest('fixture:retry:cancelled-again', 5), + MANAGE, + 'job_cancelled_retry_second', + 1_060, + ); + assert.equal(retriedAgain.status, 'queued'); + assert.equal(retriedAgain.operation, 'retry'); + assert.equal(retriedAgain.installationState, 'provisioning'); +}); + +test('repeated worker loss stops at the immutable attempt bound', t => { + const { stateRoot } = fixture(t); + const selected = manifest('attempt_bound'); + const store = newStore(stateRoot); + t.after(() => store.close()); + store.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION, 900); + store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:attempt-bound', 1), + MANAGE, + 'job_attempt_bound', + 950, + ); + let claim = store.claimNext('worker_attempt_0', 1_000, 100); + for (let index = 1; index < INSTALLATION_JOB_MAX_ATTEMPTS; index += 1) { + claim = store.claimNext(`worker_attempt_${index}`, 1_000 + index * 100, 100); + assert.equal(Object.hasOwn(claim, 'terminalJob'), false); + } + const exhausted = store.claimNext( + 'worker_attempt_terminal', + 1_000 + INSTALLATION_JOB_MAX_ATTEMPTS * 100, + 100, + ); + assert.equal(exhausted.terminalJob.status, 'failed'); + assert.deepEqual(exhausted.terminalJob.failure, { + code: 'installation_operation_failed', category: 'infrastructure', recoverable: false, + }); + assert.deepEqual(store.progress(selected, authority(selected), READ, 'job_attempt_bound'), { + completedStages: 0, + totalStages: INSTALLATION_JOB_STAGES.length, + attempts: INSTALLATION_JOB_MAX_ATTEMPTS, + }); + assert.throws(() => store.work(claim, 2_000), isCode('installation_operation_in_progress')); +}); + +test('resumed completion revalidates checkpointed external state', t => { + const { stateRoot, installationsRoot } = fixture(t); + const selected = manifest('final_revalidation'); + const manager = createInstallationLayoutManager({ installationsRoot }); + let store = newStore(stateRoot); + store.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION, 900); + store.request( + selected, + authority(selected), + provisionRequest('fixture:provision:final-revalidation', 1), + MANAGE, + 'job_final_revalidation', + 950, + ); + const claim = store.claimNext('worker_before_exit', 1_000, 100); + let work = store.work(claim, 1_010); + store.completeStage( + claim, + work.stage, + manager.materialize( + work.manifest, + work.authority, + mutation => store.mutateClaim(claim, 1_015, mutation), + ), + 1_020, + ); + work = store.work(claim, 1_030); + store.completeStage( + claim, + work.stage, + manager.inspect(work.manifest, work.authority), + 1_040, + ); + store.close(); + store = null; + const layout = manager.derive(selected, authority(selected)); + fs.chmodSync(layout.directories.configRoot, 0o755); + + let now = 1_120; + const provisioner = createDurableInstallationProvisioner({ + stateRoot, + installationsRoot, + clock: () => now++, + leaseMs: 100, + }); + t.after(() => provisioner.close()); + const failed = provisioner.runNext('worker_after_exit'); + assert.equal(failed.status, 'failed'); + assert.deepEqual(failed.failure, { + code: 'runtime_layout_failed', category: 'infrastructure', recoverable: true, + }); +}); + +test('the sanitized provisioner API exercises two isolated durable fixture jobs', t => { + const { root, stateRoot, installationsRoot } = fixture(t); + let now = 1_000; + const ids = ['job_api_alpha', 'job_api_bravo', 'job_api_unused']; + let provisioner = createDurableInstallationProvisioner({ + stateRoot, + installationsRoot, + clock: () => now++, + idFactory: () => ids.shift(), + leaseMs: 100, + }); + const alpha = manifest('api_alpha'); + const bravo = manifest('api_bravo'); + provisioner.registerFixture(alpha, authority(alpha), FIXTURE_REGISTRATION); + provisioner.registerFixture(bravo, authority(bravo), FIXTURE_REGISTRATION); + const alphaRequest = provisionRequest('fixture:provision:api-alpha', 1); + const alphaJob = provisioner.request(alpha, authority(alpha), alphaRequest, MANAGE); + assert.equal(provisioner.request(alpha, authority(alpha), alphaRequest, MANAGE).replayed, true); + const bravoJob = provisioner.request( + bravo, + authority(bravo), + provisionRequest('fixture:provision:api-bravo', 1), + MANAGE, + ); + assert.notEqual(alphaJob.id, bravoJob.id); + assert.equal(provisioner.runNext('worker_api_one').status, 'succeeded'); + assert.equal(provisioner.runNext('worker_api_two').status, 'succeeded'); + assert.deepEqual(provisioner.runNext('worker_api_idle'), { ok: true, status: 'idle' }); + assert.equal(provisioner.health().jobs.succeeded, 2); + assert.equal(JSON.stringify(provisioner.inspect(alpha, authority(alpha), READ)).includes(root), false); + provisioner.close(); + + provisioner = createDurableInstallationProvisioner({ + stateRoot, + installationsRoot, + clock: () => now++, + idFactory: () => 'job_api_reopened', + leaseMs: 100, + }); + t.after(() => provisioner.close()); + provisioner.registerFixture(alpha, authority(alpha), FIXTURE_REGISTRATION); + provisioner.registerFixture(bravo, authority(bravo), FIXTURE_REGISTRATION); + assert.equal(provisioner.inspect(alpha, authority(alpha), READ).status, 'succeeded'); + assert.equal(provisioner.inspect(bravo, authority(bravo), READ).status, 'succeeded'); +}); + +test('durable provisioning selects the immutable OCI pipeline and fences every host mutation', t => { + const { stateRoot, installationsRoot } = fixture(t); + const calls = []; + const mutate = (name, capability) => { + capability(() => { calls.push(name); }); + return { ociDeploymentPlanVersion: 1, status: name, changed: true }; + }; + const adapter = Object.freeze({ + plan: () => Object.freeze({ selected: true }), + reconcileHostAccount: (manifestValue, authorityValue, options, capability) => + mutate('host_account_ready', capability), + reconcileImage: (planValue, claim, capability) => mutate('image_ready', capability), + reconcileBridge: (planValue, claim, capability) => mutate('bridge_ready', capability), + reconcileContainer: (planValue, claim, capability) => mutate('container_ready', capability), + verify: () => ({ ociDeploymentPlanVersion: 1, status: 'healthy', changed: false }), + commit: (planValue, claim, capability) => { capability(() => { calls.push('committed'); }); }, + rollback: () => { throw new Error('unexpected_rollback'); }, + }); + let now = 2_000; + const provisioner = createDurableInstallationProvisioner({ + stateRoot, + installationsRoot, + clock: () => now++, + idFactory: () => 'job_oci_pipeline', + leaseMs: 100, + ociAdapter: adapter, + }); + t.after(() => provisioner.close()); + const selected = manifest('oci_pipeline'); + provisioner.registerFixture(selected, authority(selected), FIXTURE_REGISTRATION); + const requested = provisioner.request( + selected, + authority(selected), + provisionRequest('fixture:provision:oci-pipeline', 1), + MANAGE, + 'oci_container_v1', + ); + assert.deepEqual(provisioner.progress(selected, authority(selected), READ, requested.id), { + completedStages: 0, totalStages: 5, attempts: 0, + }); + assert.equal(provisioner.runNext('worker_oci_pipeline').status, 'succeeded'); + assert.deepEqual(calls, [ + 'host_account_ready', 'image_ready', 'bridge_ready', 'container_ready', 'committed', + ]); +}); diff --git a/core/core/installations/tests/layout.test.js b/core/core/installations/tests/layout.test.js new file mode 100644 index 0000000..b0be90b --- /dev/null +++ b/core/core/installations/tests/layout.test.js @@ -0,0 +1,333 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { + INSTALLATION_LAYOUT_TEMPLATE, + INSTALLATION_LAYOUT_VERSION, + PRIVATE_DIRECTORY_MODE, + RELATIVE_DIRECTORIES, + createInstallationLayoutManager, +} = require('../src'); +const { + resolveManagedInstallationRuntimePaths, + managedInstallationRuntimeEnvironment, + managedRuntimeEnvironmentFromProcess, +} = require('../../../shared/paths/runtime-paths'); + +const EMPTY_FIXTURE_CLEANUP = Object.freeze({ + fixture: true, + installationState: 'failed', + retainedData: false, +}); + +function manifest(id = 'alpha', overrides = {}) { + const value = { + manifestVersion: 1, + revision: 1, + organization: { id: `org_${id}`, stationCode: 'TST1', timezone: 'America/Los_Angeles' }, + runtime: { key: `fixture_${id}`, templateId: INSTALLATION_LAYOUT_TEMPLATE, releaseId: 'dispatch_fixture_1' }, + }; + return { + ...value, + ...overrides, + organization: { ...value.organization, ...(overrides.organization || {}) }, + runtime: { ...value.runtime, ...(overrides.runtime || {}) }, + }; +} + +function authority(value) { + return { + revision: value.revision, + organization: { ...value.organization }, + runtime: { ...value.runtime }, + }; +} + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-layout-test-')); + fs.chmodSync(root, PRIVATE_DIRECTORY_MODE); + const installationsRoot = path.join(root, 'installations'); + fs.mkdirSync(installationsRoot, { mode: PRIVATE_DIRECTORY_MODE }); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return { root, installationsRoot, manager: createInstallationLayoutManager({ installationsRoot }) }; +} + +function isCode(code) { + return error => error?.code === code && error.message === code; +} + +test('layout derivation is closed, server-bound, and target-safe', t => { + const { installationsRoot, manager } = fixture(t); + const selected = manifest(); + const layout = manager.derive(selected, authority(selected)); + + assert.equal(INSTALLATION_LAYOUT_VERSION, 1); + assert.equal(layout.templateId, INSTALLATION_LAYOUT_TEMPLATE); + assert.equal(layout.projectRoot, path.resolve(__dirname, "../../..")); + assert.equal(layout.installationRoot, path.join(installationsRoot, selected.runtime.key)); + assert.equal(path.dirname(layout.installationRoot), installationsRoot); + assert.equal(Object.keys(layout.directories).length, RELATIVE_DIRECTORIES.length); + assert.equal(Object.isFrozen(layout), true); + assert.equal(Object.isFrozen(layout.directories), true); + for (const target of Object.values(layout.directories)) { + assert.equal(path.relative(layout.installationRoot, target).startsWith('..'), false); + } + + const wrongAuthority = authority(selected); + wrongAuthority.runtime.key = 'fixture_bravo'; + assert.throws(() => manager.derive(selected, wrongAuthority), isCode('runtime_identity_mismatch')); + assert.equal(fs.existsSync(layout.installationRoot), false); + + const unsupported = manifest('unsupported', { runtime: { templateId: 'isolated_dsp_v2' } }); + assert.throws(() => manager.materialize(unsupported, authority(unsupported)), isCode('runtime_boundary_violation')); + assert.equal(fs.existsSync(path.join(installationsRoot, unsupported.runtime.key)), false); + assert.throws(() => createInstallationLayoutManager({ installationsRoot, unexpected: true }), + isCode('runtime_boundary_violation')); + for (const projectRoot of ['', null, false, 0]) { + assert.throws(() => createInstallationLayoutManager({ installationsRoot, projectRoot }), + isCode('runtime_boundary_violation')); + } +}); + +test('two fixture layouts materialize privately, remain isolated, and replay idempotently', t => { + const { installationsRoot, manager } = fixture(t); + const first = manifest('alpha'); + const second = manifest('bravo'); + const firstLayout = manager.derive(first, authority(first)); + const secondLayout = manager.derive(second, authority(second)); + + const firstReceipt = manager.materialize(first, authority(first)); + const secondReceipt = manager.materialize(second, authority(second)); + assert.deepEqual(firstReceipt, { + layoutVersion: 1, + status: 'verified', + directoryCount: RELATIVE_DIRECTORIES.length, + changed: true, + }); + assert.equal(secondReceipt.changed, true); + assert.equal(manager.materialize(first, authority(first)).changed, false); + assert.equal(manager.inspect(first, authority(first)).status, 'verified'); + assert.equal(Object.isFrozen(firstReceipt), true); + assert.equal(JSON.stringify(firstReceipt).includes(installationsRoot), false); + assert.equal(JSON.stringify(firstReceipt).includes(first.runtime.key), false); + + assert.notEqual(firstLayout.installationRoot, secondLayout.installationRoot); + assert.equal(path.relative(firstLayout.installationRoot, secondLayout.installationRoot).startsWith('..'), true); + assert.equal(path.relative(secondLayout.installationRoot, firstLayout.installationRoot).startsWith('..'), true); + const installationsDevice = fs.lstatSync(installationsRoot).dev; + for (const target of [ + firstLayout.installationRoot, + secondLayout.installationRoot, + ...Object.values(firstLayout.directories), + ...Object.values(secondLayout.directories), + ]) { + const info = fs.lstatSync(target); + assert.equal(info.isDirectory(), true); + assert.equal(info.isSymbolicLink(), false); + assert.equal(info.uid, process.geteuid()); + assert.equal(info.dev, installationsDevice); + assert.equal(info.mode & 0o7777, PRIVATE_DIRECTORY_MODE); + assert.equal(fs.realpathSync(target), target); + } +}); + +test('managed runtime projection supplies isolated component paths without Access Control', t => { + const { installationsRoot, manager } = fixture(t); + const selected = manifest('projection'); + manager.materialize(selected, authority(selected)); + const layout = manager.derive(selected, authority(selected)); + const runtimePaths = manager.runtimePaths(selected, authority(selected)); + const environment = manager.runtimeEnvironment(selected, authority(selected)); + + assert.equal(runtimePaths.auth.databaseRoot, layout.directories.authDataRoot); + assert.equal(runtimePaths.auth.secretRoot, layout.directories.authSecretsRoot); + assert.equal(runtimePaths.collection.databaseRoot, layout.directories.collectionDataRoot); + assert.equal(runtimePaths.paycom.dataRoot, path.join(layout.directories.providerDataRoot, 'paycom')); + assert.equal(runtimePaths.cdf.dataRoot, path.join(layout.directories.providerDataRoot, 'cdf')); + assert.equal(runtimePaths.paycom.stagingRoot, path.join(layout.directories.providerStagingRoot, 'paycom')); + assert.equal(runtimePaths.cdf.stagingRoot, path.join(layout.directories.providerStagingRoot, 'cdf')); + assert.equal(Object.hasOwn(runtimePaths, 'accessControl'), false); + assert.equal(Object.hasOwn(environment, 'DISPATCH_ACCESS_CONTROL_DATABASE_ROOT'), false); + assert.equal(Object.hasOwn(environment, 'DISPATCH_LOCAL_ROOT'), false); + assert.equal(environment.DISPATCH_AUTH_SOCKET, runtimePaths.auth.socket); + assert.equal(Object.isFrozen(runtimePaths), true); + assert.equal(Object.isFrozen(environment), true); + + const managedProcessEnvironment = { ...environment, DISPATCH_MANAGED_RUNTIME: '1' }; + assert.deepEqual(managedRuntimeEnvironmentFromProcess(managedProcessEnvironment), + managedInstallationRuntimeEnvironment(layout)); + assert.throws(() => managedRuntimeEnvironmentFromProcess({ ...managedProcessEnvironment, DISPATCH_LOCAL_ROOT: installationsRoot }), + isCode('unsafe_runtime_config')); + const missingManagedPath = { ...managedProcessEnvironment }; + delete missingManagedPath.DISPATCH_AUTH_SOCKET; + assert.throws(() => managedRuntimeEnvironmentFromProcess(missingManagedPath), + isCode('unsafe_runtime_config')); + + const tampered = { + ...layout, + directories: { ...layout.directories, dataRoot: path.join(installationsRoot, 'wrong') }, + }; + assert.throws(() => resolveManagedInstallationRuntimePaths(tampered), + isCode('unsafe_runtime_config')); + + const projectRoot = path.resolve(__dirname, "../../.."); + const script = ` + const auth = require(${JSON.stringify(path.join(projectRoot, 'runtime/auth-broker/src/paths.js'))}).defaultPaths(); + const collection = require(${JSON.stringify(path.join(projectRoot, 'runtime/collection-manager/src/paths.js'))}).defaultPaths(); + const paycom = require(${JSON.stringify(path.join(projectRoot, 'plugins/paycom/backend/src/paths.js'))}); + const cdf = require(${JSON.stringify(path.join(projectRoot, 'compatibility/cdf/src/paths.js'))}); + process.stdout.write(JSON.stringify({ + auth: { databaseRoot: auth.databaseRoot, secretRoot: auth.secretRoot, stateRoot: auth.stateRoot, + runtimeRoot: auth.runtimeRoot, socket: auth.socket }, + collection: { databaseRoot: collection.databaseRoot, stateRoot: collection.stateRoot }, + paycom: { dataRoot: paycom.DATA_ROOT, stagingRoot: paycom.STAGING_ROOT, authSocket: paycom.AUTH_SOCKET }, + cdf: { dataRoot: cdf.DATA_ROOT, stagingRoot: cdf.STAGING_ROOT, authSocket: cdf.AUTH_SOCKET }, + })); + `; + const cleanEnvironment = Object.fromEntries( + Object.entries(process.env).filter(([key]) => !key.startsWith('DISPATCH_')), + ); + const child = spawnSync(process.execPath, ['--no-warnings', '-e', script], { + encoding: 'utf8', + env: { ...cleanEnvironment, ...environment }, + }); + assert.equal(child.status, 0, child.stderr); + assert.deepEqual(JSON.parse(child.stdout), { + auth: { + databaseRoot: runtimePaths.auth.databaseRoot, + secretRoot: runtimePaths.auth.secretRoot, + stateRoot: runtimePaths.auth.stateRoot, + runtimeRoot: runtimePaths.auth.runtimeRoot, + socket: runtimePaths.auth.socket, + }, + collection: { + databaseRoot: runtimePaths.collection.databaseRoot, + stateRoot: runtimePaths.collection.stateRoot, + }, + paycom: { + dataRoot: runtimePaths.paycom.dataRoot, + stagingRoot: runtimePaths.paycom.stagingRoot, + authSocket: runtimePaths.paycom.authSocket, + }, + cdf: { + dataRoot: runtimePaths.cdf.dataRoot, + stagingRoot: runtimePaths.cdf.stagingRoot, + authSocket: runtimePaths.auth.socket, + }, + }); +}); + +test('unsafe roots and replaced root identities fail before layout mutation', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-layout-root-')); + fs.chmodSync(root, PRIVATE_DIRECTORY_MODE); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + + const unsafeMode = path.join(root, 'unsafe-mode'); + fs.mkdirSync(unsafeMode, { mode: 0o755 }); + assert.throws(() => createInstallationLayoutManager({ installationsRoot: unsafeMode }), + isCode('runtime_boundary_violation')); + + const specialMode = path.join(root, 'special-mode'); + fs.mkdirSync(specialMode, { mode: PRIVATE_DIRECTORY_MODE }); + fs.chmodSync(specialMode, 0o1700); + assert.throws(() => createInstallationLayoutManager({ installationsRoot: specialMode }), + isCode('runtime_boundary_violation')); + + const target = path.join(root, 'target'); + fs.mkdirSync(target, { mode: PRIVATE_DIRECTORY_MODE }); + const alias = path.join(root, 'alias'); + fs.symlinkSync(target, alias, 'dir'); + assert.throws(() => createInstallationLayoutManager({ installationsRoot: alias }), + isCode('runtime_boundary_violation')); + + const source = path.join(root, 'source'); + fs.mkdirSync(source, { mode: PRIVATE_DIRECTORY_MODE }); + const nestedInstallations = path.join(source, 'installations'); + fs.mkdirSync(nestedInstallations, { mode: PRIVATE_DIRECTORY_MODE }); + assert.throws(() => createInstallationLayoutManager({ + projectRoot: source, + installationsRoot: nestedInstallations, + }), isCode('runtime_boundary_violation')); + + const enclosingInstallations = path.join(root, 'enclosing'); + fs.mkdirSync(enclosingInstallations, { mode: PRIVATE_DIRECTORY_MODE }); + const nestedSource = path.join(enclosingInstallations, 'source'); + fs.mkdirSync(nestedSource, { mode: PRIVATE_DIRECTORY_MODE }); + assert.throws(() => createInstallationLayoutManager({ + projectRoot: nestedSource, + installationsRoot: enclosingInstallations, + }), isCode('runtime_boundary_violation')); + + const pinned = path.join(root, 'pinned'); + fs.mkdirSync(pinned, { mode: PRIVATE_DIRECTORY_MODE }); + const manager = createInstallationLayoutManager({ installationsRoot: pinned }); + fs.renameSync(pinned, `${pinned}-old`); + fs.mkdirSync(pinned, { mode: PRIVATE_DIRECTORY_MODE }); + const selected = manifest('pinned'); + assert.throws(() => manager.materialize(selected, authority(selected)), + isCode('runtime_boundary_violation')); + assert.equal(fs.existsSync(path.join(pinned, selected.runtime.key)), false); +}); + +test('unsafe partial layouts fail before missing directories are added', t => { + const { root, installationsRoot, manager } = fixture(t); + const selected = manifest('partial'); + const layout = manager.derive(selected, authority(selected)); + fs.mkdirSync(layout.installationRoot, { mode: PRIVATE_DIRECTORY_MODE }); + fs.mkdirSync(layout.directories.dataRoot, { mode: 0o755 }); + + assert.throws(() => manager.materialize(selected, authority(selected)), isCode('runtime_layout_failed')); + assert.equal(fs.existsSync(layout.directories.configRoot), false); + + fs.chmodSync(layout.directories.dataRoot, PRIVATE_DIRECTORY_MODE); + fs.writeFileSync(path.join(layout.directories.dataRoot, 'unexpected'), '', { mode: 0o600 }); + assert.throws(() => manager.materialize(selected, authority(selected)), isCode('runtime_layout_failed')); + assert.equal(fs.existsSync(layout.directories.configRoot), false); + + fs.rmSync(layout.installationRoot, { recursive: true, force: true }); + const outside = path.join(root, 'outside'); + fs.mkdirSync(outside, { mode: PRIVATE_DIRECTORY_MODE }); + fs.symlinkSync(outside, layout.installationRoot, 'dir'); + assert.throws(() => manager.materialize(selected, authority(selected)), isCode('runtime_layout_failed')); + assert.deepEqual(fs.readdirSync(outside), []); + assert.equal(fs.lstatSync(layout.installationRoot).isSymbolicLink(), true); + assert.equal(path.dirname(layout.installationRoot), installationsRoot); +}); + +test('cleanup is non-recursive, refuses retained content, and removes empty layouts only', t => { + const { manager } = fixture(t); + const selected = manifest('cleanup'); + const layout = manager.derive(selected, authority(selected)); + manager.materialize(selected, authority(selected)); + const retained = path.join(layout.directories.logsRoot, 'retained.log'); + fs.writeFileSync(retained, 'fixture', { mode: 0o600 }); + + assert.throws(() => manager.removeEmpty(selected, authority(selected), EMPTY_FIXTURE_CLEANUP), + isCode('runtime_layout_failed')); + assert.equal(fs.readFileSync(retained, 'utf8'), 'fixture'); + assert.equal(fs.existsSync(layout.directories.configRoot), true); + + fs.unlinkSync(retained); + assert.throws(() => manager.removeEmpty(selected, authority(selected)), + isCode('runtime_boundary_violation')); + assert.throws(() => manager.removeEmpty(selected, authority(selected), { + fixture: true, + installationState: 'ready', + retainedData: false, + }), isCode('runtime_boundary_violation')); + assert.equal(fs.existsSync(layout.installationRoot), true); + + assert.deepEqual(manager.removeEmpty(selected, authority(selected), EMPTY_FIXTURE_CLEANUP), { + layoutVersion: 1, + status: 'removed', + directoryCount: 0, + changed: true, + }); + assert.equal(fs.existsSync(layout.installationRoot), false); + assert.equal(manager.removeEmpty(selected, authority(selected), EMPTY_FIXTURE_CLEANUP).changed, false); +}); diff --git a/core/core/installations/tests/lifecycle.test.js b/core/core/installations/tests/lifecycle.test.js new file mode 100644 index 0000000..ee6e762 --- /dev/null +++ b/core/core/installations/tests/lifecycle.test.js @@ -0,0 +1,634 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { + INSTALLATION_ACTIVATION_RUNS, + installationActivationEvidenceDigest, +} = require('../../../shared/contracts/src'); +const { AccessStore } = require('../../accounts/src/store'); +const { createAccessInstallationLifecycleAuthority } = require('../../accounts/src/installation-lifecycle'); +const { createInstallationLayoutManager } = require('../src/layout'); +const { createInstallationBackupManager } = require('../src/backups'); +const { createManagedInstallationLifecycle } = require('../src/lifecycle'); +const { createInstallationLifecycleReconciler } = require('../src/lifecycle-reconcile'); + +function evidence(jobId, runtimeKey, capturedAt, manifestRevision = 1) { + const runs = INSTALLATION_ACTIVATION_RUNS.map((run, index) => ({ + id: `run_lifecycle_${index + 1}`, + taskId: run.taskId, + plan: run.plan, + method: run.method, + })); + const payload = { + schemaVersion: 1, + manifestRevision, + jobId, + runtimeKey, + definitionDigest: 'a'.repeat(64), + requestDigest: 'b'.repeat(64), + previewDigest: 'c'.repeat(64), + batchId: 'batch_lifecycle', + preparationRunId: 'run_periods_lifecycle', + target: '2026-09-05', + runs, + publications: { + payPeriods: { id: 'pub_periods_lifecycle', runId: 'run_periods_lifecycle', originRunId: 'run_periods_lifecycle', contentSha256: '1'.repeat(64), batchBound: false }, + roster: { id: 'pub_roster_lifecycle', runId: runs[0].id, originRunId: runs[0].id, contentSha256: '2'.repeat(64), batchBound: true }, + timecards: { id: 'pub_timecards_lifecycle', runId: runs[1].id, originRunId: runs[1].id, contentSha256: '3'.repeat(64), batchBound: true }, + resourceLinks: { id: 'pub_links_lifecycle', runId: runs[3].id, originRunId: runs[3].id, contentSha256: '4'.repeat(64), batchBound: true }, + }, + capturedAt, + }; + return { ...payload, evidenceDigest: installationActivationEvidenceDigest(payload) }; +} + +function fakeSupervisor(state) { + function receipt(status, changed = false) { + return { servicePlanVersion: 2, status, serviceCount: 3, changed }; + } + function snapshot(plan) { + return plan.units.map(unit => ({ + id: unit.id, name: unit.name, enabled: state.enabled, active: state.active, + enableMode: state.enabled ? 'persistent' : 'none', + })); + } + return { + snapshot, + reload: () => receipt('reloaded', true), + enable: () => { state.enabled = true; return receipt('enabled', true); }, + disable: () => { state.enabled = false; return receipt('disabled', true); }, + start: () => { + if (state.failStart) { + state.failStart = false; + throw Object.assign(new Error('service_installation_failed'), { code: 'service_installation_failed' }); + } + state.active = true; + if (state.startMutation) { + const mutation = state.startMutation; + state.startMutation = null; + mutation(); + } + return receipt('started', true); + }, + stop: () => { state.active = false; return receipt('stopped', true); }, + resetFailed: () => receipt('failure_state_reset', true), + restoreState: (plan, selected) => { + state.active = selected.some(unit => unit.active); + state.enabled = selected.some(unit => unit.enabled); + return receipt('state_restored', true); + }, + inspect: () => { if (!state.active) throw Object.assign(new Error('runtime_health_failed'), { code: 'runtime_health_failed' }); return receipt('active'); }, + health: () => { + if (state.failHealthOnce) { + state.failHealthOnce = false; + throw Object.assign(new Error('runtime_health_failed'), { code: 'runtime_health_failed' }); + } + if (!state.active) throw Object.assign(new Error('runtime_health_failed'), { code: 'runtime_health_failed' }); + return receipt('healthy'); + }, + }; +} + +function fakeServiceFactory(state) { + return () => ({ + plan: manifest => ({ + runtimeKey: manifest.runtime.key, + units: ['auth_broker', 'collection_manager', 'runtime_gateway'].map(id => ({ + id, name: `${id}.service`, + })), + }), + inspectInstalled: () => { if (state.servicesRemoved) throw Object.assign(new Error('service_installation_failed'), { code: 'service_installation_failed' }); return { status: 'installed' }; }, + finalizeSettled: () => ({ status: 'settled', changed: false }), + render: () => ({ status: 'rendered', changed: true }), + validate: () => ({ status: 'validated', changed: false }), + install: () => { state.servicesRemoved = false; state.targetInstalled = true; return { status: 'installed', changed: true }; }, + markVerified: () => ({ status: 'verified', changed: true }), + rollbackState: plan => state.targetInstalled ? plan.units.map(unit => ({ + id: unit.id, name: unit.name, enabled: true, active: false, enableMode: 'persistent', + })) : null, + commit: () => { state.targetInstalled = false; return { status: 'committed', changed: true }; }, + restoreFiles: () => { state.targetInstalled = false; state.servicesRemoved = false; return { status: 'restored', changed: true }; }, + finishRollback: () => ({ status: 'rolled_back', changed: true }), + removeInstalled: () => { state.servicesRemoved = true; return { status: 'removed', changed: true, serviceCount: 3 }; }, + inspectAbsent: () => { if (!state.servicesRemoved) throw Object.assign(new Error('decommission_failed'), { code: 'decommission_failed' }); return { status: 'absent' }; }, + }); +} + +function setUp(t, { backend = 'systemd_user' } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-lifecycle-')); + fs.chmodSync(root, 0o700); + const accessRoot = path.join(root, 'access'); + const installationsRoot = path.join(root, 'installations'); + const unitRoot = path.join(root, 'units'); + fs.mkdirSync(installationsRoot, { mode: 0o700 }); + fs.mkdirSync(unitRoot, { mode: 0o700 }); + const store = new AccessStore({ + databaseRoot: accessRoot, + database: path.join(accessRoot, 'access-control.sqlite3'), + }); + store.transaction(() => { + store.createOrganization({ + id: 'org_lifecycle', name: 'Lifecycle DSP', abbreviation: 'LIFE', + timezone: 'America/Los_Angeles', status: 'active', createdBy: null, timestamp: 1_000, + }); + store.insertStation('org_lifecycle', 'TST1', true, 1_000); + store.createInstallation('org_lifecycle', 'runtime_lifecycle', 'ready', 1_000, 'dispatch_current_1', backend); + }); + const manifest = { + manifestVersion: 1, + revision: 1, + organization: { id: 'org_lifecycle', stationCode: 'TST1', timezone: 'America/Los_Angeles' }, + runtime: { key: 'runtime_lifecycle', templateId: 'isolated_dsp_v1', releaseId: 'dispatch_current_1' }, + }; + const manifestAuthority = { + revision: 1, organization: { ...manifest.organization }, runtime: { ...manifest.runtime }, + }; + const layoutManager = createInstallationLayoutManager({ installationsRoot, projectRoot: process.cwd() }); + layoutManager.materialize(manifest, manifestAuthority); + const layout = layoutManager.derive(manifest, manifestAuthority); + const dataFile = path.join(layout.directories.providerDataRoot, 'fixture.sqlite3'); + fs.writeFileSync(dataFile, 'before', { mode: 0o600 }); + const otherManifest = { + manifestVersion: 1, + revision: 1, + organization: { id: 'org_lifecycle_other', stationCode: 'TST2', timezone: 'America/Los_Angeles' }, + runtime: { key: 'runtime_lifecycle_other', templateId: 'isolated_dsp_v1', releaseId: 'dispatch_current_1' }, + }; + const otherAuthority = { + revision: 1, + organization: { ...otherManifest.organization }, + runtime: { ...otherManifest.runtime }, + }; + layoutManager.materialize(otherManifest, otherAuthority); + const otherFile = path.join( + layoutManager.derive(otherManifest, otherAuthority).directories.providerDataRoot, + 'other.sqlite3', + ); + fs.writeFileSync(otherFile, 'other', { mode: 0o600 }); + const prior = evidence('job_prior_activation', 'runtime_lifecycle', '2026-09-03T00:00:00.000Z'); + store.db.prepare(`INSERT INTO installation_activation_jobs( + id,organization_id,operation,status,installation_state,installation_revision,manifest_revision, + runtime_key,authority_scope,idempotency_key,worker_id,fence,lease_expires_at,provider,profile_id, + provider_tested_at,evidence_json,evidence_digest,failure_code,created_at,started_at,finished_at,updated_at + ) VALUES(?,'org_lifecycle','resume','succeeded','ready',1,1,'runtime_lifecycle','fixture_scope', + 'fixture:activation:prior','worker_prior',1,NULL,'paycom','paycom-main',?,?,?,?,?,?,?,?)`).run( + prior.jobId, Date.parse(prior.capturedAt), JSON.stringify(prior), prior.evidenceDigest, null, + 1_000, 1_000, 1_000, 1_000, + ); + store.db.prepare("UPDATE installations SET current_job_id='job_prior_activation' WHERE organization_id='org_lifecycle'") + .run(); + let now = 2_000; + let jobs = 0; + let backups = 0; + const makeAuthority = authorityScope => createAccessInstallationLifecycleAuthority({ + store, + organizationId: 'org_lifecycle', + authorityScope, + clock: () => ++now, + jobFactory: () => `life_fixture_${++jobs}`, + backupFactory: () => `backup_fixture_${++backups}`, + releaseCatalog: ['dispatch_fixture_2'], + destructionEnabled: true, + }); + const authority = makeAuthority('platform_lifecycle'); + const state = { + active: true, enabled: true, servicesRemoved: false, targetInstalled: false, + failHealthOnce: false, failRestoredOnce: false, startMutation: null, syncRunning: true, + }; + const supervisor = fakeSupervisor(state); + const makeRuntime = selectedAuthority => createManagedInstallationLifecycle({ + authority: selectedAuthority, + waitForBackupDeletion: async () => {}, + installationsRoot, + unitRoot, + supervisor, + projectRoot: process.cwd(), + releaseCatalog: { dispatch_fixture_2: process.cwd() }, + serviceManagerFactory: fakeServiceFactory(state), + backupManagerFactory: settings => { + const manager = createInstallationBackupManager(settings); + return Object.freeze({ + ...manager, + inspectRestored: source => { + if (state.failRestoredOnce) { + state.failRestoredOnce = false; + throw Object.assign(new Error('restore_failed'), { code: 'restore_failed' }); + } + return manager.inspectRestored(source); + }, + }); + }, + activationRuntimeFactory: context => ({ + inspectSchedule: async () => ({ syncWasRunning: state.syncRunning }), + quiesceSchedule: async syncWasRunning => { + assert.equal(syncWasRunning, state.syncRunning); + state.syncRunning = false; + return { syncWasRunning }; + }, + restoreSchedule: async syncWasRunning => { + state.syncRunning = syncWasRunning; + return { syncWasRunning }; + }, + verifyInfrastructure: async () => ({ ok: true }), + verifyPublication: async () => { + const selected = evidence( + context.job.id, + 'runtime_lifecycle', + '2026-09-03T00:01:00.000Z', + context.manifest.revision, + ); + const { schemaVersion, manifestRevision, jobId, runtimeKey, evidenceDigest, ...raw } = selected; + return raw; + }, + }), + }); + const runtime = makeRuntime(authority); + t.after(() => { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + }); + return { + store, authority, runtime, makeAuthority, makeRuntime, state, dataFile, otherFile, installationsRoot, + }; +} + +async function requestAndRun(context, operation) { + const requested = context.authority.request(operation); + return context.runtime.run(requested.id, `worker_${requested.id}`); +} + +test('managed lifecycle completes backup, restore, suspension, resume, upgrade, and retained destruction', async t => { + const context = setUp(t); + let revision = 1; + const backupOperation = { + operation: 'backup', idempotencyKey: 'lifecycle:backup:fixture', expectedRevision: revision, + }; + const backupRequest = context.authority.request(backupOperation); + assert.equal(context.store.installationControl('org_lifecycle').status, 'verifying'); + assert.equal(context.authority.request(backupOperation).id, backupRequest.id); + let result = await context.runtime.run(backupRequest.id, `worker_${backupRequest.id}`); + assert.equal(result.status, 'succeeded', JSON.stringify({ + result, + job: context.store.lifecycleJob(result.id), + })); + revision = context.store.installationControl('org_lifecycle').revision; + const backup = context.authority.backups()[0]; + assert.equal(fs.readFileSync(context.dataFile, 'utf8'), 'before'); + assert.equal(context.state.syncRunning, true); + + result = await requestAndRun(context, { + operation: 'suspend', idempotencyKey: 'lifecycle:suspend:fixture', expectedRevision: revision, + }); + assert.equal(result.installationState, 'suspended'); + assert.equal(context.state.active, false); + assert.equal(context.state.syncRunning, false); + revision = context.store.installationControl('org_lifecycle').revision; + fs.writeFileSync(context.dataFile, 'after', { mode: 0o600 }); + + result = await requestAndRun(context, { + operation: 'restore', idempotencyKey: 'lifecycle:restore:fixture', expectedRevision: revision, + backupId: backup.id, + }); + assert.equal(result.installationState, 'suspended'); + assert.equal(fs.readFileSync(context.dataFile, 'utf8'), 'before'); + revision = context.store.installationControl('org_lifecycle').revision; + + result = await requestAndRun(context, { + operation: 'resume', idempotencyKey: 'lifecycle:resume:fixture', expectedRevision: revision, + }); + assert.equal(result.installationState, 'ready'); + assert.equal(context.state.active, true); + assert.equal(context.state.syncRunning, true); + revision = context.store.installationControl('org_lifecycle').revision; + + fs.writeFileSync(context.dataFile, 'pre-upgrade', { mode: 0o600 }); + context.state.startMutation = () => fs.writeFileSync(context.dataFile, 'target-write', { mode: 0o600 }); + context.state.failHealthOnce = true; + result = await requestAndRun(context, { + operation: 'upgrade', idempotencyKey: 'lifecycle:upgrade:rollback', expectedRevision: revision, + releaseId: 'dispatch_fixture_2', + }); + assert.equal(result.status, 'failed'); + assert.equal(context.state.targetInstalled, false); + assert.equal(context.state.active, true); + assert.equal(context.state.syncRunning, true); + assert.equal(fs.readFileSync(context.dataFile, 'utf8'), 'pre-upgrade'); + assert.equal(context.store.installationControl('org_lifecycle').releaseId, 'dispatch_current_1'); + revision = context.store.installationControl('org_lifecycle').revision; + + result = await requestAndRun(context, { + operation: 'upgrade', idempotencyKey: 'lifecycle:upgrade:fixture', expectedRevision: revision, + releaseId: 'dispatch_fixture_2', + }); + assert.equal(result.installationState, 'ready'); + const upgraded = context.store.installationControl('org_lifecycle'); + assert.equal(upgraded.releaseId, 'dispatch_fixture_2'); + assert.equal(upgraded.manifestRevision, 2); + assert.equal(context.state.syncRunning, true); + + result = await requestAndRun(context, { + operation: 'decommission', idempotencyKey: 'lifecycle:decommission:fixture', + expectedRevision: upgraded.revision, + }); + assert.equal(result.installationState, 'decommissioned'); + assert.equal(context.state.active, false); + assert.equal(context.state.enabled, false); + assert.equal(context.state.syncRunning, false); + assert.equal(fs.existsSync(path.join(context.installationsRoot, 'runtime_lifecycle')), true); + + assert.equal(context.state.servicesRemoved, false); + const backupCount = context.store.installationBackups('org_lifecycle').length; + result = await requestAndRun(context, { + operation: 'resume', idempotencyKey: 'lifecycle:restore-removed:fixture', + expectedRevision: context.store.installationControl('org_lifecycle').revision, + }); + assert.equal(result.status, 'succeeded'); + assert.equal(result.installationState, 'ready'); + assert.equal(context.state.active, true); + assert.equal(context.state.enabled, true); + assert.equal(context.state.syncRunning, true); + assert.equal(context.store.organization('org_lifecycle').status, 'active'); + assert.equal(context.store.db.prepare('SELECT count(*) n FROM dsp_removals').get().n, 0); + result = await requestAndRun(context, { + operation: 'decommission', idempotencyKey: 'lifecycle:remove-again:fixture', + expectedRevision: context.store.installationControl('org_lifecycle').revision, + }); + assert.equal(result.status, 'succeeded'); + assert.equal(context.store.installationBackups('org_lifecycle').length, backupCount); + + result = await requestAndRun(context, { + operation: 'destroy', idempotencyKey: 'lifecycle:destroy:fixture', + expectedRevision: context.store.installationControl('org_lifecycle').revision, + }); + assert.equal(result.installationState, 'decommissioned'); + assert.equal(fs.existsSync(path.join(context.installationsRoot, 'runtime_lifecycle')), false); + assert.equal(fs.readFileSync(context.otherFile, 'utf8'), 'other'); + assert.deepEqual(context.authority.backups(), []); +}); + +test('failed restore verification reinstates the safety snapshot', async t => { + const context = setUp(t); + let result = await requestAndRun(context, { + operation: 'backup', idempotencyKey: 'lifecycle:backup:restore-compensation', expectedRevision: 1, + }); + const backup = context.authority.backups()[0]; + result = await requestAndRun(context, { + operation: 'suspend', idempotencyKey: 'lifecycle:suspend:restore-compensation', + expectedRevision: result.installationRevision, + }); + fs.writeFileSync(context.dataFile, 'safety-state', { mode: 0o600 }); + context.state.failRestoredOnce = true; + result = await requestAndRun(context, { + operation: 'restore', idempotencyKey: 'lifecycle:restore:compensate', + expectedRevision: result.installationRevision, backupId: backup.id, + }); + assert.equal(result.status, 'failed'); + assert.equal(result.installationState, 'suspended'); + assert.equal(fs.readFileSync(context.dataFile, 'utf8'), 'safety-state'); + assert.equal(context.store.installationControl('org_lifecycle').currentJobId, null); +}); + +test('reconciliation turns organization suspension and resumption into runtime work', async t => { + const context = setUp(t); + const reconciler = createInstallationLifecycleReconciler({ + store: context.store, + authorityFactory: (organizationId, authorityScope) => { + assert.equal(organizationId, 'org_lifecycle'); + return context.makeAuthority(authorityScope); + }, + runtimeFactory: (organizationId, authority) => { + assert.equal(organizationId, 'org_lifecycle'); + return context.makeRuntime(authority); + }, + clock: () => 10_000, + }); + + context.store.updateOrganizationStatus('org_lifecycle', 'suspended', 3_000); + let result = await reconciler.runPending('worker_status_suspend', 5); + assert.deepEqual(result, { + requested: 1, processed: 1, completed: 1, failed: 0, exhausted: 0, pending: false, + }); + assert.equal(context.store.installationControl('org_lifecycle').status, 'suspended'); + assert.equal(context.state.active, false); + assert.equal(context.state.syncRunning, false); + + context.store.updateOrganizationStatus('org_lifecycle', 'active', 4_000); + result = await reconciler.runPending('worker_status_resume', 5); + assert.deepEqual(result, { + requested: 1, processed: 1, completed: 1, failed: 0, exhausted: 0, pending: false, + }); + assert.equal(context.store.installationControl('org_lifecycle').status, 'ready'); + assert.equal(context.state.active, true); + assert.equal(context.state.syncRunning, true); +}); + +test('expired lifecycle attempts stop automatic reclaim until explicit same-job retry', async t => { + const context = setUp(t); + const requested = context.authority.request({ + operation: 'backup', idempotencyKey: 'lifecycle:backup:attempt-bound', expectedRevision: 1, + }); + for (const worker of ['worker_attempt_a', 'worker_attempt_b', 'worker_attempt_c']) { + context.authority.claim(requested.id, worker); + context.store.db.prepare('UPDATE installation_lifecycle_jobs SET lease_expires_at=0 WHERE id=?') + .run(requested.id); + } + const reconciler = createInstallationLifecycleReconciler({ + store: context.store, + authorityFactory: () => context.authority, + runtimeFactory: () => { throw new Error('exhausted_job_must_not_execute'); }, + clock: () => 10_000, + }); + assert.deepEqual(await reconciler.runPending('worker_attempt_reconcile', 5), { + requested: 0, processed: 0, completed: 0, failed: 0, exhausted: 1, pending: true, + }); + assert.equal(context.store.lifecycleJob(requested.id).status, 'running'); + assert.equal(context.store.installationControl('org_lifecycle').status, 'verifying'); + const reopened = context.authority.retryExhausted(requested.id); + assert.equal(reopened.status, 'queued'); + assert.equal(reopened.attempt, 0); + const completed = await context.runtime.run(requested.id, 'worker_attempt_operator'); + assert.equal(completed.status, 'succeeded'); + assert.equal(context.store.installationControl('org_lifecycle').status, 'ready'); + assert.equal(context.authority.backups().length, 1); +}); + +test('lifecycle claim capabilities cannot cross organization authorities', t => { + const context = setUp(t); + context.store.transaction(() => { + context.store.createOrganization({ + id: 'org_lifecycle_beta', name: 'Lifecycle Beta', abbreviation: 'LFB', + timezone: 'America/Los_Angeles', status: 'active', createdBy: null, timestamp: 4_000, + }); + context.store.insertStation('org_lifecycle_beta', 'TST2', true, 4_000); + context.store.createInstallation('org_lifecycle_beta', 'runtime_lifecycle_beta', 'ready', 4_000); + }); + let now = 5_000; + const beta = createAccessInstallationLifecycleAuthority({ + store: context.store, + organizationId: 'org_lifecycle_beta', + authorityScope: 'platform_lifecycle', + clock: () => ++now, + jobFactory: () => 'life_beta_claim', + backupFactory: () => 'backup_beta_claim', + }); + const requested = beta.request({ + operation: 'backup', idempotencyKey: 'lifecycle:beta:claim-boundary', expectedRevision: 1, + }); + const claim = beta.claim(requested.id, 'worker_beta_claim'); + assert.throws( + () => beta.checkpoint(claim.claim, 'inspect_schedule', { status: 'destroyed' }), + error => error?.code === 'installation_operation_failed', + ); + assert.throws( + () => context.authority.renew(claim.claim), + error => error?.code === 'installation_operation_not_found', + ); + const wrongScope = createAccessInstallationLifecycleAuthority({ + store: context.store, organizationId: 'org_lifecycle_beta', authorityScope: 'other_lifecycle_scope', + clock: () => ++now, + }); + assert.throws( + () => wrongScope.renew(claim.claim), + error => error?.code === 'installation_operation_not_found', + ); + assert.equal(context.store.lifecycleJob(requested.id).fence, 1); + assert.equal(context.store.lifecycleJob(requested.id).worker_id, 'worker_beta_claim'); +}); + +test('failed compensation keeps the installation non-routable and linked to the failed job', async t => { + const context = setUp(t); + fs.symlinkSync(context.dataFile, path.join(path.dirname(context.dataFile), 'unsafe-link')); + context.state.failStart = true; + const requested = context.authority.request({ + operation: 'backup', idempotencyKey: 'lifecycle:backup:compensation-failure', expectedRevision: 1, + }); + const result = await context.runtime.run(requested.id, 'worker_compensation_failure'); + assert.equal(result.status, 'failed'); + assert.equal(result.failure.code, 'lifecycle_compensation_failed'); + const control = context.store.installationControl('org_lifecycle'); + assert.equal(control.status, 'failed'); + assert.equal(control.currentJobId, requested.id); + assert.equal(context.state.active, false); + assert.equal(context.state.syncRunning, false); +}); + +for (const matches of [true, false]) test(`OCI resume requires its durable current publication baseline: match ${matches}`, t => { + const context = setUp(t, { backend: 'oci_container_v1' }); + const { authority, store } = context; + const suspend = authority.request({ operation: 'suspend', idempotencyKey: 'baseline:suspend:fixture', expectedRevision: 1 }); + const stopped = authority.claim(suspend.id, 'worker_baseline_suspend'); + for (const [stage, status] of [['inspect_schedule', 'verified'], ['quiesce_schedule', 'stopped'], ['stop_runtime', 'stopped'], ['verify_stopped', 'inactive']]) { + authority.checkpoint(stopped.claim, stage, { status, ...(stage === 'inspect_schedule' ? { syncWasRunning: true } : {}) }); + } + authority.succeed(stopped.claim); + const resume = authority.request({ operation: 'resume', idempotencyKey: 'baseline:resume:fixture', expectedRevision: store.installationControl('org_lifecycle').revision }); + const claimed = authority.claim(resume.id, 'worker_baseline_resume'); + assert.deepEqual(claimed.stages, ['capture_publication', 'start_runtime', 'verify_infrastructure', 'verify_publication', 'restore_schedule']); + const { createPublicationBaseline } = require('../../../shared/contracts/src/publication-baseline'); + const publications = Object.fromEntries(['payPeriods', 'roster', 'timecards', 'resourceLinks'].map(name => [name, + { id: `pub_current_${name}`, originRunId: `run_current_${name}`, contentSha256: 'e'.repeat(64) }])); + const baseline = createPublicationBaseline('2026-09-19', publications); + authority.checkpoint(claimed.claim, 'capture_publication', { status: 'verified', publicationBaseline: baseline }); + authority.checkpoint(claimed.claim, 'start_runtime', { status: 'started' }); + authority.checkpoint(claimed.claim, 'verify_infrastructure', { status: 'verified' }); + authority.checkpoint(claimed.claim, 'verify_publication', { status: 'verified', + publicationBaselineDigest: matches ? baseline.digest : '0'.repeat(64), + activationEvidence: evidence(resume.id, 'runtime_lifecycle', '2026-09-03T00:01:00.000Z') }); + authority.checkpoint(claimed.claim, 'restore_schedule', { status: 'started', syncWasRunning: true }); + if (!matches) assert.throws(() => authority.succeed(claimed.claim), /first_publication_failed/); + else { + assert.equal(authority.succeed(claimed.claim).status, 'succeeded'); + const result = JSON.parse(store.lifecycleJob(resume.id).result_json); + assert.equal(result.publicationBaseline.target, '2026-09-19'); + assert.equal(result.activationEvidence.target, '2026-09-05'); + } +}); + +test('DSP restore checkpoint persists across authority recreation and rejects unfenced use', t => { + const f = setUp(t, { backend: 'oci_container_v1' }); + const job = f.authority.request({ operation: 'upgrade', releaseId: 'dispatch_fixture_2', expectedRevision: f.store.installationControl('org_lifecycle').revision, idempotencyKey: 'fixture:recovery:checkpoint' }); + const claimed = f.authority.claim(job.id, 'worker_recovery'); + assert.throws(() => f.authority.checkpointCompensationRestore(claimed.claim)); + f.authority.beginCompensation(claimed.claim, Object.assign(new Error('upgrade_failed'), { code: 'upgrade_failed' })); + f.authority.checkpointCompensationRestore(claimed.claim); + assert.equal(JSON.parse(f.store.lifecycleJob(job.id).stage_receipts_json).__compensationRestored, true); + const recreated = f.makeAuthority('platform_lifecycle'); + assert.throws(() => recreated.checkpointCompensationRestore({ ...claimed.claim, fence: claimed.claim.fence + 1 })); + recreated.checkpointCompensationRestore(claimed.claim); + assert.equal(JSON.parse(f.store.lifecycleJob(job.id).stage_receipts_json).__compensationRestored, true); +}); + +test('removal fences an interrupted backup and preserves its original schedule intent', t => { + const f = setUp(t); + const backup = f.authority.request({ operation: 'backup', expectedRevision: 1, idempotencyKey: 'cancel:backup:fixture' }); + const claim = f.authority.claim(backup.id, 'worker_cancel_backup'); + f.authority.checkpoint(claim.claim, 'inspect_schedule', { status: 'verified', syncWasRunning: true }); + f.authority.checkpoint(claim.claim, 'quiesce_schedule', { status: 'stopped' }); + const removal = f.authority.request({ operation: 'decommission', expectedRevision: f.store.installationControl('org_lifecycle').revision, idempotencyKey: 'cancel:remove:fixture' }); + assert.equal(f.store.lifecycleJob(backup.id).status, 'failed'); + assert.throws(() => f.authority.mutate(claim.claim, () => assert.fail('stale worker must not start runtime')), /installation_operation_in_progress/); + assert.equal(f.store.db.prepare('SELECT sync_running FROM dsp_removals').get().sync_running, 1); + assert.equal(f.store.lifecycleJob(removal.id).backup_id, null); +}); + +for (const backend of ['oci_container_v1', 'native_service_v1']) test(`${backend} restores a removed DSP only after publication and runtime verification`, t => { + const f = setUp(t, { backend }); + const { store, authority } = f; + const removal = authority.request({ operation: 'decommission', expectedRevision: 1, idempotencyKey: 'removed:verified:remove' }); + const stopped = authority.claim(removal.id, 'worker_verified_remove'); + for (const [stage, status] of [['inspect_schedule', 'verified'], ['quiesce_schedule', 'stopped'], ['stop_runtime', 'stopped'], ['disable_runtime', 'disabled'], ['verify_retained', 'retained']]) { + authority.checkpoint(stopped.claim, stage, { status, ...(stage === 'inspect_schedule' ? { syncWasRunning: true } : {}) }); + } + authority.succeed(stopped.claim); + assert.equal(store.organization('org_lifecycle').status, 'suspended'); + assert.equal(store.installationBackups('org_lifecycle').length, 0); + const failedRestore = authority.request({ operation: 'resume', expectedRevision: store.installationControl('org_lifecycle').revision, idempotencyKey: 'removed:verified:failed' }); + const failure = authority.claim(failedRestore.id, 'worker_failed_restore'); + authority.failed(failure.claim, 'installation_not_ready'); + assert.equal(store.installationControl('org_lifecycle').status, 'decommissioned'); + assert.equal(store.organization('org_lifecycle').status, 'suspended'); + const job = authority.request({ operation: 'resume', expectedRevision: store.installationControl('org_lifecycle').revision, idempotencyKey: 'removed:verified:restore' }); + const restored = authority.claim(job.id, 'worker_verified_restore'); + assert.equal(restored.resumeSync, true); + assert.equal(authority.desiredRuntimeState(restored.claim), 'active'); + assert.equal(store.organization('org_lifecycle').status, 'suspended'); + assert.throws(() => authority.succeed(restored.claim), /installation_operation_failed/); + const { createPublicationBaseline } = require('../../../shared/contracts/src/publication-baseline'); + const baseline = createPublicationBaseline('2026-09-19', Object.fromEntries(['payPeriods', 'roster', 'timecards', 'resourceLinks'].map(name => [name, + { id: `pub_current_${name}`, originRunId: `run_current_${name}`, contentSha256: 'e'.repeat(64) }]))); + authority.checkpoint(restored.claim, 'capture_publication', { status: 'verified', publicationBaseline: baseline }); + authority.checkpoint(restored.claim, 'start_runtime', { status: 'started' }); + authority.checkpoint(restored.claim, 'verify_infrastructure', { status: 'verified' }); + authority.checkpoint(restored.claim, 'verify_publication', { status: 'verified', publicationBaselineDigest: baseline.digest, + activationEvidence: evidence(job.id, 'runtime_lifecycle', '2026-09-03T00:01:00.000Z') }); + authority.checkpoint(restored.claim, 'restore_schedule', { status: 'started', syncWasRunning: true }); + assert.equal(authority.succeed(restored.claim).installationState, 'ready'); + assert.equal(store.organization('org_lifecycle').status, 'active'); + assert.equal(store.db.prepare('SELECT count(*) n FROM dsp_removals').get().n, 0); + assert.equal(authority.request({ operation: 'resume', expectedRevision: job.installationRevision - 1, idempotencyKey: 'removed:verified:restore' }).replayed, true); +}); + +test('older removed DSPs recover their saved schedule and reinstall service definitions on restore', async t => { + const f = setUp(t, { backend: 'native_service_v1' }); + const job = f.authority.request({ operation: 'decommission', expectedRevision: 1, idempotencyKey: 'legacy:removed:fixture' }); + const claim = f.authority.claim(job.id, 'worker_legacy_remove'); + for (const [stage, status] of [['inspect_schedule', 'verified'], ['quiesce_schedule', 'stopped'], ['stop_runtime', 'stopped'], ['disable_runtime', 'disabled'], ['verify_retained', 'retained']]) + f.authority.checkpoint(claim.claim, stage, { status, ...(stage === 'inspect_schedule' ? { syncWasRunning: true } : {}) }); + f.authority.succeed(claim.claim); + f.store.db.exec('DELETE FROM dsp_removals; UPDATE installations SET current_job_id=NULL'); + const legacyStages = ['inspect_schedule', 'quiesce_schedule', 'stop_runtime', 'final_backup', 'disable_runtime', 'remove_services', 'verify_retained']; + f.store.db.prepare('UPDATE installation_lifecycle_jobs SET stages_json=? WHERE id=?').run(JSON.stringify(legacyStages), job.id); + require('../../accounts/src/schema').initializeAccessSchema(f.store.db, require('../../accounts/src/schema').SCHEMA_VERSION); + const saved = f.store.db.prepare('SELECT * FROM dsp_removals').get(); + assert.equal(saved.legacy_services, 1); + assert.equal(saved.installation_state, 'ready'); + assert.equal(saved.sync_running, 1); + const restore = f.authority.request({ operation: 'resume', expectedRevision: f.store.installationControl('org_lifecycle').revision, idempotencyKey: 'legacy:restore:fixture' }); + const restored = f.authority.claim(restore.id, 'worker_legacy_restore'); + assert.deepEqual(restored.stages.slice(0, 3), ['restore_services', 'capture_publication', 'start_runtime']); + assert.equal(restored.resumeSync, true); +}); diff --git a/core/core/installations/tests/live-dsps.test.js b/core/core/installations/tests/live-dsps.test.js new file mode 100644 index 0000000..1a2fc22 --- /dev/null +++ b/core/core/installations/tests/live-dsps.test.js @@ -0,0 +1,63 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const { validateTarget } = require('./live-dsps/runner'); +const { main } = require('./live-dsps/operator'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const { AccessStore, AccessControlService } = require('../../accounts/src'); +test('live-test targeting rejects existing DSPs, changed identities and arbitrary cleanup targets', () => { + const id = 'live_' + 'a'.repeat(32), organizationId = 'org_' + 'b'.repeat(32); + const target = { organizationId, index: 0, email: `${id}-0@dispatch-test.invalid` }; + const state = { id, targets: [target] }, row = { organization_id: organizationId, backend: 'native_service_v1', owner_email: target.email, name: 'TEST aaaaaaaa DSP 1' }; + validateTarget(state, target, row); + for (const patch of [{ organization_id: 'org_' + 'c'.repeat(32) }, { backend: 'oci_container_v1' }, { owner_email: 'real@example.com' }, { name: 'Real DSP' }]) + assert.throws(() => validateTarget(state, target, { ...row, ...patch }, true)); + assert.throws(() => validateTarget(state, { ...target }, row)); + validateTarget(state, target, { ...row, name: 'New DSP' }, true); + target.profileApplied = true; + assert.throws(() => validateTarget(state, target, { ...row, name: 'New DSP' }, true)); +}); +test('private test creation uses normal native provisioning and a single-use invitation without email', async t => { + const localRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-live-runner-test-')); fs.chmodSync(localRoot, 0o700); + fs.mkdirSync(path.join(localRoot, 'data'), { mode: 0o700 }); + const store = new AccessStore({ databaseRoot: localRoot + '/data/access-control', database: localRoot + '/data/access-control/access-control.sqlite3' }); + t.after(() => { store.close(); fs.rmSync(localRoot, { recursive: true, force: true }); }); + store.insertUser({ id: 'user_owner', email: 'owner@example.test', firstName: 'Owner', lastName: 'Test', passwordHash: 'unused', platformRole: 'owner', timestamp: Date.now() }); + const session = await main({ localRoot, action: 'session' }); + const request = { localRoot, action: 'create', session, runId: 'live_' + 'a'.repeat(32), index: 0 }; + const result = await main(request); + assert.equal(store.installationBackend(result.organizationId), 'native_service_v1'); + assert.equal(store.db.prepare('SELECT count(*) n FROM installation_provisioning_requests').get().n, 1); + const access = new AccessControlService(store); + const accepted = await access.acceptNewUser({ token: result.invitationToken, firstName: 'Test', lastName: 'Owner', password: 'Synthetic test password 123!', confirmPassword: 'Synthetic test password 123!' }); + assert.ok(accepted); + await assert.rejects(main(request), /test_creation_already_exists/); + assert.equal(store.db.prepare('SELECT count(*) n FROM organizations').get().n, 1); + const cleanup = { ...request, action: 'destroy', organizationId: result.organizationId, + expectedRevision: store.installationControl(result.organizationId).revision }; + await assert.rejects(main({ ...cleanup, organizationId: 'org_unrelated' }), /invalid_test_request/); + await assert.rejects(main({ ...cleanup, index: 1 }), /invalid_test_request/); + await assert.rejects(main(cleanup), /installation_operation_not_allowed/); + store.db.prepare("UPDATE installations SET status='decommissioned' WHERE organization_id=?").run(result.organizationId); + const destruction = await main(cleanup); + assert.equal(destruction.operation, 'destroy'); + assert.equal(store.lifecycleJob(destruction.id).organization_id, result.organizationId); + await main({ localRoot, action: 'logout', session }); + assert.equal(access.session(session.token), null); +}); + +test('local API transport preserves public-origin checks, CSRF and secure cookies', async t => { + const server = require('node:http').createServer((req, res) => { + assert.equal(req.headers.host, 'dispatch.example.test'); + assert.equal(req.headers.origin, 'https://dispatch.example.test'); + assert.equal(req.headers['cf-visitor'], '{"scheme":"https"}'); + assert.equal(req.headers.cookie, '__Host-dispatch_session=synthetic'); + assert.equal(req.headers['x-dispatch-csrf'], 'csrf'); + assert.equal(req.method, 'POST'); + res.writeHead(202, { 'Content-Type': 'application/json', 'Set-Cookie': '__Host-dispatch_session=next; Secure; HttpOnly' }); + res.end(JSON.stringify({ ok: true })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => server.close(resolve))); + const result = await require('./live-dsps/runner').requestApi({ port: server.address().port, publicOrigin: 'https://dispatch.example.test' }, { token: 'synthetic', csrf: 'csrf' }, '/test', {}); + assert.equal(result.status, 202); assert.equal(result.token, 'next'); assert.equal(result.value.ok, true); +}); diff --git a/core/core/installations/tests/live-dsps/README.md b/core/core/installations/tests/live-dsps/README.md new file mode 100644 index 0000000..336e80a --- /dev/null +++ b/core/core/installations/tests/live-dsps/README.md @@ -0,0 +1,41 @@ +# Live DSP lifecycle verification + +Run from the clean source checkout matching the deployed native Core commit: + +```sh +sudo ./core/installations/scripts/verify-live-dsps run +``` + +The command uses the existing Core service identity and a one-hour session for +an existing platform owner. It creates two clearly named `TEST … DSP …` tenants, +registers generated owners through private single-use invitations (no email), +and exercises the real provisioning, backup, suspend, restore, resume and removal +paths. Cleanup first removes each DSP through the API, then uses the protected local operator to permanently delete only fixtures created by that run. Browser password confirmation is covered by the dashboard and disposable lab tests. While waiting for operations, it also checks the real session and DSP listing APIs so database contention cannot silently leave the dashboard unavailable. It creates no VM and does not replace Core or reboot the host. + +Synthetic provider publications are inserted only into the recorded test DSPs, +as their own Linux users in a temporary systemd namespace with networking disabled. +Provider schedules stay stopped. Activation uses explicit synthetic authentication +and publication receipts; this checks DSP lifecycle, not real provider login or +collection. The shared immutable runtime package is not modified. + +The test requires the matching native Core release, connected encrypted backups, +a finished rollout, and automatic fleet backups disabled for the test window. +It does not change shared backup settings. Existing DSP identities and existing +verified archives are checked afterward. New Core archives containing test DSPs +would necessarily be deleted during test cleanup; historical Core backups proven +not to contain the test DSPs are preserved. + +Progress streams one scenario at a time. Private state and a credential-free JSON +report live under `/var/lib/dispatch-live-tests/live_/`. A failed run +leaves its DSPs available for diagnosis, prints its cleanup command and exits nonzero. +Cleanup only accepts identities recorded by that run and verifies their native +backend, generated owner address and test name; it does not accept arbitrary DSP IDs. + +```sh +sudo ./core/installations/scripts/verify-live-dsps status live_ +sudo ./core/installations/scripts/verify-live-dsps cleanup live_ +``` + +The completed report remains after cleanup. Treat `state.json` as private: it can +contain generated test account credentials. The current owner session is revoked +when the command exits normally or handles SIGINT/SIGTERM. A forced kill leaves it to expire within one hour. An interrupted run can be cleaned up after its process exits. diff --git a/core/core/installations/tests/live-dsps/activate.js b/core/core/installations/tests/live-dsps/activate.js new file mode 100644 index 0000000..0d6c3af --- /dev/null +++ b/core/core/installations/tests/live-dsps/activate.js @@ -0,0 +1,125 @@ +"use strict"; +// Provider authentication/publication receipts are synthetic in this mode. +// The activation authority, transitions, evidence validation and health calls are real. +const fs = require("node:fs"); +const assert = require("node:assert/strict"); +const { AccessStore } = require("../../../accounts/src"); +const { + runManagedPaycomActivation, + ACTIVATION_INFRASTRUCTURE_GATES, +} = require("../../src/activation"); +const { + runtimeAgentControlInvoke, +} = require("../../../agents/src/control"); +const { + INSTALLATION_ACTIVATION_RUNS, +} = require("../../../../shared/contracts/src"); +async function main() { + const input = JSON.parse(fs.readFileSync(0, 'utf8')), organizationId = input.organizationId, + root = input.localRoot; + const store = new AccessStore({ + databaseRoot: root + "/data/access-control", + database: root + "/data/access-control/access-control.sqlite3", + }); + try { + // DSP onboarding is complete before optional provider fixture activation. + assert.equal(store.installationControl(organizationId).status, "ready"); + const requests = require("../../../accounts/src/onboarding-store").createOnboardingStore(store); + const actor = store.db.prepare("SELECT m.user_id FROM memberships m JOIN roles r ON r.id=m.role_id WHERE m.organization_id=? AND m.status='active' AND r.key='owner'").get(organizationId).user_id; + const row = store.transaction(() => { + const request = requests.begin(organizationId, actor, "lab:optional:" + require("node:crypto").randomUUID(), "create", store.installationControl(organizationId).manifestRevision); + requests.enrolled(request.id); + return requests.claim(request.id, "worker_lab_activation"); + }); + const authority = require("../../../accounts/src/optional-paycom-activation").createOptionalPaycomActivation({ store, row, requests }); + const context = authority.inspect(), + key = context.manifest.runtime.key; + const seed = input.seed; + const runs = INSTALLATION_ACTIVATION_RUNS.map((run, i) => ({ + id: `run_fixture_${i}`, + taskId: run.taskId, + plan: run.plan, + method: run.method, + })); + const pub = (value, runId, originRunId) => ({ + id: value.publicationId, + runId, + originRunId, + contentSha256: value.contentSha256, + batchBound: true, + }); + const audited = { + definitionDigest: "a".repeat(64), + requestDigest: "b".repeat(64), + previewDigest: "c".repeat(64), + batchId: "batch_lab_" + organizationId, + preparationRunId: "run_periods_fixture", + target: "2026-09-05", + runs, + publications: { + payPeriods: { + id: seed.payPeriods.publicationId, + runId: "run_periods_fixture", + originRunId: "run_periods_fixture", + contentSha256: seed.payPeriods.contentSha256, + batchBound: false, + }, + roster: pub(seed.roster, runs[0].id, "run_fixture_roster"), + timecards: pub(seed.timecards, runs[1].id, "run_fixture_timecards"), + resourceLinks: pub(seed.links, runs[3].id, "run_fixture_links"), + }, + capturedAt: new Date().toISOString(), + }; + const result = await runManagedPaycomActivation({ + authority, + runtime: { + verifyInfrastructure: async () => { + const health = await runtimeAgentControlInvoke( + root + "/run/runtime-agent-control.sock", + key, + "health", + {}, + ); + assert.equal(health.ok, true); + return { + runtimeKey: key, + ...Object.fromEntries( + ACTIVATION_INFRASTRUCTURE_GATES.map((k) => [k, true]), + ), + }; + }, + testProvider: async () => ({ + profileId: "paycom-main", + provider: "paycom", + status: "authenticated", + testedAt: new Date().toISOString(), + }), + configure: async () => ({ + digest: audited.definitionDigest, + collectors: 1, + sources: 1, + plans: 15, + syncs: 1, + }), + publishFirst: async () => ({ + batchId: audited.batchId, + preparationRunId: audited.preparationRunId, + status: "succeeded", + runCount: runs.length, + succeededRuns: runs.length, + failedRuns: 0, + cancelledRuns: 0, + }), + verifyPublication: async () => audited, + }, + }); + assert.equal(result.ok, true, JSON.stringify(result)); + console.log(JSON.stringify(result)); + } finally { + store.close(); + } +} +main().catch((e) => { + console.error(e.stack); + process.exitCode = 1; +}); diff --git a/core/core/installations/tests/live-dsps/operator.js b/core/core/installations/tests/live-dsps/operator.js new file mode 100644 index 0000000..e5d801f --- /dev/null +++ b/core/core/installations/tests/live-dsps/operator.js @@ -0,0 +1,43 @@ +'use strict'; +// Run by the local root operator as the existing Core service user. Credentials +// travel through protected pipes, never arguments or the public test report. +const fs = require('node:fs'); +const { AccessStore, AccessControlService } = require('../../../accounts/src'); +async function main(input) { + const store = new AccessStore({ databaseRoot: input.localRoot + '/data/access-control', database: input.localRoot + '/data/access-control/access-control.sqlite3' }); + const access = new AccessControlService(store, { installationOperatorEnabled: true, installationBackend: 'native_service_v1', sessionTtlMs: 60 * 60 * 1000 }); + const sessionView = value => ({ token: value.token, csrf: value.session.csrfToken }); + try { + if (input.action === 'session') { + const owner = store.db.prepare("SELECT id FROM users WHERE platform_role='owner' AND status='active' ORDER BY id LIMIT 1").get(); + if (!owner) throw Error('platform_owner_required'); + return sessionView(access.createSession(owner.id)); + } + const session = access.requireSession(input.session.token); + if (input.action === 'logout') { access.signOut(session); return {}; } + if (input.action === 'destroy') { + access.requirePlatform(session, 'platform.installations.manage'); + if (!/^live_[a-f0-9]{32}$/.test(input.runId) || ![0, 1].includes(input.index)) throw Error('invalid_test_request'); + const created = store.db.prepare("SELECT organization_id FROM platform_mutation_requests WHERE actor_user_id=? AND action='organization.create' AND idempotency_key=?") + .get(session.user.id, `${input.runId}:create:${input.index}`); + const profile = created && store.db.prepare('SELECT owner_email FROM organization_profiles WHERE organization_id=?').get(created.organization_id); + if (!created || created.organization_id !== input.organizationId || profile?.owner_email !== `${input.runId}-${input.index}@dispatch-test.invalid` + || store.installationBackend(created.organization_id) !== 'native_service_v1') throw Error('invalid_test_request'); + // This protected local operator already has host authority. It can retire + // only its own recorded fixtures; browser deletion always needs a password. + return require('../../../accounts/src/installation-lifecycle').createAccessInstallationLifecycleAuthority({ + store, organizationId: created.organization_id, authorityScope: 'platform_removal', actorUserId: session.user.id, destructionEnabled: true, + }).request({ operation: 'destroy', expectedRevision: input.expectedRevision, idempotencyKey: `${input.runId}:delete:${input.index}` }); + } + if (input.action !== 'create' || !/^live_[a-f0-9]{32}$/.test(input.runId) || ![0, 1].includes(input.index)) throw Error('invalid_test_request'); + const email = `${input.runId}-${input.index}@dispatch-test.invalid`; + const result = access.createOrganization(session, { ownerEmail: email, idempotencyKey: `${input.runId}:create:${input.index}` }); + // A retry must never reset an existing account or recover somebody's login. + if (!result.token) throw Error('test_creation_already_exists'); + return { organizationId: result.organization.id, invitationToken: result.token, email }; + } finally { store.close(); } +} +if (require.main === module) main(JSON.parse(fs.readFileSync(0, 'utf8'))).then(v => process.stdout.write(JSON.stringify(v))).catch(e => { + process.stderr.write(JSON.stringify({ code: e.code || e.message })); process.exitCode = 1; +}); +module.exports = { main }; diff --git a/core/core/installations/tests/live-dsps/runner.js b/core/core/installations/tests/live-dsps/runner.js new file mode 100644 index 0000000..8f8bc8c --- /dev/null +++ b/core/core/installations/tests/live-dsps/runner.js @@ -0,0 +1,290 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto'), assert = require('node:assert/strict'); +const { execFileSync } = require('node:child_process'); +const { DatabaseSync } = require('node:sqlite'); +const { publicRootJson } = require('../../src/offsite-policy'); +const { privateJson, atomic } = require('../../src/release-delivery-files'); +const { createPlan } = require('../../src/native-deployment'); +const PROJECT = path.resolve(__dirname, "../../../.."); +const ROOT = '/var/lib/dispatch-live-tests'; +function validateTarget(state, target, row, allowIncomplete = false) { + assert.match(state.id, /^live_[a-f0-9]{32}$/); + assert.ok(state.targets.includes(target)); + assert.match(target.organizationId, /^org_[a-f0-9]{32}$/); + assert.equal(row.organization_id, target.organizationId); + assert.equal(row.backend, 'native_service_v1'); + assert.equal(row.owner_email, target.email); + assert.equal(target.email, `${state.id}-${target.index}@dispatch-test.invalid`); + assert.ok(row.name === `TEST ${state.id.slice(5, 13)} DSP ${target.index + 1}` || allowIncomplete && !target.profileApplied && row.name === 'New DSP'); +} +function requestApi(config, session, endpoint, body) { + return new Promise((resolve, reject) => { + const request = require('node:http').request({ hostname: '127.0.0.1', port: config.port, path: endpoint, + method: body === undefined ? 'GET' : 'POST', timeout: 30000, + headers: { Host: new URL(config.publicOrigin).host, Origin: config.publicOrigin, 'CF-Visitor': '{"scheme":"https"}', + ...(session ? { Cookie: `__Host-dispatch_session=${session.token}` } : {}), + ...(body === undefined ? {} : { 'Content-Type': 'application/json', 'X-Dispatch-CSRF': session?.csrf || '' }) }, + }, response => { + const chunks = []; let bytes = 0; + response.on('data', chunk => { bytes += chunk.length; if (bytes > 2 * 1024 * 1024) request.destroy(Error('test_response_too_large')); else chunks.push(chunk); }); + response.on('error', reject); + response.on('end', () => { try { + const value = JSON.parse(Buffer.concat(chunks)); + resolve({ status: response.statusCode, value, token: response.headers['set-cookie']?.[0]?.split(';')[0]?.split('=')[1] }); + } catch (error) { reject(error); } }); + }); + request.on('timeout', () => request.destroy(Error('test_request_timeout'))); request.on('error', reject); + request.end(body === undefined ? undefined : JSON.stringify(body)); + }); +} +async function main(argv) { + assert.equal(process.geteuid(), 0, 'Run as the server administrator with sudo'); + assert.ok(['run', 'cleanup', 'status'].includes(argv[0])); + assert.equal(argv.length, argv[0] === 'run' ? 1 : 2); + process.umask(0o077); + const config = privateJson('/etc/dispatch/release-delivery.json', 0); + const source = path.join(config.localRoot, 'data/access-control/access-control.sqlite3'); + const read = fn => { const db = new DatabaseSync(source, { readOnly: true }); try { return fn(db); } finally { db.close(); } }; + fs.mkdirSync(ROOT, { recursive: true, mode: 0o700 }); + assert.equal(fs.realpathSync(ROOT), ROOT); + const stat = fs.statSync(ROOT); assert.equal(stat.uid, 0); assert.equal(stat.mode & 0o077, 0); + const id = argv[0] === 'run' ? 'live_' + crypto.randomBytes(16).toString('hex') : argv[1]; + assert.match(id, /^live_[a-f0-9]{32}$/); + const directory = path.join(ROOT, id), file = path.join(directory, 'state.json'); + if (argv[0] === 'run') fs.mkdirSync(directory, { mode: 0o700 }); + let state = argv[0] === 'run' ? { schemaVersion: 1, id, status: 'running', targets: [], cases: [], boundaries: { + provider: 'Synthetic publication records; provider collection remains stopped', + email: 'Private invitation registration; no email sent', + backup: 'Live encrypted backup worker and R2 storage', + } } : privateJson(file, 0); + const save = () => { atomic(file, state); atomic(path.join(directory, 'report.json'), { schemaVersion: 1, id, status: state.status, cases: state.cases, boundaries: state.boundaries, targets: state.targets.map(t => ({ organizationId: t.organizationId, name: t.name, deleted: t.deleted || false })) }); }; + if (argv[0] === 'status') { console.log(JSON.stringify(privateJson(path.join(directory, 'report.json'), 0))); return; } + const lock = path.join(ROOT, 'active.lock'); + if (fs.existsSync(lock)) { + const previous = privateJson(lock, 0); assert.ok(Number.isSafeInteger(previous.pid) && previous.pid > 1); + try { process.kill(previous.pid, 0); throw Error('live_test_already_running'); } + catch (error) { if (error.code !== 'ESRCH') throw error; } + fs.unlinkSync(lock); + } + const fd = fs.openSync(lock, 'wx', 0o600); fs.writeSync(fd, JSON.stringify({ pid: process.pid, id })); fs.closeSync(fd); + let interrupted = false; + const interrupt = () => { interrupted = true; }; + process.on('SIGINT', interrupt); process.on('SIGTERM', interrupt); + const active = () => { if (interrupted) throw Error('test_interrupted'); }; + const invoke = (script, data) => JSON.parse(execFileSync('/usr/bin/node', ['--no-warnings', path.join(__dirname, script)], { + uid: config.uid, gid: config.gid, input: JSON.stringify({ ...data, localRoot: config.localRoot }), encoding: 'utf8', timeout: 120000, + env: { PATH: '/usr/bin:/bin', HOME: execFileSync('/usr/bin/getent', ['passwd', String(config.uid)], { encoding: 'utf8' }).trim().split(':')[5], DISPATCH_LOCAL_ROOT: config.localRoot }, + })); + async function api(session, endpoint, body, expected = 200) { + active(); + const result = await requestApi(config, session, endpoint, body); + assert.equal(result.status, expected, `${endpoint}: ${result.value.error?.code || result.value.status}`); + return result; + } + async function until(fn, timeout = 3600000) { + const deadline = Date.now() + timeout; + while (Date.now() < deadline) { + active(); const value = await fn(); if (value) return value; + // Exercise the browser's read paths while host operations hold fences. + const session = await api(state.platform, '/api/auth/session'); + assert.equal(session.value.data.authenticated, true, 'Platform session must remain available during DSP operations'); + await api(state.platform, '/api/platform/organizations'); + await new Promise(r => setTimeout(r, 2000)); + } + throw Error('test_operation_timeout'); + } + async function check(name, action) { + active(); console.log('START ' + name); const started = Date.now(); + try { await action(); state.cases.push({ name, status: 'passed', durationMs: Date.now() - started }); console.log('PASS ' + name); } + catch (e) { state.cases.push({ name, status: 'failed', code: e.code === 'ERR_ASSERTION' ? e.message : e.code || e.message }); throw e; } + finally { save(); } + } + const rowFor = target => read(db => db.prepare(`SELECT i.*,o.name,p.owner_email,s.code station,o.timezone FROM installations i JOIN organizations o ON o.id=i.organization_id + JOIN organization_profiles p ON p.organization_id=o.id JOIN stations s ON s.organization_id=o.id AND s.is_primary=1 WHERE o.id=?`).get(target.organizationId)); + const owned = target => { const row = rowFor(target); validateTarget(state, target, row); return row; }; + const controls = async (target, allowIncomplete = false) => { + const current = rowFor(target); validateTarget(state, target, current, allowIncomplete); + const rows = (await api(state.platform, '/api/platform/organizations')).value.data; + const matches = rows.filter(r => r.name === current.name && r.ownerEmail === target.email); + assert.equal(matches.length, 1); return matches[0]; + }; + async function settleStatus(target, operation, startingRevision, expectedStatus) { + await until(() => read(db => { + const job = db.prepare(`SELECT status,failure_code FROM installation_lifecycle_jobs + WHERE organization_id=? AND operation=? AND installation_revision>? + ORDER BY created_at DESC,rowid DESC LIMIT 1`).get(target.organizationId, operation, startingRevision); + assert.notEqual(job?.status, 'failed', `${operation} lifecycle failed: ${job?.failure_code}`); + return job?.status === 'succeeded' && owned(target).status === expectedStatus; + }), 300000); + } + async function drain() { + await until(() => read(db => { + const rows = db.prepare("SELECT status,failure_code FROM platform_backup_requests WHERE idempotency_key LIKE ?").all(`${id}:%`); + assert.ok(!rows.some(r => r.status === 'failed'), JSON.stringify(rows)); + return rows.length && rows.every(r => r.status === 'completed'); + })); + } + function planFor(target) { + const row = owned(target), registry = new DatabaseSync('/var/lib/dispatch-host/state/oci-host.sqlite3', { readOnly: true }); + let allocation; try { allocation = registry.prepare('SELECT * FROM allocations WHERE runtime_key=?').get(row.runtime_key); } finally { registry.close(); } + assert.ok(allocation); assert.equal(allocation.account_name.startsWith('dsp-'), true); + const release = JSON.parse(fs.readFileSync(path.join(config.localRoot, 'config/oci-releases.json'))).releases[row.release_id]; + const manifest = { manifestVersion: 1, revision: row.manifest_revision, organization: { id: row.organization_id, stationCode: row.station, timezone: row.timezone }, runtime: { key: row.runtime_key, templateId: 'isolated_dsp_v1', releaseId: row.release_id } }; + return createPlan(manifest, { revision: manifest.revision, organization: manifest.organization, runtime: manifest.runtime }, release, + { name: allocation.account_name, uid: allocation.uid, gid: allocation.gid, subuidStart: allocation.subuid_start, subgidStart: allocation.subgid_start, subidCount: 65536 }, + { version: 1, backend: 'native_service_v1', channel: 'production', organizationId: row.organization_id, runtimeKey: row.runtime_key, manifestRevision: row.manifest_revision, releaseId: row.release_id }); + } + const systemState = unit => execFileSync('/usr/bin/systemctl', ['show', unit, '--property=ActiveState', '--value'], { encoding: 'utf8' }).trim(); + function marker(target, content) { + const plan = planFor(target), markerFile = path.join(plan.host.installationRoot, 'data/live-test-marker'); + if (content === undefined) return execFileSync('/usr/sbin/runuser', ['--user', plan.account.name, '--', '/usr/bin/cat', '--', markerFile], { encoding: 'utf8' }); + execFileSync('/usr/sbin/runuser', ['--user', plan.account.name, '--', '/usr/bin/python3', '-c', + 'import os,sys; f=os.open(sys.argv[1],os.O_WRONLY|os.O_CREAT|os.O_TRUNC|os.O_NOFOLLOW,0o600); os.write(f,sys.stdin.buffer.read()); os.close(f)', markerFile], { input: content }); + } + async function cleanup() { + for (const target of state.targets) { + if (!rowFor(target)) { target.deleted = true; save(); continue; } + let row = await controls(target, true); + const allocation = target.account; + if (row.installation.state !== 'decommissioned' && row.installation.operation?.kind !== 'destroy') { + await api(state.platform, '/api/platform/installation/remove', { controlRef: row.controlRef, expectedRevision: row.installation.revision, + idempotencyKey: `${id}:remove:${target.index}` }, 202); + await until(() => rowFor(target)?.status === 'decommissioned'); + row = await controls(target, true); + } + invoke('operator.js', { action: 'destroy', session: state.platform, runId: id, index: target.index, + organizationId: target.organizationId, expectedRevision: row.installation.revision }); + await until(() => !rowFor(target)); target.deleted = true; save(); + if (target.backupId) { + const proof = privateJson(`/var/lib/dispatch-backup/archives/${target.backupId}.json`, 0); + assert.equal(proof.status, 'destroyed'); assert.ok(proof.deletedAt); + } + if (target.session) await api(target.session, '/api/paycom/daily?date=2026-09-05', undefined, 401); + if (allocation) { + assert.throws(() => execFileSync('/usr/bin/id', [allocation], { stdio: 'pipe' })); + assert.equal(fs.existsSync(target.hostRoot), false); + } + } + } + try { + await check('verify installed native release', async () => { + const health = (await api(null, '/api/platform/core-health')).value; + assert.equal(health.ok, true); + const command = execFileSync('/usr/bin/systemctl', ['--user', 'show', 'dispatch-dashboard.service', '--property=ExecStart', '--value'], { + uid: config.uid, gid: config.gid, encoding: 'utf8', env: { PATH: '/usr/bin:/bin', XDG_RUNTIME_DIR: `/run/user/${config.uid}` }, + }); + assert.match(command, /--installation-backend native_service_v1/, 'Live Core must use the native backend'); + const commit = execFileSync('/usr/bin/git', ['rev-parse', 'HEAD'], { cwd: PROJECT, uid: config.uid, gid: config.gid, encoding: 'utf8' }).trim(); + assert.equal(health.data.sourceCommit, commit, 'Run the tool from the deployed source commit'); + }); + state.platform = invoke('operator.js', { action: 'session' }); save(); + if (argv[0] === 'cleanup') { await check('delete recorded test DSPs', cleanup); state.status = 'cleaned'; save(); return; } + await check('live native preflight', async () => { + assert.equal(read(db => !!db.prepare("SELECT 1 FROM platform_rollouts WHERE status!='completed'").get()), false, 'Finish the rollout first'); + const backupStatus = publicRootJson('/var/lib/dispatch-backup-receipts/catalog.json', false, 0, 1024 * 1024); + assert.ok(backupStatus.backups); + const policy = require('../../src/offsite-policy'); + assert.equal(policy.offsiteRequired(), true, 'Verified offsite protection must be enabled'); policy.assertOffsiteReady(); + for (const name of fs.readdirSync('/var/lib/dispatch-backup/archives')) { + if (!/^(backup|breq)_[a-f0-9]{32}\.json$/.test(name)) continue; + const proof = privateJson('/var/lib/dispatch-backup/archives/' + name, 0); + if (proof.kind === 'core' && !proof.deletedAt) assert.equal(proof.organizationInventoryVersion, 1, 'Existing Core backup needs a complete inventory before test deletion'); + } + for (const name of fs.readdirSync('/var/lib/dispatch-backup-receipts')) { + if (!/^[a-f0-9]{64}\.json$/.test(name)) continue; + const proof = publicRootJson('/var/lib/dispatch-backup-receipts/' + name, false, 0, 1024 * 1024); + if (Array.isArray(proof.organizationIds)) assert.equal(proof.organizationInventoryVersion, 1, 'Existing Core snapshot needs a complete inventory before test deletion'); + } + state.baselineArchives = Object.entries(backupStatus.backups).filter(([, proof]) => proof.status === 'verified').map(([id]) => id); + const backupSettings = read(db => db.prepare('SELECT settings_json FROM platform_backup_settings WHERE id=1').get()); + assert.ok(!backupSettings || !JSON.parse(backupSettings.settings_json).enabled, 'Disable automatic fleet backups for the test window; the runner does not change shared settings'); + state.baseline = read(db => db.prepare("SELECT organization_id,runtime_key,release_id,status FROM installations WHERE status!='decommissioned' ORDER BY organization_id").all()); + }); + await check('create and register two native test DSPs', async () => { + for (let index = 0; index < 2; index++) { + const created = invoke('operator.js', { action: 'create', session: state.platform, runId: id, index }); + const target = { ...created, index, name: `TEST ${id.slice(5, 13)} DSP ${index + 1}`, password: crypto.randomBytes(24).toString('base64url') }; + state.targets.push(target); save(); + const result = await api(null, '/api/auth/register', { token: target.invitationToken, firstName: 'Synthetic', lastName: 'Test Owner', password: target.password, confirmPassword: target.password }, 201); + target.session = { token: result.token, csrf: result.value.data.csrfToken }; delete target.invitationToken; save(); + await api(target.session, '/api/organization/profile', { name: target.name, abbreviation: `T${index + 1}`, stationCode: 'TST1', timezone: 'UTC' }); + target.profileSubmitted = true; save(); + await until(() => { const row = rowFor(target); assert.notEqual(row?.status, 'failed'); return row?.name === target.name && row.status === 'ready'; }, 300000); + target.profileApplied = true; + const plan = planFor(target); target.account = plan.account.name; target.hostRoot = path.dirname(path.dirname(plan.host.installationRoot)); save(); + assert.equal(systemState(plan.identity.unitName), 'active'); + } + assert.notEqual(state.targets[0].account, state.targets[1].account); + }); + await check('seed only recorded DSPs with synthetic publication data', async () => { + const tools = path.join(directory, 'tools'); fs.mkdirSync(tools, { mode: 0o755 }); fs.chmodSync(tools, 0o755); + fs.writeFileSync(path.join(tools, 'helpers.js'), fs.readFileSync(path.join(PROJECT, 'plugins/paycom/backend/tests/helpers.js'), 'utf8').replace("require('../src/timecard-period')", "require('/opt/dispatch/plugins/paycom/backend/src/timecard-period')"), { mode: 0o444 }); + fs.chmodSync(path.join(tools, 'helpers.js'), 0o444); + fs.copyFileSync(path.join(__dirname, "./seed.js"), path.join(tools, 'seed.js')); fs.chmodSync(path.join(tools, 'seed.js'), 0o444); + for (const target of state.targets) { + const plan = planFor(target), artifact = `/opt/dispatch-runtime/releases/${plan.release.releaseId}/runtime-artifact`; + const properties = { User: plan.account.name, Group: plan.account.name, ProtectSystem: 'strict', ProtectHome: 'true', PrivateTmp: 'true', PrivateNetwork: 'true', NoNewPrivileges: 'true', + BindReadOnlyPaths: `${artifact}:/opt/dispatch ${tools}:/run/dispatch-test-tools`, BindPaths: plan.host.installationRoot + ':' + plan.guest.installationRoot, WorkingDirectory: '/opt/dispatch', UMask: '0077' }; + const seed = JSON.parse(execFileSync('/usr/bin/systemd-run', ['--quiet', '--wait', '--pipe', '--collect', '--unit=dispatch-live-seed-' + crypto.randomBytes(8).toString('hex'), + ...Object.entries(properties).flatMap(([k, v]) => ['--property', `${k}=${v}`]), + ...Object.entries({ ...plan.guest.environment, PATH: '/opt/dispatch/dependencies/node/bin:/usr/bin:/bin' }).flatMap(([k, v]) => ['--setenv', `${k}=${v}`]), + artifact + '/dependencies/node/bin/node', '--no-warnings', '/run/dispatch-test-tools/seed.js'], { encoding: 'utf8', timeout: 120000 })); + invoke('activate.js', { organizationId: target.organizationId, seed }); + marker(target, `${id}:${target.index}:before`); + } + }); + await check('back up both DSPs through the live encrypted backup worker', async () => { + const body = { action: 'backup', scope: 'dsps', organizationIds: state.targets.map(t => owned(t).organization_id), idempotencyKey: `${id}:backup` }; + await api(state.platform, '/api/platform/backups', body); await api(state.platform, '/api/platform/backups', body); await drain(); + for (const target of state.targets) { + target.backupId = read(db => db.prepare("SELECT id FROM platform_backup_records WHERE organization_id=? AND kind='dsp' ORDER BY created_at DESC LIMIT 1").get(target.organizationId)).id; + const proof = privateJson(`/var/lib/dispatch-backup/archives/${target.backupId}.json`, 0); + assert.equal(proof.status, 'verified'); assert.match(proof.recoveryDigest, /^[a-f0-9]{64}$/); save(); + } + }); + const target = state.targets[0], peer = state.targets[1]; + await check('suspend only the target DSP and deny its existing session', async () => { + const control = await controls(target); + await api(state.platform, '/api/platform/organization/status', { controlRef: control.controlRef, suspended: true, idempotencyKey: `${id}:suspend` }); + await settleStatus(target, 'suspend', control.installation.revision, 'suspended'); + assert.equal(systemState(planFor(target).identity.unitName), 'inactive'); + await api(target.session, '/api/paycom/daily?date=2026-09-05', undefined, 401); + assert.equal(systemState(planFor(peer).identity.unitName), 'active'); + }); + await check('restore backed-up data while keeping the DSP suspended', async () => { + await api(state.platform, '/api/platform/backups', { action: 'restore', organizationId: peer.organizationId, backupId: target.backupId, confirmation: peer.name, idempotencyKey: `${id}:wrong-restore` }, 409); + marker(target, 'changed after backup'); + await api(state.platform, '/api/platform/backups', { action: 'restore', organizationId: target.organizationId, backupId: target.backupId, confirmation: target.name, idempotencyKey: `${id}:restore` }); + await drain(); assert.equal(marker(target), `${id}:0:before`); assert.equal(marker(peer), `${id}:1:before`); + assert.equal(owned(target).status, 'suspended'); assert.equal(systemState(planFor(target).identity.unitName), 'inactive'); + }); + await check('resume the restored DSP', async () => { + const control = await controls(target); + await api(state.platform, '/api/platform/organization/status', { controlRef: control.controlRef, suspended: false, idempotencyKey: `${id}:resume` }); + await settleStatus(target, 'resume', control.installation.revision, 'ready'); + assert.equal(systemState(planFor(target).identity.unitName), 'active'); + const result = await api(null, '/api/auth/login', { email: target.email, password: target.password }); + target.session = { token: result.token, csrf: result.value.data.csrfToken }; save(); + await api(target.session, '/api/paycom/daily?date=2026-09-05'); + }); + await check('delete test DSPs and verify account and data removal', cleanup); + await check('preserve pre-existing DSP installations', async () => { + const current = read(db => db.prepare("SELECT organization_id,runtime_key,release_id,status FROM installations WHERE status!='decommissioned' ORDER BY organization_id").all()); + assert.deepEqual(current, state.baseline); + await until(() => { + const catalog = publicRootJson('/var/lib/dispatch-backup-receipts/catalog.json', false, 0, 1024 * 1024); + return state.baselineArchives.every(id => catalog.backups[id]?.status === 'verified'); + }, 120000); + }); + state.status = 'passed'; save(); + } catch (e) { state.status = 'failed'; save(); throw e; } + finally { + if (state.platform) { try { invoke('operator.js', { action: 'logout', session: state.platform }); } catch {} delete state.platform; save(); } + process.removeListener('SIGINT', interrupt); process.removeListener('SIGTERM', interrupt); + fs.unlinkSync(lock); + console.log(`Report: ${directory}/report.json`); + if (state.targets.some(t => !t.deleted)) console.log(`Cleanup: sudo ${PROJECT}/core/installations/scripts/verify-live-dsps cleanup ${id}`); + } +} +if (require.main === module) main(process.argv.slice(2)).catch(e => { console.error(e.code === 'ERR_ASSERTION' ? e.message : e.code || e.message); process.exitCode = 1; }); +module.exports = { main, validateTarget, requestApi }; diff --git a/core/core/installations/tests/live-dsps/seed.js b/core/core/installations/tests/live-dsps/seed.js new file mode 100644 index 0000000..cfef971 --- /dev/null +++ b/core/core/installations/tests/live-dsps/seed.js @@ -0,0 +1,46 @@ +#!/usr/local/bin/node +'use strict'; +// Invoked only by the root-owned live-test runner in its own recorded DSP namespace. +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { PaycomStore, stageCandidate, cleanupStage } = require('/opt/dispatch/plugins/paycom/backend/src/store'); +const { DATABASE, STAGING_ROOT } = require('/opt/dispatch/plugins/paycom/backend/src/paths'); +const { periodFromEnd } = require('/opt/dispatch/plugins/paycom/backend/src/timecard-period'); +const { TIMECARD_SUMMARY, ROUTE_VERSION, linkRows } = require('/opt/dispatch/plugins/paycom/backend/src/resource-links'); +const { timecardRecord, rosterRow } = require('/run/dispatch-test-tools/helpers'); +const PAYCOM_SYNC_ID = 'paycom-main-workforce'; +process.umask(0o077); +const store = new PaycomStore(DATABASE); +const changed = process.argv[2] === '--changed'; +const period = periodFromEnd(changed ? '2026-09-19' : '2026-09-05'); +const collectedAt = new Date().toISOString(); +function publish(candidate) { + const stage = stageCandidate(STAGING_ROOT, { attempt: 1, collectedAt, ...candidate, runId: candidate.runId + (changed ? '_changed' : '') }); + try { return store.publish(stage); } finally { cleanupStage(stage, STAGING_ROOT); } +} +const payPeriods = publish({ kind: 'pay_periods', target: period.end, runId: 'run_periods_fixture', metadata: {}, + rows: [{ start: period.start, end: period.end, key: period.key, relation: 'current' }] }); +const rows = [rosterRow('Z999', changed ? 'Synthetic Updated Employee' : 'Synthetic Fixture Employee')]; +const roster = publish({ kind: 'roster', target: period.end, runId: 'run_fixture_roster', metadata: {}, rows }); +const timecards = publish({ kind: 'timecards', target: period.end, periodKey: period.key, + runId: 'run_fixture_timecards', metadata: { periodStart: period.start, periodEnd: period.end, mode: 'full', + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256 }, + rows: [{ employeeCode: rows[0].employeeCode, employeeName: rows[0].employeeName, + record: timecardRecord(rows[0].employeeCode, period.end), sourceSha256: 'b'.repeat(64) }] }); +const links = publish({ kind: 'resource_links', target: period.end, periodKey: period.key, + runId: 'run_fixture_links', metadata: { resourceType: TIMECARD_SUMMARY, periodStart: period.start, + periodEnd: period.end, rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256, + routeVersion: ROUTE_VERSION }, rows: linkRows(TIMECARD_SUMMARY, rows, period) }); +store.close(); +if (!changed) { + const { CollectionStore } = require('/opt/dispatch/runtime/collection-manager/src/store'); + const { defaultPaths } = require('/opt/dispatch/runtime/collection-manager/src/paths'); + const { materializeSpec } = require('/opt/dispatch/runtime/collection-manager/src/control-cli'); + const spec = JSON.parse(fs.readFileSync('/opt/dispatch/plugins/paycom/backend/config/collection-manager.json')); + for (const sync of spec.syncs || []) sync.desiredState = 'stopped'; + for (const plan of spec.plans) plan.schedule = { type: 'manual' }; + const collection = new CollectionStore(defaultPaths()); + try { collection.applySpec(materializeSpec(spec, '/opt/dispatch')); } finally { collection.close(); } +} +process.stdout.write(`${JSON.stringify({ payPeriods, roster, timecards, links })}\n`); diff --git a/core/core/installations/tests/managed-activation-evidence.test.js b/core/core/installations/tests/managed-activation-evidence.test.js new file mode 100644 index 0000000..c48466a --- /dev/null +++ b/core/core/installations/tests/managed-activation-evidence.test.js @@ -0,0 +1,195 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { CollectionManager } = require('dispatch-dsp/runtime/collection-manager/src/manager.js'); +const { LocalCollectionAdminPort } = require('dispatch-dsp/runtime/adapters/local/collection-admin-port.js'); +const { LocalCollectionManagerPort } = require('dispatch-dsp/runtime/adapters/local/collection-manager-port.js'); +const { CollectionClient } = require('dispatch-runtime-kit/sdk/src/collection-client'); +const { + MANAGED_RUNTIME_ENVIRONMENT_KEYS, +} = require('../../../shared/paths/runtime-paths'); +const { createInstallationLayoutManager } = require('../src/layout'); +const { + PAYCOM_FIRST_PUBLICATION_TASKS, + managedPaycomDefinition, + managedPaycomFirstPublicationRequest, +} = require('../../../compatibility/provisioner/src/managed-paycom.js'); +const { + createManagedPaycomActivationEvidenceVerifier, +} = require('../../../compatibility/provisioner/src/managed-activation-evidence.js'); + +const PROJECT_ROOT = path.resolve(__dirname, "../../.."); +const FIXTURE_COLLECTOR = path.join(__dirname, "./fixture-paycom-collector.js"); +const FORBIDDEN_ENV = Object.freeze([ + 'DISPATCH_LOCAL_ROOT', + 'DISPATCH_ACCESS_CONTROL_DATA_ROOT', + 'DISPATCH_ACCESS_CONTROL_DATABASE_ROOT', +]); + +function manifest(suffix) { + return { + manifestVersion: 1, + revision: 1, + organization: { + id: `org_evidence_${suffix}`, + stationCode: 'TST1', + timezone: 'America/Los_Angeles', + }, + runtime: { + key: `runtime_evidence_${suffix}`, + templateId: 'isolated_dsp_v1', + releaseId: 'dispatch_fixture_1', + }, + }; +} +function authority(selected) { + return { + revision: selected.revision, + organization: { ...selected.organization }, + runtime: { ...selected.runtime }, + }; +} +function applyEnvironment(values) { + const keys = [...MANAGED_RUNTIME_ENVIRONMENT_KEYS, 'DISPATCH_MANAGED_RUNTIME', ...FORBIDDEN_ENV]; + const previous = Object.fromEntries(keys.map(key => [key, process.env[key]])); + for (const key of FORBIDDEN_ENV) delete process.env[key]; + Object.assign(process.env, values, { DISPATCH_MANAGED_RUNTIME: '1' }); + return () => { + for (const key of keys) { + if (previous[key] === undefined) delete process.env[key]; + else process.env[key] = previous[key]; + } + }; +} + +async function collectFixtureRuntime(installationsRoot, suffix) { + const selected = manifest(suffix); + const selectedAuthority = authority(selected); + const layoutManager = createInstallationLayoutManager({ + installationsRoot, + projectRoot: PROJECT_ROOT, + }); + layoutManager.materialize(selected, selectedAuthority); + const paths = layoutManager.runtimePaths(selected, selectedAuthority); + const environment = layoutManager.runtimeEnvironment(selected, selectedAuthority); + const restoreEnvironment = applyEnvironment(environment); + const managerStore = new CollectionStore(paths.collection); + require('dispatch-runtime-kit/collection-manager/src/plugin-state').applyState(managerStore, { command: 'apply', pluginId: 'paycom', version: '0.18.7', state: 'enabled', revision: 1 }); + const manager = new CollectionManager(managerStore, { tickMs: 5 }); + let started = false; + try { + const definition = managedPaycomDefinition(selected, selectedAuthority, { projectRoot: PROJECT_ROOT }); + const fixtureSpec = JSON.parse(JSON.stringify(definition.specification)); + const fixtureExecutable = path.join(paths.configRoot, 'fixture-paycom-collector'); + const fixtureSource = fs.readFileSync(FIXTURE_COLLECTOR, 'utf8'); + assert.equal(path.isAbsolute(process.execPath), true); + assert.doesNotMatch(process.execPath, /[\0\r\n ]/); + fs.writeFileSync(fixtureExecutable, + fixtureSource.replace(/^#![^\n]+/, `#!${process.execPath}`), { mode: 0o700, flag: 'wx' }); + fixtureSpec.collectors[0].command = fixtureExecutable; + const admin = new LocalCollectionAdminPort({ paths: paths.collection }); + admin.apply(fixtureSpec); + assert.deepEqual(admin.inspect().counts, { collectors: 1, sources: 1, plans: 15, syncs: 1 }); + assert.deepEqual(admin.attest(fixtureSpec), { matched: true }); + const driftedSpec = JSON.parse(JSON.stringify(fixtureSpec)); + driftedSpec.sources[0].config.timezone = 'America/New_York'; + assert.deepEqual(admin.attest(driftedSpec), { matched: false }); + await manager.start(); + started = true; + + const client = new CollectionClient({ port: new LocalCollectionManagerPort({ paths: paths.collection }) }); + const periods = await client.startRun('paycom-periods', {}, { + idempotencyKey: `fixture-periods-${suffix}`, + }); + assert.equal(periods.ok, true, JSON.stringify(periods)); + assert.equal((await manager.runUntilIdle({ timeoutMs: 10_000 })).idle, true); + + const batch = await client.enqueue(managedPaycomFirstPublicationRequest(), { + idempotencyKey: `fixture-activation-${suffix}`, + }); + assert.equal(batch.ok, true, JSON.stringify(batch)); + assert.equal((await manager.runUntilIdle({ timeoutMs: 20_000 })).idle, true); + const terminal = await client.batchStatus(batch.data.id, { limit: 50, offset: 0 }); + assert.equal(terminal.status, 'succeeded', JSON.stringify(terminal)); + assert.deepEqual(terminal.data.runPage.items.map(item => item.run.plan).sort(), + Object.keys(PAYCOM_FIRST_PUBLICATION_TASKS).sort()); + + const verifier = createManagedPaycomActivationEvidenceVerifier({ environment }); + const evidence = verifier.verify({ + batchId: batch.data.id, + preparationRunId: periods.data.id, + definitionDigest: definition.digest, + }); + assert.equal(evidence.batchId, batch.data.id); + assert.equal(evidence.target, '2026-09-05'); + assert.equal(evidence.runs.length, 5); + assert.equal(evidence.publications.payPeriods.batchBound, false); + for (const key of ['roster', 'timecards', 'resourceLinks']) { + assert.equal(evidence.publications[key].batchBound, true); + assert.match(evidence.publications[key].contentSha256, /^[a-f0-9]{64}$/); + } + + const retryPeriods = await client.startRun('paycom-periods', {}, { + idempotencyKey: `fixture-periods-retry-${suffix}`, + }); + assert.equal(retryPeriods.ok, true); + assert.equal((await manager.runUntilIdle({ timeoutMs: 10_000 })).idle, true); + const retryBatch = await client.enqueue(managedPaycomFirstPublicationRequest(), { + idempotencyKey: `fixture-batch-retry-${suffix}`, + }); + assert.equal(retryBatch.ok, true); + assert.equal((await manager.runUntilIdle({ timeoutMs: 20_000 })).idle, true); + + const retryEvidence = verifier.verify({ + batchId: retryBatch.data.id, + preparationRunId: retryPeriods.data.id, + definitionDigest: definition.digest, + }); + for (const key of ['payPeriods', 'roster', 'timecards', 'resourceLinks']) { + assert.equal(retryEvidence.publications[key].id, evidence.publications[key].id); + assert.equal(retryEvidence.publications[key].originRunId, + evidence.publications[key].originRunId); + assert.notEqual(retryEvidence.publications[key].runId, evidence.publications[key].runId); + } + return Object.freeze({ paths, environment, evidence, retryEvidence }); + } finally { + if (started) await manager.stop(); + managerStore.close(); + restoreEnvironment(); + } +} + +test('two isolated real managers produce independently bound first-publication evidence', async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-managed-evidence-')); + fs.chmodSync(root, 0o700); + const installationsRoot = path.join(root, 'installations'); + fs.mkdirSync(installationsRoot, { mode: 0o700 }); + try { + const alpha = await collectFixtureRuntime(installationsRoot, 'alpha'); + const bravo = await collectFixtureRuntime(installationsRoot, 'bravo'); + assert.notEqual(alpha.paths.collection.database, bravo.paths.collection.database); + assert.notEqual(alpha.paths.paycom.database, bravo.paths.paycom.database); + for (const key of ['payPeriods', 'roster', 'timecards', 'resourceLinks']) { + assert.notEqual(alpha.evidence.publications[key].id, bravo.evidence.publications[key].id); + assert.notEqual(alpha.evidence.publications[key].runId, bravo.evidence.publications[key].runId); + } + const crossedEnvironment = { + ...alpha.environment, + ...Object.fromEntries(Object.entries(bravo.environment) + .filter(([key]) => key.startsWith('DISPATCH_PAYCOM_'))), + }; + const crossed = createManagedPaycomActivationEvidenceVerifier({ environment: crossedEnvironment }); + assert.throws(() => crossed.verify({ + batchId: alpha.evidence.batchId, + preparationRunId: alpha.evidence.preparationRunId, + definitionDigest: alpha.evidence.definitionDigest, + }), error => error.code === 'first_publication_failed'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/core/core/installations/tests/managed-activation-runtime.test.js b/core/core/installations/tests/managed-activation-runtime.test.js new file mode 100644 index 0000000..ec87d5b --- /dev/null +++ b/core/core/installations/tests/managed-activation-runtime.test.js @@ -0,0 +1,367 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const test = require('node:test'); +const { success } = require('../../../shared/contracts/src'); +const { + PAYCOM_FIRST_PUBLICATION_TASKS, + managedPaycomDefinition, + managedPaycomFirstPublicationRequest, +} = require('../../../compatibility/provisioner/src/managed-paycom.js'); +const { + EXPECTED_FIRST_PUBLICATION_PLANS, + createManagedPaycomActivationRuntime, +} = require('../../../compatibility/provisioner/src/managed-activation-runtime.js'); + +function manifest() { + return { + manifestVersion: 1, + revision: 1, + organization: { id: 'org_activation_fixture', stationCode: 'TST1', timezone: 'America/Chicago' }, + runtime: { key: 'fixture_activation', templateId: 'isolated_dsp_v1', releaseId: 'dispatch_fixture_1' }, + }; +} + +function authority(value) { + return { + revision: value.revision, + organization: { ...value.organization }, + runtime: { ...value.runtime }, + }; +} + +function fixture() { + const selected = manifest(); + const selectedAuthority = authority(selected); + const definition = managedPaycomDefinition(selected, selectedAuthority); + const calls = []; + let batchReads = 0; + let now = Date.parse('2026-09-02T21:30:00.000Z'); + let syncRunning = false; + const batchData = status => ({ + id: 'batch_activation_001', + status, + counts: { + queued: status === 'queued' ? 5 : 0, + running: 0, + succeeded: status === 'succeeded' ? 5 : 0, + failed: 0, + cancelled: 0, + }, + runCount: 5, + runPage: { + items: Object.entries(EXPECTED_FIRST_PUBLICATION_PLANS).map(([plan, method]) => ({ + targetKey: '2026-09-05', + taskId: PAYCOM_FIRST_PUBLICATION_TASKS[plan].taskId, + run: { plan, method, source: 'paycom-main' }, + })), + total: 5, + limit: 50, + offset: 0, + hasMore: false, + }, + }); + const options = { + manifest: selected, + manifestAuthority: selectedAuthority, + layout: { + inspect: () => { calls.push('layout'); return { runtimeKey: selected.runtime.key }; }, + }, + serviceManager: { + plan: () => { calls.push('plan'); return { runtimeKey: selected.runtime.key }; }, + inspectInstalled: () => { calls.push('installed'); }, + }, + supervisor: { + inspect: () => { calls.push('supervisor'); }, + health: () => { calls.push('health'); }, + }, + client: { + auth: { + health: async () => success('ready', { vault: { verified: true } }), + profileStatus: async profile => success('configured', { + profile: { configured: true, profile, provider: 'paycom' }, session: 'stopped', + }), + testProfile: async profile => success('authenticated', { + profile, provider: 'paycom', testedAt: new Date(now).toISOString(), + }), + }, + collections: { + health: async () => success('ready', { + databaseIntegrity: 'ok', + manager: { running: true }, + counts: { queued: 0, running: 0 }, + syncAlerts: { critical: 0 }, + }), + source: async () => success('found', { authProfile: 'paycom-main' }), + startRun: async (plan, input, operation) => { + calls.push(['periods', plan, Object.keys(input).length, operation.idempotencyKey]); + return success('succeeded', { id: 'run_periods' }); + }, + runStatus: async () => success('succeeded', { id: 'run_periods' }), + cancelRun: async () => success('cancelled', { id: 'run_periods' }), + enqueue: async (request, operation) => { + calls.push(['enqueue', request.scope, operation.idempotencyKey]); + return success('queued', batchData('queued')); + }, + batchStatus: async () => { + batchReads += 1; + return success(batchReads === 1 ? 'queued' : 'succeeded', batchData(batchReads === 1 ? 'queued' : 'succeeded')); + }, + cancelBatch: async () => success('cancelled', batchData('cancelled')), + }, + sync: { + status: async () => success('found', { + desiredState: syncRunning ? 'running' : 'stopped', activity: 'idle', + activeRun: null, queuedRunCount: 0, + }), + stop: async () => { + calls.push('sync-stop'); + syncRunning = false; + return success('stopped', { + desiredState: 'stopped', activity: 'idle', activeRun: null, queuedRunCount: 0, + }); + }, + start: async () => { + calls.push('sync-start'); + syncRunning = true; + return success('started', { sync: { desiredState: 'running' } }); + }, + }, + paycom: { + health: async () => success('ready', { + ready: true, + publicationStatus: 'ready', + payPeriods: { verified: true, target: '2026-09-02', projectionValid: true }, + roster: { verified: true, target: '2026-09-05' }, + timecards: { verified: true, target: '2026-09-05' }, + resourceLinks: { verified: true, target: '2026-09-05' }, + }), + }, + }, + collectionAdmin: { + preview: () => ({ valid: true }), + apply: () => ({ collectors: 1, sources: 1, plans: 15, syncs: 1 }), + inspect: () => ({ initialized: true, counts: { collectors: 1, sources: 1, plans: 15, syncs: 1 } }), + attest: () => ({ matched: true }), + }, + gateway: { + health: async () => success('ready', { runtimeIdentity: 'matched' }), + }, + evidenceVerifier: { + verify: async ({ batchId, definitionDigest, preparationRunId }) => ({ + definitionDigest, + requestDigest: crypto.createHash('sha256') + .update(JSON.stringify(managedPaycomFirstPublicationRequest())).digest('hex'), + previewDigest: 'a'.repeat(64), + batchId, + preparationRunId, + target: '2026-09-05', + runs: Object.entries(PAYCOM_FIRST_PUBLICATION_TASKS).map(([plan, item], index) => ({ + id: `run_${index}`, + taskId: item.taskId, + plan, + method: item.method, + })), + publications: { + payPeriods: { id: 'pub_periods', runId: 'run_periods', originRunId: 'run_periods', contentSha256: '1'.repeat(64), batchBound: false }, + roster: { id: 'pub_roster', runId: 'run_0', originRunId: 'run_0', contentSha256: '2'.repeat(64), batchBound: true }, + timecards: { id: 'pub_timecards', runId: 'run_1', originRunId: 'run_1', contentSha256: '3'.repeat(64), batchBound: true }, + resourceLinks: { id: 'pub_links', runId: 'run_3', originRunId: 'run_3', contentSha256: '4'.repeat(64), batchBound: true }, + }, + capturedAt: new Date(now).toISOString(), + }), + }, + clock: () => now, + delay: async milliseconds => { now += milliseconds; }, + publicationTimeoutMs: 10_000, + publicationPollMs: 10, + }; + return { selected, selectedAuthority, definition, calls, options, batchData }; +} + +test('managed activation runtime applies the fixed Paycom definition and verifies one complete publication', async () => { + const context = fixture(); + const runtime = createManagedPaycomActivationRuntime(context.options); + const infrastructure = await runtime.verifyInfrastructure(context.selected); + assert.deepEqual(infrastructure, { + runtimeKey: 'fixture_activation', + runtime_layout: true, + service_supervision: true, + auth_broker: true, + collection_manager: true, + runtime_gateway: true, + }); + assert.deepEqual(await runtime.configure(context.definition), { + digest: context.definition.digest, + collectors: 1, + sources: 1, + plans: 15, + syncs: 1, + }); + assert.equal((await runtime.testProvider('paycom-main')).status, 'authenticated'); + const publication = await runtime.publishFirst(managedPaycomFirstPublicationRequest(), { + idempotencyKey: 'activation:job_activation_001', + heartbeat: async () => {}, + }); + assert.deepEqual(publication, { + batchId: 'batch_activation_001', + preparationRunId: 'run_periods', + status: 'succeeded', + runCount: 5, + succeededRuns: 5, + failedRuns: 0, + cancelledRuns: 0, + }); + const evidence = await runtime.verifyPublication(publication.batchId, publication.preparationRunId); + assert.equal(evidence.batchId, publication.batchId); + assert.equal(evidence.target, '2026-09-05'); + assert.equal(evidence.publications.roster.batchBound, true); + assert.deepEqual(await runtime.inspectSchedule(), { syncWasRunning: false }); + assert.deepEqual(await runtime.restoreSchedule(true), { syncWasRunning: true }); + assert.deepEqual(await runtime.inspectSchedule(), { syncWasRunning: true }); + assert.deepEqual(await runtime.quiesceSchedule(true), { syncWasRunning: true }); + assert.deepEqual(context.calls.filter(value => typeof value === 'string' && value.startsWith('sync-')), + ['sync-start', 'sync-stop']); + assert.deepEqual(context.calls.find(value => Array.isArray(value) && value[0] === 'periods'), + ['periods', 'paycom-periods', 0, 'activation:job_activation_001:periods']); + assert.deepEqual(context.calls.find(value => Array.isArray(value) && value[0] === 'enqueue'), + ['enqueue', 'full', 'activation:job_activation_001']); +}); + +test('managed activation reuses an already-terminal idempotent batch', async () => { + const context = fixture(); + context.options.client.collections.enqueue = async () => success( + 'succeeded', context.batchData('succeeded'), + ); + context.options.client.collections.batchStatus = async () => success( + 'succeeded', context.batchData('succeeded'), + ); + const runtime = createManagedPaycomActivationRuntime(context.options); + const result = await runtime.publishFirst(managedPaycomFirstPublicationRequest(), { + idempotencyKey: 'activation:job_activation_001', + heartbeat: async () => {}, + }); + assert.equal(result.status, 'succeeded'); + assert.equal(result.batchId, 'batch_activation_001'); +}); + +test('a stale activation worker never cancels the shared idempotent batch', async () => { + const context = fixture(); + let beats = 0; + let batchCancels = 0; + let runCancels = 0; + context.options.client.collections.cancelBatch = async () => { + batchCancels += 1; + return success('cancelled', context.batchData('cancelled')); + }; + context.options.client.collections.cancelRun = async () => { + runCancels += 1; + return success('cancelled', { id: 'run_periods' }); + }; + const runtime = createManagedPaycomActivationRuntime(context.options); + await assert.rejects(runtime.publishFirst(managedPaycomFirstPublicationRequest(), { + idempotencyKey: 'activation:job_activation_001', + heartbeat: async () => { + beats += 1; + if (beats === 2) throw Object.assign(new Error('installation_operation_in_progress'), { + code: 'installation_operation_in_progress', + }); + }, + }), /installation_operation_in_progress/); + assert.equal(batchCancels, 0); + assert.equal(runCancels, 0); +}); + +test('managed activation cancels and drains its exact batch at the publication deadline', async () => { + const context = fixture(); + let cancelled = 0; + context.options.publicationTimeoutMs = 1000; + context.options.publicationPollMs = 1000; + context.options.client.collections.batchStatus = async () => success('queued', context.batchData('queued')); + context.options.client.collections.cancelBatch = async () => { + cancelled += 1; + return success('cancelled', context.batchData('cancelled')); + }; + const runtime = createManagedPaycomActivationRuntime(context.options); + await assert.rejects(runtime.publishFirst(managedPaycomFirstPublicationRequest(), { + idempotencyKey: 'activation:job_activation_001', + heartbeat: async () => {}, + }), /first_publication_failed/); + assert.equal(cancelled, 1); +}); + +test('managed activation runtime rejects runtime identity, definition, and publication drift', async () => { + const context = fixture(); + const runtime = createManagedPaycomActivationRuntime(context.options); + const mismatched = manifest(); + mismatched.runtime.key = 'fixture_other'; + await assert.rejects(runtime.verifyInfrastructure(mismatched), /runtime_identity_mismatch/); + await assert.rejects(runtime.configure({ ...context.definition, digest: '0'.repeat(64) }), /runtime_boundary_violation/); + + const failedContext = fixture(); + failedContext.options.client.paycom.health = async () => success('ready', { + ready: true, + publicationStatus: 'ready', + payPeriods: { verified: true, target: '2026-09-02', projectionValid: true }, + roster: { verified: true, target: '2026-09-05' }, + timecards: { verified: true, target: '2026-09-05' }, + resourceLinks: { verified: true, target: '2026-09-12' }, + }); + const failedRuntime = createManagedPaycomActivationRuntime(failedContext.options); + await assert.rejects(failedRuntime.verifyPublication('batch_activation_001', 'run_periods'), + /first_publication_failed/); +}); + +test('hourly setup initializes a real manager once and preserves runs and schedule on reconnection', async () => { + const fs = require('node:fs'); + const { failure } = require('../../../shared/contracts/src'); + const { fixture: managerFixture } = require('dispatch-dsp/runtime/collection-manager/tests/helpers.js'); + const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); + const { SyncService } = require('dispatch-runtime-kit/collection-manager/src/syncs'); + const { LocalCollectionAdminPort } = require('dispatch-dsp/runtime/adapters/local/collection-admin-port.js'); + const context = fixture(); + const manager = managerFixture(); + const store = new CollectionStore(manager.paths); + require('dispatch-runtime-kit/collection-manager/src/plugin-state').applyState(store, { command: 'apply', pluginId: 'paycom', version: '0.18.7', state: 'enabled', revision: 1 }); + let now = Date.now(); + const service = new SyncService(store, { clock: () => now }); + context.options.collectionAdmin = new LocalCollectionAdminPort({ paths: manager.paths }); + context.options.client.collections.source = async id => success('found', store.source(id)); + context.options.client.sync = { + status: async id => { + try { return success('found', service.status(id)); } + catch (error) { return failure(error.code); } + }, + start: async id => success('started', service.start(id)), + edit: async (id, patch) => success('updated', await service.edit(id, patch)), + }; + const runtime = createManagedPaycomActivationRuntime(context.options); + try { + assert.deepEqual(await runtime.startWorkforceSync(), { + syncId: 'paycom-main-workforce', intervalSeconds: 3600, desiredState: 'running', + }); + const first = store.sync('paycom-main-workforce'); + assert.equal(first.intervalSeconds, 3600); + assert.equal(first.jitterSeconds, 300); + assert.ok(first.nextDueAt >= now + 3600_000 && first.nextDueAt <= now + 3900_000); + assert.equal(first.activity, 'queued'); + assert.equal(store.syncHistory(first.id).total, 1); + now += 30_000; + await createManagedPaycomActivationRuntime(context.options).startWorkforceSync(); + assert.equal(store.sync(first.id).nextDueAt, first.nextDueAt); + assert.equal(store.syncHistory(first.id).total, 1); + service.runNow(first.id, { idempotencyKey: 'another-user-click' }); + assert.equal(store.syncHistory(first.id).total, 1); + assert.equal(store.sync(first.id).nextDueAt, first.nextDueAt); + await service.edit(first.id,{intervalSeconds:7200,jitterSeconds:120}); + const customized=store.sync(first.id); + await createManagedPaycomActivationRuntime(context.options).startWorkforceSync(); + assert.equal(store.sync(first.id).intervalSeconds,7200); + assert.equal(store.sync(first.id).jitterSeconds,120); + assert.equal(store.sync(first.id).nextDueAt,customized.nextDueAt); + context.options.collectionAdmin.attest = () => ({ matched: false }); + await assert.rejects(runtime.startWorkforceSync(), /runtime_health_failed/); + assert.equal(store.sync(first.id).nextDueAt, customized.nextDueAt); + } finally { store.close(); fs.rmSync(manager.root, { recursive: true, force: true }); } +}); diff --git a/core/core/installations/tests/managed-auth-setup.test.js b/core/core/installations/tests/managed-auth-setup.test.js new file mode 100644 index 0000000..6fe7fe7 --- /dev/null +++ b/core/core/installations/tests/managed-auth-setup.test.js @@ -0,0 +1,105 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const path = require('node:path'); +const test = require('node:test'); +const { ManagedRuntimeServicePort, createManagedPaycomAuthSetup } = require('../../../compatibility/provisioner/src/managed-auth-setup.js'); + +function fixture() { + const plan = { units: [{ id: 'auth' }, { id: 'manager' }, { id: 'gateway' }] }; + let active = true; + let guards = 0; + let health = 0; + const authority = { + guard(mutation) { + guards += 1; + return mutation(); + }, + }; + const supervisor = { + snapshot: () => plan.units.map(unit => ({ id: unit.id, active })), + health: () => { health += 1; }, + stop: (_plan, mutate) => mutate(() => { active = false; }), + start: (_plan, mutate) => mutate(() => { active = true; }), + }; + return { plan, authority, supervisor, values: () => ({ active, guards, health }) }; +} + +test('managed credential setup controls the complete server-owned runtime service set through the authority guard', async () => { + const context = fixture(); + const service = new ManagedRuntimeServicePort(context); + assert.deepEqual(await service.status(), { status: 'ready', managed: true }); + assert.deepEqual(await service.stop(), { status: 'stopped', managed: true, stopped: true }); + assert.deepEqual(await service.status(), { status: 'stopped', managed: true }); + assert.deepEqual(await service.start(), { status: 'ready', managed: true, started: true }); + assert.deepEqual(context.values(), { active: true, guards: 2, health: 2 }); +}); + +test('managed credential setup refuses a partially active runtime', async () => { + const context = fixture(); + context.supervisor.snapshot = () => [ + { id: 'auth', active: false }, + { id: 'manager', active: true }, + { id: 'gateway', active: false }, + ]; + const service = new ManagedRuntimeServicePort(context); + await assert.rejects(service.status(), /broker_state_unknown/); +}); + +test('managed activation CLI requires explicit create or replace credential intent', () => { + const executable = path.resolve(__dirname, "../../../compatibility/provisioner/bin/dispatch-managed-activation"); + for (const argv of [ + ['setup-auth', 'org_activation_fixture'], + ['setup-auth', 'org_activation_fixture', 'auto'], + ['setup-auth', 'org_activation_fixture', 'keep'], + ]) { + const result = spawnSync(process.execPath, ['--no-warnings', executable, ...argv], { + encoding: 'utf8', + env: {}, + }); + assert.equal(result.status, 1); + assert.equal(JSON.parse(result.stdout).status, 'invalid_input'); + assert.equal(result.stderr, ''); + } +}); + +test('managed credential setup rejects a crossed Access Control manifest before reading runtime paths', () => { + const manifest = { + manifestVersion: 1, + revision: 1, + organization: { id: 'org_alpha', stationCode: 'TST1', timezone: 'America/Chicago' }, + runtime: { key: 'runtime_alpha', templateId: 'isolated_dsp_v1', releaseId: 'dispatch_current_1' }, + }; + const manifestAuthority = { + revision: 1, + organizationId: 'org_alpha', + stationCode: 'TST1', + timezone: 'America/Chicago', + runtimeKey: 'runtime_alpha', + templateId: 'isolated_dsp_v1', + releaseId: 'dispatch_current_1', + }; + const crossed = structuredClone(manifest); + crossed.organization.id = 'org_bravo'; + crossed.runtime.key = 'runtime_bravo'; + assert.throws(() => createManagedPaycomAuthSetup({ + manifest, + manifestAuthority, + authority: { + peek: () => ({ + manifest: crossed, + manifestAuthority: { + ...manifestAuthority, organizationId: 'org_bravo', runtimeKey: 'runtime_bravo', + }, + installation: { state: 'waiting_for_provider_auth' }, + }), + beginSetup: () => null, + endSetup: () => null, + guard: mutation => mutation(), + }, + installationsRoot: '/unread-installations-root', + unitRoot: '/unread-unit-root', + supervisor: {}, + }), error => error?.code === 'runtime_identity_mismatch'); +}); diff --git a/core/core/installations/tests/managed-paycom.test.js b/core/core/installations/tests/managed-paycom.test.js new file mode 100644 index 0000000..2739ffe --- /dev/null +++ b/core/core/installations/tests/managed-paycom.test.js @@ -0,0 +1,87 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { fixture } = require('dispatch-dsp/runtime/collection-manager/tests/helpers.js'); +const { + managedPaycomDefinition, + managedPaycomFirstPublicationRequest, +} = require('../../../compatibility/provisioner/src/managed-paycom.js'); + +function manifest(timezone = 'America/Chicago') { + return { + manifestVersion: 1, + revision: 1, + organization: { id: 'org_activation_fixture', stationCode: 'TST1', timezone }, + runtime: { key: 'fixture_activation', templateId: 'isolated_dsp_v1', releaseId: 'dispatch_fixture_1' }, + }; +} + +function authority(value) { + return { + revision: value.revision, + organization: { ...value.organization }, + runtime: { ...value.runtime }, + }; +} + +test('managed Paycom configuration is server-owned, tenant-timezone-bound, and manager-valid', () => { + const selected = manifest(); + const definition = managedPaycomDefinition(selected, authority(selected)); + assert.equal(definition.profileId, 'paycom-main'); + assert.equal(definition.sourceId, 'paycom-main'); + assert.equal(definition.syncId, 'paycom-main-workforce'); + assert.match(definition.digest, /^[a-f0-9]{64}$/); + assert.equal(definition.specification.sources[0].config.timezone, 'America/Chicago'); + assert.equal(definition.specification.sources[0].authProfile, 'paycom-main'); + assert.equal(definition.specification.syncs[0].desiredState, 'stopped'); + assert.equal(Object.isFrozen(definition.specification), true); + + const context = fixture(); + const store = new CollectionStore(context.paths); + require('dispatch-runtime-kit/collection-manager/src/plugin-state').applyState(store, { command: 'apply', pluginId: 'paycom', version: '0.18.7', state: 'enabled', revision: 1 }); + try { + assert.deepEqual(store.applySpec(definition.specification), { + collectors: 1, + sources: 1, + plans: 15, + syncs: 1, + }); + assert.equal(store.source('paycom-main').authProfile, 'paycom-main'); + assert.equal(store.sourceRuntime('paycom-main').config.timezone, 'America/Chicago'); + assert.equal(store.sync('paycom-main-workforce').desiredState, 'stopped'); + } finally { + store.close(); + fs.rmSync(context.root, { recursive: true, force: true }); + } +}); + +test('managed Paycom configuration rejects authority drift and exposes one fixed activation request', () => { + const selected = manifest(); + const mismatched = authority(selected); + mismatched.organization.timezone = 'UTC'; + assert.throws(() => managedPaycomDefinition(selected, mismatched), /runtime_boundary_violation/); + assert.throws(() => managedPaycomDefinition(selected, authority(selected), { projectRoot: '' }), /runtime_boundary_violation/); + assert.deepEqual(managedPaycomFirstPublicationRequest(), { + source: 'paycom-main', + scope: 'full', + selector: { kind: 'current' }, + mode: 'refresh', + }); +}); + +test('managed Paycom source validation accepts the reviewed tree and rejects changed collector bytes', t => { + const { verifyManagedPaycomSource } = require('dispatch-dsp/plugins/paycom/backend/runtime/definition.js'); + const root = path.resolve(__dirname, '../../..'); + assert.equal(verifyManagedPaycomSource(root), root); + const read = fs.readFileSync; + const changed = path.join(root, 'plugins/paycom/backend/bin/dispatch-paycom-collector'); + t.mock.method(fs, 'readFileSync', (file, ...args) => { + const value = read(file, ...args); + return file === changed ? Buffer.concat([Buffer.from(value), Buffer.from('\n// changed collector\n')]) : value; + }); + assert.throws(() => verifyManagedPaycomSource(root), error => error.code === 'runtime_boundary_violation'); +}); diff --git a/core/core/installations/tests/native-deployment.test.js b/core/core/installations/tests/native-deployment.test.js new file mode 100644 index 0000000..cbc31ab --- /dev/null +++ b/core/core/installations/tests/native-deployment.test.js @@ -0,0 +1,37 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const { createOciDeploymentPlan, validateOciDeploymentPlan, renderOciSystemUnit, hostAccountName, podmanArguments } = require('../src/oci-deployment'); +const { configuration, assertMountBoundary } = require('dispatch-dsp/runtime/supervisor/src/supervisor.js'); +function plan(key) { + const manifest = { manifestVersion: 1, revision: 1, + organization: { id: `org_${key}`, stationCode: 'DXX1', timezone: 'America/Chicago' }, + runtime: { key, templateId: 'isolated_dsp_v1', releaseId: 'dispatch_1.0.0' } }; + const authority = { revision: 1, organization: manifest.organization, runtime: manifest.runtime }; + const release = { version: 1, backend: 'native_service_v1', releaseId: 'dispatch_1.0.0', channel: 'production', sourceCommit: 'a'.repeat(40), + platform: 'linux/amd64', runtimeAgentProtocol: 1, runtimeGatewayProtocol: 1, artifactSha256: 'b'.repeat(64), + embeddedManifestSha256: 'c'.repeat(64), bridgeManifestSha256: 'd'.repeat(64) }; + const account = { name: hostAccountName(key), uid: key.endsWith('alpha') ? 20001 : 20002, gid: key.endsWith('alpha') ? 20001 : 20002, + subuidStart: 300000, subgidStart: 300000, subidCount: 65536 }; + return createOciDeploymentPlan(manifest, authority, release, account, { version: 1, backend: release.backend, channel: release.channel, + organizationId: manifest.organization.id, runtimeKey: key, manifestRevision: 1, releaseId: release.releaseId }); +} +test('native DSP services share pinned code and keep independent accounts, data and private browser transport', () => { + const a = plan('runtime_alpha'), b = plan('runtime_beta'); + assert.notEqual(a.account.uid, b.account.uid); + assert.notEqual(a.host.installationRoot, b.host.installationRoot); + const unit = renderOciSystemUnit(a); + assert.match(unit, /\/opt\/dispatch-runtime\/releases\/dispatch_1.0.0\/runtime-artifact:/); + assert.match(unit, /DISPATCH_RUNTIME_BACKEND=native_service_v1/); + assert.match(unit, /^ExecStart=\/opt\/dispatch-runtime\/releases\/dispatch_1\.0\.0\/runtime-artifact\/dependencies\/node\/bin\/node /m); + assert.match(unit, /TemporaryFileSystem=\/tmp:rw,noexec,nosuid,nodev/); + assert.doesNotMatch(unit, /Exec\w*=.*(?:podman|docker)|RootImage|remote-debugging-port/); + assert.match(unit, /InaccessiblePaths=-\/run\/docker.sock -\/run\/podman/); + assert.deepEqual(validateOciDeploymentPlan(JSON.parse(JSON.stringify(a))), a); + assert.throws(() => validateOciDeploymentPlan({ ...a, host: b.host }), /runtime_boundary_violation/); + assert.throws(() => podmanArguments(a), /runtime_boundary_violation/); + const config = configuration(a.guest.environment); + const mount = (point, options) => `1 0 0:1 / ${point} ${options} - tmpfs tmpfs rw`; + const mounts = [['/', 'ro'], ['/opt/dispatch', 'ro'], [a.guest.installationRoot, 'rw'], ['/run/dispatch-agent', 'ro'], ['/tmp', 'rw,noexec,nosuid,nodev']]; + assert.equal(assertMountBoundary(config, mounts.map(([p, m]) => mount(p, m)).join('\n'), () => false), true); + assert.throws(() => assertMountBoundary(config, mounts.filter(([p]) => p !== '/opt/dispatch').map(([p, m]) => mount(p, m)).join('\n'), () => false)); +}); diff --git a/core/core/installations/tests/native-lab/README.md b/core/core/installations/tests/native-lab/README.md new file mode 100644 index 0000000..5b85ef2 --- /dev/null +++ b/core/core/installations/tests/native-lab/README.md @@ -0,0 +1,73 @@ +# Historical native DSP VM lab + +This VM harness is no longer run by CI or the current testing workflow. Operational testing uses disposable test DSPs in the existing live setup. The instructions below document the historical isolated recovery drill; do not run it as part of normal verification, and never run its destructive recovery steps on the live host. + +Run from a clean checkout: + +```sh +./core/installations/scripts/verify-native-dsp-lab +``` + +This creates a disposable Ubuntu 24.04 VM, installs the current native Dispatch +code, exercises real dashboard HTTP APIs, the real reconciliation worker, fenced +root helpers, Linux accounts, systemd services and encrypted Restic repositories. +It destroys and restores the application host, reboots it, uses the real browser +UI, provisions another DSP, then permanently deletes the synthetic DSPs. + +The runner needs Linux/KVM, QEMU, passwordless sudo, Node 22, Python 3.11+, patchelf, +Chrome, at least 15 GiB free storage and enough memory for a 4 GiB VM. Install the +dashboard's locked dependencies with `npm ci --prefix dashboard` and +its Playwright Chromium browser before running. The native package builder also +requires a clean Git checkout. `--package /path/to/package` reuses a previously +built native runtime package for debugging; use a fresh build for acceptance. + +Production accounts, configuration, providers, email and backups are never +selected. Test invitations go to a private file inbox. Provider authentication +and initial-publication receipts are synthetic; provider stores, runtime health, +manual collection jobs and the tenant API are real. The offsite adapter runs real +encrypted Restic with separate local repositories. Actual Cloudflare transport, +retention locks, email delivery and third-party provider accounts require separate +integration checks; this lab does not report those as tested. + +A JSON report is written to `/tmp/dispatch-native-dsp-lab-report.json`; override +with `--report`. A failure returns a nonzero exit status and preserves diagnostics. +The VM, temporary SSH key, runtime packages and all synthetic archives are removed +on exit. `--keep-on-failure` retains the stopped VM only when debugging a failure; +its directory contains synthetic secrets and must be deleted after investigation. +No account or Dispatch service is created on the runner host. Downloads use the +official Ubuntu image with its published SHA-256 checksum; SSH tunnels bind only +to loopback. + +## Coverage + +- Invitations, registration, duplicate create requests, native provisioning and owner details. +- Real native health, synthetic workforce publication, tenant API access and manual sync idempotency. +- Separate OS identities, peer-data and Core-secret denial, platform authorization and CSRF denial. +- Verified encrypted DSP backups, remote hydration, restore confirmation and wrong-tenant refusal. +- Restored data and session invalidation; peer data and service continuity. +- Suspension blocks existing sessions and new logins; resumption requires a new session. +- Full host recovery recreates code, accounts, secrets, data and services; metadata tampering and occupied destinations fail closed. +- Archive rediscovery after rollback, real reboot, then new provisioning using restored host permissions. +- Browser creation, fleet and backup pages, custom DSP roles, member invitation and revocation. +- Permanent deletion including individual and containing Core archives, exclusive users and pending invitations; deleted credentials and archive IDs cannot restore access. + +Run `./tooling/verify` as well for the larger contract, failure, authorization, +provider-fixture, backup scheduling, rollout, compensation and retention suites. +`./runtime/tooling/verify` checks the native Chrome pipe and service sandbox. +The lab complements these tests; a passing report is not a claim of exhaustive +coverage of every possible state or live provider behavior. + +## Live Cloudflare transport canary + +On the configured VPS: + +```sh +sudo node --no-warnings core/installations/tests/native-lab/live-r2.js --empty-bucket +``` + +This exercises the actual R2 credentials, encrypted upload, independent restore, recovery capsule +verification, archive deletion and restoration of retention locks. It refuses a +bucket containing existing managed archives, writes only synthetic data under a +new random archive ID, and deletes that archive and its local scratch directory. +It does not provision DSPs or send email. Run this separately from the VM suite; +CI uses the local encrypted storage adapter and has no production credentials. diff --git a/core/core/installations/tests/native-lab/activate.js b/core/core/installations/tests/native-lab/activate.js new file mode 100644 index 0000000..19f18ff --- /dev/null +++ b/core/core/installations/tests/native-lab/activate.js @@ -0,0 +1,135 @@ +"use strict"; +// Provider authentication/publication receipts are synthetic in this mode. +// The activation authority, transitions, evidence validation and health calls are real. +const fs = require("node:fs"); +const assert = require("node:assert/strict"); +const { AccessStore } = require("../../../accounts/src"); +const { + runManagedPaycomActivation, + ACTIVATION_INFRASTRUCTURE_GATES, +} = require("../../src/activation"); +const { + runtimeAgentControlInvoke, +} = require("../../../agents/src/control"); +const { + INSTALLATION_ACTIVATION_RUNS, +} = require("../../../../shared/contracts/src"); +async function main() { + const organizationId = process.argv[2], + root = process.env.DISPATCH_LOCAL_ROOT; + const store = new AccessStore({ + databaseRoot: root + "/data/access-control", + database: root + "/data/access-control/access-control.sqlite3", + }); + try { + // DSP onboarding is complete before optional provider fixture activation. + assert.equal(store.installationControl(organizationId).status, "ready"); + const requests = require("../../../accounts/src/onboarding-store").createOnboardingStore(store); + const actor = store.db.prepare("SELECT m.user_id FROM memberships m JOIN roles r ON r.id=m.role_id WHERE m.organization_id=? AND m.status='active' AND r.key='owner'").get(organizationId).user_id; + const row = store.transaction(() => { + const request = requests.begin(organizationId, actor, "lab:optional:" + require("node:crypto").randomUUID(), "create", store.installationControl(organizationId).manifestRevision); + requests.enrolled(request.id); + return requests.claim(request.id, "worker_lab_activation"); + }); + const authority = require("../../../accounts/src/optional-paycom-activation").createOptionalPaycomActivation({ store, row, requests }); + const context = authority.inspect(), + key = context.manifest.runtime.key; + const seed = JSON.parse( + fs.readFileSync(root + "/seed-" + organizationId + ".json"), + ); + const runs = INSTALLATION_ACTIVATION_RUNS.map((run, i) => ({ + id: `run_fixture_${i}`, + taskId: run.taskId, + plan: run.plan, + method: run.method, + })); + const pub = (value, runId, originRunId) => ({ + id: value.publicationId, + runId, + originRunId, + contentSha256: value.contentSha256, + batchBound: true, + }); + const audited = { + definitionDigest: "a".repeat(64), + requestDigest: "b".repeat(64), + previewDigest: "c".repeat(64), + batchId: "batch_lab_" + organizationId, + preparationRunId: "run_periods_fixture", + target: "2026-09-05", + runs, + publications: { + payPeriods: { + id: seed.payPeriods.publicationId, + runId: "run_periods_fixture", + originRunId: "run_periods_fixture", + contentSha256: seed.payPeriods.contentSha256, + batchBound: false, + }, + roster: pub(seed.roster, runs[0].id, "run_fixture_roster"), + timecards: pub(seed.timecards, runs[1].id, "run_fixture_timecards"), + resourceLinks: pub(seed.links, runs[3].id, "run_fixture_links"), + }, + capturedAt: new Date().toISOString(), + }; + const result = await runManagedPaycomActivation({ + authority, + runtime: { + verifyInfrastructure: async () => { + const health = await runtimeAgentControlInvoke( + root + "/run/runtime-agent-control.sock", + key, + "health", + {}, + ); + assert.equal(health.ok, true); + return { + runtimeKey: key, + ...Object.fromEntries( + ACTIVATION_INFRASTRUCTURE_GATES.map((k) => [k, true]), + ), + }; + }, + testProvider: async () => ({ + profileId: "paycom-main", + provider: "paycom", + status: "authenticated", + testedAt: new Date().toISOString(), + }), + configure: async () => ({ + digest: audited.definitionDigest, + collectors: 1, + sources: 1, + plans: 15, + syncs: 1, + }), + publishFirst: async () => ({ + batchId: audited.batchId, + preparationRunId: audited.preparationRunId, + status: "succeeded", + runCount: runs.length, + succeededRuns: runs.length, + failedRuns: 0, + cancelledRuns: 0, + }), + verifyPublication: async () => audited, + }, + }); + assert.equal(result.ok, true, JSON.stringify(result)); + const started = await runtimeAgentControlInvoke( + root + "/run/runtime-agent-control.sock", + key, + "sync.start", + { id: "paycom-main-workforce" }, + ); + assert.equal(started.ok, true); + requests.finish(row); + console.log(JSON.stringify(result)); + } finally { + store.close(); + } +} +main().catch((e) => { + console.error(e.stack); + process.exitCode = 1; +}); diff --git a/core/core/installations/tests/native-lab/after-reboot.js b/core/core/installations/tests/native-lab/after-reboot.js new file mode 100644 index 0000000..4e680f9 --- /dev/null +++ b/core/core/installations/tests/native-lab/after-reboot.js @@ -0,0 +1,288 @@ +"use strict"; +const fs = require("node:fs"), + os = require("node:os"), + assert = require("node:assert/strict"); +const { execFileSync } = require("node:child_process"); +const { api, login, tick, read, check, until, report } = require("./scenarios"); +const { controls } = require("./exercise"); +const call = (cmd, args) => + execFileSync(cmd, args, { encoding: "utf8", timeout: 120000 }).trim(); +async function main() { + assert.equal(os.hostname(), "dispatch-dsp-lab"); + assert.equal(process.geteuid(), 0); + Object.assign(report, JSON.parse(fs.readFileSync("/root/lab-report.json"))); + const { rows, plans, records } = JSON.parse( + fs.readFileSync("/root/lab-state.json"), + ); + await check( + "restored Core and DSPs restart automatically after a real reboot", + async () => { + assert.notEqual( + fs.readFileSync("/proc/sys/kernel/random/boot_id", "utf8"), + fs.readFileSync("/root/lab-boot-id", "utf8"), + ); + for (let i = 0; i < 2; i++) { + await until(async () => { + try { + const owner = await login(`owner${i}@example.test`); + return (await api(owner, "/api/paycom/daily?date=2026-09-05")).value + .ok; + } catch { + return false; + } + }, 120000); + assert.equal( + call("/usr/bin/systemctl", ["is-active", plans[i].identity.unitName]), + "active", + ); + } + }, + ); + await check("reboot removes interrupted backup transfer history", async () => { + const interrupted = fs.readFileSync("/root/lab-interrupted-transfer", "utf8"); + await until(async () => !fs.existsSync(interrupted)); + }); + await check( + "restored DSP Chrome uses a private pipe and cannot read peer data", + async () => { + const first = plans[0], + second = plans[1]; + const pid = call("/usr/bin/systemctl", [ + "show", + first.identity.unitName, + "--property=MainPID", + "--value", + ]); + assert.match(pid, /^[1-9][0-9]*$/); + const probe = `const fs=require('node:fs');const {ChromeBrowserRuntime}=require('/opt/dispatch/runtime/auth-broker/src/browser-runtime');const {createTarget,CdpConnection}=require('/opt/dispatch/runtime/auth-broker/src/cdp');(async()=>{const browser=await new ChromeBrowserRuntime({stateRoot:process.env.DISPATCH_AUTH_STATE_ROOT,socketRoot:process.env.DISPATCH_RUNTIME_ROOT,executable:'/opt/dispatch/dependencies/browser/chrome',transport:'pipe'}).launch();try{const target=await createTarget(browser.endpoint,'about:blank');const c=await CdpConnection.connect(target.webSocketDebuggerUrl);try{if(await c.evaluate('6*7')!==42)throw Error('browser_evaluation_failed');}finally{c.close();}if(fs.existsSync(${JSON.stringify(second.host.installationRoot)}))throw Error('peer_data_visible');console.log('private_chrome_passed');}finally{await browser.close();}})().catch(e=>{console.error(e.message);process.exitCode=1});`; + assert.equal( + call("/usr/bin/nsenter", [ + "--target", + pid, + "--mount", + `--setuid=${first.account.uid}`, + `--setgid=${first.account.gid}`, + "--", + "/usr/bin/env", + "-i", + "PATH=/opt/dispatch/dependencies/node/bin:/usr/bin:/bin", + "HOME=/tmp", + ...Object.entries(first.guest.environment).map( + ([k, v]) => `${k}=${v}`, + ), + "/opt/dispatch/dependencies/node/bin/node", + "--no-warnings", + "-e", + probe, + ]), + "private_chrome_passed", + ); + }, + ); + const platform = await login("platform@example.test"); + async function removeForDeletion(row) { + if (row.installation.state !== 'decommissioned') { + await api(platform, '/api/platform/installation/remove', { + controlRef: row.controlRef, expectedRevision: row.installation.revision, + idempotencyKey: 'lab:acceptance:remove:' + row.continuityRef, + }, 202); + await until(async () => { + await tick(); + return (await controls(platform)).find(r => r.continuityRef === row.continuityRef)?.installation.state === 'decommissioned'; + }, 600000); + } + return (await controls(platform)).find(r => r.continuityRef === row.continuityRef); + } + + await check( + "a new invitation provisions a third DSP using restored host permissions", + async () => { + await tick(); + const row = read((db) => + db + .prepare( + "SELECT * FROM installations WHERE organization_id NOT IN (?,?)", + ) + .get(...rows.map((r) => r.organization_id)), + ); + assert.ok(row); + assert.equal(row.status, "waiting_for_owner"); + }, + ); + await check( + "permanent deletion requires removal, the administrator password and current revision", + async () => { + const row = (await controls(platform)).find( + (r) => r.name === rows[0].name, + ); + await api( + platform, + "/api/platform/installation/delete", + { + controlRef: row.controlRef, + expectedRevision: row.installation.revision, + idempotencyKey: "lab:acceptance:delete:bad-password", + password: "wrong", + }, + 403, + ); + await api( + platform, + "/api/platform/installation/delete", + { + controlRef: row.controlRef, + expectedRevision: row.installation.revision - 1, + idempotencyKey: "lab:acceptance:delete:stale", + password: "disposable lab password 123", + }, + 409, + ); + }, + ); + await check( + "deleting a DSP erases its data, account, secrets, metadata and every containing archive", + async () => { + const row = await removeForDeletion((await controls(platform)).find( + (r) => r.name === rows[0].name, + )); + const body = { + controlRef: row.controlRef, + expectedRevision: row.installation.revision, + idempotencyKey: "lab:acceptance:delete:one", + password: "disposable lab password 123", + }; + await api(platform, "/api/platform/installation/delete", body, 202); + await until(async () => { + await tick(); + return !read((db) => + db + .prepare("SELECT 1 FROM organizations WHERE id=?") + .get(rows[0].organization_id), + ); + }, 600000); + assert.equal(fs.existsSync(plans[0].host.tenantRoot), false); + assert.equal(fs.existsSync(plans[0].host.bridgeRoot), false); + assert.equal(fs.existsSync(plans[0].host.unitPath), false); + assert.throws(() => + call("/usr/bin/getent", ["passwd", plans[0].account.name]), + ); + for (const email of ["owner0@example.test", "member0@example.test"]) + assert.equal( + read((db) => + db.prepare("SELECT 1 FROM users WHERE email=?").get(email), + ), + undefined, + ); + for (const r of records.filter( + (r) => r.organization_id === rows[0].organization_id, + )) + for (const tier of ["all", "7", "30", "90", "365"]) + assert.equal( + fs.existsSync(`/srv/dispatch-lab-remote/archives/${tier}/${r.id}`), + false, + ); + const state = JSON.parse( + fs.readFileSync("/root/lab-recovery-state.json"), + ); + for (const tier of ["all", "7", "30", "90", "365"]) + assert.equal( + fs.existsSync( + `/srv/dispatch-lab-remote/archives/${tier}/${state.row.id}`, + ), + false, + ); + const peer = await login("owner1@example.test"); + await api(peer, "/api/paycom/daily?date=2026-09-05"); + assert.equal( + fs.readFileSync( + plans[1].host.installationRoot + "/data/lab-marker", + "utf8", + ), + "tenant-1-before", + ); + }, + ); + await check( + "deleted DSP credentials and archive IDs cannot recreate or restore it", + async () => { + await api( + null, + "/api/auth/login", + { + email: "owner0@example.test", + password: "disposable lab password 123", + }, + 401, + ); + const record = records.find( + (r) => r.organization_id === rows[0].organization_id, + ); + await api( + platform, + "/api/platform/backups", + { + action: "restore", + organizationId: rows[0].organization_id, + backupId: record.id, + confirmation: rows[0].name, + idempotencyKey: "lab:acceptance:deleted:restore", + }, + 409, + ); + }, + ); + await check( + "all remaining test DSPs can be deleted, including an unaccepted invitation", + async () => { + for (const original of await controls(platform)) { + const row = await removeForDeletion(original); + await api( + platform, + "/api/platform/installation/delete", + { + controlRef: row.controlRef, + expectedRevision: row.installation.revision, + idempotencyKey: "lab:acceptance:cleanup:" + row.continuityRef, + password: "disposable lab password 123", + }, + 202, + ); + await until(async () => { + await tick(); + return !(await controls(platform)).some( + (r) => r.controlRef === row.controlRef, + ); + }, 600000); + } + assert.equal( + read( + (db) => db.prepare("SELECT count(*) AS n FROM organizations").get().n, + ), + 0, + ); + }, + ); + await check("cleanup leaves no tenant identities, remote archives or host allocations", async () => { + await until(async () => { + if ((await require("./offsite").storage.listArchives()).length) return false; + const { DatabaseSync } = require("node:sqlite"); + const host = JSON.parse(fs.readFileSync("/etc/dispatch/oci-host.json")); + const db = new DatabaseSync(host.stateRoot + "/oci-host.sqlite3", { readOnly: true }); + try { return db.prepare("SELECT count(*) AS n FROM allocations").get().n === 0; } + finally { db.close(); } + }, 120000); + for (const table of ["installations", "roles", "memberships"]) + assert.equal(read(db => db.prepare(`SELECT count(*) AS n FROM ${table}`).get().n), 0); + assert.equal(read(db => db.prepare("SELECT count(*) AS n FROM invitations WHERE organization_id IS NOT NULL").get().n), 0); + assert.equal(read(db => db.prepare("SELECT count(*) AS n FROM users").get().n), 1); + }); + report.status = report.cases.every(c => c.status === "passed") ? "passed" : "failed"; + if (report.status !== "passed") process.exitCode = 1; + fs.writeFileSync("/root/lab-report.json", JSON.stringify(report, null, 2)); +} +main().catch((e) => { + report.status = "failed"; + fs.writeFileSync("/root/lab-report.json", JSON.stringify(report, null, 2)); + console.error(e.stack); + process.exitCode = 1; +}); diff --git a/core/core/installations/tests/native-lab/bootstrap.js b/core/core/installations/tests/native-lab/bootstrap.js new file mode 100644 index 0000000..d588ccb --- /dev/null +++ b/core/core/installations/tests/native-lab/bootstrap.js @@ -0,0 +1,31 @@ +"use strict"; +const fs = require("node:fs"); +const { + AccessStore, + AccessControlService, +} = require("../../../accounts/src"); +const root = process.env.DISPATCH_LOCAL_ROOT; +(async () => { + const store = new AccessStore({ + databaseRoot: root + "/data/access-control", + database: root + "/data/access-control/access-control.sqlite3", + }); + const access = new AccessControlService(store, { + installationOperatorEnabled: true, + installationBackend: "native_service_v1", + }); + const invite = access.createPlatformBootstrap({ + email: "platform@example.test", + }); + await access.acceptNewUser({ + token: invite.token, + firstName: "Lab", + lastName: "Platform", + password: "disposable lab password 123", + confirmPassword: "disposable lab password 123", + }); + store.close(); +})().catch((e) => { + console.error(e.stack); + process.exitCode = 1; +}); diff --git a/core/core/installations/tests/native-lab/browser.js b/core/core/installations/tests/native-lab/browser.js new file mode 100644 index 0000000..e811698 --- /dev/null +++ b/core/core/installations/tests/native-lab/browser.js @@ -0,0 +1,132 @@ +"use strict"; +// Real UI and real HTTP server in the VM. No route interception or fake API. +const fs = require("node:fs"), + assert = require("node:assert/strict"); +const { + chromium, + expect, +} = require("../../../../dashboard/node_modules/@playwright/test"); +async function main() { + const baseURL = process.argv[2], + output = process.argv[3]; + assert.match(baseURL, /^http:\/\/127\.0\.0\.1:\d+$/); + const browser = await chromium.launch({ headless: true }); + const cases = []; + try { + const context = await browser.newContext({ baseURL }); + const page = await context.newPage(); + const errors = []; + page.on("pageerror", (e) => errors.push(e.message)); + const login = async (email) => { + await page.goto("/"); + await page.getByLabel("Email address").fill(email); + await page + .getByLabel("Password", { exact: true }) + .fill("disposable lab password 123"); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect( + page.getByRole("navigation", { name: "Primary navigation" }), + ).toBeVisible(); + }; + await login("platform@example.test"); + await expect( + page.getByRole("button", { name: "Lab DSP 0 L0", exact: true }), + ).toBeVisible(); + await expect( + page.getByRole("button", { name: "Lab DSP 1 L1", exact: true }), + ).toBeVisible(); + await page + .getByRole("button", { name: "Create new DSP", exact: true }) + .click(); + await page + .getByLabel("Owner email", { exact: true }) + .fill("after-restore@example.test"); + await page + .getByRole("button", { name: "Create DSP & send invite", exact: true }) + .click(); + await expect(page.getByRole("dialog")).toHaveCount(0); + await page.getByRole("tab", { name: "Onboarding", exact: true }).click(); + await expect( + page.getByRole("button", { + name: "after-restore@example.test Awaiting DSP details", + exact: true, + }), + ).toBeVisible(); + cases.push({ + name: "browser creates a DSP invitation against restored dashboard", + status: "passed", + }); + await page + .locator(".desktop-sidebar") + .getByRole("link", { name: "Backups", exact: true }) + .click(); + await expect( + page.getByRole("heading", { level: 1, name: "Backups", exact: true }), + ).toBeVisible(); + cases.push({ + name: "browser opens restored backup catalog", + status: "passed", + }); + await context.clearCookies(); + await login("owner1@example.test"); + await page + .locator(".desktop-sidebar") + .getByRole("link", { name: "Team & Roles", exact: true }) + .click(); + await page.getByRole("tab", { name: "Roles", exact: true }).click(); + await expect(page.locator('.role-row h2')).toHaveText(['Owner', 'Manager', 'Dispatcher', 'Driver']); + await expect(page.getByRole('button', { name: 'Create role', exact: true })).toHaveCount(0); + await page + .getByRole("button", { name: "Invite member", exact: true }) + .click(); + await page.getByLabel("Email address").fill("reviewer@example.test"); + await page + .getByLabel("Role", { exact: true }) + .selectOption({ label: "Dispatcher" }); + await page + .getByRole("button", { name: "Send invitation", exact: true }) + .click(); + await expect(page.getByRole("dialog")).toHaveCount(0); + await page.getByRole("tab", { name: /^Invitations/ }).click(); + await expect( + page.getByRole("cell", { name: "reviewer@example.test", exact: true }), + ).toBeVisible(); + await page + .getByRole("button", { + name: "Revoke invitation for reviewer@example.test", + }) + .click(); + await page + .getByRole("dialog") + .getByRole("button", { name: "Revoke invitation", exact: true }) + .click(); + await expect( + page.getByRole("cell", { name: "reviewer@example.test", exact: true }), + ).toHaveCount(0); + cases.push({ + name: "browser creates a DSP role and invites and revokes a member", + status: "passed", + }); + assert.deepEqual(errors, []); + cases.push({ + name: "real dashboard browser session has no uncaught JavaScript errors", + status: "passed", + }); + fs.writeFileSync( + output, + JSON.stringify({ status: "passed", cases }, null, 2), + ); + } catch (e) { + fs.writeFileSync( + output, + JSON.stringify({ status: "failed", cases, error: e.message }, null, 2), + ); + throw e; + } finally { + await browser.close(); + } +} +main().catch((e) => { + console.error(e.stack); + process.exitCode = 1; +}); diff --git a/core/core/installations/tests/native-lab/dashboard.js b/core/core/installations/tests/native-lab/dashboard.js new file mode 100644 index 0000000..ac8e444 --- /dev/null +++ b/core/core/installations/tests/native-lab/dashboard.js @@ -0,0 +1,20 @@ +"use strict"; +// Test-only email boundary: no message can leave this VM. +const fs = require("node:fs"); +require("../../../../dashboard/server/main") + .main(["--installation-operator", "--operator"], { + invitationDelivery: { + send: async (invitation) => { + fs.appendFileSync( + process.env.DISPATCH_LOCAL_ROOT + "/inbox.jsonl", + JSON.stringify(invitation) + "\n", + { mode: 0o600 }, + ); + return { status: "accepted" }; + }, + }, + }) + .catch((e) => { + console.error(e.stack); + process.exitCode = 1; + }); diff --git a/core/core/installations/tests/native-lab/exercise.js b/core/core/installations/tests/native-lab/exercise.js new file mode 100644 index 0000000..4755a76 --- /dev/null +++ b/core/core/installations/tests/native-lab/exercise.js @@ -0,0 +1,403 @@ +"use strict"; +const fs = require("node:fs"), + assert = require("node:assert/strict"), + crypto = require("node:crypto"); +const { execFile } = require("node:child_process"); +const execute = require("node:util").promisify(execFile); +const { api, login, tick, read, until, check } = require("./scenarios"); +const { seed } = require("./seed"); +const LOCAL = "/home/dispatchlab/local"; +const user = (script, args = []) => + execute( + "/usr/sbin/runuser", + [ + "--user", + "dispatchlab", + "--", + "/usr/bin/env", + "DISPATCH_LOCAL_ROOT=" + LOCAL, + "/usr/bin/node", + "--no-warnings", + script, + ...args, + ], + { timeout: 120000 }, + ); +const controls = async (platform) => + (await api(platform, "/api/platform/organizations")).value.data; +async function drained() { + await until(async () => { + await tick(); + const rows = read((db) => + db + .prepare( + "SELECT status,failure_code FROM platform_backup_requests WHERE status IN ('queued','running','failed')", + ) + .all(), + ); + assert.ok(!rows.some((r) => r.status === "failed"), JSON.stringify(rows)); + return !rows.length; + }, 600000); +} +async function exercise({ platform, owners }) { + const rows = read((db) => + db + .prepare( + "SELECT i.*,o.name FROM installations i JOIN organizations o ON o.id=i.organization_id ORDER BY o.name", + ) + .all(), + ); + const plans = []; + await check( + "native health and synthetic provider publication activation", + async () => { + for (const row of rows) { + const { plan } = seed(row.organization_id); + plans.push(plan); + await user( + "/work/core/installations/tests/native-lab/activate.js", + [row.organization_id], + ); + } + for (const owner of owners) { + const daily = await api(owner, "/api/paycom/daily?date=2026-09-05"); + assert.equal(daily.value.status, "found"); + assert.match( + JSON.stringify(daily.value.data), + /Synthetic Fixture Employee/, + ); + await api(owner, "/api/integrations"); + } + }, + ); + await check( + "tenant OS identities cannot read peer data or Core secrets", + async () => { + for (let i = 0; i < 2; i++) { + const file = plans[i].host.installationRoot + "/data/lab-marker"; + fs.writeFileSync(file, "tenant-" + i + "-before", { mode: 0o600 }); + fs.chownSync(file, plans[i].account.uid, plans[i].account.gid); + await assert.rejects( + execute("/usr/sbin/runuser", [ + "--user", + plans[i].account.name, + "--", + "/usr/bin/cat", + plans[1 - i].host.installationRoot + + "/secrets/runtime-agent/registration-token", + ]), + ); + await assert.rejects( + execute("/usr/sbin/runuser", [ + "--user", + plans[i].account.name, + "--", + "/usr/bin/cat", + LOCAL + "/config/provisioning.env", + ]), + ); + } + }, + ); + await check( + "anonymous and DSP owner cannot administer platform or peer DSP", + async () => { + await api(null, "/api/platform/organizations", undefined, 401); + await api(owners[0], "/api/platform/organizations", undefined, 403); + const target = (await controls(platform))[1]; + await api( + owners[0], + "/api/platform/organization/status", + { + controlRef: target.controlRef, + suspended: true, + idempotencyKey: "lab:acceptance:forbidden:suspend", + }, + 403, + ); + await api( + { ...platform, csrf: "wrong" }, + "/api/platform/organization/status", + { + controlRef: target.controlRef, + suspended: true, + idempotencyKey: "lab:acceptance:csrf:suspend", + }, + 403, + ); + }, + ); + await check( + "manual sync executes a real collection job and preserves peer service", + async () => { + const before = await execute("/usr/bin/systemctl", [ + "show", + plans[1].identity.unitName, + "--property=MainPID", + "--value", + ]); + const queued = await api( + owners[0], + "/api/paycom/sync", + { idempotencyKey: "lab_sync_manual_001" }, + 202, + ); + await until(async () => { + const again = await api( + owners[0], + "/api/paycom/sync", + { idempotencyKey: "lab_sync_manual_001" }, + 202, + ); + assert.equal(again.value.data.run.id, queued.value.data.run.id); + assert.notEqual(again.value.data.run.status, "failed"); + return again.value.data.run.status === "succeeded"; + }, 30000); + assert.equal( + ( + await execute("/usr/bin/systemctl", [ + "show", + plans[1].identity.unitName, + "--property=MainPID", + "--value", + ]) + ).stdout, + before.stdout, + ); + }, + ); + let member; + await check( + "custom-role member registers, reads permitted data and cannot administer the DSP", + async () => { + const role = ( + await api( + owners[0], + "/api/organization/roles", + { + name: "Read-only lab member", + description: "Synthetic role", + permissions: ["dashboard.view", "workforce.read"], + }, + 201, + ) + ).value.data; + await api( + owners[0], + "/api/organization/invitations", + { email: "member0@example.test", roleId: role.id }, + 201, + ); + const message = fs + .readFileSync(LOCAL + "/inbox.jsonl", "utf8") + .trim() + .split("\n") + .map(JSON.parse) + .find((m) => m.email === "member0@example.test"); + assert.ok(message); + const registered = await api( + null, + "/api/auth/register", + { + token: message.token, + firstName: "Lab", + lastName: "Member", + password: "disposable lab password 123", + confirmPassword: "disposable lab password 123", + }, + 201, + ); + member = { + cookie: registered.cookie, + csrf: registered.value.data.csrfToken, + }; + await api(member, "/api/paycom/daily?date=2026-09-05"); + await api( + member, + "/api/organization/roles", + { name: "Forbidden", description: "", permissions: ["members.manage"] }, + 403, + ); + await api( + member, + "/api/paycom/sync", + { idempotencyKey: "lab_member_forbidden_sync" }, + 403, + ); + }, + ); + await check( + "DSP backup requests are idempotent and produce verified encrypted archives", + async () => { + const request = { + action: "backup", + scope: "dsps", + organizationIds: rows.map((r) => r.organization_id), + idempotencyKey: "lab:acceptance:backup:two", + }; + await api(platform, "/api/platform/backups", request); + await api(platform, "/api/platform/backups", request); + await drained(); + assert.equal( + read( + (db) => + db + .prepare( + "SELECT count(*) AS n FROM platform_backup_requests WHERE kind='backup'", + ) + .get().n, + ), + 2, + ); + const catalog = JSON.parse( + fs.readFileSync("/var/lib/dispatch-backup-receipts/catalog.json"), + ); + assert.equal( + Object.values(catalog.backups).filter((r) => r.status === "verified") + .length, + 2, + ); + }, + ); + const records = read((db) => + db + .prepare( + "SELECT * FROM platform_backup_records WHERE kind='dsp' ORDER BY created_at", + ) + .all(), + ); + const saved = records.find( + (r) => r.organization_id === rows[0].organization_id, + ); + await check( + "restore rejects the wrong DSP and requires named confirmation", + async () => { + await api( + platform, + "/api/platform/backups", + { + action: "restore", + organizationId: rows[1].organization_id, + backupId: saved.id, + confirmation: rows[1].name, + idempotencyKey: "lab:acceptance:restore:wrong-tenant", + }, + 409, + ); + await api( + platform, + "/api/platform/backups", + { + action: "restore", + organizationId: rows[0].organization_id, + backupId: saved.id, + confirmation: "wrong", + idempotencyKey: "lab:acceptance:restore:wrong-name", + }, + 409, + ); + }, + ); + await check( + "DSP restore hydrates remote archive and restores data without altering peer", + async () => { + fs.writeFileSync( + plans[0].host.installationRoot + "/data/lab-marker", + "changed after backup", + ); + await api(platform, "/api/platform/backups", { + action: "restore", + organizationId: rows[0].organization_id, + backupId: saved.id, + confirmation: rows[0].name, + idempotencyKey: "lab:acceptance:restore:valid", + }); + await drained(); + assert.equal( + fs.readFileSync( + plans[0].host.installationRoot + "/data/lab-marker", + "utf8", + ), + "tenant-0-before", + ); + member = await login("member0@example.test"); + await api(member, "/api/paycom/daily?date=2026-09-05"); + assert.equal( + fs.readFileSync( + plans[1].host.installationRoot + "/data/lab-marker", + "utf8", + ), + "tenant-1-before", + ); + await api(owners[0], "/api/paycom/daily?date=2026-09-05", undefined, 401); + owners[0] = await login("owner0@example.test"); + owners[0].email = "owner0@example.test"; + await api(owners[0], "/api/paycom/daily?date=2026-09-05"); + await api(owners[1], "/api/paycom/daily?date=2026-09-05"); + }, + ); + await check( + "suspension denies old sessions and new logins and stops only its DSP", + async () => { + const target = (await controls(platform)).find( + (r) => r.name === rows[0].name, + ); + await api(platform, "/api/platform/organization/status", { + controlRef: target.controlRef, + suspended: true, + idempotencyKey: "lab:acceptance:suspend:one", + }); + await api(owners[0], "/api/paycom/daily?date=2026-09-05", undefined, 401); + await api( + null, + "/api/auth/login", + { email: owners[0].email, password: "disposable lab password 123" }, + 403, + ); + await api(member, "/api/paycom/daily?date=2026-09-05", undefined, 401); + await api( + null, + "/api/auth/login", + { + email: "member0@example.test", + password: "disposable lab password 123", + }, + 403, + ); + await tick(); + assert.equal( + ( + await execute("/usr/bin/systemctl", [ + "show", + plans[0].identity.unitName, + "--property=UnitFileState", + "--value", + ]) + ).stdout.trim(), + "disabled", + ); + await api(owners[1], "/api/paycom/daily?date=2026-09-05"); + }, + ); + await check("resume restores runtime access with a new session", async () => { + const target = (await controls(platform)).find( + (r) => r.name === rows[0].name, + ); + await api(platform, "/api/platform/organization/status", { + controlRef: target.controlRef, + suspended: false, + idempotencyKey: "lab:acceptance:resume:one", + }); + await tick(); + await api(owners[0], "/api/paycom/daily?date=2026-09-05", undefined, 401); + owners[0] = await login("owner0@example.test"); + await api(owners[0], "/api/paycom/daily?date=2026-09-05"); + }); + // Further destructive checks use these verified identities and archive IDs. + fs.writeFileSync( + "/root/lab-state.json", + JSON.stringify({ rows, plans, records }), + { mode: 0o600 }, + ); +} +module.exports = { exercise, drained, controls }; diff --git a/core/core/installations/tests/native-lab/live-r2.js b/core/core/installations/tests/native-lab/live-r2.js new file mode 100644 index 0000000..4596699 --- /dev/null +++ b/core/core/installations/tests/native-lab/live-r2.js @@ -0,0 +1,118 @@ +"use strict"; +// Explicit live transport check. Uses synthetic data and refuses a bucket with +// existing managed archives. Never reads a DSP or Core application database. +const fs = require("node:fs"), + path = require("node:path"), + crypto = require("node:crypto"); +const assert = require("node:assert/strict"); +const { DatabaseSync } = require("node:sqlite"); +const { loadConfig } = require("../../src/offsite-backup"); +const { createR2BackupStorage } = require("../../src/r2-backup-storage"); +const { createBackupArchives } = require("../../src/backup-archives"); +const { atomic, hashFileSync } = require("../../src/release-delivery-files"); +async function main() { + assert.equal(process.geteuid(), 0); + assert.deepEqual(process.argv.slice(2), ["--empty-bucket"]); + process.umask(0o077); + const config = loadConfig(), + root = fs.mkdtempSync("/var/tmp/dispatch-r2-lab-"); + const storage = createR2BackupStorage(config, { + journalFile: path.join(root, "deletion-locks.json"), + }); + const id = "breq_" + crypto.randomBytes(16).toString("hex"); + let touched = false, + cleaned = false; + try { + assert.equal( + (await storage.listArchives()).length, + 0, + "Live canary requires no existing managed archives", + ); + const source = path.join(root, "source"), + workRoot = path.join(root, "work"), + receiptRoot = path.join(root, "receipts"); + for (const p of [source, workRoot, receiptRoot]) + fs.mkdirSync(p, { mode: 0o700 }); + const file = path.join(source, "access-control-before.sqlite3"), + db = new DatabaseSync(file); + db.exec( + "CREATE TABLE synthetic(value TEXT); INSERT INTO synthetic VALUES('Dispatch disposable R2 acceptance data')", + ); + db.close(); + fs.chmodSync(file, 0o600); + atomic(path.join(source, "manifest.json"), { + version: 1, + kind: "core", + sha256: hashFileSync(file), + size: fs.statSync(file).size, + }); + const archives = createBackupArchives(config, { + storage, + workRoot, + receiptRoot, + pathResolver: () => ({ source, uid: 0 }), + recoveryCapture: ({ destination }) => ({ + organizationIds: [], + ...require("../../src/recovery-capsule").capture( + destination, + [{ source, target: "/home/fixture/local/data" }], + { + kind: "core", + platform: "ubuntu-24.04-amd64", + localRoot: "/home/fixture/local", + accounts: [ + { name: "fixture", uid: 1001, gid: 1001, home: "/home/fixture" }, + ], + services: [], + installations: [], + }, + new Set([0]), + ), + }), + }); + await storage.ensureLocks(); + touched = true; + const receipt = await archives.exportRecord({ + id, + kind: "core", + organization_id: null, + metadata_json: JSON.stringify({ name: "Disposable R2 acceptance" }), + retention_days: null, + created_at: Date.now(), + }); + assert.equal(receipt.status, "verified"); + assert.match(receipt.recoveryDigest, /^[a-f0-9]{64}$/); + assert.ok((await storage.listArchives()).some((row) => row.id === id)); + await storage.withDeletionAccess(["archives/all/"], () => + storage.removePermanent({ id, retentionDays: null }), + ); + await storage.ensureLocks(); + assert.equal((await storage.listArchives()).length, 0); + cleaned = true; + console.log( + JSON.stringify({ + status: "passed", + encryptedUpload: true, + independentRestoreVerified: true, + recoveryCapsuleVerified: true, + archiveDeleted: true, + retentionLocksRestored: true, + }), + ); + } finally { + if (touched && !cleaned) { + await storage.withDeletionAccess(["archives/all/"], () => + storage.removePermanent({ id, retentionDays: null }), + ); + await storage.ensureLocks(); + assert.ok(!(await storage.listArchives()).some((row) => row.id === id)); + } + fs.rmSync(root, { recursive: true, force: true }); + } +} +main().catch((error) => { + console.error( + JSON.stringify({ status: "failed", code: error.code || error.message }), + ); + process.exitCode = 1; +}); diff --git a/core/core/installations/tests/native-lab/offsite.js b/core/core/installations/tests/native-lab/offsite.js new file mode 100644 index 0000000..b0fa755 --- /dev/null +++ b/core/core/installations/tests/native-lab/offsite.js @@ -0,0 +1,104 @@ +"use strict"; +const fs = require("node:fs"), + path = require("node:path"), + os = require("node:os"); +const { createRestic } = require("../../src/offsite-backup"); +const { createBackupArchives } = require("../../src/backup-archives"); +const { atomic } = require("../../src/release-delivery-files"); +const { RECEIPTS } = require("../../src/offsite-policy"); +const root = "/srv/dispatch-lab-remote", + localRoot = "/home/dispatchlab/local"; +const config = { + localRoot, + coreUid: 1001, + accountId: "a".repeat(32), + bucket: "dispatch-lab", + prefix: "dispatch", + environment: { + PATH: "/usr/bin:/bin", + RESTIC_PASSWORD_FILE: "/etc/dispatch/offsite-backup-password", + RESTIC_REPOSITORY: "s3:https://fixture/dispatch-lab/dispatch", + }, +}; +function repository(env) { + const relative = env.RESTIC_REPOSITORY.split("/dispatch-lab/")[1]; + if ( + !/^(dispatch|archives\/(all|7|30|90|365)\/(breq|backup)_[a-f0-9]{32})$/.test( + relative, + ) + ) + throw Error("unsafe_test_repository"); + return path.join(root, relative); +} +function runFactory(env) { + const repo = repository(env), + run = createRestic({ ...env, RESTIC_REPOSITORY: repo }); + if (!fs.existsSync(repo + "/config")) { + fs.mkdirSync(repo, { recursive: true, mode: 0o700 }); + run(["init"]); + } + return run; +} +const storage = { + listArchives: async () => + [null, 7, 30, 90, 365].flatMap((retentionDays) => { + const dir = + root + "/archives/" + (retentionDays === null ? "all" : retentionDays); + return fs.existsSync(dir) + ? fs + .readdirSync(dir) + .filter((id) => fs.existsSync(dir + "/" + id + "/config")) + .map((id) => ({ id, retentionDays })) + : []; + }), + ensureLocks: async () => {}, + withDeletionAccess: async (prefixes, action) => action(), + removePermanent: async ({ id, retentionDays }) => { + if (!/^(breq|backup)_[a-f0-9]{32}$/.test(id)) + throw Error("bad_test_archive"); + fs.rmSync( + `${root}/archives/${retentionDays === null ? "all" : retentionDays}/${id}`, + { recursive: true, force: true }, + ); + }, + removeExpired: async (row) => storage.removePermanent(row), +}; +async function main() { + if (os.hostname() !== "dispatch-dsp-lab" || process.geteuid() !== 0) + throw Error("disposable_vm_required"); + process.umask(0o077); + // This fixture runs one serialized worker process, like the locked production job. + require("../../src/backup-scratch").cleanupBackupScratch("/var/lib/dispatch-backup"); + require("../../src/backup-scratch").cleanupRestoreStaging(); + const archives = createBackupArchives(config, { runFactory, storage }); + for (;;) { + try { + const result = await archives.scan(); + atomic( + RECEIPTS + "/status.json", + { + schemaVersion: 1, + status: result.failed ? "attention" : "verified", + checkedAt: Date.now(), + }, + 0o644, + ); + fs.chmodSync(RECEIPTS + "/status.json", 0o644); + if (result.failed) console.log(JSON.stringify(result)); + try { + require("../../src/retired-dsp-metadata").purgeRetiredMetadata(config); + } catch (e) { + console.error("retired_metadata:" + e.message); + } + } catch (e) { + console.error(e.stack); + } + await new Promise((r) => setTimeout(r, 1000)); + } +} +if (require.main === module) + main().catch((e) => { + console.error(e.stack); + process.exitCode = 1; + }); +module.exports = { config, runFactory, storage }; diff --git a/core/core/installations/tests/native-lab/recovery.js b/core/core/installations/tests/native-lab/recovery.js new file mode 100644 index 0000000..390d614 --- /dev/null +++ b/core/core/installations/tests/native-lab/recovery.js @@ -0,0 +1,141 @@ +"use strict"; +const fs = require("node:fs"), + assert = require("node:assert/strict"); +const { execFileSync } = require("node:child_process"); +const { api, tick, read, check, until } = require("./scenarios"); +const recovery = require("../../src/host-recovery-bundle"); +const { runFactory, config } = require("./offsite"); +const call = (cmd, args) => + execFileSync(cmd, args, { encoding: "utf8", timeout: 120000 }); +async function disaster({ platform, rows, plans }) { + await check( + "full platform backup is exported and independently restored from encrypted storage", + async () => { + await api(platform, "/api/platform/backups", { + action: "backup", + scope: "core", + idempotencyKey: "lab:acceptance:backup:core", + }); + await until(async () => { + try { + await tick(); + } catch {} + const r = read((db) => + db + .prepare("SELECT * FROM platform_backup_requests WHERE kind='core'") + .get(), + ); + assert.notEqual(r.status, "failed", JSON.stringify(r)); + return r.status === "completed"; + }, 600000); + const row = read((db) => + db + .prepare("SELECT * FROM platform_backup_records WHERE kind='core'") + .get(), + ); + const rec = JSON.parse( + fs.readFileSync( + "/var/lib/dispatch-backup/archives/" + row.id + ".json", + ), + ); + runFactory({ + ...config.environment, + RESTIC_REPOSITORY: `s3:https://fixture/dispatch-lab/archives/${rec.retentionDays === null ? "all" : rec.retentionDays}/${row.id}`, + })([ + "restore", + rec.snapshotId, + "--target", + "/root/lab-download", + "--verify", + ]); + fs.writeFileSync( + "/root/lab-recovery-state.json", + JSON.stringify({ row, rec, rows, plans }), + ); + }, + ); + const directory = "/root/lab-download/bundle/recovery", + proof = JSON.parse( + fs.readFileSync("/root/lab-download/bundle/recovery-proof.json"), + ); + const manifest = JSON.parse(fs.readFileSync(directory + "/recovery.json")); + await check( + "full recovery includes the privileged-operation caller account", + async () => + assert.ok( + manifest.metadata.accounts.some((a) => a.name === "dispatchhelper"), + ), + ); + await check( + "corrupt recovery metadata and occupied destination fail before changing live data", + async () => { + await assert.rejects( + recovery.restoreHostRecovery({ + directory, + digest: "0".repeat(64), + installPackages: false, + }), + ); + await assert.rejects( + recovery.restoreHostRecovery({ + directory, + digest: proof.sha256, + installPackages: false, + }), + ); + for (let i = 0; i < plans.length; i++) + assert.equal( + fs.readFileSync( + plans[i].host.installationRoot + "/data/lab-marker", + "utf8", + ), + "tenant-" + i + "-before", + ); + }, + ); + await check( + "complete host loss restores accounts, code, secrets, DSP data and running services", + async () => { + for (const service of manifest.metadata.services) + recovery.systemctl(service, ["disable", "--now", service.name]); + call("/usr/bin/loginctl", ["disable-linger", "dispatchlab"]); + call("/usr/bin/systemctl", ["stop", "user@1001.service"]); + for (const a of manifest.metadata.accounts) { + call("/usr/sbin/userdel", [a.name]); + try { + call("/usr/sbin/groupdel", [a.name]); + } catch {} + } + for (const root of manifest.roots) + fs.rmSync(root, { recursive: true, force: true }); + fs.rmSync("/home/dispatchlab", { recursive: true, force: true }); + fs.rmSync("/var/lib/dispatch-backup", { recursive: true, force: true }); + for (const parent of ["/opt/dispatch-platform", "/opt/dispatch-runtime", "/opt/dispatch-control", "/var/lib/dispatch/tenants"]) + fs.rmSync(parent, { recursive: true, force: true }); + + const priorUmask = process.umask(0o077); + try { + await recovery.restoreHostRecovery({ + directory, + digest: proof.sha256, + installPackages: false, + }); + } finally { process.umask(priorUmask); } + for (let i = 0; i < plans.length; i++) + assert.equal( + fs.readFileSync( + plans[i].host.installationRoot + "/data/lab-marker", + "utf8", + ), + "tenant-" + i + "-before", + ); + }, + ); + // Recovery download is sensitive history too; erase it before DSP deletion. + fs.rmSync("/root/lab-download", { recursive: true, force: true }); + // Model a transfer whose process was killed before its finally cleanup ran. + const interrupted = fs.mkdtempSync("/var/lib/dispatch-backup/archive-transfer-"); + fs.writeFileSync(interrupted + "/tenant-history", "synthetic interrupted backup"); + fs.writeFileSync("/root/lab-interrupted-transfer", interrupted); +} +module.exports = { disaster }; diff --git a/core/core/installations/tests/native-lab/run.py b/core/core/installations/tests/native-lab/run.py new file mode 100644 index 0000000..1703872 --- /dev/null +++ b/core/core/installations/tests/native-lab/run.py @@ -0,0 +1,124 @@ +"""Disposable native DSP lab; never installs Dispatch on the runner host.""" +import argparse, hashlib, http.server, json, os, pathlib, shutil, socket, subprocess, tarfile, tempfile, threading, time, urllib.request +ROOT = pathlib.Path(__file__).resolve().parents[4] +IMAGE = 'https://cloud-images.ubuntu.com/releases/noble/release/' +NAME = 'ubuntu-24.04-server-cloudimg-amd64.img' +def run(args, **kw): + return subprocess.run([str(x) for x in args], check=True, **kw) +def port(): + with socket.socket() as s: + s.bind(('127.0.0.1', 0)); return s.getsockname()[1] +def main(): + p = argparse.ArgumentParser(); p.add_argument('--report', default='/tmp/dispatch-native-dsp-lab-report.json'); p.add_argument('--package'); p.add_argument('--keep-on-failure', action='store_true'); args = p.parse_args() + report = pathlib.Path(args.report).resolve(); report.parent.mkdir(parents=True, exist_ok=True) + report.write_text(json.dumps({'status':'running','cases':[]})) + try: + if shutil.disk_usage('/var/tmp').free < 15 * 1024**3: raise RuntimeError('15 GiB free space required') + if not os.access('/dev/kvm', os.R_OK | os.W_OK): run(['sudo','-n','test','-r','/dev/kvm']) + run(['sudo','-n','true']) + for tool in ['qemu-system-x86_64','qemu-img','ssh','scp','node','patchelf']: + if not shutil.which(tool): raise RuntimeError('Missing lab prerequisite: '+tool) + if not (ROOT/'dashboard/node_modules/@playwright/test').exists(): raise RuntimeError('Install dashboard dependencies before running the lab') + except Exception as error: + report.write_text(json.dumps({'status':'failed','cases':[],'error':str(error)},indent=2)) + raise + work = pathlib.Path(tempfile.mkdtemp(prefix='dispatch-dsp-lab-', dir='/var/tmp')); os.chmod(work, 0o700) + vm = None; server = None; tunnel = None; success = False; ssh = None; scp = None + def cleanup(): + if vm and vm.poll() is None and ssh: + try: + subprocess.run([str(x) for x in ssh]+['sync && systemctl poweroff'],timeout=15,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + vm.wait(timeout=30) + except (subprocess.TimeoutExpired, OSError): pass + if vm and vm.poll() is None: + subprocess.run(['sudo','-n','kill','-TERM',str(vm.pid)], check=False, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: vm.wait(timeout=20) + except subprocess.TimeoutExpired: subprocess.run(['sudo','-n','kill','-KILL',str(vm.pid)], check=False); vm.wait(timeout=10) + if tunnel and tunnel.poll() is None: + tunnel.terminate(); tunnel.wait(timeout=10) + if server: server.shutdown() + if not success and args.keep_on_failure: + print('Disposable lab retained for debugging:', work, flush=True) + else: + # Root-owned guest/image files are contained below this generated directory. + run(['sudo','-n','python3','-c','import shutil,sys; shutil.rmtree(sys.argv[1])',work]) + try: + print('Downloading and verifying disposable Ubuntu image', flush=True) + urllib.request.urlretrieve(IMAGE+NAME, work/'disk.qcow2') + sums=urllib.request.urlopen(IMAGE+'SHA256SUMS', timeout=30).read().decode() + expected=next(line.split()[0] for line in sums.splitlines() if line.split()[-1].lstrip('*') == NAME) + with open(work/'disk.qcow2','rb') as f: actual=hashlib.file_digest(f,'sha256').hexdigest() + if actual != expected: raise RuntimeError('Ubuntu image checksum mismatch') + run(['qemu-img','resize',work/'disk.qcow2','24G'], stdout=subprocess.DEVNULL) + run(['ssh-keygen','-q','-t','ed25519','-N','','-f',work/'key']) + seed=work/'seed';seed.mkdir(); (seed/'meta-data').write_text('instance-id: dispatch-dsp-lab\nlocal-hostname: dispatch-dsp-lab\n') + (seed/'user-data').write_text('#cloud-config\ndisable_root: false\nssh_pwauth: false\nusers:\n - name: root\n ssh_authorized_keys:\n - '+(work/'key.pub').read_text().strip()+'\n') + class Handler(http.server.SimpleHTTPRequestHandler): + def __init__(self,*a,**kw): super().__init__(*a,directory=str(seed),**kw) + def log_message(self,*a): pass + server=http.server.ThreadingHTTPServer(('127.0.0.1',0),Handler); threading.Thread(target=server.serve_forever,daemon=True).start() + ssh_port=port(); app_port=port() + command=['sudo','-n','qemu-system-x86_64','-enable-kvm','-cpu','host','-m','4096','-smp','2','-nographic','-drive',f'file={work}/disk.qcow2,format=qcow2,if=virtio','-netdev',f'user,id=net0,hostfwd=tcp:127.0.0.1:{ssh_port}-:22','-device','virtio-net-pci,netdev=net0','-smbios',f'type=1,serial=ds=nocloud-net;s=http://10.0.2.2:{server.server_port}/'] + with open(work/'console.log','wb') as console: vm=subprocess.Popen(command,stdout=console,stderr=subprocess.STDOUT) + ssh=['ssh','-i',work/'key','-p',str(ssh_port),'-o',f'UserKnownHostsFile={work}/known_hosts','-o','StrictHostKeyChecking=accept-new','-o','ConnectTimeout=3','root@127.0.0.1'] + (work/'connection.json').write_text(json.dumps({'sshPort':ssh_port,'appPort':app_port})) + deadline=time.monotonic()+180 + while time.monotonic()/dev/null'],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + if probe.returncode==0: break + time.sleep(2) + else: raise RuntimeError('Restored VM reboot readiness timeout') + tunnel=subprocess.Popen([str(x) for x in ssh[:-1]]+['-N','-L',f'127.0.0.1:{app_port}:127.0.0.1:4310','-o','ExitOnForwardFailure=yes',ssh[-1]],stdout=subprocess.DEVNULL,stderr=subprocess.PIPE) + deadline=time.monotonic()+15 + while time.monotonic() new Promise((r) => setTimeout(r, ms)); +async function check(name, action) { + const started = Date.now(); + try { + await action(); + report.cases.push({ + name, + status: "passed", + durationMs: Date.now() - started, + }); + console.log("PASS " + name); + } catch (e) { + report.cases.push({ name, status: "failed", error: e.message }); + throw e; + } finally { + fs.writeFileSync("/root/lab-report.json", JSON.stringify(report, null, 2)); + } +} +async function api(session, endpoint, body, status = 200) { + const response = await fetch(BASE + endpoint, { + method: body === undefined ? "GET" : "POST", + headers: { + ...(session?.cookie ? { Cookie: session.cookie } : {}), + ...(body === undefined + ? {} + : { + "Content-Type": "application/json", + "X-Dispatch-CSRF": session?.csrf || "", + }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + const value = await response.json(); + assert.equal(response.status, status, JSON.stringify(value)); + return { value, cookie: response.headers.get("set-cookie")?.split(";")[0] }; +} +async function login(email) { + const { value, cookie } = await api(null, "/api/auth/login", { + email, + password: "disposable lab password 123", + }); + return { cookie, csrf: value.data.csrfToken }; +} +async function tick() { + await execute( + "/usr/sbin/runuser", + [ + "--user", + "dispatchlab", + "--", + "/usr/bin/env", + "XDG_RUNTIME_DIR=/run/user/1001", + "/usr/bin/systemctl", + "--user", + "start", + "dispatch-installation-reconcile.service", + ], + { timeout: 600000 }, + ); +} +const read = (fn) => { + const db = new DatabaseSync( + LOCAL + "/data/access-control/access-control.sqlite3", + { readOnly: true }, + ); + try { + return fn(db); + } finally { + db.close(); + } +}; +async function until(fn, timeout = 120000) { + const end = Date.now() + timeout; + while (Date.now() < end) { + const result = await fn(); + if (result) return result; + await sleep(500); + } + throw Error("condition_timeout"); +} +async function main() { + assert.equal(os.hostname(), "dispatch-dsp-lab"); + assert.equal(process.geteuid(), 0); + await until(async () => { + try { + return (await fetch(BASE + "/api/auth/session")).status < 500; + } catch { + return false; + } + }); + const platform = await login("platform@example.test"); + const owners = []; + await check( + "dashboard invitation creates exactly one queued native DSP", + async () => { + for (let i = 0; i < 2; i++) { + const body = { + ownerEmail: `owner${i}@example.test`, + idempotencyKey: `lab:acceptance:create:owner:${i}`, + }; + await api(platform, "/api/platform/organizations", body, 201); + await api(platform, "/api/platform/organizations", body, 200); + const messages = fs + .readFileSync(LOCAL + "/inbox.jsonl", "utf8") + .trim() + .split("\n") + .map(JSON.parse); + const message = messages.find( + (m) => + m.email === body.ownerEmail || + m.recipientEmail === body.ownerEmail || + m.ownerEmail === body.ownerEmail, + ); + assert.ok(message, JSON.stringify(messages)); + const { value, cookie } = await api( + null, + "/api/auth/register", + { + token: message.token, + firstName: "Lab", + lastName: `Owner${i}`, + password: "disposable lab password 123", + confirmPassword: "disposable lab password 123", + }, + 201, + ); + owners.push({ + cookie, + csrf: value.data.csrfToken, + email: body.ownerEmail, + }); + } + assert.equal( + read( + (db) => db.prepare("SELECT count(*) AS n FROM organizations").get().n, + ), + 2, + ); + assert.equal( + read( + (db) => + db + .prepare( + "SELECT count(*) AS n FROM installations WHERE backend='native_service_v1'", + ) + .get().n, + ), + 2, + ); + }, + ); + await check( + "real reconciler provisions native accounts and healthy services", + async () => { + await tick(); + const rows = read((db) => + db.prepare("SELECT * FROM installations").all(), + ); + assert.ok( + rows.every((r) => + ["waiting_for_owner", "waiting_for_provider_auth"].includes(r.status), + ), + JSON.stringify(rows), + ); + }, + ); + await check("owners complete DSP details through dashboard API", async () => { + for (let i = 0; i < owners.length; i++) + await api(owners[i], "/api/organization/profile", { + name: `Lab DSP ${i}`, + abbreviation: `L${i}`, + stationCode: "DXX1", + timezone: "UTC", + }); + await tick(); + for (const owner of owners) { + const r = await api(owner, "/api/organization/profile"); + assert.equal(r.value.data.status, "complete"); + const setup = await api(owner, "/api/organization/setup"); + assert.equal(setup.value.data.installationState, "ready"); + assert.equal(setup.value.data.operationalAccess, "available"); + } + }); + await require("./exercise").exercise({ platform, owners }); + await require("./recovery").disaster({ + platform, + ...JSON.parse(fs.readFileSync("/root/lab-state.json")), + }); + fs.writeFileSync( + "/root/lab-boot-id", + fs.readFileSync("/proc/sys/kernel/random/boot_id"), + ); + report.status = "awaiting_reboot"; + fs.writeFileSync("/root/lab-report.json", JSON.stringify(report, null, 2)); +} +if (require.main === module) + main().catch((e) => { + report.status = "failed"; + fs.writeFileSync("/root/lab-report.json", JSON.stringify(report, null, 2)); + console.error(e.stack); + process.exitCode = 1; + }); +module.exports = { api, login, tick, read, until, check, report }; diff --git a/core/core/installations/tests/native-lab/seed.js b/core/core/installations/tests/native-lab/seed.js new file mode 100644 index 0000000..06c1a84 --- /dev/null +++ b/core/core/installations/tests/native-lab/seed.js @@ -0,0 +1,120 @@ +"use strict"; +// Root-only, VM-only fixture boundary. Uses the DSP identity and real provider stores. +const fs = require("node:fs"), + os = require("node:os"), + path = require("node:path"); +const { execFileSync } = require("node:child_process"); +const { DatabaseSync } = require("node:sqlite"); +const { createPlan } = require("../../src/native-deployment"); +function seed(organizationId, changed = false) { + if (os.hostname() !== "dispatch-dsp-lab" || process.geteuid() !== 0) + throw Error("disposable_vm_required"); + const config = JSON.parse(fs.readFileSync("/root/lab-config.json")); + const db = new DatabaseSync( + config.localRoot + "/data/access-control/access-control.sqlite3", + { readOnly: true }, + ); + let row; + try { + row = db + .prepare( + "SELECT i.*,o.timezone,s.code AS station FROM installations i JOIN organizations o ON o.id=i.organization_id JOIN stations s ON s.organization_id=o.id AND s.is_primary=1 WHERE o.id=?", + ) + .get(organizationId); + } finally { + db.close(); + } + const registry = new DatabaseSync( + "/var/lib/dispatch-host/state/oci-host.sqlite3", + { readOnly: true }, + ); + let a; + try { + a = registry + .prepare("SELECT * FROM allocations WHERE runtime_key=?") + .get(row.runtime_key); + } finally { + registry.close(); + } + const manifest = { + manifestVersion: 1, + revision: row.manifest_revision, + organization: { + id: organizationId, + stationCode: row.station, + timezone: row.timezone, + }, + runtime: { + key: row.runtime_key, + templateId: "isolated_dsp_v1", + releaseId: row.release_id, + }, + }; + const plan = createPlan( + manifest, + { + revision: manifest.revision, + organization: manifest.organization, + runtime: manifest.runtime, + }, + config.release, + { + name: a.account_name, + uid: a.uid, + gid: a.gid, + subuidStart: a.subuid_start, + subgidStart: a.subgid_start, + subidCount: 65536, + }, + { + version: 1, + backend: "native_service_v1", + channel: "production", + organizationId, + runtimeKey: row.runtime_key, + manifestRevision: manifest.revision, + releaseId: row.release_id, + }, + ); + const artifact = `/opt/dispatch-runtime/releases/${row.release_id}/runtime-artifact`; + const args = [ + "--quiet", + "--wait", + "--pipe", + "--collect", + `--unit=dispatch-lab-seed-${Date.now()}`, + ...Object.entries({ + User: plan.account.name, + Group: plan.account.name, + ProtectSystem: "strict", + ProtectHome: "true", + PrivateTmp: "true", + BindReadOnlyPaths: artifact + ":/opt/dispatch", + BindPaths: plan.host.installationRoot + ":" + plan.guest.installationRoot, + WorkingDirectory: "/opt/dispatch", + UMask: "0077", + }).flatMap(([k, v]) => ["--property", k + "=" + v]), + ...Object.entries({ + ...plan.guest.environment, + PATH: "/opt/dispatch/dependencies/node/bin:/usr/bin:/bin", + }).flatMap(([k, v]) => ["--setenv", k + "=" + v]), + artifact + "/dependencies/node/bin/node", + "--no-warnings", + "/opt/dispatch/fixture-seed.js", + ...(changed ? ["--changed"] : []), + ]; + const output = execFileSync("/usr/bin/systemd-run", args, { + encoding: "utf8", + timeout: 60000, + }); + const proof = JSON.parse(output.trim()); + const file = config.localRoot + "/seed-" + organizationId + ".json"; + fs.writeFileSync(file, JSON.stringify(proof), { mode: 0o600 }); + fs.chownSync(file, 1001, 1001); + return { plan, proof }; +} +if (require.main === module) + console.log( + JSON.stringify(seed(process.argv[2], process.argv[3] === "changed")), + ); +module.exports = { seed }; diff --git a/core/core/installations/tests/native-lab/setup.js b/core/core/installations/tests/native-lab/setup.js new file mode 100644 index 0000000..1865af3 --- /dev/null +++ b/core/core/installations/tests/native-lab/setup.js @@ -0,0 +1,346 @@ +"use strict"; +const fs = require("node:fs"), + path = require("node:path"), + os = require("node:os"), + crypto = require("node:crypto"); +const { execFileSync } = require("node:child_process"); +const assert = require("node:assert/strict"); +const ROOT = "/work", + LOCAL = "/home/dispatchlab/local", + UID = 1001, + RELEASE = "dispatch_current_1"; +const run = (file, args, options = {}) => + execFileSync(file, args, { stdio: "inherit", timeout: 900000, ...options }); +const write = (file, value, mode = 0o600, uid = 0) => { + fs.mkdirSync(path.dirname(file), { recursive: true, mode: 0o755 }); + require("../../src/release-delivery-files").atomic(file, value, mode); + fs.chmodSync(file, mode); + fs.chownSync(file, uid, uid); +}; +const hash = (file) => + crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); +async function main() { + assert.equal(os.hostname(), "dispatch-dsp-lab"); + assert.equal(process.geteuid(), 0); + assert.equal(fs.existsSync("/etc/dispatch"), false); + if (!process.argv.includes("--skip-packages")) { + run("/usr/bin/apt-get", ["update"]); + run("/usr/bin/apt-get", [ + "install", + "-y", + "--no-install-recommends", + "restic", + "patchelf", + "sudo", + "dbus-user-session", + "apparmor-utils", + "libnss3", + "libatk-bridge2.0-0t64", + "libx11-xcb1", + "libxcomposite1", + "libxdamage1", + "libxrandr2", + "libgbm1", + "libasound2t64", + "libcups2t64", + "libgtk-3-0t64", + "fonts-liberation", + ]); + } + run("/usr/sbin/groupadd", ["--gid", String(UID), "dispatchlab"]); + run("/usr/sbin/useradd", [ + "--uid", + String(UID), + "--gid", + String(UID), + "--create-home", + "dispatchlab", + ]); + run("/usr/sbin/useradd", [ + "--system", + "--user-group", + "--no-create-home", + "--shell", + "/usr/sbin/nologin", + "dispatchhelper", + ]); + const helperUid = Number( + execFileSync("/usr/bin/id", ["-u", "dispatchhelper"]).toString().trim(), + ); + const helperGid = Number( + execFileSync("/usr/bin/id", ["-g", "dispatchhelper"]).toString().trim(), + ); + for (const child of [ + "data", + "state/provisioner", + "config", + "secrets/oci-runtime-agents", + "run", + "installations", + "backups", + "logs", + ]) + fs.mkdirSync(path.join(LOCAL, child), { recursive: true, mode: 0o700 }); + const units = "/home/dispatchlab/.config/systemd/user"; + fs.mkdirSync(units, { recursive: true, mode: 0o700 }); + for (const [directory, mode] of [ + ["/etc/dispatch", 0o755], + ["/var/lib/dispatch/tenants", 0o755], + ["/run/dispatch-runtime-agents", 0o711], + ["/var/lib/dispatch-host/state", 0o700], + ["/var/lib/dispatch-host/authority", 0o700], + ["/var/lib/dispatch-backup", 0o700], + ["/var/lib/dispatch-backup-receipts", 0o755], + ["/srv/dispatch-lab-remote", 0o700], + ]) { + fs.mkdirSync(directory, { recursive: true, mode }); + fs.chmodSync(directory, mode); + } + const runtime = `/opt/dispatch-runtime/releases/${RELEASE}`, + core = `/opt/dispatch-platform/releases/${RELEASE}/core-artifact`, + control = `/opt/dispatch-control/releases/${RELEASE}`; + for (const directory of [runtime, core, control]) + fs.mkdirSync(directory, { recursive: true, mode: 0o755 }); + const original = JSON.parse(fs.readFileSync("/root/descriptor.json")); + require("../../src/native-runtime-artifact").unpackNativeRuntime( + "/root/runtime.tar.gz", + runtime + "/runtime-artifact", + original, + ); + // Only this disposable package contains the fixed synthetic data/collector fixture. + const artifact = runtime + "/runtime-artifact"; + const add = (relative, source, executable = false) => + write( + artifact + "/" + relative, + fs + .readFileSync(source, "utf8") + .replaceAll( + "/usr/local/bin/node", + "/opt/dispatch/dependencies/node/bin/node", + ), + executable ? 0o555 : 0o444, + ); + add( + "fixture-seed.js", + "/work/plugins/paycom/backend/tests/oci-lifecycle-seed.js", + ); + add( + "fixture-collector", + "/work/runtime/collection-manager/tests/fixture-collector.js", + true, + ); + add( + "plugins/paycom/backend/tests/helpers.js", + "/work/plugins/paycom/backend/tests/helpers.js", + ); + const inventory = []; + const walk = (directory) => { + for (const name of fs.readdirSync(directory).sort()) { + const f = path.join(directory, name), + stat = fs.statSync(f); + if (stat.isDirectory()) walk(f); + else if (path.relative(artifact, f) !== "runtime-release-manifest.json") + inventory.push({ + path: path.relative(artifact, f), + mode: (stat.mode & 0o777).toString(8), + size: stat.size, + sha256: hash(f), + }); + } + fs.chmodSync(directory, 0o555); + }; + walk(artifact); + write( + artifact + "/runtime-release-manifest.json", + { + schemaVersion: 1, + backend: "native_service_v1", + sourceCommit: original.sourceCommit, + platform: "linux/amd64", + files: inventory, + }, + 0o444, + ); + run("/usr/bin/python3", [ + "-I", + "/work/core/installations/src/native-runtime-archive.py", + "pack", + artifact, + "/root/lab-runtime.tar.gz", + ]); + require("../../src/create-bridge-artifact").main([ + runtime + "/bridge-artifact", + ]); + require("../../src/create-host-helper-artifact").main([ + control + "/host-helper-artifact", + ]); + require("../../src/release-delivery-install").seal(control); + fs.symlinkSync(control, "/opt/dispatch-control/current"); + require("../../src/release-delivery-install").installBrowserSandboxProfile(); + const release = { + ...original, + releaseId: RELEASE, + channel: "production", + artifactSha256: hash("/root/lab-runtime.tar.gz"), + embeddedManifestSha256: hash(artifact + "/runtime-release-manifest.json"), + }; + // Host verification reads the exact immutable package prepared above. + fs.copyFileSync("/root/lab-runtime.tar.gz", runtime + "/runtime.tar.gz"); + fs.chmodSync(runtime + "/runtime.tar.gz", 0o444); + require("../../src/release-delivery-install").seal(runtime); + fs.cpSync(ROOT, core + "/code", { recursive: true }); + const immutable = (d) => { + for (const entry of fs.readdirSync(d, { withFileTypes: true })) { + const f = path.join(d, entry.name); + if (entry.isDirectory()) immutable(f); + else fs.chmodSync(f, fs.statSync(f).mode & 0o111 ? 0o555 : 0o444); + } + fs.chmodSync(d, 0o555); + }; + immutable(core + "/code"); + require("../../src/core-artifact-layout").finishCoreArtifact(core, { + releaseId: RELEASE, + version: "lab", + sourceCommit: original.sourceCommit, + localRoot: LOCAL, + unitRoot: units, + publicOrigin: "https://dispatch.example.test", + port: 4310, + }); + const helper = + control + + "/host-helper-artifact/core/installations/bin/dispatch-oci-host-helper", + issuer = + control + + "/host-helper-artifact/core/installations/bin/dispatch-oci-host-issuer"; + write("/etc/dispatch/oci-host.json", { + stateRoot: "/var/lib/dispatch-host/state", + authorityRoot: "/var/lib/dispatch-host/authority", + unitRoot: "/etc/systemd/system", + releaseRoot: "/opt/dispatch-runtime/releases", + centralSocket: LOCAL + "/run/runtime-agent-hub.sock", + centralUid: UID, + controllerUid: 0, + authorityUid: UID, + helperCallerUid: helperUid, + helperCallerGid: helperGid, + controlReleaseId: RELEASE, + helperManifestSha256: hash(control + "/host-helper-artifact/manifest.json"), + }); + for (const [name, executable] of [ + ["dispatchlab", issuer], + ["dispatchhelper", helper], + ]) + write( + "/etc/sudoers.d/dispatch-lab-" + name, + `Defaults:${name} env_reset,!setenv,secure_path="/usr/bin:/bin"\nDefaults:${name} env_delete += "NODE_OPTIONS NODE_PATH LD_PRELOAD LD_LIBRARY_PATH"\n${name} ALL=(root) NOPASSWD: NOSETENV: ${executable} ""\n`, + 0o440, + ); + write( + LOCAL + "/config/oci-releases.json", + { schemaVersion: 1, releases: { [RELEASE]: release } }, + 0o600, + UID, + ); + const env = { + DISPATCH_LOCAL_ROOT: LOCAL, + DISPATCH_ACCESS_CONTROL_DATABASE_ROOT: LOCAL + "/data/access-control", + DISPATCH_PROVISIONER_STATE_ROOT: LOCAL + "/state/provisioner", + DISPATCH_INSTALLATIONS_ROOT: LOCAL + "/installations", + DISPATCH_SYSTEMD_UNIT_ROOT: units, + DISPATCH_RUNTIME_AGENT_HUB_SOCKET: LOCAL + "/run/runtime-agent-hub.sock", + DISPATCH_RUNTIME_AGENT_CONTROL_SOCKET: + LOCAL + "/run/runtime-agent-control.sock", + DISPATCH_OCI_RELEASE_CATALOG_FILE: LOCAL + "/config/oci-releases.json", + DISPATCH_OCI_RUNTIME_AGENT_CREDENTIAL_ROOT: + LOCAL + "/secrets/oci-runtime-agents", + }; + write( + LOCAL + "/config/provisioning.env", + Object.entries(env) + .map(([k, v]) => k + "=" + v) + .join("\n") + "\n", + 0o600, + UID, + ); + write("/root/lab-config.json", { + localRoot: LOCAL, + coreUid: UID, + release, + environment: env, + }); + write( + "/etc/dispatch/offsite-backup-password", + crypto.randomBytes(32).toString("hex"), + ); + write( + "/etc/dispatch/offsite-backup-policy.json", + { schemaVersion: 1, required: true }, + 0o644, + ); + run("/usr/bin/chown", ["-R", `${UID}:${UID}`, "/home/dispatchlab"]); + run("/usr/sbin/runuser", [ + "--user", + "dispatchlab", + "--", + "/usr/bin/env", + ...Object.entries(env).map(([k, v]) => k + "=" + v), + "/usr/bin/node", + "--no-warnings", + ROOT + "/core/installations/tests/native-lab/bootstrap.js", + ]); + const code = core + "/code"; + write( + units + "/dispatch-dashboard.service", + `[Unit]\nDescription=Dispatch acceptance dashboard\n[Service]\nEnvironmentFile=${LOCAL}/config/provisioning.env\nExecStart=/usr/bin/node --no-warnings ${code}/core/installations/tests/native-lab/dashboard.js\nRestart=always\nRestartSec=3\nUMask=0077\n[Install]\nWantedBy=default.target\n`, + 0o600, + UID, + ); + write( + units + "/dispatch-installation-reconcile.service", + `[Service]\nType=oneshot\nEnvironmentFile=${LOCAL}/config/provisioning.env\nExecStart=/usr/bin/node --no-warnings ${code}/core/installations/bin/dispatch-installation-reconcile\nTimeoutStartSec=30min\nUMask=0077\n`, + 0o600, + UID, + ); + write( + "/etc/systemd/system/dispatch-offsite-backup.service", + `[Service]\nExecStart=/usr/bin/node --no-warnings ${code}/core/installations/tests/native-lab/offsite.js\nRestart=always\nRestartSec=3\nUMask=0077\n[Install]\nWantedBy=multi-user.target\n`, + 0o644, + ); + run("/usr/bin/loginctl", ["enable-linger", "dispatchlab"]); + run("/usr/bin/systemctl", ["start", `user@${UID}.service`]); + run("/usr/bin/systemctl", ["daemon-reload"]); + run("/usr/bin/systemctl", [ + "enable", + "--now", + "dispatch-offsite-backup.service", + ]); + run("/usr/sbin/runuser", [ + "--user", + "dispatchlab", + "--", + "/usr/bin/env", + `XDG_RUNTIME_DIR=/run/user/${UID}`, + "/usr/bin/systemctl", + "--user", + "daemon-reload", + ]); + run("/usr/sbin/runuser", [ + "--user", + "dispatchlab", + "--", + "/usr/bin/env", + `XDG_RUNTIME_DIR=/run/user/${UID}`, + "/usr/bin/systemctl", + "--user", + "enable", + "--now", + "dispatch-dashboard.service", + ]); + run("/usr/bin/sync", []); + console.log("native_lab_ready"); +} +main().catch((e) => { + console.error(e.stack); + process.exitCode = 1; +}); diff --git a/core/core/installations/tests/native-runtime-archive.test.py b/core/core/installations/tests/native-runtime-archive.test.py new file mode 100644 index 0000000..049bd5f --- /dev/null +++ b/core/core/installations/tests/native-runtime-archive.test.py @@ -0,0 +1,56 @@ +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import tarfile +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('archive', Path(__file__).parents[1] / 'src/native-runtime-archive.py') +archive = importlib.util.module_from_spec(spec) +spec.loader.exec_module(archive) + + +class ArchiveTests(unittest.TestCase): + def make_archive(self, root, extra=None, content=b'original'): + payload = b'original' + manifest = json.dumps({'schemaVersion': 1, 'backend': 'native_service_v1', 'sourceCommit': 'a' * 40, + 'platform': 'linux/amd64', 'files': [{'path': 'app/libstdc++.so.6', 'mode': '444', + 'size': len(payload), 'sha256': hashlib.sha256(payload).hexdigest()}]}).encode() + file = root / 'runtime.tar.gz' + with tarfile.open(file, 'w:gz') as target: + for name, data in [('runtime-release-manifest.json', manifest), ('app/libstdc++.so.6', content)]: + member = tarfile.TarInfo(name) + member.mode = 0o444 + member.size = len(data) + target.addfile(member, io.BytesIO(data)) + if extra: + member = tarfile.TarInfo(extra) + member.type = tarfile.SYMTYPE + member.linkname = '/etc/passwd' + target.addfile(member) + return file, hashlib.sha256(manifest).hexdigest() + + def test_roundtrip_preserves_code_and_rejects_unexpected_file(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + file, digest = self.make_archive(root) + archive.unpack(str(file), str(root / 'out'), digest, 'a' * 40, archive.digest_file(file)) + self.assertEqual((root / 'out/app/libstdc++.so.6').read_bytes(), b'original') + self.assertEqual((root / 'out/app/libstdc++.so.6').stat().st_mode & 0o777, 0o444) + (root / 'out').chmod(0o700) + (root / 'out/app').chmod(0o700) + + def test_rejects_traversal_links_and_modified_payload(self): + for extra, content in [('../escape', b'original'), ('app/link', b'original'), (None, b'modified')]: + with self.subTest(extra=extra), tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + file, digest = self.make_archive(root, extra, content) + with self.assertRaises(ValueError): + archive.unpack(str(file), str(root / 'out'), digest, 'a' * 40, archive.digest_file(file)) + self.assertFalse((root / 'escape').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/core/core/installations/tests/oci-adapter.test.js b/core/core/installations/tests/oci-adapter.test.js new file mode 100644 index 0000000..5a09408 --- /dev/null +++ b/core/core/installations/tests/oci-adapter.test.js @@ -0,0 +1,99 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { INSTALLATION_MANIFEST_VERSION } = require('../../../shared/contracts/src/installation'); +const { hostAccountName } = require('../../runtime-host-identity'); +const { OCI_BACKEND, SUBID_COUNT } = require('../src/oci-deployment'); +const { createOciContainerAdapter } = require('../src/oci-adapter'); + +function fixture() { + const runtimeKey = 'runtime_oci_adapter'; + const manifest = { + manifestVersion: INSTALLATION_MANIFEST_VERSION, + revision: 1, + organization: { id: 'organization_adapter', stationCode: 'DXX1', timezone: 'America/Chicago' }, + runtime: { key: runtimeKey, templateId: 'isolated_dsp_v1', releaseId: 'dispatch-runtime-fixture' }, + }; + const authority = { + revision: manifest.revision, + organization: { ...manifest.organization }, + runtime: { ...manifest.runtime }, + }; + const account = { + runtimeKey, + name: hostAccountName(runtimeKey), + uid: 20_001, + gid: 20_001, + subuidStart: 1_048_576, + subgidStart: 1_048_576, + subidCount: SUBID_COUNT, + status: 'reserved', + }; + const release = { + version: 2, + backend: OCI_BACKEND, + releaseId: manifest.runtime.releaseId, + channel: 'fixture', + image: `localhost/dispatch-runtime@sha256:${'a'.repeat(64)}`, + imageDigest: `sha256:${'a'.repeat(64)}`, + imageId: 'f'.repeat(64), + sourceCommit: 'b'.repeat(40), + platform: 'linux/amd64', + runtimeAgentProtocol: 1, + runtimeGatewayProtocol: 1, + embeddedManifestSha256: 'c'.repeat(64), + imageArchiveSha256: 'd'.repeat(64), + bridgeManifestSha256: 'e'.repeat(64), + }; + return { manifest, authority, account, release }; +} + +test('OCI adapter maps the durable pipeline to closed host operations without leaking its token', () => { + const values = fixture(); + let allocated = null; + const calls = []; + const hostRegistry = { + reserve: () => { allocated = values.account; return allocated; }, + inspect: () => allocated, + }; + const hostExecutor = { + prepareAccount: () => ({ changed: true }), + materializeLayout: (plan, token) => { calls.push(['layout', token]); return { changed: true }; }, + prepareImage: () => ({ changed: true }), + render: () => ({ changed: true }), + validate: () => ({ changed: false }), + install: () => ({ changed: true }), + start: () => ({ changed: true }), + health: () => ({ changed: false }), + commit: () => ({ changed: true }), + rollback: () => ({ changed: true }), + }; + const adapter = createOciContainerAdapter({ + hostRegistry, + hostExecutor, + releaseResolver: (releaseId, fixtureChannel) => { + assert.equal(releaseId, values.release.releaseId); + assert.equal(fixtureChannel, true); + return values.release; + }, + credentialPort: { read: () => 'A'.repeat(43) }, + }); + const guard = callback => callback(); + const claim = { jobId: 'job_adapter', workerId: 'worker_adapter', fence: 1, generation: 1 }; + assert.deepEqual(adapter.reconcileHostAccount( + values.manifest, values.authority, { fixture: true, claim }, guard, + ), { ociDeploymentPlanVersion: 1, status: 'host_account_ready', changed: true }); + const plan = adapter.plan(values.manifest, values.authority, { fixture: true, claim }); + assert.equal(plan.account.name, values.account.name); + assert.deepEqual(adapter.reconcileImage(plan, claim, guard), + { ociDeploymentPlanVersion: 1, status: 'image_ready', changed: true }); + assert.deepEqual(adapter.reconcileBridge(plan, claim, guard), + { ociDeploymentPlanVersion: 1, status: 'bridge_ready', changed: true }); + assert.deepEqual(adapter.reconcileContainer(plan, claim, guard), + { ociDeploymentPlanVersion: 1, status: 'container_ready', changed: true }); + assert.deepEqual(adapter.verify(plan, claim), + { ociDeploymentPlanVersion: 1, status: 'healthy', changed: false }); + assert.equal(JSON.stringify(adapter.verify(plan, claim)).includes('AAAA'), false); + assert.deepEqual(calls, [['layout', 'A'.repeat(43)]]); +}); diff --git a/core/core/installations/tests/oci-deployment.test.js b/core/core/installations/tests/oci-deployment.test.js new file mode 100644 index 0000000..c807fa1 --- /dev/null +++ b/core/core/installations/tests/oci-deployment.test.js @@ -0,0 +1,176 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { INSTALLATION_MANIFEST_VERSION } = require('../../../shared/contracts/src'); +const { + OCI_DEPLOYMENT_PLAN_VERSION, + OCI_BACKEND, + SUBID_COUNT, + hostAccountName, + createOciDeploymentPlan, + createOciFixtureDeploymentPlan, + validateOciDeploymentPlan, + podmanArguments, + renderOciSystemUnit, + renderOciBridgeSystemUnit, +} = require('../src/oci-deployment'); + +function inputs() { + const runtimeKey = 'runtime_oci_alpha'; + const manifest = { + manifestVersion: INSTALLATION_MANIFEST_VERSION, + revision: 7, + organization: { id: 'organization_alpha', stationCode: 'DXX1', timezone: 'America/Chicago' }, + runtime: { key: runtimeKey, templateId: 'isolated_dsp_v1', releaseId: 'dispatch-runtime-v1.0.0' }, + }; + const authority = { + revision: manifest.revision, + organization: { ...manifest.organization }, + runtime: { ...manifest.runtime }, + }; + const release = { + version: 2, + backend: OCI_BACKEND, + releaseId: manifest.runtime.releaseId, + imageDigest: `sha256:${'a'.repeat(64)}`, + imageId: 'f'.repeat(64), + channel: 'production', + image: `ghcr.io/example-organization/dispatch-runtime@sha256:${'a'.repeat(64)}`, + sourceCommit: 'b'.repeat(40), + platform: 'linux/amd64', + runtimeAgentProtocol: 1, + runtimeGatewayProtocol: 1, + embeddedManifestSha256: 'c'.repeat(64), + imageArchiveSha256: 'd'.repeat(64), + bridgeManifestSha256: 'e'.repeat(64), + }; + const account = { + name: hostAccountName(runtimeKey), + uid: 951, + gid: 951, + subuidStart: 300_000, + subgidStart: 400_000, + subidCount: SUBID_COUNT, + }; + const deployment = { + version: 1, + backend: OCI_BACKEND, + channel: 'production', + organizationId: manifest.organization.id, + runtimeKey, + manifestRevision: manifest.revision, + releaseId: manifest.runtime.releaseId, + }; + return { manifest, authority, release, account, deployment }; +} + +test('OCI plan derives one closed non-root container from server-bound authority', () => { + const values = inputs(); + const plan = createOciDeploymentPlan(values.manifest, values.authority, values.release, values.account, values.deployment); + assert.equal(plan.version, OCI_DEPLOYMENT_PLAN_VERSION); + assert.equal(plan.backend, OCI_BACKEND); + assert.equal(plan.runtimeKey, values.manifest.runtime.key); + assert.equal(plan.deployment.runtimeKey, values.manifest.runtime.key); + assert.equal(plan.identity.containerName.startsWith('dispatch-dsp-'), true); + assert.equal(plan.identity.unitName.endsWith('.service'), true); + assert.equal(plan.host.installationRoot.endsWith(`/runtime/${plan.runtimeKey}`), true); + assert.equal(plan.guest.installationRoot, `/var/lib/dispatch/${plan.runtimeKey}`); + assert.equal(plan.security.readOnlyRoot, true); + assert.equal(plan.security.engineSocketMounted, false); + assert.deepEqual(plan.security.capabilities, ['SYS_CHROOT']); + assert.equal(Object.hasOwn(plan.guest.environment, 'DISPATCH_LOCAL_ROOT'), false); + assert.equal(Object.hasOwn(plan.guest.environment, 'DISPATCH_ACCESS_CONTROL_DATABASE_ROOT'), false); + + const args = podmanArguments(plan); + assert.equal(args.at(-1), values.release.image); + assert.equal(args.includes('--read-only'), true); + assert.equal(args.includes('--http-proxy=false'), true); + assert.equal(args.includes('--pid=private'), true); + assert.equal(args.includes('--ipc=private'), true); + assert.equal(args.includes('--uts=private'), true); + assert.equal(args.includes('ALL'), true); + assert.equal(args.includes('SYS_CHROOT'), true); + assert.equal(args.includes('no-new-privileges'), true); + assert.equal(args.some(value => value === '-p' || value === '--publish' || value.startsWith('--publish=')), false); + assert.equal(args.some(value => value.includes('/private-host-home') || value.includes('podman.sock')), false); + assert.equal(args.filter(value => value === '--volume').length, 2); + assert.equal(args.filter(value => value === '--env').length, Object.keys(plan.guest.environment).length); + + const unit = renderOciSystemUnit(plan); + assert.match(unit, new RegExp(`User=${plan.account.name}`)); + assert.match(unit, /MemoryMax=4G/); + assert.match(unit, /CPUQuota=200%/); + assert.match(unit, /TasksMax=512/); + assert.match(unit, /Slice=dispatch-dsp\.slice/); + assert.match(unit, /--pull=never/); + assert.match(unit, /--network=pasta/); + assert.match(unit, /UnsetEnvironment=.*HTTP_PROXY.*http_proxy/); + assert.doesNotMatch(unit, /latest|docker\.sock|podman\.sock|DISPATCH_LOCAL_ROOT=/); + const bridge = renderOciBridgeSystemUnit(plan, { + bridgeExecutable: '/opt/dispatch/runtime-agent-bridge/releases/test/service-cli.js', + centralSocket: '/run/user/1000/dispatch/runtime-agent-hub.sock', + centralUid: 1000, + controllerUid: 0, + }); + assert.match(bridge, /User=0/); + assert.match(bridge, new RegExp(`DISPATCH_RUNTIME_BRIDGE_TENANT_UID=${plan.account.uid}`)); + assert.match(bridge, /RestrictAddressFamilies=AF_UNIX/); + assert.doesNotMatch(bridge, /podman|docker|ExecStart=.*\.\.\//); +}); + +test('OCI plan rejects crossed releases, forged plans, arbitrary fields, and account identities', () => { + const values = inputs(); + const wrongTemplate = { + ...values.manifest, + runtime: { ...values.manifest.runtime, templateId: 'systemd_user_v1' }, + }; + assert.throws(() => createOciDeploymentPlan( + wrongTemplate, + { ...values.authority, runtime: { ...wrongTemplate.runtime } }, + values.release, + values.account, + values.deployment, + ), error => error.code === 'runtime_boundary_violation'); + assert.throws(() => createOciDeploymentPlan(values.manifest, values.authority, values.release, values.account), + error => error.code === 'runtime_boundary_violation'); + assert.throws(() => createOciDeploymentPlan(values.manifest, values.authority, values.release, values.account, { + ...values.deployment, organizationId: 'organization_crossed', + }), error => error.code === 'runtime_identity_mismatch'); + assert.throws(() => createOciDeploymentPlan(values.manifest, values.authority, { + ...values.release, releaseId: 'dispatch-runtime-v2.0.0', + }, values.account, values.deployment), error => error.code === 'runtime_identity_mismatch'); + assert.throws(() => createOciDeploymentPlan(values.manifest, values.authority, { + ...values.release, registryPassword: 'forbidden', + }, values.account, values.deployment), error => error.code === 'runtime_boundary_violation'); + assert.throws(() => createOciDeploymentPlan(values.manifest, values.authority, values.release, { + ...values.account, name: 'caller-selected', + }, values.deployment), error => error.code === 'runtime_boundary_violation'); + const plan = createOciDeploymentPlan(values.manifest, values.authority, values.release, values.account, values.deployment); + const serialized = validateOciDeploymentPlan(JSON.parse(JSON.stringify(plan))); + assert.deepEqual(serialized, plan); + assert.equal(podmanArguments(serialized).at(-1), values.release.image); + assert.throws(() => podmanArguments({ ...plan }), error => error.code === 'runtime_boundary_violation'); + assert.throws(() => validateOciDeploymentPlan({ + ...JSON.parse(JSON.stringify(plan)), + host: { ...plan.host, tenantRoot: '/var/lib/dispatch/tenants/crossed' }, + }), error => error.code === 'runtime_boundary_violation'); + assert.throws(() => renderOciSystemUnit({ ...plan }), error => error.code === 'runtime_boundary_violation'); +}); + +test('production and fixture image channels cannot be confused', () => { + const values = inputs(); + const local = { + ...values.release, + channel: 'fixture', + image: `localhost/dispatch-runtime@${values.release.imageDigest}`, + }; + assert.throws(() => createOciDeploymentPlan(values.manifest, values.authority, local, values.account, values.deployment), + error => error.code === 'invalid_runtime_release'); + assert.equal( + createOciFixtureDeploymentPlan(values.manifest, values.authority, local, values.account, { + ...values.deployment, channel: 'fixture', + }).release.channel, + 'fixture', + ); +}); diff --git a/core/core/installations/tests/oci-helper-input.test.js b/core/core/installations/tests/oci-helper-input.test.js new file mode 100644 index 0000000..446bebc --- /dev/null +++ b/core/core/installations/tests/oci-helper-input.test.js @@ -0,0 +1,24 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { readHelperRequest } = require('../src/oci-helper-input'); + +test('helper input is bounded before allocation and rejects ambiguous JSON and invalid UTF-8', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-helper-input-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const file = path.join(root, 'input'); + const read = input => { + fs.writeFileSync(file, input); + const fd = fs.openSync(file, 'r'); + try { return readHelperRequest(fd, 64); } finally { fs.closeSync(fd); } + }; + assert.deepEqual(read('{"ok":true}\n'), { ok: true }); + for (const input of ['x'.repeat(65), '{"a":1,"a":2}\n', '{}\n{}\n', '{}', '{}\r\n', + Buffer.from([0x7b, 0x22, 0xff, 0x22, 0x3a, 0x31, 0x7d, 0x0a])]) { + assert.throws(() => read(input), { code: 'runtime_boundary_violation' }); + } +}); diff --git a/core/core/installations/tests/oci-host-account-registry.test.js b/core/core/installations/tests/oci-host-account-registry.test.js new file mode 100644 index 0000000..affd06d --- /dev/null +++ b/core/core/installations/tests/oci-host-account-registry.test.js @@ -0,0 +1,82 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { + OCI_HOST_REGISTRY_SCHEMA_VERSION, + createOciHostAccountRegistry, +} = require('../src/oci-host-account-registry'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-oci-host-registry-')); + fs.chmodSync(root, 0o700); + const stateRoot = path.join(root, 'state'); + fs.mkdirSync(stateRoot, { mode: 0o700 }); + const subuidFile = path.join(root, 'subuid'); + const subgidFile = path.join(root, 'subgid'); + fs.writeFileSync(subuidFile, 'existing:1000000:65536\n', { mode: 0o600 }); + fs.writeFileSync(subgidFile, 'existing:1000000:65536\n', { mode: 0o600 }); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return { root, stateRoot, subuidFile, subgidFile }; +} + +function manager(paths, clock = () => 1_000) { + return createOciHostAccountRegistry({ + stateRoot: paths.stateRoot, + subuidFile: paths.subuidFile, + subgidFile: paths.subgidFile, + uidMinimum: 20_000, + uidMaximum: 20_010, + identityAvailable: value => value !== 20_001, + clock, + }); +} + +test('OCI host account registry durably reserves distinct identities and never reuses retired allocations', t => { + const paths = fixture(t); + let now = 1_000; + let selected = manager(paths, () => ++now); + const alpha = selected.reserve('runtime_oci_alpha'); + const beta = selected.reserve('runtime_oci_beta'); + assert.deepEqual(selected.reserve('runtime_oci_alpha'), alpha); + assert.equal(alpha.uid, 20_000); + assert.equal(beta.uid, 20_002); + assert.equal(alpha.subuidStart, 1_114_112); + assert.equal(beta.subuidStart, alpha.subuidStart + 65_536); + assert.equal(alpha.name.startsWith('dsp-'), true); + assert.equal(selected.activate(alpha.runtimeKey).status, 'active'); + assert.equal(selected.activate(alpha.runtimeKey).status, 'active'); + assert.equal(selected.retire(alpha.runtimeKey).status, 'retired'); + selected.close(); + + selected = manager(paths, () => ++now); + assert.equal(selected.inspect(alpha.runtimeKey).status, 'retired'); + assert.deepEqual(selected.inspect(beta.runtimeKey), beta); + assert.equal(selected.reserve('runtime_oci_gamma').uid, 20_003); + assert.equal(selected.reserve('runtime_oci_gamma').subuidStart, beta.subuidStart + 65_536); + assert.equal(selected.inspect('runtime_unknown'), null); + assert.equal(OCI_HOST_REGISTRY_SCHEMA_VERSION, 1); + selected.close(); +}); + +test('OCI host account registry fails closed on unsafe roots and identity exhaustion', t => { + const paths = fixture(t); + fs.chmodSync(paths.stateRoot, 0o755); + assert.throws(() => manager(paths), error => error.code === 'runtime_boundary_violation'); + fs.chmodSync(paths.stateRoot, 0o700); + const selected = createOciHostAccountRegistry({ + stateRoot: paths.stateRoot, + subuidFile: paths.subuidFile, + subgidFile: paths.subgidFile, + uidMinimum: 20_000, + uidMaximum: 20_000, + identityAvailable: () => false, + }); + assert.throws(() => selected.reserve('runtime_exhausted'), + error => error.code === 'service_installation_failed'); + assert.throws(() => selected.reserve('../unsafe'), error => error.code === 'runtime_boundary_violation'); + selected.close(); +}); diff --git a/core/core/installations/tests/oci-host-artifact.test.js b/core/core/installations/tests/oci-host-artifact.test.js new file mode 100644 index 0000000..ac09f63 --- /dev/null +++ b/core/core/installations/tests/oci-host-artifact.test.js @@ -0,0 +1,230 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const test = require('node:test'); +const { verifyHostArtifact, verifyPreparedHostArtifact } = require('../src/oci-host-artifact'); +const { createOciHostAuthority, ISSUER, AUDIENCE } = require('../src/oci-host-authority'); + +function removeTree(root) { + const writable = directory => { + fs.chmodSync(directory, 0o700); + for (const name of fs.readdirSync(directory)) { + const child = path.join(directory, name); + if (fs.lstatSync(child).isDirectory()) writable(child); + } + }; + writable(root); + fs.rmSync(root, { recursive: true }); +} + +if (process.geteuid() !== 0) { + test('production-style root-owned helper artifact verifies its complete tree and current pointer', t => { + if (fs.existsSync('/opt/dispatch-control')) return t.skip('existing control installation is preserved'); + if (spawnSync('/usr/bin/sudo', ['-n', '/usr/bin/true']).status !== 0) return t.skip('requires noninteractive sudo'); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-artifact-test-')); + t.after(() => removeTree(root)); + const target = path.join(root, 'artifact'); + const built = spawnSync('/usr/bin/node', [path.resolve(__dirname, "../src/create-host-helper-artifact.js"), target], + { encoding: 'utf8' }); + assert.equal(built.status, 0, built.stderr); + const result = spawnSync('/usr/bin/sudo', ['-n', '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin', + `DISPATCH_SYNTHETIC_ARTIFACT=${target}`, '/usr/bin/node', '--no-warnings', '--test', __filename], + { encoding: 'utf8', timeout: 90_000 }); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.equal(fs.existsSync('/opt/dispatch-control'), false); + }); +} else { + test('immutable helper rejects altered content, modes, links and current redirection', t => { + const control = '/opt/dispatch-control'; + // mkdir is exclusive: never adopt or remove an existing installation. + fs.mkdirSync(control, { mode: 0o755 }); + const identity = fs.lstatSync(control); + t.after(() => { + const current = fs.lstatSync(control); + assert.equal(current.ino, identity.ino); + assert.equal(current.dev, identity.dev); + removeTree(control); + }); + fs.mkdirSync(path.join(control, 'releases'), { mode: 0o755 }); + const releaseId = 'synthetic-artifact-verification'; + const release = path.join(control, 'releases', releaseId); + fs.mkdirSync(release, { mode: 0o755 }); + const artifact = path.join(release, 'host-helper-artifact'); + fs.cpSync(process.env.DISPATCH_SYNTHETIC_ARTIFACT, artifact, { recursive: true, preserveTimestamps: false }); + const seal = directory => { + for (const name of fs.readdirSync(directory)) { + const child = path.join(directory, name); + if (fs.lstatSync(child).isDirectory()) seal(child); + } + fs.chmodSync(directory, 0o555); + }; + seal(artifact); + fs.chmodSync(release, 0o555); + const current = path.join(control, 'current'); + const manifest = fs.readFileSync(path.join(artifact, 'manifest.json')); + const digest = crypto.createHash('sha256').update(manifest).digest('hex'); + const helper = path.join(artifact, 'core/installations/bin/dispatch-oci-host-helper'); + // The candidate must verify before it becomes current; active execution must not. + assert.equal(verifyPreparedHostArtifact(helper, releaseId, digest).artifactRoot, artifact); + assert.throws(() => verifyHostArtifact(helper, releaseId, digest)); + fs.symlinkSync(release, current); + const check = () => verifyHostArtifact(helper, releaseId, digest); + assert.equal(check().artifactRoot, artifact); + const source = path.join(artifact, 'core/installations/src/oci-host-helper.js'); + const content = fs.readFileSync(source); + fs.chmodSync(source, 0o644); + assert.throws(check, { code: 'runtime_boundary_violation' }); + fs.writeFileSync(source, Buffer.concat([content, Buffer.from('\n// altered\n')])); + fs.chmodSync(source, 0o444); + assert.throws(() => verifyPreparedHostArtifact(helper, releaseId, digest), { code: 'runtime_boundary_violation' }); + assert.throws(check, { code: 'runtime_boundary_violation' }); + fs.chmodSync(source, 0o644); fs.writeFileSync(source, content); fs.chmodSync(source, 0o444); + assert.equal(check().artifactRoot, artifact); + fs.unlinkSync(current); fs.symlinkSync('/tmp', current); + assert.throws(check, { code: 'runtime_boundary_violation' }); + fs.unlinkSync(current); fs.symlinkSync(release, current); + fs.chmodSync(artifact, 0o755); + assert.throws(check, { code: 'runtime_boundary_violation' }); + fs.chmodSync(artifact, 0o555); + assert.equal(check().artifactRoot, artifact); + + // Exercise the real no-argument sudo command from a user with no other + // sudo authorization. All state is synthetic and removed in finally. + const caller = 'dispatch-helper-fixture'; + const issuerCaller = 'dispatch-issuer-fixture'; + const issuerRule = `/etc/sudoers.d/${issuerCaller}`; + const issuerCommand = commandPlaceholder(); + function commandPlaceholder() { return '/opt/dispatch-control/current/host-helper-artifact/core/installations/bin/dispatch-oci-host-issuer'; } + const rule = `/etc/sudoers.d/${caller}`; + const config = '/etc/dispatch/oci-host.json'; + const command = '/opt/dispatch-control/current/host-helper-artifact/core/installations/bin/dispatch-oci-host-helper'; + assert.equal(spawnSync('/usr/bin/getent', ['passwd', caller]).status, 2); + assert.equal(spawnSync('/usr/bin/getent', ['group', caller]).status, 2); + for (const file of [rule, config]) assert.throws(() => fs.lstatSync(file), { code: 'ENOENT' }); + const privateRoot = fs.mkdtempSync('/run/dispatch-helper-boundary-test-'); + const authorityRoot = path.join(privateRoot, 'authority'); + const stateRoot = path.join(privateRoot, 'state'); + fs.mkdirSync(authorityRoot, { mode: 0o700 }); fs.mkdirSync(stateRoot, { mode: 0o700 }); + let configParentCreated = false; + let callerCreated = false; + let issuerCreated = false; + let ruleCreated = false; + let configCreated = false; + let authority; + let recoveryUnit; + try { + try { fs.mkdirSync('/etc/dispatch', { mode: 0o755 }); configParentCreated = true; } + catch (error) { if (error.code !== 'EEXIST') throw error; } + const created = spawnSync('/usr/sbin/useradd', ['--system', '--user-group', '--no-create-home', + '--shell', '/usr/sbin/nologin', caller], { encoding: 'utf8' }); + assert.equal(created.status, 0, created.stderr); callerCreated = true; + assert.equal(spawnSync('/usr/bin/getent', ['passwd', issuerCaller]).status, 2); + assert.equal(spawnSync('/usr/sbin/useradd', ['--system', '--user-group', '--no-create-home', + '--shell', '/usr/sbin/nologin', issuerCaller]).status, 0); + issuerCreated = true; + const callerUid = Number(spawnSync('/usr/bin/id', ['-u', caller], { encoding: 'utf8' }).stdout.trim()); + const callerGid = Number(spawnSync('/usr/bin/id', ['-g', caller], { encoding: 'utf8' }).stdout.trim()); + const issuerUid = Number(spawnSync('/usr/bin/id', ['-u', issuerCaller], { encoding: 'utf8' }).stdout.trim()); + fs.writeFileSync(config, `${JSON.stringify({ stateRoot, authorityRoot, unitRoot: '/etc/systemd/system', + releaseRoot: '/synthetic-unused-release-root', centralSocket: '/synthetic-unused-hub.sock', + centralUid: 1001, controllerUid: 0, authorityUid: issuerUid, helperCallerUid: callerUid, helperCallerGid: callerGid, controlReleaseId: releaseId, helperManifestSha256: digest })}\n`, + { flag: 'wx', mode: 0o600 }); + configCreated = true; + const policy = `Defaults:${caller} env_reset,!setenv,secure_path="/usr/bin:/bin"\n` + + `Defaults:${caller} env_delete += "NODE_OPTIONS NODE_PATH LD_PRELOAD LD_LIBRARY_PATH"\n` + + `${caller} ALL=(root) NOPASSWD: NOSETENV: ${command} ""\n`; + fs.writeFileSync(rule, policy, { flag: 'wx', mode: 0o440 }); ruleCreated = true; + fs.writeFileSync(issuerRule, `Defaults:${issuerCaller} env_reset,!setenv,secure_path="/usr/bin:/bin"\n` + + `Defaults:${issuerCaller} env_delete += "NODE_OPTIONS NODE_PATH LD_PRELOAD LD_LIBRARY_PATH"\n` + + `${issuerCaller} ALL=(root) NOPASSWD: NOSETENV: ${issuerCommand} ""\n`, { flag: 'wx', mode: 0o440 }); + assert.equal(spawnSync('/usr/sbin/visudo', ['-cf', issuerRule]).status, 0); + const checked = spawnSync('/usr/sbin/visudo', ['-cf', rule], { encoding: 'utf8' }); + assert.equal(checked.status, 0, checked.stderr); + const invoke = (args, input, environment = []) => spawnSync('/usr/sbin/runuser', ['--user', caller, + '--', '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin', ...environment, '/usr/bin/sudo', '-n', ...args], + { input, encoding: 'utf8', timeout: 15_000 }); + assert.notEqual(invoke(['/usr/bin/true']).status, 0); + assert.notEqual(invoke([command, '--arbitrary']).status, 0); + authority = createOciHostAuthority({ root: authorityRoot }); + const lease = { version: 1, issuer: ISSUER, audience: AUDIENCE, organizationId: 'org_helper_fixture', + runtimeKey: 'runtime_helper_fixture', installationRevision: 1, manifestRevisions: [1], + backend: 'oci_container_v1', jobKind: 'provisioning', jobId: 'job_helper_fixture', + workerId: 'worker_helper_fixture', generation: 1, fence: 1, expiresAt: Date.now() + 60_000 }; + authority.issueLease(lease); + const request = { version: 2, operation: 'inspect_account', runtimeKey: lease.runtimeKey, + claim: { jobId: lease.jobId, workerId: lease.workerId, generation: 1, fence: 1 } }; + const forged = invoke([command], `${JSON.stringify({ ...request, authorization: '0'.repeat(64) })}\n`); + assert.notEqual(forged.status, 0); + assert.deepEqual(fs.readdirSync(stateRoot), []); + request.authorization = authority.issueAction(request); + const accepted = invoke([command], `${JSON.stringify(request)}\n`, ['NODE_OPTIONS=--invalid-fixture-option']); + assert.equal(accepted.status, 0, accepted.stdout + accepted.stderr); + assert.deepEqual(JSON.parse(accepted.stdout), { ok: true, result: null }); + const replay = invoke([command], `${JSON.stringify(request)}\n`); + assert.notEqual(replay.status, 0); + assert.notEqual(invoke([issuerCommand], '{}\n').status, 0); + const invokeIssuer = (args, input) => spawnSync('/usr/sbin/runuser', ['--user', issuerCaller, + '--', '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin', '/usr/bin/sudo', '-n', ...args], + { input, encoding: 'utf8', timeout: 70_000 }); + assert.notEqual(invokeIssuer([command], '{}\n').status, 0); + assert.notEqual(invokeIssuer(['/usr/bin/true']).status, 0); + assert.notEqual(invokeIssuer([issuerCommand, '--extra'], '{}\n').status, 0); + const { authorization, ...body } = request; + const dispatched = invokeIssuer([issuerCommand], `${JSON.stringify({ version: 1, lease, request: body })}\n`); + assert.equal(dispatched.status, 0, dispatched.stdout + dispatched.stderr); + assert.deepEqual(JSON.parse(dispatched.stdout), { ok: true, result: null }); + assert.throws(() => authority.issueAction(body), { code: 'runtime_boundary_violation' }); + const again = invokeIssuer([issuerCommand], `${JSON.stringify({ version: 1, lease, request: body })}\n`); + assert.equal(again.status, 0, again.stdout + again.stderr); + assert.equal(fs.readdirSync(stateRoot).includes('candidates'), false); + // A lost helper leaves its durable gate running while a process survives + // in the exact supervised action unit. The issuer must stop it before it + // can authorize another request for this runtime. + authority.synchronizeLease(lease); + const lost = { ...body, authorization: authority.issueAction(body) }; + recoveryUnit = `dispatch-host-action-${lost.authorization}.service`; + const started = spawnSync('/usr/bin/systemd-run', ['--quiet', '--collect', `--unit=${recoveryUnit}`, + `--property=User=${callerUid}`, `--property=Group=${callerGid}`, '--property=KillMode=control-group', + '/usr/bin/sleep', '60'], { encoding: 'utf8' }); + assert.equal(started.status, 0, started.stderr); + assert.throws(() => authority.execute(lost, () => { throw new Error('synthetic_interrupted_helper'); }), /synthetic_interrupted_helper/); + assert.throws(() => authority.issueAction(body), /runtime_boundary_violation/); + const recovered = invokeIssuer([issuerCommand], `${JSON.stringify({ version: 1, lease, request: body })}\n`); + assert.equal(recovered.status, 0, recovered.stdout + recovered.stderr); + const observed = spawnSync('/usr/bin/systemctl', ['show', recoveryUnit, '--property=ActiveState,SubState,Job'], { encoding: 'utf8' }); + assert.match(observed.stdout, /ActiveState=inactive/); + assert.match(observed.stdout, /SubState=dead/); + assert.match(observed.stdout, /^Job=$/m); + assert.throws(() => authority.issueAction(body), /runtime_boundary_violation/); + + } finally { + if (recoveryUnit) spawnSync('/usr/bin/systemctl', ['stop', recoveryUnit]); + authority?.close(); + if (fs.existsSync(issuerRule)) fs.unlinkSync(issuerRule); + if (issuerCreated) { + assert.equal(spawnSync('/usr/sbin/userdel', [issuerCaller]).status, 0); + if (spawnSync('/usr/bin/getent', ['group', issuerCaller]).status === 0) { + assert.equal(spawnSync('/usr/sbin/groupdel', [issuerCaller]).status, 0); + } + } + if (ruleCreated) fs.unlinkSync(rule); + if (configCreated) fs.unlinkSync(config); + if (configParentCreated) fs.rmdirSync('/etc/dispatch'); + if (callerCreated) { + const removed = spawnSync('/usr/sbin/userdel', [caller], { encoding: 'utf8' }); + assert.equal(removed.status, 0, removed.stderr); + if (spawnSync('/usr/bin/getent', ['group', caller]).status === 0) { + assert.equal(spawnSync('/usr/sbin/groupdel', [caller]).status, 0); + } + } + fs.rmSync(privateRoot, { recursive: true }); + assert.equal(spawnSync('/usr/bin/getent', ['passwd', caller]).status, 2); + assert.equal(spawnSync('/usr/bin/getent', ['group', caller]).status, 2); + } + }); +} diff --git a/core/core/installations/tests/oci-host-authority.test.js b/core/core/installations/tests/oci-host-authority.test.js new file mode 100644 index 0000000..13e567e --- /dev/null +++ b/core/core/installations/tests/oci-host-authority.test.js @@ -0,0 +1,181 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const test = require('node:test'); +const { createOciHostAuthority, ISSUER, AUDIENCE } = require('../src/oci-host-authority'); + +if (process.geteuid() !== 0) { + test('root-owned OCI action authority rejects an ordinary process', () => { + assert.throws(() => createOciHostAuthority({ root: '/run/unused' }), { code: 'runtime_boundary_violation' }); + }); + test('root-owned OCI action authority adversarial fixture', t => { + const available = spawnSync('/usr/bin/sudo', ['-n', '/usr/bin/true']); + if (available.status !== 0) return t.skip('requires noninteractive sudo for disposable /run authority fixture'); + const result = spawnSync('/usr/bin/sudo', ['-n', '/usr/bin/env', '-i', 'PATH=/usr/bin:/bin', + '/usr/bin/node', '--no-warnings', '--test', __filename], { encoding: 'utf8', timeout: 30_000 }); + assert.equal(result.status, 0, result.stdout + result.stderr); + }); +} else { + function fixture(t) { + process.umask(0o077); + const root = fs.mkdtempSync('/run/dispatch-authority-test-'); + let now = 1000; + const authority = createOciHostAuthority({ root, clock: () => now }); + t.after(() => { authority.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const lease = { version: 1, issuer: ISSUER, audience: AUDIENCE, organizationId: 'org_alpha', + runtimeKey: 'runtime_alpha', installationRevision: 7, manifestRevisions: [1], backend: 'oci_container_v1', + jobKind: 'provisioning', jobId: 'job_alpha', workerId: 'worker_alpha', generation: 1, fence: 1, expiresAt: 10_000 }; + authority.issueLease(lease); + const request = { version: 2, operation: 'reserve_account', runtimeKey: lease.runtimeKey, + claim: { jobId: lease.jobId, workerId: lease.workerId, generation: 1, fence: 1 } }; + const issued = () => ({ ...structuredClone(request), authorization: authority.issueAction(request) }); + let mutations = 0; + const execute = value => authority.execute(value, guard => guard(() => { mutations += 1; return 'ok'; })); + return { root, authority, lease, request, issued, execute, mutations: () => mutations, time: value => { now = value; } }; + } + test('invented, substituted and replayed action claims fail before host effects', t => { + const f = fixture(t); + assert.throws(() => f.execute({ ...f.request, authorization: '0'.repeat(64) }), { code: 'runtime_boundary_violation' }); + const original = f.issued(); + for (const alter of [ + value => { value.operation = 'destroy_account'; }, + value => { value.runtimeKey = 'runtime_beta'; }, + value => { value.claim.workerId = 'worker_beta'; }, + value => { value.claim.jobId = 'job_beta'; }, + value => { value.claim.fence = 2; }, + value => { value.claim.generation = 2; }, + value => { value.payload = { arbitrary: true }; }, + value => { value.claim.extra = true; }, + ]) { + const changed = structuredClone(original); alter(changed); + assert.throws(() => f.execute(changed), { code: 'runtime_boundary_violation' }); + } + assert.equal(f.mutations(), 0); + assert.equal(f.execute(original), 'ok'); + assert.throws(() => f.execute(original), { code: 'runtime_boundary_violation' }); + assert.equal(f.mutations(), 1); + }); + test('expiry, takeover, terminal revocation and permanent retirement invalidate issued actions', t => { + const f = fixture(t); + const expired = f.issued(); + f.time(10_000); + assert.throws(() => f.execute(expired), { code: 'runtime_boundary_violation' }); + f.time(2000); + const stale = f.issued(); + f.authority.issueLease({ ...f.lease, fence: 2 }); + assert.throws(() => f.execute(stale), { code: 'runtime_boundary_violation' }); + f.request.claim.fence = 2; + const revoked = f.issued(); + f.authority.revoke(f.lease.runtimeKey); + assert.throws(() => f.execute(revoked), { code: 'runtime_boundary_violation' }); + assert.throws(() => f.authority.issueLease({ ...f.lease, fence: 2 }), { code: 'runtime_boundary_violation' }); + f.authority.issueLease({ ...f.lease, fence: 3 }); + f.request.claim.fence = 3; + const retired = f.issued(); + f.authority.revoke(f.lease.runtimeKey, true); + assert.throws(() => f.execute(retired), { code: 'runtime_boundary_violation' }); + assert.throws(() => f.authority.issueLease({ ...f.lease, generation: 2 }), { code: 'runtime_boundary_violation' }); + assert.equal(f.mutations(), 0); + }); + test('retired authority permits current destruction read-back without restoring mutation authority', t => { + const f = fixture(t); + f.authority.revoke(f.lease.runtimeKey, true); + const lease = { ...f.lease, jobKind: 'lifecycle', installationRevision: 8, generation: 8 }; + for (const operation of ['reserve_account', 'prepare_account', 'destroy_account', 'backup_destroy']) { + assert.throws(() => f.authority.synchronizeLease(lease, operation), /runtime_boundary_violation/); + } + f.authority.synchronizeLease(lease, 'verify_destroyed'); + const request = { version: 2, operation: 'verify_destroyed', runtimeKey: lease.runtimeKey, + claim: { jobId: lease.jobId, workerId: lease.workerId, fence: lease.fence } }; + assert.equal(f.authority.execute({ ...request, authorization: f.authority.issueAction(request) }, () => 'absent'), 'absent'); + f.authority.revoke(lease.runtimeKey); + assert.throws(() => f.authority.issueAction({ ...request, operation: 'destroy_account' }), /runtime_boundary_violation/); + assert.throws(() => f.authority.issueLease({ ...lease, fence: 2 }), /runtime_boundary_violation/); + }); + test('authority binds exact plan, installation, backend, token and backup payload', t => { + const f = fixture(t); + const request = { version: 2, operation: 'materialize_layout', claim: f.request.claim, token: 'synthetic', + plan: { runtimeKey: 'runtime_alpha', backend: 'oci_container_v1', planDigest: 'a'.repeat(64), + deployment: { organizationId: 'org_alpha', manifestRevision: 1 } } }; + const issued = { ...request, authorization: f.authority.issueAction(request) }; + for (const alter of [ + value => { value.plan.planDigest = 'b'.repeat(64); }, + value => { value.plan.backend = 'systemd_user'; }, + value => { value.plan.deployment.organizationId = 'org_beta'; }, + value => { value.plan.deployment.manifestRevision = 2; }, + value => { value.token = 'substituted'; }, + value => { value.payload = { destructionApproved: true }; }, + ]) { + const changed = structuredClone(issued); alter(changed); + assert.throws(() => f.execute(changed), { code: 'runtime_boundary_violation' }); + } + assert.equal(f.mutations(), 0); + }); + test('failed host effects consume their action durably and expiry is checked at each mutation', t => { + const f = fixture(t); + const request = f.issued(); + assert.throws(() => f.authority.execute(request, guard => { + f.time(10_000); + guard(() => { throw new Error('must not run'); }); + }), { code: 'runtime_boundary_violation' }); + f.time(2000); + assert.throws(() => f.execute(request), { code: 'runtime_boundary_violation' }); + assert.throws(() => f.authority.issueLease({ ...f.lease, fence: 2 }), { code: 'runtime_boundary_violation' }); + assert.throws(() => f.issued(), { code: 'runtime_boundary_violation' }); + assert.equal(f.mutations(), 0); + }); + test('renewal preserves the same action scope and fence', t => { + const f = fixture(t); + const request = f.issued(); + f.authority.renewLease({ ...f.lease, expiresAt: 20_000 }); + assert.throws(() => f.authority.renewLease({ ...f.lease, workerId: 'worker_beta', expiresAt: 20_000 }), + { code: 'runtime_boundary_violation' }); + f.time(11_000); + assert.equal(f.execute(request), 'ok'); + }); + test('lifecycle claims use the server-owned generation without fabricating a caller generation', t => { + const f = fixture(t); + f.authority.issueLease({ ...f.lease, jobKind: 'lifecycle', generation: 2 }); + delete f.request.claim.generation; + const request = f.issued(); + assert.equal(f.execute(request), 'ok'); + f.authority.issueLease({ ...f.lease, jobKind: 'lifecycle', generation: 3 }); + assert.throws(() => f.execute(request), { code: 'runtime_boundary_violation' }); + }); + test('root and database replacement and unsafe modes fail closed', t => { + const f = fixture(t); + const request = f.issued(); + fs.chmodSync(f.root, 0o750); + assert.throws(() => f.execute(request), { code: 'runtime_boundary_violation' }); + fs.chmodSync(f.root, 0o700); + const file = path.join(f.root, 'authority.sqlite3'); + fs.renameSync(file, `${file}.old`); + fs.writeFileSync(file, '', { mode: 0o600 }); + assert.throws(() => f.execute(request), { code: 'runtime_boundary_violation' }); + assert.equal(f.mutations(), 0); + }); + test('supervised recovery retains the in-flight gate until quiescence is proven', t => { + const f = fixture(t); + const request = f.issued(); + assert.throws(() => f.authority.execute(request, () => { throw new Error('interrupted'); }), /interrupted/); + assert.throws(() => f.authority.recover(f.lease.runtimeKey, () => false), /runtime_boundary_violation/); + assert.throws(() => f.authority.synchronizeLease(f.lease), /runtime_boundary_violation/); + assert.equal(f.authority.recover(f.lease.runtimeKey, ids => { + assert.deepEqual(ids, [request.authorization]); return true; + }), true); + assert.throws(() => f.execute(request), /runtime_boundary_violation/); + f.authority.synchronizeLease(f.lease); + assert.equal(f.execute(f.issued()), 'ok'); + }); + test('Access Control operation revisions order provisioning and lifecycle epochs', t => { + const f = fixture(t); + f.authority.issueLease({ ...f.lease, installationRevision: 8, jobKind: 'lifecycle', generation: 8, fence: 1 }); + f.authority.revoke(f.lease.runtimeKey); + f.authority.issueLease({ ...f.lease, installationRevision: 9, generation: 2, fence: 1 }); + assert.throws(() => f.authority.synchronizeLease({ ...f.lease, installationRevision: 8, + jobKind: 'lifecycle', generation: 99 }), /runtime_boundary_violation/); + }); +} diff --git a/core/core/installations/tests/oci-host-executor.test.js b/core/core/installations/tests/oci-host-executor.test.js new file mode 100644 index 0000000..24da206 --- /dev/null +++ b/core/core/installations/tests/oci-host-executor.test.js @@ -0,0 +1,217 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const Module = require('node:module'); +const path = require('node:path'); +const test = require('node:test'); +const { HOST_TENANT_ROOT, HOST_BRIDGE_ROOT } = require('../../runtime-host-identity'); + +// Isolate fixed host I/O; deployment-plan validation has its own real-contract +// tests. No account, systemd or Podman command is run by this unit harness. +function harness({ status = 'retired', occupied = false, redirectLayout = false, wrongRunningImage = false, starting = false, interruptedRemoval = false } = {}) { + const root = '/synthetic-executor-state'; + const files = new Map(); + const commands = []; + let containerInspections = 0, reloaded = false; + const info = (mode, uid = process.geteuid(), directory = true) => ({ + uid, gid: uid, mode, nlink: 1, isDirectory: () => directory, + isFile: () => !directory, isSymbolicLink: () => false, + }); + for (const [file, mode, owner] of [[root, 0o700], ['/etc/systemd/system', 0o755], + ['/synthetic-releases', 0o755], [HOST_TENANT_ROOT, 0o755, 0], [HOST_BRIDGE_ROOT, 0o711, 0]]) { + files.set(file, { info: info(mode, owner) }); + } + const fakeFs = { + ...fs, + lstatSync(file) { + if (!files.has(file)) throw Object.assign(new Error('absent'), { code: 'ENOENT' }); + return files.get(file).info; + }, + realpathSync: file => redirectLayout && file.includes('/runtime/runtime_executor_synthetic') ? '/outside/tenant-boundary' : file, + mkdirSync: (file, options) => { files.set(file, { info: info(options.mode) }); }, + readFileSync: file => { + if (['/etc/subuid', '/etc/subgid'].includes(file)) return ''; + return files.get(file)?.content || ''; + }, + writeFileSync: (file, content, options) => { files.set(file, { info: info(options.mode, undefined, false), content }); }, + }; + const file = path.resolve(__dirname, "../src/oci-host-executor.js"); + const loaded = new Module(file, module); + loaded.filename = file; + loaded.paths = Module._nodeModulePaths(path.dirname(file)); + const realRequire = Module.createRequire(file); + loaded.require = name => name === 'node:fs' ? fakeFs : name === './oci-host-artifact' ? { readRootFile: file => Buffer.from(fakeFs.readFileSync(file)) } : realRequire(name); + loaded._compile(fs.readFileSync(file, 'utf8'), file); + const runtimeKey = 'runtime_executor_synthetic'; + const { createOciDeploymentPlan, hostAccountName } = realRequire('./oci-deployment'); + const manifest = { manifestVersion: 1, revision: 1, + organization: { id: 'org_executor', stationCode: 'SITE', timezone: 'UTC' }, + runtime: { key: runtimeKey, templateId: 'isolated_dsp_v1', releaseId: 'release_executor' } }; + const account = { name: hostAccountName(runtimeKey), uid: 29999, gid: 29999, + subuidStart: 300000, subgidStart: 400000, subidCount: 65536 }; + files.set(`/run/user/${account.uid}`, { info: info(0o700, account.uid) }); + const release = { version: 2, backend: 'oci_container_v1', releaseId: 'release_executor', + imageDigest: `sha256:${'a'.repeat(64)}`, imageId: 'f'.repeat(64), channel: 'production', + image: `ghcr.io/example-organization/dispatch-runtime@sha256:${'a'.repeat(64)}`, sourceCommit: 'b'.repeat(40), + platform: 'linux/amd64', runtimeAgentProtocol: 1, runtimeGatewayProtocol: 1, + embeddedManifestSha256: 'c'.repeat(64), imageArchiveSha256: 'd'.repeat(64), bridgeManifestSha256: 'e'.repeat(64) }; + const plan = createOciDeploymentPlan(manifest, + { revision: 1, organization: manifest.organization, runtime: manifest.runtime }, release, account, + { version: 1, backend: 'oci_container_v1', channel: 'production', organizationId: manifest.organization.id, + runtimeKey, manifestRevision: 1, releaseId: release.releaseId }); + const { renderOciSystemUnit, renderOciBridgeSystemUnit } = realRequire('./oci-deployment'); + files.set(`/etc/systemd/system/${plan.identity.unitName}`, { info: info(0o644, 0, false), content: renderOciSystemUnit(plan) }); + files.set(`/etc/systemd/system/${plan.identity.bridgeUnitName}`, { info: info(0o644, 0, false), content: renderOciBridgeSystemUnit(plan, { bridgeExecutable: '/synthetic-releases/release_executor/bridge-artifact/core/agent-bridge/src/service-cli.js', centralSocket: '/synthetic-hub.sock', centralUid: 1001, controllerUid: 0 }) }); + if (interruptedRemoval) { + files.set(`${root}/journals/${plan.identity.suffix}.json`, { info: info(0o600, process.geteuid(), false), + content: JSON.stringify({ version: 2, planDigest: plan.planDigest, + units: [plan.identity.bridgeUnitName, plan.identity.unitName].map(name => ({ name, existed: true, + content: Buffer.from(files.get(`/etc/systemd/system/${name}`).content).toString('base64'), + active: false, enabled: 'disabled' })) }) }); + files.delete(`/etc/systemd/system/${plan.identity.unitName}`); + } + if (status === 'retired') { files.delete(`/etc/systemd/system/${plan.identity.unitName}`); files.delete(`/etc/systemd/system/${plan.identity.bridgeUnitName}`); } + const registry = { reserve() {}, inspect: () => ({ ...account, runtimeKey, status }), activate() {}, retire() {} }; + const execute = (command, args, settings) => { + commands.push([command, args]); + if (command === '/usr/bin/getent') { + if (occupied && args[0] === 'passwd') return { status: 0, stdout: `${account.name}:x:123:123::/foreign:/bin/bash\n` }; + return { status: 2, stdout: '' }; + } + if (command === '/usr/bin/systemctl' && args[0] === 'show') { + const bridge = args[1] === plan.identity.bridgeUnitName; + if (interruptedRemoval) return { status: 0, stdout: + `LoadState=${!bridge && reloaded ? 'not-found' : 'loaded'}\nActiveState=inactive\nSubState=dead\nMainPID=0\nNeedDaemonReload=no\nDropInPaths=\nJob=\nFragmentPath=/etc/systemd/system/${args[1]}\n` }; + + return { status: 0, stdout: `LoadState=loaded\nActiveState=active\nSubState=running\nMainPID=123\nNeedDaemonReload=no\nDropInPaths=\nJob=\n` + + `User=${bridge ? '0' : account.name}\nFragmentPath=/etc/systemd/system/${args[1]}\n` }; + } + if (command === '/usr/bin/systemctl' && args[0] === 'daemon-reload') { reloaded = true; return { status: 0, stdout: '' }; } + if (command === '/usr/bin/systemctl' && ['stop', 'enable'].includes(args[0])) return { status: 0, stdout: '' }; + if (command === '/usr/sbin/runuser') { + assert.deepEqual(args.slice(0, 3), ['--user', account.name, '--']); + if (args.includes('/usr/bin/install')) { + const target = args.at(-1); + const directory = args.includes('-d'); + files.set(target, { info: info(directory ? 0o700 : 0o600, account.uid, directory), content: settings.input }); + return { status: 0, stdout: '' }; + } + if (args.includes('/usr/bin/dd')) { + const target = args.find(arg => arg.startsWith('of=')).slice(3); + files.set(target, { info: info(0o600, account.uid, false), content: settings.input }); + return { status: 0, stdout: '' }; + } + if (args.includes('/usr/bin/cmp')) { + const target = args.at(-2); + return { status: files.get(target)?.content === settings.input ? 0 : 1, stdout: '' }; + } + const podman = args.indexOf('/usr/bin/podman'); + if (podman !== -1) { + const operation = args.slice(podman + 1, podman + 3).join(' '); + if (operation === 'image inspect') return { status: 0, stdout: JSON.stringify([{ + Id: release.imageId, Digest: release.imageDigest, Architecture: 'amd64', Os: 'linux', + Config: { User: '10001:10001' }, RepoDigests: [release.image], + Labels: { 'org.opencontainers.image.revision': release.sourceCommit }, + }]) }; + if (operation === 'container inspect') return { status: 0, stdout: JSON.stringify([{ + Image: wrongRunningImage ? '0'.repeat(64) : release.imageId, + Name: plan.identity.containerName, State: { Running: !starting || ++containerInspections > 1 }, + Config: { User: plan.security.user, Labels: { 'io.dispatch.runtime-key': runtimeKey, + 'io.dispatch.release-id': release.releaseId, 'io.dispatch.plan-digest': plan.planDigest } }, + }]) }; + if (args.includes('/opt/dispatch/runtime/supervisor/src/health.js')) return { status: 0, stdout: '' }; + } + } + throw new Error(`unexpected host command: ${command}`); + }; + const executor = loaded.exports.createOciHostExecutor({ registry, stateRoot: root, + unitRoot: '/etc/systemd/system', releaseRoot: '/synthetic-releases', centralSocket: '/synthetic-hub.sock', + centralUid: 1001, controllerUid: 0, execute }); + return { executor, plan, files, commands }; +} + +test('destruction verification accepts the retired allocation and performs only absence queries', () => { + const f = harness(); + assert.equal(f.executor.verifyDestroyed(f.plan).status, 'absent'); + assert.deepEqual(f.commands.map(([, args]) => args[0]), ['passwd', 'group']); +}); + +test('destruction verification cannot accept a still-active allocation', () => { + const f = harness({ status: 'active' }); + assert.throws(() => f.executor.verifyDestroyed(f.plan), { code: 'runtime_identity_mismatch' }); + assert.equal(f.commands.length, 0); +}); + +test('account reconciliation refuses a foreign account before any group or filesystem mutation', () => { + const f = harness({ status: 'reserved', occupied: true }); + let mutations = 0; + assert.throws(() => f.executor.prepareAccount(f.plan, callback => { mutations += 1; return callback(); }), + { code: 'runtime_identity_mismatch' }); + assert.equal(mutations, 0); + assert.equal(f.commands.every(([command]) => command === '/usr/bin/getent'), true); +}); + +test('destruction rejects a substituted OS account before any destructive command', () => { + const f = harness({ status: 'active', occupied: true }); + let mutations = 0; + assert.throws(() => f.executor.destroyAccount(f.plan, callback => { mutations += 1; return callback(); }), + { code: 'runtime_identity_mismatch' }); + assert.equal(mutations, 0); + assert.equal(f.commands.every(([command]) => command === '/usr/bin/getent'), true); +}); + +test('destruction replay after retirement verifies absence without running commands as the deleted user', () => { + const f = harness(); + let mutations = 0; + assert.equal(f.executor.destroyAccount(f.plan, callback => { mutations += 1; return callback(); }).status, 'destroyed'); + assert.equal(mutations, 0); +}); + +test('layout directories and credentials are created only after dropping to the tenant identity', () => { + const f = harness({ status: 'active' }); + const token = 'A'.repeat(43); + assert.equal(f.executor.materializeLayout(f.plan, token, callback => callback()).changed, true); + assert.equal(f.executor.materializeLayout(f.plan, token, callback => callback()).changed, false); + assert.equal(f.commands.every(([command, args]) => command === '/usr/sbin/runuser' && args[1] === f.plan.account.name), true); +}); + +test('a redirected layout ancestor cannot cause root mkdir/chown outside the tenant boundary', () => { + const f = harness({ status: 'active', redirectLayout: true }); + assert.throws(() => f.executor.materializeLayout(f.plan, 'A'.repeat(43), callback => callback()), + { code: 'runtime_boundary_violation' }); + assert.equal(f.commands.length, 1); + assert.equal(f.commands[0][0], '/usr/sbin/runuser'); + assert.equal(f.commands[0][1][1], f.plan.account.name); +}); + +test('health rejects a different running release even when the requested image is cached', () => { + const good = harness({ status: 'active' }); + assert.equal(good.executor.health(good.plan).status, 'healthy'); + const wrong = harness({ status: 'active', wrongRunningImage: true }); + assert.throws(() => wrong.executor.health(wrong.plan), { code: 'runtime_identity_mismatch' }); + assert.equal(wrong.commands.some(([, args]) => args.includes('/opt/dispatch/runtime/supervisor/src/health.js')), false); +}); + + +test('health waits for a correctly identified container to enter running state', () => { + const f = harness({ status: 'active', starting: true }); + assert.equal(f.executor.health(f.plan).status, 'healthy'); + assert.equal(f.commands.filter(([, args]) => args.includes('container')).length, 2); +}); + +test('a journal-authorized interrupted unlink reloads the inactive cached unit before recovery', () => { + const f = harness({ status: 'active', interruptedRemoval: true }); + assert.equal(f.executor.stop(f.plan, callback => callback()).status, 'stopped'); + assert.equal(f.commands.filter(([, args]) => args[0] === 'daemon-reload').length, 1); + assert.equal(f.commands.some(([, args]) => args[0] === 'stop' && args[1] === f.plan.identity.unitName), false); +}); + + +test('initial start attests and enables both current units without reading a rollback snapshot', () => { + const f = harness({ status: 'active' }); + assert.equal(f.executor.start(f.plan, callback => callback()).status, 'started'); + assert.deepEqual(f.commands.filter(([, args]) => args[0] === 'enable').map(([, args]) => args), [ + ['enable', '--now', f.plan.identity.bridgeUnitName], ['enable', '--now', f.plan.identity.unitName], + ]); +}); diff --git a/core/core/installations/tests/oci-host-helper-client.test.js b/core/core/installations/tests/oci-host-helper-client.test.js new file mode 100644 index 0000000..371b767 --- /dev/null +++ b/core/core/installations/tests/oci-host-helper-client.test.js @@ -0,0 +1,47 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { createOciHostHelperClient } = require('../src/oci-host-helper-client'); + +const CLAIM = Object.freeze({ jobId: 'job_helper', workerId: 'worker_helper', fence: 1, generation: 1 }); + +test('OCI host-helper client invokes only the fixed privileged executable with one closed request', () => { + const requests = []; + const execute = (command, args, options) => { + assert.equal(command, '/usr/bin/sudo'); + assert.deepEqual(args, ['-n', '/opt/dispatch-control/current/host-helper-artifact/core/installations/bin/dispatch-oci-host-helper']); + const request = JSON.parse(options.input.slice(0, -1)); + requests.push(request); + return { status: 0, signal: null, stdout: `${JSON.stringify({ ok: true, result: { status: 'ok' } })}\n`, stderr: '' }; + }; + const client = createOciHostHelperClient({ execute, authorizeRequest: () => 'a'.repeat(64) }); + assert.deepEqual(client.hostRegistry.reserve('runtime_helper', CLAIM), { status: 'ok' }); + let guarded = 0; + assert.deepEqual(client.hostExecutor.materializeLayout( + { plan: true }, + 'A'.repeat(43), + CLAIM, + callback => { guarded += 1; return callback(); }, + ), { status: 'ok' }); + assert.equal(guarded, 1); + assert.deepEqual(requests.map(value => value.operation), ['reserve_account', 'materialize_layout']); + assert.equal(requests[1].token, 'A'.repeat(43)); + assert.equal(JSON.stringify(requests[0]).includes('token'), false); +}); + +test('OCI host-helper client sanitizes helper failures', () => { + const client = createOciHostHelperClient({ + authorizeRequest: () => 'a'.repeat(64), + execute: () => ({ status: 0, signal: null, stdout: '{"ok":false,"status":"runtime_boundary_violation"}\n', stderr: '' }), + }); + assert.throws(() => client.hostRegistry.inspect('runtime_helper', CLAIM), + error => error.code === 'runtime_boundary_violation' && !JSON.stringify(error).includes('runtime_helper')); +}); + +test('OCI helper client cannot fall back to caller-generated claims without an authority action', () => { + let invoked = false; + assert.throws(() => createOciHostHelperClient({ execute: () => { invoked = true; } }), + { code: 'runtime_boundary_violation' }); + assert.equal(invoked, false); +}); diff --git a/core/core/installations/tests/oci-host-permissions.test.js b/core/core/installations/tests/oci-host-permissions.test.js new file mode 100644 index 0000000..f9b0dbd --- /dev/null +++ b/core/core/installations/tests/oci-host-permissions.test.js @@ -0,0 +1,34 @@ +'use strict'; +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { authorizeHostRequest } = require('../src/oci-host-permissions'); +const claim = { jobId: 'job_permissions', workerId: 'worker_permissions', generation: 1, fence: 2 }; +const manifest = { revision: 3, organization: { id: 'org_permissions' }, runtime: { key: 'runtime_permissions', releaseId: 'release_current' } }; +const snapshot = { kind: 'provisioning', claim, manifest, backend: 'oci_container_v1', + stage: 'runtime_oci_host_account', compensation: false, operation: 'provision', installationRevision: 7, expiresAt: Date.now() + 60000 }; +const request = { version: 2, operation: 'reserve_account', runtimeKey: manifest.runtime.key, claim }; +test('host permission uses the authoritative epoch and exact native claim', () => { + const lease = authorizeHostRequest(snapshot, request); + assert.equal(lease.installationRevision, 7); + assert.deepEqual(lease.manifestRevisions, [3]); + assert.equal(lease.generation, 1); + assert.throws(() => authorizeHostRequest(snapshot, { ...request, claim: { ...claim, fence: 3 } }), /runtime_boundary_violation/); + assert.throws(() => authorizeHostRequest(snapshot, { ...request, runtimeKey: 'runtime_other' }), /runtime_boundary_violation/); +}); +test('provisioning stage and compensation forbid forward or destructive substitutions', () => { + for (const operation of ['start', 'destroy_account', 'backup_destroy', 'rollback', 'settle_committed']) { + assert.throws(() => authorizeHostRequest(snapshot, { ...request, operation }), /runtime_boundary_violation/); + } + assert.throws(() => authorizeHostRequest({ ...snapshot, compensation: true }, request), /runtime_boundary_violation/); + assert.equal(authorizeHostRequest({ ...snapshot, compensation: true }, { ...request, operation: 'rollback' }).jobId, claim.jobId); +}); +test('lifecycle grants target release only in release stages and compensation', () => { + const { generation, ...native } = claim; + const s = { ...snapshot, kind: 'lifecycle', claim: native, operation: 'upgrade', stage: 'install_release', targetReleaseId: 'release_target' }; + const r = { version: 2, claim: native, operation: 'install', plan: { runtimeKey: manifest.runtime.key, + backend: s.backend, deployment: { organizationId: manifest.organization.id, manifestRevision: 4 }, release: { releaseId: 'release_target' } } }; + assert.deepEqual(authorizeHostRequest(s, r).manifestRevisions, [3, 4]); + assert.equal(authorizeHostRequest(s, r).generation, 7); + assert.throws(() => authorizeHostRequest({ ...s, stage: 'stop_runtime' }, { ...r, operation: 'stop' }), /runtime_boundary_violation/); + assert.throws(() => authorizeHostRequest(s, { ...r, plan: { ...r.plan, release: { releaseId: 'release_forged' } } }), /runtime_boundary_violation/); +}); diff --git a/core/core/installations/tests/oci-host-recovery.test.js b/core/core/installations/tests/oci-host-recovery.test.js new file mode 100644 index 0000000..e760e8b --- /dev/null +++ b/core/core/installations/tests/oci-host-recovery.test.js @@ -0,0 +1,41 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const Module = require('node:module'); +const test = require('node:test'); +const { opaqueRuntimeSuffix, hostAccountName } = require('../../runtime-host-identity'); +function fixture({ passwdPresent = true, liveSubuid = false } = {}) { + const key = 'runtime_interrupted_destroy', suffix = opaqueRuntimeSuffix(key), calls = []; + const account = { name: hostAccountName(key), uid: 29998, gid: 29998, subuidStart: 500000, subidCount: 65536, status: 'active' }; + const file = path.resolve(__dirname, "../src/oci-host-issuer.js"); + const loaded = new Module(file, module); loaded.filename = file; loaded.paths = Module._nodeModulePaths(path.dirname(file)); + const realRequire = Module.createRequire(file); + loaded.require = name => { + if (name === 'node:fs') return { + lstatSync() { throw Object.assign(new Error('missing'), { code: 'ENOENT' }); }, + readdirSync: () => ['123'], + readFileSync: () => `State:\tS (sleeping)\nUid:\t${liveSubuid ? '500001' : '0'} 0 0 0\n`, + }; + if (name === './oci-host-account-registry') return { createOciHostAccountRegistry: () => ({ inspect: () => account, close() {} }) }; + if (name === 'node:child_process') return { spawnSync: (file, args) => { + calls.push([file, args]); + if (file === '/usr/bin/getent') return { status: passwdPresent ? 0 : 2, + stdout: passwdPresent ? `${account.name}:x:${account.uid}:${account.gid}::/var/lib/dispatch/tenants/${suffix}/home:/usr/sbin/nologin\n` : '' }; + return { status: 0, stdout: args[0] === 'show' ? 'LoadState=not-found\nActiveState=inactive\nSubState=dead\nJob=\nControlGroup=\n' : '' }; + } }; + return realRequire(name); + }; + loaded._compile(fs.readFileSync(file, 'utf8') + '\nmodule.exports.recoverHost = recoverHost;\n', file); + return { recover: () => loaded.exports.recoverHost(key, ['a'.repeat(64)], { stateRoot: '/unused' }), calls }; +} +for (const passwdPresent of [true, false]) test(`interrupted destroy recovers absent runtime directory with passwd ${passwdPresent}`, () => { + const f = fixture({ passwdPresent }); + assert.equal(f.recover(), true); + assert.equal(f.calls.some(([, args]) => args.includes('/usr/bin/podman')), false); + assert.equal(f.calls.some(([, args]) => args.includes('user@29998.service')), true); +}); +test('recovery still refuses live subordinate processes after passwd deletion', () => { + const f = fixture({ passwdPresent: false, liveSubuid: true }); + assert.throws(f.recover, /runtime_boundary_violation/); +}); diff --git a/core/core/installations/tests/oci-lifecycle.test.js b/core/core/installations/tests/oci-lifecycle.test.js new file mode 100644 index 0000000..ca8c81a --- /dev/null +++ b/core/core/installations/tests/oci-lifecycle.test.js @@ -0,0 +1,244 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { createOciInstallationLifecycle } = require('../src/oci-lifecycle'); + +function values(operation, stages) { + const manifest = { + manifestVersion: 1, revision: 3, + organization: { id: 'org_oci_lifecycle', stationCode: 'SITE', timezone: 'UTC' }, + runtime: { key: 'runtime_oci_lifecycle', templateId: 'isolated_dsp_v1', releaseId: 'release_current' }, + }; + return { + backend: 'oci_container_v1', operation, startingState: 'ready', manifest, + manifestAuthority: { revision: 3, organization: { ...manifest.organization }, runtime: { ...manifest.runtime } }, + targetManifest: null, targetManifestAuthority: null, + job: { id: `job_${operation}` }, claim: { jobId: `job_${operation}`, workerId: 'worker_oci', fence: 1, generation: 1 }, + nextStage: 0, stages, stageReceipts: {}, backup: null, safetyBackup: null, + sourceBackup: null, priorEvidence: null, resumeSync: true, + }; +} + +function harness(context, { failTargetHealth = false } = {}) { + const state = { active: true, installed: true, targetInstalled: false, restored: false, rolledBack: false, calls: [] }; + const authority = { + claim: () => context, + renew: () => {}, + checkpointCompensationRestore: () => { context.stageReceipts.__compensationRestored = true; }, + desiredRuntimeState: () => context.startingState === 'suspended' ? 'suspended' : 'active', + mutate: (claim, callback) => callback(), + checkpoint: (claim, stage, receipt) => { context.stageReceipts[stage] = receipt; }, + succeed: () => ({ status: 'succeeded' }), + failed: (claim, error) => ({ status: 'failed', code: error.code }), + }; + const adapter = { plan: manifest => ({ host: { installationRoot: '/tmp/native-lifecycle-fixture' }, backend: context.backend, runtimeKey: manifest.runtime.key, releaseId: manifest.runtime.releaseId }) }; + adapter.destructionPlan = adapter.plan; + adapter.destructionContext = manifest => ({ plan: adapter.plan(manifest), retired: context.retired === true }); + const hostExecutor = { + start: plan => { state.active = true; state.calls.push(`start:${plan.releaseId}`); return { changed: true }; }, + stop: plan => { state.active = false; state.calls.push(`stop:${plan.releaseId}`); return { changed: true }; }, + disable: () => ({ changed: true }), + health: plan => { + if (!state.active || failTargetHealth && plan.releaseId === 'release_target') throw Object.assign(new Error('runtime_health_failed'), { code: 'runtime_health_failed' }); + }, + inspectInactive: () => { if (state.active) throw new Error('active'); }, + render: () => ({ changed: true }), validate: () => {}, + install: plan => { state.targetInstalled = plan.releaseId === 'release_target'; return { changed: true }; }, + commit: () => ({ changed: true }), + rollback: () => { state.rolledBack = true; state.active = false; return { changed: true }; }, + rollbackStopped: () => { state.rolledBack = true; state.active = false; state.calls.push('rollback-stopped'); return { changed: true }; }, + settleRollback: () => { state.calls.push('settle-rollback'); return { changed: true }; }, + removeServices: () => { state.installed = false; return { changed: true }; }, + inspectRemoved: () => { if (state.installed) throw new Error('installed'); }, + settleRemoved: () => ({ changed: true }), + verifyPublication: () => ({ status: 'verified' }), + destroyAccount: () => ({ changed: true }), verifyDestroyed: () => {}, + }; + const backup = { + snapshot: spec => ({ ...spec, status: 'snapshot', treeDigest: 'a'.repeat(64), fileCount: 0, totalBytes: 0 }), + inspect: () => ({ status: 'snapshot' }), + restore: () => { assert.equal(state.active, false); state.restored = true; state.calls.push('restore-data'); return { status: 'restored', changed: true }; }, + inspectRestored: () => ({ status: 'restored' }), destroy: () => { state.calls.push('destroy-data'); return { status: 'destroyed' }; }, + }; + const runtime = { + inspectSchedule: async () => ({ syncWasRunning: true }), + quiesceSchedule: async () => {}, restoreSchedule: async () => {}, + verifyInfrastructure: async () => {}, verifyPublication: async () => { throw new Error('unused'); }, + }; + return { + state, backup, runtime, + lifecycle: createOciInstallationLifecycle({ + authority, adapter, hostExecutor, backupManagerFactory: () => backup, runtimeFactory: () => runtime, + offsitePolicy: { assertOffsiteReady: () => {}, offsiteRequired: () => false, waitForOffsiteBackup: async () => {}, + waitForDspBackupDeletion: async () => { state.calls.push('purge-offsite'); } }, + }), + }; +} + +test('OCI lifecycle suspends through the closed host and Runtime Agent ports', async () => { + const context = values('suspend', ['inspect_schedule', 'quiesce_schedule', 'stop_runtime', 'verify_stopped']); + const { lifecycle, state } = harness(context); + assert.deepEqual(await lifecycle.run(context.job.id, 'worker_oci'), { status: 'succeeded' }); + assert.equal(state.active, false); + assert.deepEqual(state.calls, ['stop:release_current']); +}); + +test('OCI upgrade failure restores the pre-upgrade snapshot and prior units', async () => { + const context = values('upgrade', [ + 'inspect_schedule', 'quiesce_schedule', 'stop_runtime', 'upgrade_backup', + 'install_release', 'start_release', 'verify_release', + ]); + context.backup = { id: 'backup_upgrade', purpose: 'upgrade', manifestRevision: 3, releaseId: 'release_current', status: 'reserved' }; + context.targetManifest = { ...context.manifest, revision: 4, runtime: { ...context.manifest.runtime, releaseId: 'release_target' } }; + context.targetManifestAuthority = { + revision: 4, organization: { ...context.manifest.organization }, runtime: { ...context.targetManifest.runtime }, + }; + const { lifecycle, state } = harness(context, { failTargetHealth: true }); + const result = await lifecycle.run(context.job.id, 'worker_oci'); + assert.equal(result.status, 'failed'); + assert.equal(result.code, 'runtime_health_failed'); + assert.equal(state.rolledBack, true); + assert.equal(state.restored, true); + assert.equal(state.active, true); + assert.deepEqual(state.calls.slice(-4), ['rollback-stopped', 'restore-data', 'start:release_current', 'settle-rollback']); +}); + + +test('completed compensation replay settles the journal without restoring data or starting again', async () => { + const context = values('upgrade', ['install_release']); + context.targetManifest = { ...context.manifest, revision: 4, + runtime: { ...context.manifest.runtime, releaseId: 'release_target' } }; + context.targetManifestAuthority = { revision: 4, organization: context.manifest.organization, + runtime: context.targetManifest.runtime }; + context.stageReceipts = { __compensating: true, __compensated: true, __compensationFailure: 'upgrade_failed' }; + const { lifecycle, state } = harness(context); + assert.deepEqual(await lifecycle.run(context.job.id, 'worker_oci'), { status: 'failed', code: 'upgrade_failed' }); + assert.deepEqual(state.calls, ['settle-rollback']); + assert.equal(state.restored, false); +}); + + +test('new destruction retry after retirement verifies absence without invoking the deleted account', async () => { + const context = values('destroy', ['destroy_runtime', 'verify_destroyed']); + context.retired = true; + const { lifecycle, state } = harness(context); + assert.equal((await lifecycle.run(context.job.id, 'worker_oci')).status, 'succeeded'); + assert.equal(context.stageReceipts.destroy_runtime.changed, false); + assert.equal(state.calls.includes('destroy-data'), false); +}); + + +test('restored DSP compensation replay preserves records created by resumed background work', async () => { + const context = values('upgrade', ['install_release']); + context.targetManifest = { ...context.manifest, revision: 4, + runtime: { ...context.manifest.runtime, releaseId: 'release_target' } }; + context.targetManifestAuthority = { revision: 4, organization: context.manifest.organization, + runtime: context.targetManifest.runtime }; + context.stageReceipts = { __compensating: true, __compensationRestored: true, + __compensationFailure: 'upgrade_failed', inspect_schedule: { syncWasRunning: true }, + upgrade_backup: { status: 'snapshot', treeDigest: 'a'.repeat(64), fileCount: 1, totalBytes: 42 } }; + context.backup = { id: 'backup_upgrade', status: 'available' }; + const { lifecycle, state } = harness(context); + assert.deepEqual(await lifecycle.run(context.job.id, 'worker_oci'), { status: 'failed', code: 'upgrade_failed' }); + assert.equal(state.restored, false); + assert.equal(state.active, true); + assert.deepEqual(state.calls, ['start:release_current', 'settle-rollback']); +}); + +test('direct deletion stops the DSP and purges offsite backups before removing local data', async () => { + const context = values('destroy', ['destroy_runtime', 'verify_destroyed']); + const { lifecycle, state } = harness(context); + assert.equal((await lifecycle.run(context.job.id, 'worker_oci')).status, 'succeeded'); + assert.equal(state.active, false); + assert.equal(state.installed, false); + assert.ok(state.calls.indexOf('stop:release_current') < state.calls.indexOf('purge-offsite')); + assert.ok(state.calls.indexOf('purge-offsite') < state.calls.indexOf('destroy-data')); +}); + +for (const startingState of ['suspended', 'waiting_for_owner', 'waiting_for_provider_auth']) { + test(`native ${startingState} upgrade preserves whether the runtime is running`, async () => { + const context = values('upgrade', ['stop_runtime', 'upgrade_backup', 'install_release', + ...(startingState === 'suspended' ? ['verify_stopped_release'] : ['start_release', 'verify_release']), 'commit_release']); + context.backend = 'native_service_v1'; context.startingState = startingState; + context.backup = { id: 'backup_upgrade', status: 'reserved' }; + context.targetManifest = { ...context.manifest, revision: 4, runtime: { ...context.manifest.runtime, releaseId: 'release_target' } }; + context.targetManifestAuthority = { revision: 4, organization: context.manifest.organization, runtime: context.targetManifest.runtime }; + const { lifecycle, state } = harness(context); + state.active = startingState !== 'suspended'; + assert.deepEqual(await lifecycle.run(context.job.id, 'worker_native'), { status: 'succeeded' }); + assert.equal(state.active, startingState !== 'suspended'); + assert.equal(state.targetInstalled, true); + assert.equal(state.calls.includes('start:release_target'), startingState !== 'suspended'); + }); +} + +test('removal disables the DSP while retaining its service definitions and data', async () => { + const context = values('decommission', ['inspect_schedule', 'quiesce_schedule', 'stop_runtime', 'disable_runtime', 'verify_retained']); + context.removal = { sync_running: null, installation_state: 'ready', legacy_services: 0 }; + const { lifecycle, state } = harness(context); + assert.equal((await lifecycle.run(context.job.id, 'worker_remove_retained')).status, 'succeeded'); + assert.equal(state.active, false); + assert.equal(state.installed, true); + assert.ok(!state.calls.includes('destroy-data')); + assert.equal(context.stageReceipts.final_backup, undefined); +}); + +test('deleting a removed DSP still removes its retained service definitions before erasing data', async () => { + const context = values('destroy', ['destroy_runtime', 'verify_destroyed']); + context.startingState = 'decommissioned'; + const { lifecycle, state } = harness(context); + state.active = false; + assert.equal((await lifecycle.run(context.job.id, 'worker_destroy_removed')).status, 'succeeded'); + assert.equal(state.installed, false); + assert.equal(state.active, false); + assert.ok(state.calls.indexOf('purge-offsite') < state.calls.indexOf('destroy-data')); +}); + +test('pre-update snapshot is inspected and reused without making or uploading another backup', async () => { + const context = values('upgrade', ['inspect_schedule', 'quiesce_schedule', 'stop_runtime', 'upgrade_backup', 'install_release', 'start_release', 'verify_release', 'restore_schedule']); + context.backup = { id: 'backup_prior', purpose: 'manual', manifestRevision: 3, releaseId: 'release_current', + status: 'available', treeDigest: 'a'.repeat(64), fileCount: 2, totalBytes: 100 }; + context.stageReceipts.__preUpdateBackup = context.backup.id; + context.targetManifest = { ...context.manifest, revision: 4, runtime: { ...context.manifest.runtime, releaseId: 'release_target' } }; + context.targetManifestAuthority = { revision: 4, organization: context.manifest.organization, runtime: context.targetManifest.runtime }; + const { lifecycle, state, backup } = harness(context); + backup.snapshot = () => assert.fail('must reuse the existing backup'); + let inspected = false; + backup.inspect = value => { assert.equal(value.id, 'backup_prior'); inspected = true; }; + assert.deepEqual(await lifecycle.run(context.job.id, 'worker_oci'), { status: 'succeeded' }); + assert.equal(inspected, true); + assert.equal(state.restored, false); + assert.equal(context.stageReceipts.upgrade_backup.treeDigest, context.backup.treeDigest); +}); + +for (const operation of ['backup', 'upgrade', 'suspend', 'resume']) test(`unconnected DSP ${operation} never requires a Paycom schedule or publication`, async () => { + const { lifecycleStages } = require('../../accounts/src/installation-lifecycle'); + const context = values(operation, lifecycleStages(operation, 'native_service_v1', operation === 'resume' ? 'suspended' : 'ready', null, true)); + context.backend = 'native_service_v1'; + context.withoutPaycom = true; + context.resumeSync = false; + if (operation === 'resume') context.startingState = 'suspended'; + if (['backup', 'upgrade'].includes(operation)) context.backup = { id: 'backup_optional', purpose: operation === 'upgrade' ? 'upgrade' : 'manual', manifestRevision: 3, releaseId: 'release_current', status: 'reserved' }; + if (operation === 'upgrade') { + context.targetManifest = { ...context.manifest, revision: 4, runtime: { ...context.manifest.runtime, releaseId: 'release_target' } }; + context.targetManifestAuthority = { revision: 4, organization: context.manifest.organization, runtime: context.targetManifest.runtime }; + } + const { lifecycle, runtime } = harness(context); + for (const method of ['inspectSchedule', 'quiesceSchedule', 'restoreSchedule', 'verifyPublication']) runtime[method] = async () => { throw new Error('Paycom must remain optional'); }; + assert.equal((await lifecycle.run(context.job.id, 'worker_oci')).status, 'succeeded'); +}); + +test('unconnected DSP upgrade rollback does not try to restore a nonexistent Paycom schedule', async () => { + const { lifecycleStages } = require('../../accounts/src/installation-lifecycle'); + const context = values('upgrade', lifecycleStages('upgrade', 'native_service_v1', 'ready', null, true)); + context.backend = 'native_service_v1'; context.withoutPaycom = true; + context.backup = { id: 'backup_optional', purpose: 'upgrade', manifestRevision: 3, releaseId: 'release_current', status: 'reserved' }; + context.targetManifest = { ...context.manifest, revision: 4, runtime: { ...context.manifest.runtime, releaseId: 'release_target' } }; + context.targetManifestAuthority = { revision: 4, organization: context.manifest.organization, runtime: context.targetManifest.runtime }; + const { lifecycle, runtime, state } = harness(context, { failTargetHealth: true }); + runtime.restoreSchedule = async () => { throw new Error('no schedule exists'); }; + const result = await lifecycle.run(context.job.id, 'worker_oci'); + assert.equal(result.code, 'runtime_health_failed'); + assert.equal(state.rolledBack, true); assert.equal(state.active, true); +}); diff --git a/core/core/installations/tests/oci-runtime-agent-credential.test.js b/core/core/installations/tests/oci-runtime-agent-credential.test.js new file mode 100644 index 0000000..be7d009 --- /dev/null +++ b/core/core/installations/tests/oci-runtime-agent-credential.test.js @@ -0,0 +1,27 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { createOciRuntimeAgentCredentialPort } = require('../src/oci-runtime-agent-credential'); + +test('OCI Runtime Agent credentials remain in the central private handoff root', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-oci-agent-credential-')); + fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const port = createOciRuntimeAgentCredentialPort({ credentialRoot: root }); + const first = port.issue('runtime_oci_credential'); + assert.match(first.tokenHash, /^[a-f0-9]{64}$/); + assert.equal(first.tokenChanged, true); + assert.equal(port.issue('runtime_oci_credential').tokenHash, first.tokenHash); + const file = path.join(root, 'runtime_oci_credential.token'); + assert.equal(fs.lstatSync(file).mode & 0o7777, 0o600); + assert.equal(port.read('runtime_oci_credential').includes('\n'), false); + const rotated = port.issue('runtime_oci_credential', { rotate: true }); + assert.notEqual(rotated.tokenHash, first.tokenHash); + assert.equal(port.revoke('runtime_oci_credential', first.tokenHash), false); + assert.equal(port.revoke('runtime_oci_credential', rotated.tokenHash), true); + assert.equal(port.revoke('runtime_oci_credential', rotated.tokenHash), false); +}); diff --git a/core/core/installations/tests/offsite-backup.test.js b/core/core/installations/tests/offsite-backup.test.js new file mode 100644 index 0000000..9d0d788 --- /dev/null +++ b/core/core/installations/tests/offsite-backup.test.js @@ -0,0 +1,139 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { DatabaseSync } = require('node:sqlite'); +const { createRestic, exportSnapshot, verifySnapshot } = require('../src/offsite-backup'); +const { receiptKey, hasVerifiedReceipt } = require('../src/offsite-policy'); +const { atomic, hashFileSync } = require('../src/release-delivery-files'); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-offsite-')); fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const source = path.join(root, 'source'), workRoot = path.join(root, 'work'), receiptRoot = path.join(root, 'receipts'); + for (const p of [source, workRoot, receiptRoot]) fs.mkdirSync(p, { mode: 0o700 }); + const file = path.join(source, 'access-control-before.sqlite3'), secret = 'business-record-' + crypto.randomBytes(32).toString('hex'); + const db = new DatabaseSync(file); db.exec('CREATE TABLE business(value TEXT)'); db.prepare('INSERT INTO business VALUES(?)').run(secret); db.close(); fs.chmodSync(file, 0o600); + atomic(path.join(source, 'manifest.json'), { version: 1, kind: 'core', sha256: hashFileSync(file), size: fs.statSync(file).size }); + return { root, source, workRoot, receiptRoot, uid: process.geteuid(), secret }; +} +const binary = process.env.DISPATCH_RESTIC_TEST_BINARY || '/usr/bin/restic'; +test('real encrypted repository upload, download and restore preserve the full Core snapshot', { skip: !fs.existsSync(binary) }, async t => { + const f = fixture(t), password = path.join(f.root, 'password'); + fs.writeFileSync(password, crypto.randomBytes(32).toString('hex'), { mode: 0o600 }); + const repo = path.join(f.root, 'repository'); + const run = createRestic({ PATH: '/usr/bin:/bin', RESTIC_REPOSITORY: repo, RESTIC_PASSWORD_FILE: password }, { binary }); + run(['init', '--repository-version', '2']); + // Use the actual WAL-backed Core store and the production snapshot path. + const liveRoot = path.join(f.root, 'live'); + const live = new (require('../../accounts/src/store').AccessStore)({ databaseRoot: liveRoot, database: path.join(liveRoot, 'access-control.sqlite3') }); + try { + live.db.exec('CREATE TABLE business(value TEXT)'); live.db.prepare('INSERT INTO business VALUES(?)').run(f.secret); + await require('../src/core-recovery-host').snapshotDatabase(path.join(liveRoot, 'access-control.sqlite3'), f.source); + } finally { live.close(); } + const receipt = exportSnapshot({ ...f, run }); + assert.equal(receipt.status, 'verified'); + assert.equal(hasVerifiedReceipt(f.source, receipt.digest, { root: f.receiptRoot, uid: f.uid }), true); + run(['check', '--read-data']); + const restored = path.join(f.root, 'independent-restore'); + run(['restore', receipt.snapshotId, '--target', restored, '--verify']); + const db = new DatabaseSync(path.join(restored, 'snapshot/access-control-before.sqlite3'), { readOnly: true }); + assert.equal(db.prepare('SELECT value FROM business').get().value, f.secret); db.close(); + const walk = dir => fs.readdirSync(dir, { withFileTypes: true }).flatMap(e => e.isDirectory() ? walk(path.join(dir, e.name)) : [path.join(dir, e.name)]); + for (const file of walk(repo)) assert.equal(fs.readFileSync(file).includes(Buffer.from(f.secret)), false); + assert.deepEqual(exportSnapshot({ ...f, run: () => { throw Error('already verified'); } }), receipt); + // Wrong keys cannot decrypt the same repository. + fs.writeFileSync(password, 'a different password which cannot recover anything'); + assert.throws(() => run(['restore', receipt.snapshotId, '--target', path.join(f.root, 'wrong-key')]), { code: 'offsite_transfer_failed' }); +}); +test('failed or unconfirmed uploads never issue a completion receipt', t => { + const f = fixture(t); + for (const result of [null, [], [{ message_type: 'summary', snapshot_id: 'invalid' }]]) { + assert.throws(() => exportSnapshot({ ...f, run() { + if (result === null) throw Error('upload failed'); + return result; + } })); + assert.equal(fs.existsSync(path.join(f.receiptRoot, receiptKey(f.source) + '.json')), false); + assert.deepEqual(fs.readdirSync(f.workRoot), []); + } +}); +test('upload completion does not download, restore, or scan the remote repository', t => { + const f = fixture(t), calls = []; + const receipt = exportSnapshot({ ...f, run(args) { + calls.push(args[0]); + assert.equal(args[0], 'backup'); + return [{ message_type: 'summary', snapshot_id: 'a'.repeat(64) }]; + } }); + assert.equal(receipt.verification, 'upload'); + assert.deepEqual(calls, ['backup']); +}); +test('snapshot export rejects symlinks and corrupt local database backups before invoking restic', t => { + const f = fixture(t); + fs.symlinkSync('/etc/passwd', path.join(f.source, 'outside')); + assert.throws(() => exportSnapshot({ ...f, run: () => assert.fail('must not upload') })); + fs.unlinkSync(path.join(f.source, 'outside')); + fs.appendFileSync(path.join(f.source, 'access-control-before.sqlite3'), 'corruption'); + assert.throws(() => verifySnapshot(f.source, f.uid), { code: 'backup_corrupt' }); +}); +test('DSP snapshot export verifies every payload entry against its existing backup manifest', t => { + const f = fixture(t); + fs.rmSync(f.source, { recursive: true }); fs.mkdirSync(f.source, { mode: 0o700 }); + for (const dir of ['payload', 'payload/data', 'payload/state']) fs.mkdirSync(path.join(f.source, dir), { mode: 0o700 }); + const contents = 'tenant business data'; + fs.writeFileSync(path.join(f.source, 'payload/data/record'), contents, { mode: 0o600 }); + const entries = [{ path: 'data/record', type: 'file', size: contents.length, sha256: crypto.createHash('sha256').update(contents).digest('hex') }]; + const treeDigest = crypto.createHash('sha256').update(JSON.stringify(entries)).digest('hex'); + atomic(path.join(f.source, 'manifest.json'), { version: 1, treeDigest, entries }); + assert.equal(verifySnapshot(f.source, f.uid).digest, treeDigest); + fs.writeFileSync(path.join(f.source, 'payload/data/record'), 'tampered'); + assert.throws(() => verifySnapshot(f.source, f.uid), { code: 'backup_corrupt' }); +}); + +test('required-backup gate renews its lease and opens only for its exact verified digest', async t => { + const f = fixture(t); const policyFile = path.join(f.root, 'policy.json'); + atomic(policyFile, { schemaVersion: 1, required: true }); + let now = 1000, polls = 0, renewals = 0; + const file = path.join(f.receiptRoot, receiptKey(f.source) + '.json'); + const policy = require('../src/offsite-policy').createOffsitePolicy({ policyFile, receiptRoot: f.receiptRoot, uid: f.uid, + clock: () => now, sleep: async ms => { + now += ms; polls++; + atomic(file, { schemaVersion: 1, status: 'verified', snapshotId: 'b'.repeat(64), verifiedAt: now, + digest: (polls === 1 ? 'c' : 'a').repeat(64) }); + } }); + assert.throws(policy.assertOffsiteReady, { code: 'offsite_backup_unavailable' }); + atomic(path.join(f.receiptRoot, 'status.json'), { status: 'verified', checkedAt: now }); + policy.assertOffsiteReady(); + await policy.waitForOffsiteBackup(f.source, 'a'.repeat(64), () => renewals++); + assert.equal(polls, 2); assert.equal(renewals, 2); + now += 300001; + assert.throws(policy.assertOffsiteReady, { code: 'offsite_backup_unavailable' }); + const missing = require('../src/offsite-policy').createOffsitePolicy({ policyFile, receiptRoot: f.receiptRoot, uid: f.uid, + clock: () => now, sleep: async () => { now += 300001; } }); + await assert.rejects(missing.waitForOffsiteBackup(f.source, 'd'.repeat(64)), { code: 'offsite_backup_unavailable' }); + fs.writeFileSync(policyFile, 'malformed'); + assert.throws(policy.offsiteRequired); +}); + +test('dashboard restore safety cannot bypass offsite verification before global policy activation', async t => { + const f=fixture(t);let now=1000,renewals=0; + const policy=require('../src/offsite-policy').createOffsitePolicy({policyFile:path.join(f.root,'not-enabled.json'),receiptRoot:f.receiptRoot,uid:f.uid,clock:()=>now,sleep:async()=>{now+=300001;}}); + assert.equal(policy.offsiteRequired(),false); + await assert.rejects(policy.waitForOffsiteBackup(f.source,'d'.repeat(64),()=>renewals++,{required:true}),{code:'offsite_backup_unavailable'}); + assert.ok(renewals>0); +}); + +test('DSP deletion waits for a root-owned receipt bound to the exact job and DSP', async t => { + const f = fixture(t), policyFile = path.join(f.root, 'deletion-policy.json'); + atomic(policyFile, {schemaVersion:1,required:true}); + const jobId='life_'+'a'.repeat(32), proofFile=path.join(f.receiptRoot,`deleted-${jobId}.json`); + let now=1000, renewals=0; + const policy=require('../src/offsite-policy').createOffsitePolicy({policyFile,receiptRoot:f.receiptRoot,uid:f.uid, + clock:()=>now,sleep:async()=>{now+=300001}}); + atomic(proofFile,{schemaVersion:1,status:'destroyed',jobId,organizationId:'org_other',runtimeKey:'runtime_target'}); + await assert.rejects(policy.waitForDspBackupDeletion(jobId,'org_target','runtime_target',()=>renewals++),/offsite_backup_unavailable/); + assert.ok(renewals>0); + atomic(proofFile,{schemaVersion:1,status:'destroyed',jobId,organizationId:'org_target',runtimeKey:'runtime_target'}); + await policy.waitForDspBackupDeletion(jobId,'org_target','runtime_target',()=>assert.fail('should already be complete')); +}); diff --git a/core/core/installations/tests/platform-backup-worker.test.js b/core/core/installations/tests/platform-backup-worker.test.js new file mode 100644 index 0000000..a7bba85 --- /dev/null +++ b/core/core/installations/tests/platform-backup-worker.test.js @@ -0,0 +1,414 @@ +'use strict'; +const test = require('node:test'), + assert = require('node:assert/strict'), + fs = require('node:fs'), + os = require('node:os'), + path = require('node:path'), + crypto = require('node:crypto'); +const { AccessStore } = require('../../accounts/src/store'); +const { createPlatformBackups } = require('../../accounts/src/platform-backups'); +const { createPlatformBackupWorker } = require('../src/platform-backup-worker'); +function fixture(t, backend = 'oci_container_v1') { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-backup-worker-')); + fs.chmodSync(root, 0o700); + fs.mkdirSync(path.join(root, 'data'), { mode: 0o700 }); + const store = new AccessStore({ + databaseRoot: path.join(root, 'data/access-control'), + database: path.join(root, 'data/access-control/access-control.sqlite3'), + }); + t.after(() => { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + }); + store.insertUser({ + id: 'user_platform', + email: 'owner@example.test', + firstName: 'Platform', + lastName: 'Owner', + passwordHash: 'synthetic', + platformRole: 'owner', + timestamp: 1000, + }); + store.createOrganization({ + id: 'org_dsp', + name: 'Example DSP', + abbreviation: null, + timezone: 'UTC', + status: 'active', + createdBy: null, + timestamp: 1000, + }); + store.insertStation('org_dsp', 'TST1', true, 1000); + store.createInstallation( + 'org_dsp', + 'runtime_dsp', + 'ready', + 1000, + 'dispatch_current_1', + backend, + ); + const remote = { status: 'connected', backups: {} }; + let now = 100000; + const options = { store, localRoot: root, archive: () => remote, clock: () => now }; + const manager = createPlatformBackups({ ...options, enabled: true }), + session = { user: { id: 'user_platform', platformRole: 'owner' } }; + const request = () => + store.db + .prepare('SELECT * FROM platform_backup_requests ORDER BY created_at DESC,rowid DESC LIMIT 1') + .get(); + // Fake only the existing lifecycle executor boundary: the real coordinator, + // lifecycle request validation, persistence and metadata transactions run. + function finish(status = 'succeeded') { + const job = store.activeLifecycleJob('org_dsp'); + assert.ok(job); + store.db + .prepare( + 'UPDATE installation_lifecycle_jobs SET status=?,failure_code=?,finished_at=100000,result_json=? WHERE id=?', + ) + .run( + status, + status === 'failed' ? 'runtime_health_failed' : null, + status === 'succeeded' ? '{}' : null, + job.id, + ); + const state = + job.operation === 'resume' + ? status === 'failed' + ? 'suspended' + : 'ready' + : job.operation === 'backup' + ? 'ready' + : 'suspended'; + store.db + .prepare('UPDATE installations SET status=? WHERE organization_id=?') + .run(state, 'org_dsp'); + for (const id of [job.backup_id, job.safety_backup_id].filter(Boolean)) { + store.db + .prepare( + "UPDATE installation_backups SET status='available',tree_digest=?,file_count=1,total_bytes=10,completed_at=100000 WHERE id=?", + ) + .run('a'.repeat(64), id); + const row = store.db.prepare('SELECT * FROM platform_backup_records WHERE id=?').get(id); + remote.backups[id] = { + status: 'verified', + localReady: true, + format: 2, + metadataDigest: crypto.createHash('sha256').update(row.metadata_json).digest('hex'), + }; + } + return job; + } + const tick = () => createPlatformBackupWorker(options).tick(); + async function backup() { + manager.command(session, { + action: 'backup', + organizationId: 'org_dsp', + idempotencyKey: 'worker:backup:123456', + }); + await tick(); + const job = finish(); + await tick(); + await tick(); + assert.equal(request().status, 'completed'); + return job.backup_id; + } + function restore(id) { + now++; + manager.command(session, { + action: 'restore', + organizationId: 'org_dsp', + backupId: id, + confirmation: 'Example DSP', + idempotencyKey: 'worker:restore:123456', + }); + } + return { + root, + store, + remote, + manager, + session, + request, + tick, + finish, + backup, + restore, + setNow: (n) => (now = n), + setRestartCore: value => {options.restartCore=value;}, + }; +} +test('backup coordinator survives reconstruction between every phase and waits for offsite verification', async (t) => { + const f = fixture(t); + f.manager.command(f.session, { + action: 'backup', + organizationId: 'org_dsp', + idempotencyKey: 'worker:manual:123456', + }); + await f.tick(); + const job = f.finish(); + delete f.remote.backups[job.backup_id]; + await f.tick(); + await f.tick(); + assert.equal(f.request().phase, 'uploading'); + f.remote.backups[job.backup_id] = { status: 'verified' }; + await f.tick(); + assert.equal(f.request().status, 'completed'); +}); +test('complete restore coordinates suspend, metadata restore and verified resume without changing release', async (t) => { + const f = fixture(t), + id = await f.backup(); + f.store.db.prepare("UPDATE organizations SET name='Renamed DSP' WHERE id='org_dsp'").run(); + // Name is business metadata; the physical/runtime identity is unchanged. + f.manager.command(f.session, { + action: 'restore', + organizationId: 'org_dsp', + backupId: id, + confirmation: 'Renamed DSP', + idempotencyKey: 'worker:restore:123456', + }); + await f.tick(); + assert.equal(f.request().phase, 'stopping'); + f.finish(); + await f.tick(); + assert.equal(f.request().phase, 'restoring'); + f.finish(); + await f.tick(); + assert.equal(f.request().phase, 'starting'); + assert.equal(f.store.organization('org_dsp').name, 'Example DSP'); + f.finish(); + await f.tick(); + assert.equal(f.request().status, 'completed'); + assert.equal(f.store.installationControl('org_dsp').status, 'ready'); + assert.equal(f.store.installationControl('org_dsp').releaseId, 'dispatch_current_1'); +}); +test('failed restored-runtime health returns data and metadata to the safety backup and resumes previous DSP', async (t) => { + const f = fixture(t), + id = await f.backup(); + f.store.db.prepare("UPDATE organizations SET name='Working DSP' WHERE id='org_dsp'").run(); + f.manager.command(f.session, { + action: 'restore', + organizationId: 'org_dsp', + backupId: id, + confirmation: 'Working DSP', + idempotencyKey: 'worker:recover:123456', + }); + await f.tick(); + f.finish(); + await f.tick(); + f.finish(); + await f.tick(); + f.finish('failed'); + await f.tick(); + assert.equal(f.request().phase, 'recovering'); + f.finish(); + await f.tick(); + assert.equal(f.request().phase, 'restarting_previous'); + assert.equal(f.store.organization('org_dsp').name, 'Working DSP'); + f.finish(); + await f.tick(); + assert.equal(f.request().failure_code, 'restore_recovered_previous'); + assert.equal(f.store.installationControl('org_dsp').status, 'ready'); +}); +test('missing remote download times out without stopping the DSP', async (t) => { + const f = fixture(t), + id = await f.backup(); + f.remote.backups[id].localReady = false; + f.restore(id); + f.setNow(4000000); + await f.tick(); + assert.equal(f.request().failure_code, 'backup_download_timeout'); + assert.equal(f.store.installationControl('org_dsp').status, 'ready'); +}); +test('Core snapshot is a consistent SQLite backup and completion requires an archive proof', async (t) => { + const f = fixture(t); + f.manager.enqueue('core', null, 'worker:core:123456'); + await f.tick(); + const row = f.request(); + assert.equal(row.phase, 'uploading'); + const { DatabaseSync } = require('node:sqlite'), + db = new DatabaseSync( + path.join(f.root, 'backups/scheduled-core', row.id, 'access-control-before.sqlite3'), + { readOnly: true }, + ); + assert.equal(db.prepare('PRAGMA quick_check').get().quick_check, 'ok'); + assert.equal(db.prepare('SELECT name FROM organizations').get(), undefined); + assert.equal(db.prepare("SELECT email FROM users WHERE platform_role='owner'").get().email,'owner@example.test'); + db.close(); + f.remote.backups[row.id] = { status: 'verified' }; + await f.tick(); + assert.equal(f.request().status, 'completed'); +}); + +test('metadata commit failure queues safety recovery before permitting a restart', async (t) => { + const f = fixture(t), + id = await f.backup(); + f.restore(id); + await f.tick(); + f.finish(); + await f.tick(); + f.finish(); + const row = f.store.db + .prepare('SELECT metadata_json FROM platform_backup_records WHERE id=?') + .get(id); + const metadata = JSON.parse(row.metadata_json); + metadata.stations[0].unknown_column = 'invalid'; + f.store.db + .prepare('UPDATE platform_backup_records SET metadata_json=? WHERE id=?') + .run(JSON.stringify(metadata), id); + await f.tick(); + assert.equal(f.request().phase, 'recovering'); + assert.equal(f.store.organization('org_dsp').status, 'suspended'); + f.finish(); + await f.tick(); + f.finish(); + await f.tick(); + assert.equal(f.request().failure_code, 'restore_recovered_previous'); +}); + + +test('Core backup waits for native DSP erasure before copying any shared identity records', async t => { + const f = fixture(t, 'native_service_v1'); + const authority = require('../../accounts/src/installation-lifecycle').createAccessInstallationLifecycleAuthority({ + store: f.store, organizationId: 'org_dsp', authorityScope: 'platform_removal', actorUserId: 'user_platform', destructionEnabled: true, + }); + f.store.db.prepare("UPDATE installations SET status='decommissioned' WHERE organization_id='org_dsp'").run(); + const job = authority.request({ operation: 'destroy', expectedRevision: 1, idempotencyKey: 'fixture:delete:before-backup' }); + f.manager.enqueue('core', null, 'worker:core:delete-wait'); + await f.tick(); + assert.equal(f.request().status, 'queued'); + assert.equal(fs.existsSync(path.join(f.root, 'backups/scheduled-core')), false); + f.store.db.prepare("UPDATE installation_lifecycle_jobs SET status='succeeded',result_json='{}',finished_at=? WHERE id=?").run(Date.now(), job.id); + f.store.db.prepare("UPDATE installations SET status='decommissioned'").run(); + await f.tick(); assert.equal(f.request().status, 'queued'); + f.store.eraseOrganization('org_dsp'); + await f.tick(); assert.equal(f.request().phase, 'uploading'); + const copied = new (require('node:sqlite').DatabaseSync)(path.join(f.root, 'backups/scheduled-core', f.request().id, 'access-control-before.sqlite3'), { readOnly: true }); + try { assert.equal(copied.prepare('SELECT count(*) AS n FROM organizations').get().n, 0); } finally { copied.close(); } +}); + +test('only matching root deletion receipts retire shared Core backup records', async t => { + const f = fixture(t); + const metadata = JSON.stringify({ schemaVersion: 1, name: 'Platform Core' }); + for (const id of ['breq_deleted_core', 'breq_retained_core', 'breq_mismatched_core']) + f.store.db.prepare("INSERT INTO platform_backup_records VALUES(?,NULL,'core',?,NULL,1,NULL,NULL)").run(id, metadata); + f.remote.backups.breq_deleted_core = { status: 'destroyed', metadataDigest: crypto.createHash('sha256').update(metadata).digest('hex') }; + f.remote.backups.breq_mismatched_core = { status: 'destroyed', metadataDigest: 'f'.repeat(64) }; + await f.tick(); + const rows = f.store.db.prepare('SELECT id,deleted_at FROM platform_backup_records ORDER BY id').all(); + assert.notEqual(rows.find(r => r.id === 'breq_deleted_core').deleted_at, null); + assert.equal(rows.find(r => r.id === 'breq_retained_core').deleted_at, null); + assert.equal(rows.find(r => r.id === 'breq_mismatched_core').deleted_at, null); +}); + +test('Core snapshot omits tenant records and registration secrets; Core restore preserves DSP state',async t=>{ + const f=fixture(t);fs.mkdirSync(path.join(f.root,'secrets/oci-runtime-agents'),{recursive:true,mode:0o700});fs.writeFileSync(path.join(f.root,'secrets/oci-runtime-agents/runtime_dsp.token'),'DSP-SECRET-MUST-NOT-ENTER-CORE'); + f.manager.command(f.session,{action:'backup',scope:'core',idempotencyKey:'core:isolation:backup'});await f.tick(); + const backupRow=f.request(),source=path.join(f.root,'backups/scheduled-core',backupRow.id),record=f.store.db.prepare('SELECT * FROM platform_backup_records WHERE id=?').get(backupRow.id); + const bytes=fs.readFileSync(path.join(source,'access-control-before.sqlite3'));assert.equal(bytes.includes(Buffer.from('Example DSP')),false);assert.equal(fs.existsSync(path.join(source,'core-files/secrets/oci-runtime-agents')),false); + f.remote.backups[backupRow.id]={status:'verified',localReady:true,metadataDigest:crypto.createHash('sha256').update(record.metadata_json).digest('hex')};await f.tick(); + f.manager.command(f.session,{action:'settings',scope:'system',revision:1,settings:{...require('../../accounts/src/backup-schedule').DEFAULT_BACKUP_SETTINGS,enabled:true,time:'23:59',timezone:'UTC'},idempotencyKey:'core:system:unchanged'}); + f.store.db.prepare("UPDATE users SET first_name='Changed' WHERE id='user_platform'").run();f.store.db.prepare("UPDATE organizations SET name='DSP stays changed' WHERE id='org_dsp'").run(); + const dspBefore=JSON.stringify(f.store.db.prepare('SELECT * FROM installations').all()); + f.manager.command(f.session,{action:'restore',scope:'core',backupId:backupRow.id,confirmation:'Platform Core',idempotencyKey:'core:isolation:restore'});await f.tick(); + const restore=f.request();assert.equal(restore.phase,'uploading');assert.equal(f.store.userById('user_platform').first_name,'Changed'); + f.remote.backups[restore.id]={status:'verified',localReady:true};await f.tick(); + assert.equal(f.request().status,'completed');assert.equal(f.store.userById('user_platform').first_name,'Platform');assert.equal(f.store.organization('org_dsp').name,'DSP stays changed');assert.equal(JSON.stringify(f.store.db.prepare('SELECT * FROM installations').all()),dspBefore);assert.equal(f.manager.settings('system').enabled,true); +}); + +for (const shared of [false, true]) for (const connected of [true, false]) test(`full-system disaster recovery preserves isolated DSP readiness with Paycom ${connected ? 'connected' : 'unconnected'} and ${shared ? 'shared' : 'self-contained'} releases`,async t=>{ + const f=fixture(t,'native_service_v1'); + const evidence={jobId:'job_cold_activation',runtimeKey:'runtime_dsp',marker:'readiness retained'}; + if (connected) f.store.db.prepare(`INSERT INTO installation_activation_jobs VALUES('job_cold_activation','org_dsp','resume','succeeded','ready',1,1,'runtime_dsp','fixture','fixture:activation:prior','worker_prior',1,NULL,'paycom','paycom-main',1000,?,'synthetic-digest',NULL,1000,1000,1000,1000)`).run(JSON.stringify(evidence)); + f.store.db.prepare(`INSERT INTO installation_provisioning_requests(id,organization_id,authority_scope,idempotency_key,request_json,starting_state,installation_revision,manifest_revision,runtime_key,status,provisioner_job_id,created_at,updated_at,finished_at) VALUES('prq_cold_fixture','org_dsp','fixture','fixture:cold:provision',?,'pending',2,1,'runtime_dsp','completed','job_cold_provision',1000,1000,1000)`).run(JSON.stringify({operation:'provision',idempotencyKey:'fixture:cold:provision',expectedRevision:1})); + f.manager.enqueue('core',null,'system:cold:core');await f.tick(); + const row=f.request(),capsule=require('../src/recovery-capsule'),{opaqueRuntimeSuffix,hostAccountName,HOST_TENANT_ROOT}=require('../../runtime-host-identity'); + const localRoot='/home/core_fixture/dispatch',coreAccount={name:'core_fixture',uid:process.geteuid(),gid:process.getegid(),home:'/home/core_fixture'}; + const metadata={kind:'core',scope:'core',platform:'ubuntu-24.04-amd64',localRoot,accounts:[coreAccount],services:[],installations:[],packages:[]}; + const source=path.join(f.root,'core-data');fs.mkdirSync(path.join(source,'access-control'),{recursive:true}); + fs.copyFileSync(path.join(f.root,'backups/scheduled-core',row.id,'access-control-before.sqlite3'),path.join(source,'access-control/access-control.sqlite3')); + const coreBundle=path.join(f.root,'core-bundle');fs.mkdirSync(coreBundle); + const releaseSource=path.join(f.root,'release-source'),releaseTarget='/opt/dispatch-runtime/releases/dispatch_current_1'; + fs.mkdirSync(releaseSource);fs.writeFileSync(path.join(releaseSource,'runtime'),'shared immutable runtime'); + const coreProof=capsule.capture(path.join(coreBundle,'recovery'),[{source,target:localRoot+'/data'},...(shared?[{source:releaseSource,target:releaseTarget}]:[])],metadata,new Set([process.geteuid()])); + const key='runtime_dsp',suffix=opaqueRuntimeSuffix(key),runtimeRoot=HOST_TENANT_ROOT+'/'+suffix; + const dspAccount={name:hostAccountName(key),uid:20001,gid:20001,home:runtimeRoot+'/home'}; + const dspBundle=path.join(f.root,'dsp-bundle'),dspSource=path.join(f.root,'dsp-source');fs.mkdirSync(dspBundle);fs.mkdirSync(dspSource);fs.writeFileSync(path.join(dspSource,'data.txt'),'DSP data independently retained'); + const dspMetadata=require('../../accounts/src/backup-metadata').captureDspMetadata(f.store,'org_dsp'); + fs.writeFileSync(path.join(dspBundle,'dsp.json'),JSON.stringify({kind:'dsp',metadata:dspMetadata})); + const installation={organization_id:'org_dsp',runtime_key:key,release_id:'dispatch_current_1',backend:'native_service_v1',status:'ready',organization_status:'active'}; + let dspProof=capsule.capture(path.join(dspBundle,'recovery'),[{source:dspSource,target:runtimeRoot}],{...metadata,kind:'dsp',scope:undefined,organizationId:'org_dsp',accounts:[coreAccount,dspAccount],installations:[installation]},new Set([process.geteuid()])); + if(shared) { + const artifact=path.join(f.root,'release-artifact'),a=capsule.capture(artifact,[{source:releaseSource,target:releaseTarget}],{},new Set([process.geteuid()])); + const artifacts=require('../src/recovery-artifacts'); + dspProof=artifacts.append(path.join(dspBundle,'recovery'),dspProof,[{directory:artifact,root:releaseTarget,digest:a.sha256,snapshotId:'a'.repeat(64)}]); + fs.rmSync(releaseSource,{recursive:true}); + artifacts.hydrate(path.join(dspBundle,'recovery'),dspProof.sha256,(_,args)=>fs.cpSync(artifact,path.join(args[args.indexOf('--target')+1],'artifact'),{recursive:true})); + } + const result=require('../src/assemble-system-recovery').assembleSystemRecovery({components:[{kind:'core',directory:coreBundle,digest:coreProof.sha256},{kind:'dsp',organizationId:'org_dsp',directory:dspBundle,digest:dspProof.sha256}],destination:path.join(f.root,'assembled')}); + const manifest=JSON.parse(fs.readFileSync(path.join(result.directory,'recovery.json'))),entry=manifest.entries.find(e=>e.path.endsWith('access-control.sqlite3')); + const {DatabaseSync}=require('node:sqlite'),restored=new DatabaseSync(path.join(result.directory,entry.payload),{readOnly:true}); + try{assert.equal(restored.prepare('SELECT name FROM organizations').get().name,'Example DSP');assert.equal(restored.prepare('SELECT count(*) n FROM installations').get().n,1);assert.equal(restored.prepare('PRAGMA foreign_key_check').all().length,0);assert.equal(restored.prepare('SELECT status FROM installation_provisioning_requests').get().status,'completed');if(connected)assert.deepEqual(JSON.parse(restored.prepare('SELECT evidence_json FROM installation_activation_jobs').get().evidence_json),evidence);else assert.equal(restored.prepare('SELECT count(*) n FROM installation_activation_jobs').get().n,0);}finally{restored.close();} + assert.equal(manifest.metadata.installations.length,1);assert.equal(manifest.metadata.scope,'system'); + assert.equal(f.store.organization('org_dsp').name,'Example DSP'); +}); + +test('Core health failure restores its safety snapshot and is never reported as completed',async t=>{ + const f=fixture(t);f.manager.enqueue('core',null,'core:health:source');await f.tick();const backup=f.request(),record=f.store.db.prepare('SELECT * FROM platform_backup_records WHERE id=?').get(backup.id); + f.remote.backups[backup.id]={status:'verified',localReady:true,metadataDigest:crypto.createHash('sha256').update(record.metadata_json).digest('hex')};await f.tick(); + f.store.db.prepare("UPDATE users SET first_name='Before restore' WHERE id='user_platform'").run(); + f.manager.command(f.session,{action:'restore',scope:'core',backupId:backup.id,confirmation:'Platform Core',idempotencyKey:'core:health:restore'});await f.tick(); + const request=f.request();f.remote.backups[request.id]={status:'verified',localReady:true};let calls=0;f.setRestartCore(async()=>{if(++calls===1)throw Error('health_failed');}); + await f.tick();assert.equal(f.request().phase,'verifying_core');assert.equal(f.store.userById('user_platform').first_name,'Platform');await f.tick(); + assert.equal(calls,2);assert.equal(f.request().status,'failed');assert.equal(f.request().failure_code,'restore_recovered_previous');assert.equal(f.store.userById('user_platform').first_name,'Before restore');assert.equal(f.store.organization('org_dsp').status,'active'); +}); + +test('Core archive integrity covers configuration and secrets as well as its database',async t=>{ + const f=fixture(t);fs.mkdirSync(path.join(f.root,'config'));fs.writeFileSync(path.join(f.root,'config/dashboard.env'),'PORT=4100\n'); + f.manager.enqueue('core',null,'core:integrity:source');await f.tick();const directory=path.join(f.root,'backups/scheduled-core',f.request().id),{verifySnapshot}=require('../src/offsite-backup'); + assert.ok(verifySnapshot(directory,process.geteuid()));fs.writeFileSync(path.join(directory,'core-files/config/dashboard.env'),'PORT=9999\n'); + assert.throws(()=>verifySnapshot(directory,process.geteuid()),/backup_corrupt/); +}); + +test('an incomplete full-system set continues tracking late components and deletes all of them',async t=>{ + const f=fixture(t);f.manager.command(f.session,{action:'backup',scope:'system',idempotencyKey:'system:incomplete:source'}); + const set=f.manager.view().sets[0]; + f.store.db.prepare("UPDATE platform_backup_requests SET status='failed',phase='failed' WHERE kind='core'").run(); + await f.tick();assert.equal(f.manager.view().sets[0].status,'incomplete'); + assert.throws(()=>f.manager.command(f.session,{action:'delete',scope:'system',setId:set.id,confirmation:'Full system',idempotencyKey:'system:incomplete:busy'}),e=>e.code==='backup_operation_in_progress'); + const job=f.finish();await f.tick();await f.tick();await f.tick(); + const member=f.manager.view().sets[0].members.find(m=>m.organizationId==='org_dsp');assert.equal(member.backupId,job.backup_id);assert.equal(member.status,'completed'); + f.manager.command(f.session,{action:'delete',scope:'system',setId:set.id,confirmation:'Full system',idempotencyKey:'system:incomplete:delete'}); + assert.equal(f.store.db.prepare('SELECT backup_id FROM backup_deletions').get().backup_id,job.backup_id); + f.remote.backups[job.backup_id].status='destroyed';await f.tick();assert.equal(f.manager.view().sets[0].status,'deleting'); + f.remote.sets={[set.id]:{status:'deleted'}};await f.tick();assert.equal(f.manager.view().sets[0].status,'deleted'); +}); +test('permanent DSP deletion preserves an isolated Core snapshot awaiting its first upload',async t=>{ + const f=fixture(t);f.manager.enqueue('core',null,'core:pending:purge');await f.tick(); + const record=f.store.db.prepare("SELECT * FROM platform_backup_records WHERE kind='core'").get(); + const work=path.join(f.root,'offsite');fs.mkdirSync(path.join(work,'archives'),{recursive:true}); + const removed=[];const result=await require('../src/dsp-backup-deletion').purgeDspBackups({ + config:{localRoot:f.root,coreUid:process.geteuid(),prefix:'dispatch'},jobs:[{id:'life_'+'a'.repeat(32),organization_id:'org_dsp',runtime_key:'runtime_dsp',authority_scope:'platform_removal',stage_receipts_json:JSON.stringify({__request:JSON.stringify({operation:'destroy'})})}], + records:[record],backups:[],storage:{withDeletionAccess:async(_,call)=>call(),removePermanent:async row=>removed.push(row)},run:()=>[],workRoot:work,receiptRoot:work,ownerUid:process.geteuid() + }); + assert.equal(result.failed,0);assert.deepEqual(removed,[]);assert.equal(fs.existsSync(path.join(f.root,'backups/scheduled-core',record.id,'manifest.json')),true); +}); + +test('Core compensation survives a worker exit and removes files and owners introduced by the failed restore',async t=>{ + const f=fixture(t); + f.store.insertUser({id:'user_old_owner',email:'old-owner@example.test',firstName:'Old',lastName:'Owner',passwordHash:'synthetic',platformRole:'owner',timestamp:1000}); + fs.mkdirSync(path.join(f.root,'config'));const file=path.join(f.root,'config/dashboard.env');fs.writeFileSync(file,'saved-config'); + f.manager.enqueue('core',null,'core:crash:source');await f.tick();const source=f.request(),record=f.store.db.prepare('SELECT * FROM platform_backup_records WHERE id=?').get(source.id); + f.remote.backups[source.id]={status:'verified',localReady:true,metadataDigest:crypto.createHash('sha256').update(record.metadata_json).digest('hex')};await f.tick(); + f.store.db.prepare("DELETE FROM users WHERE id='user_old_owner'").run();fs.unlinkSync(file); + f.manager.command(f.session,{action:'restore',scope:'core',backupId:source.id,confirmation:'Platform Core',idempotencyKey:'core:crash:restore'});await f.tick(); + const request=f.request();f.remote.backups[request.id]={status:'verified',localReady:true};f.setRestartCore(async()=>{});await f.tick(); + assert.ok(f.store.userById('user_old_owner'));assert.equal(fs.readFileSync(file,'utf8'),'saved-config'); + const child=require('node:child_process').spawnSync(process.execPath,['--no-warnings','-e',` + const path=require('node:path'),root=process.argv[1],remote=JSON.parse(process.argv[2]); + const {AccessStore}=require('./dispatch-core/access-control/src/store'); + const store=new AccessStore({databaseRoot:path.join(root,'data/access-control'),database:path.join(root,'data/access-control/access-control.sqlite3')}); + let calls=0;require('./dispatch-core/provisioner/src/platform-backup-worker').createPlatformBackupWorker({store,localRoot:root,archive:()=>remote,clock:()=>100000,restartCore:async()=>{if(++calls===1)throw Error('health failure');process.exit(86);}}).tick().catch(()=>process.exit(87)); + `,f.root,JSON.stringify(f.remote)],{cwd:path.resolve(__dirname, "../../.."),encoding:'utf8'}); + assert.equal(child.status,86,child.stderr);assert.equal(f.request().phase,'recovering_core');assert.equal(f.store.userById('user_old_owner'),null);assert.equal(fs.existsSync(file),false); + await f.tick();assert.equal(f.request().status,'failed');assert.equal(f.request().failure_code,'restore_recovered_previous');assert.equal(f.store.organization('org_dsp').name,'Example DSP'); +}); diff --git a/core/core/installations/tests/platform-core-update.test.js b/core/core/installations/tests/platform-core-update.test.js new file mode 100644 index 0000000..7999143 --- /dev/null +++ b/core/core/installations/tests/platform-core-update.test.js @@ -0,0 +1,102 @@ +'use strict'; +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync, spawn } = require('node:child_process'); +const { once } = require('node:events'); +const test = require('node:test'); +const { AccessStore } = require('../../accounts/src/store'); +const { openPlatformUpdateStore } = require('../src/platform-update-store'); +const { verifyCoreArtifact, executeCoreStage } = require('../src/platform-core-update'); + +if (process.geteuid() !== 0) { + test('verified Core executable package rejects tampering and mismatched receipts', t => { + if (spawnSync('/usr/bin/sudo', ['-n', '/usr/bin/true']).status !== 0) return t.skip('requires noninteractive sudo for immutable fixture'); + const result = spawnSync('/usr/bin/sudo', ['-n', '/usr/bin/node', '--no-warnings', '--test', __filename], { encoding: 'utf8', timeout: 30_000 }); + assert.equal(result.status, 0, result.stdout + result.stderr); + }); +} else { + test('synthetic Core entrypoints verify identity and reject modified files, symlinks and extra files', t => { + const parents = []; + for (const dir of ['/opt/dispatch-platform', '/opt/dispatch-platform/releases']) { + if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { mode: 0o755 }); parents.push(dir); } + const stat = fs.lstatSync(dir); + assert.ok(stat.isDirectory() && !stat.isSymbolicLink() && stat.uid === 0 && !(stat.mode & 0o022)); + } + const id = `dispatch_fixture_${crypto.randomBytes(8).toString('hex')}`; + const releaseRoot = `/opt/dispatch-platform/releases/${id}`; + fs.mkdirSync(releaseRoot, { mode: 0o755 }); + t.after(() => { fs.rmSync(releaseRoot, { recursive: true }); for (const dir of parents.reverse()) fs.rmdirSync(dir); }); + const root = `${releaseRoot}/core-artifact`; fs.mkdirSync(root, { mode: 0o755 }); + const sha = data => crypto.createHash('sha256').update(data).digest('hex'); + const script = `#!/usr/bin/node\nlet data='';process.stdin.on('data',c=>data+=c).on('end',()=>{const x=JSON.parse(data);console.log(JSON.stringify({ok:true,releaseId:x.releaseId,version:'0.0.2',sourceCommit:x.sourceCommit}));});\n`; + const files = ['apply', 'verify'].map(name => { + fs.writeFileSync(`${root}/${name}`, script, { mode: 0o555 }); + return { path: name, mode: '555', sha256: sha(script) }; + }); + const manifest = JSON.stringify({ schemaVersion: 1, releaseId: id, sourceCommit: 'a'.repeat(40), files }); + fs.writeFileSync(`${root}/manifest.json`, manifest, { mode: 0o444 }); + const release = { version: '0.0.2', sourceCommit: 'a'.repeat(40), core: { artifactPath: root, manifestSha256: sha(manifest) } }; + verifyCoreArtifact(id, release); + executeCoreStage('apply', id, release, `rollout_${'b'.repeat(32)}`, 1); + executeCoreStage('verify', id, release, `rollout_${'b'.repeat(32)}`, 1); + assert.throws(() => executeCoreStage('verify', id, { ...release, version: '0.0.3' }, `rollout_${'b'.repeat(32)}`, 1)); + fs.writeFileSync(`${root}/unexpected`, 'extra', { mode: 0o444 }); + assert.throws(() => verifyCoreArtifact(id, release)); fs.unlinkSync(`${root}/unexpected`); + fs.chmodSync(`${root}/apply`, 0o755); + assert.throws(() => verifyCoreArtifact(id, release)); fs.chmodSync(`${root}/apply`, 0o555); + fs.writeFileSync(`${root}/apply`, script + '// altered'); + assert.throws(() => verifyCoreArtifact(id, release)); + fs.unlinkSync(`${root}/apply`); fs.symlinkSync(`${root}/verify`, `${root}/apply`); + assert.throws(() => verifyCoreArtifact(id, release)); + }); +} + +test('independent updater reopens stable protocol storage across application schema changes', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-core-store-')); fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const access = new AccessStore({ databaseRoot: `${root}/access`, database: `${root}/access/access-control.sqlite3` }); + access.close(); + const store = openPlatformUpdateStore(`${root}/access`); t.after(() => store.close()); + store.db.exec('PRAGMA user_version=999; CREATE TABLE future_core_data(id TEXT) STRICT;'); + store.refresh(); + assert.equal(store.db.prepare('PRAGMA user_version').get().user_version, 999); + assert.equal(store.db.prepare('SELECT count(*) n FROM platform_rollout_core').get().n, 0); + assert.throws(() => store.transaction(() => { store.db.exec("INSERT INTO future_core_data VALUES('rollback')"); throw new Error('interrupted'); }), /interrupted/); + assert.equal(store.db.prepare('SELECT count(*) n FROM future_core_data').get().n, 0); +}); + +test('updater CLI excludes concurrent processes and runs after the persistent lock is released', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-core-lock-')); fs.chmodSync(root, 0o700); + const access = new AccessStore({ databaseRoot: `${root}/access`, database: `${root}/access/access-control.sqlite3` }); access.close(); + const lock = `${root}/access/platform-update.lock`; fs.writeFileSync(lock, '', { mode: 0o600 }); + const holder = spawn('/usr/bin/flock', [lock, '/usr/bin/node', '-e', "console.log('locked');process.stdin.resume()"], { stdio: ['pipe', 'pipe', 'pipe'] }); + t.after(() => { holder.stdin.end(); fs.rmSync(root, { recursive: true, force: true }); }); + await once(holder.stdout, 'data'); + const run = () => spawnSync('/usr/bin/node', ['--no-warnings', path.resolve(__dirname, "../bin/dispatch-platform-update")], { + encoding: 'utf8', timeout: 10_000, env: { PATH: '/usr/bin:/bin', DISPATCH_ACCESS_CONTROL_DATABASE_ROOT: `${root}/access` }, + }); + const excluded = run(); assert.equal(excluded.status, 0, excluded.stderr); assert.equal(excluded.stdout, ''); + holder.stdin.end(); await once(holder, 'exit'); + const idle = run(); assert.equal(idle.status, 0, idle.stderr); assert.equal(JSON.parse(idle.stdout).status, 'idle'); +}); + +test('recovery watchdog cannot touch services or storage while the independent updater owns its lock', async t => { + if (process.geteuid() === 0) return t.skip('watchdog runs as the dashboard service account'); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-recovery-lock-')); fs.chmodSync(root, 0o700); + fs.mkdirSync(path.join(root, 'data'), { mode: 0o700 }); + const databaseRoot = path.join(root, 'data/access-control'); + const access = new AccessStore({ databaseRoot, database: path.join(databaseRoot, 'access-control.sqlite3') }); access.close(); + const lock = path.join(databaseRoot, 'platform-update.lock'); fs.writeFileSync(lock, '', { mode: 0o600 }); + const holder = spawn('/usr/bin/flock', [lock, '/usr/bin/node', '-e', "console.log('locked');process.stdin.resume()"], { stdio: ['pipe', 'pipe', 'pipe'] }); + t.after(() => { holder.stdin.end(); fs.rmSync(root, { recursive: true, force: true }); }); + await once(holder.stdout, 'data'); + const result = spawnSync('/usr/bin/node', ['--no-warnings', path.resolve(__dirname, "../bin/dispatch-core-recover"), + 'watch', root, 'rollout_' + 'a'.repeat(32)], { encoding: 'utf8', timeout: 5000 }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, ''); assert.equal(result.stderr, ''); + assert.equal(fs.existsSync(path.join(root, 'backups')), false); + holder.stdin.end(); await once(holder, 'exit'); +}); diff --git a/core/core/installations/tests/platform-release-catalog.test.js b/core/core/installations/tests/platform-release-catalog.test.js new file mode 100644 index 0000000..1c6673d --- /dev/null +++ b/core/core/installations/tests/platform-release-catalog.test.js @@ -0,0 +1,16 @@ +'use strict'; +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { platformRelease } = require('../src/platform-release-catalog'); + +test('a platform release binds readable notes and Core artifacts to the same runtime commit and digest', () => { + const runtime = { sourceCommit: 'a'.repeat(40), imageDigest: `sha256:${'b'.repeat(64)}` }; + const release = { version: '0.0.2', publishedAt: '2026-09-05T00:00:00.000Z', sourceCommit: runtime.sourceCommit, + runtimeImageDigest: runtime.imageDigest, changelog: [{ kind: 'fixed', title: 'Clearer setup messages', description: '' }], + core: { artifactPath: '/opt/dispatch-platform/releases/dispatch_update_2/core-artifact', manifestSha256: 'c'.repeat(64) } }; + assert.equal(platformRelease('dispatch_update_2', release, runtime).version, '0.0.2'); + assert.throws(() => platformRelease('dispatch_update_2', release, { ...runtime, sourceCommit: 'd'.repeat(40) }), /platform_release_invalid/); + assert.throws(() => platformRelease('dispatch_update_2', { ...release, core: { ...release.core, artifactPath: '/tmp/untrusted' } }, runtime), /platform_release_invalid/); + assert.throws(() => platformRelease('dispatch_update_2', { ...release, version: 'dispatch_current_1' }, runtime), /platform_release_invalid/); + assert.throws(() => platformRelease('dispatch_update_2', { ...release, changelog: [] }, runtime), /platform_release_invalid/); +}); diff --git a/core/core/installations/tests/portable-node.test.js b/core/core/installations/tests/portable-node.test.js new file mode 100644 index 0000000..a31546f --- /dev/null +++ b/core/core/installations/tests/portable-node.test.js @@ -0,0 +1,30 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { bundleNode } = require('../src/portable-node'); +test('portable Node executes SQLite and can be backed up again after restoration', { skip: !fs.existsSync('/usr/bin/patchelf') }, t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-portable-node-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const first = path.join(root, 'first'), second = path.join(root, 'second'); + const priorUmask = process.umask(0o077); + let version; + try { version = bundleNode(process.execPath, first, first); } + finally { process.umask(priorUmask); } + function assertDirectories(directory) { + assert.equal(fs.statSync(directory).mode & 0o777, 0o755, directory); + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const file = path.join(directory, entry.name); + assert.equal(fs.statSync(file).uid, process.geteuid(), file); + if (entry.isDirectory()) assertDirectories(file); + } + } + assertDirectories(first); + assert.equal(bundleNode(path.join(first, 'node'), second, first), version); + for (const runtime of [first, second]) { + const result = spawnSync(path.join(runtime, 'lib/ld-linux-x86-64.so.2'), ['--library-path', path.join(runtime, 'lib'), path.join(runtime, 'node'), '--no-warnings', '-e', + "const d=new (require('node:sqlite').DatabaseSync)(':memory:'); console.log(d.prepare('SELECT 42 AS answer').get().answer); d.close()"], { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, '42\n'); + } +}); diff --git a/core/core/installations/tests/r2-backup-storage.test.js b/core/core/installations/tests/r2-backup-storage.test.js new file mode 100644 index 0000000..92b0067 --- /dev/null +++ b/core/core/installations/tests/r2-backup-storage.test.js @@ -0,0 +1,166 @@ +'use strict'; +const test = require('node:test'), + assert = require('node:assert/strict'); +const { createR2BackupStorage } = require('../src/r2-backup-storage'); +const config = { accountId: 'a'.repeat(32), bucket: 'dispatch-test' }; +const credentials = { + credential: { accessKeyId: 'synthetic-id', secretAccessKey: 'synthetic-secret' }, + token: 'synthetic-token', +}; +test('storage usage sums paginated encrypted object sizes and deduplicates archive identity across tiers',async()=>{ + const id='backup_'+'a'.repeat(32),set='breq_'+'b'.repeat(32),calls=[]; + const contents=(key,size)=>`${encodeURIComponent(key)}${size}`; + const storage=createR2BackupStorage({...config,prefix:'dispatch'},{credentials,request:async(url,method)=>{ + assert.equal(method,'GET');const query=new URL(url).searchParams;calls.push(query.get('prefix'));assert.equal(query.has('delimiter'),false); + const prefix=query.get('prefix');let body; + if(prefix==='archives/'&&!query.has('continuation-token'))body=contents(`archives/all/${id}/data/abc`,100)+'truea&b'; + else if(prefix==='archives/'){assert.equal(query.get('continuation-token'),'a&b');body=contents(`archives/30/${id}/config`,25)+'false';} + else body=contents(prefix==='sets/'?`sets/${set}/config`:prefix==='recovery-artifacts/'?`recovery-artifacts/${'a'.repeat(64)}/config`:'dispatch/config',10)+'false'; + return {status:200,body:`${body}`}; + }}); + assert.deepEqual(await storage.usage(),{archives:{[id]:125},sets:{[set]:10},legacyBytes:10,artifactBytes:10});assert.equal(calls.length,5); +}); +test('storage usage rejects malformed, duplicate and incomplete listings instead of publishing partial totals',async()=>{ + const key=`archives/all/backup_${'a'.repeat(32)}/config`,item=`${key}12`; + for(const body of [item, item+item+'false',item.replace('12','')+'false',item+'true']){ + const storage=createR2BackupStorage(config,{credentials,request:async()=>({status:200,body:`${body}`})});await assert.rejects(storage.usage()); + } +}); +test('R2 retention adds isolated archive tiers without weakening legacy or other bucket locks', async () => { + let rules = [ + { id: 'legacy', enabled: true, prefix: 'dispatch/data/', condition: { type: 'Indefinite' } }, + ], + writes = 0; + const storage = createR2BackupStorage(config, { + credentials, + request: async (url, method, headers, body) => { + assert.ok(url.endsWith('/lock')); + if (method === 'PUT') { + rules = JSON.parse(body).rules; + writes++; + } + return { + status: 200, + body: JSON.stringify({ success: true, result: { rules: structuredClone(rules) } }), + }; + }, + }); + await storage.ensureLocks(); + await storage.ensureLocks(); + assert.equal(writes, 1); + assert.equal(rules.length, 7); + assert.deepEqual(rules.find(r => r.id === 'dispatch-recovery-artifacts').condition, { type: 'Indefinite' }); + assert.deepEqual(rules[0], { + id: 'legacy', + enabled: true, + prefix: 'dispatch/data/', + condition: { type: 'Indefinite' }, + }); + assert.equal(rules.find((r) => r.id === 'dispatch-archives-30').condition.maxAgeSeconds, 2592000); + rules.find((r) => r.id === 'dispatch-archives-30').enabled = false; + await assert.rejects(() => storage.ensureLocks()); +}); +test('expiration can only remove the exact finite archive after its retention deadline', async () => { + const id = `backup_${'a'.repeat(32)}`, + prefix = `archives/30/${id}/`, + requests = []; + let listed = false; + const storage = createR2BackupStorage(config, { + credentials, + request: async (url, method, headers) => { + requests.push([url, method]); + assert.ok(headers.Authorization.startsWith('AWS4-HMAC-SHA256 ')); + if (method === 'DELETE') { + assert.ok(new URL(url).pathname.startsWith('/dispatch-test/' + prefix)); + return { status: 204, body: '' }; + } + assert.equal(new URL(url).searchParams.get('prefix'), prefix); + const body = listed + ? '' + : `${encodeURIComponent(prefix + 'data/ab/abcdef')}`; + listed = true; + return { status: 200, body }; + }, + }); + await assert.rejects(() => + storage.removeExpired({ id, retentionDays: null, expiresAt: 100 }, 200), + ); + await assert.rejects(() => storage.removeExpired({ id, retentionDays: 30, expiresAt: 100 }, 99)); + assert.equal(requests.length, 0); + await storage.removeExpired({ id, retentionDays: 30, expiresAt: 100 }, 101); + assert.equal(requests.filter((r) => r[1] === 'DELETE').length, 1); +}); +test('malformed or out-of-prefix listing never triggers deletion', async () => { + let deleted = 0; + const storage = createR2BackupStorage(config, { + credentials, + request: async (_url, method) => { + if (method === 'DELETE') deleted++; + return { + status: 200, + body: 'dispatch%2Fdata%2Fprotected', + }; + }, + }); + await assert.rejects(() => + storage.removeExpired( + { id: `backup_${'b'.repeat(32)}`, retentionDays: 7, expiresAt: 100 }, + 200, + ), + ); + assert.equal(deleted, 0); +}); + +test('explicit deletion restores retention locks on failure and after process interruption', async t => { + const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-delete-locks-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const journalFile = path.join(root, 'locks.json'); + const original = [ + { id: 'archive', enabled: true, prefix: 'archives/all/', condition: { type: 'Indefinite' } }, + { id: 'unrelated', enabled: true, prefix: 'other/', condition: { type: 'Indefinite' } }, + ]; + let rules = structuredClone(original); + const storage = createR2BackupStorage({ ...config, prefix: 'dispatch' }, { + credentials, journalFile, ownerUid: process.geteuid(), + request: async (url, method, _headers, body) => { + assert.ok(url.endsWith('/lock')); + if (method === 'PUT') rules = JSON.parse(body).rules; + return { status: 200, body: JSON.stringify({ success: true, result: { rules } }) }; + }, + }); + await assert.rejects(storage.withDeletionAccess(['archives/all/'], async () => { + assert.deepEqual(rules, [original[1]]); + assert.ok(fs.existsSync(journalFile)); + throw Error('simulated storage failure'); + }), /simulated/); + assert.deepEqual(rules.sort((a,b)=>a.id.localeCompare(b.id)), original); + assert.equal(fs.existsSync(journalFile), false); + require('../src/release-delivery-files').atomic(journalFile, { accountId: config.accountId, bucket: config.bucket, rules: [original[0]] }); + rules = [original[1]]; + await storage.restoreDeletionLocks(); + assert.deepEqual(rules.sort((a,b)=>a.id.localeCompare(b.id)), original); + await assert.rejects(storage.withDeletionAccess(['other/'], async () => assert.fail('must not run'))); + rules.push({ id: 'administrator', enabled: true, prefix: 'archives/', condition: { type: 'Indefinite' } }); + await assert.rejects(storage.withDeletionAccess(['archives/all/'], async () => assert.fail('must not weaken broader lock'))); +}); + +test('permanent deletion verifies an indefinite archive is empty and refuses foreign keys', async () => { + const id = `backup_${'c'.repeat(32)}`, prefix = `archives/all/${id}/`; + let deleted = false; + const storage = createR2BackupStorage(config, { credentials, request: async (url, method) => { + if (method === 'DELETE') { assert.equal(new URL(url).pathname, `/dispatch-test/${prefix}config`); deleted = true; return { status: 204, body: '' }; } + assert.equal(new URL(url).searchParams.get('prefix'), prefix); + return { status: 200, body: `${deleted ? '' : `${encodeURIComponent(prefix+'config')}`}` }; + } }); + await storage.removePermanent({ id, retentionDays: null }); + assert.equal(deleted, true); + await assert.rejects(storage.removePermanent({ id: '../another-dsp', retentionDays: null })); +}); + +test('backup deletion cannot unlock shared recovery artifacts', async () => { + let called = false; + const storage = createR2BackupStorage(config, { credentials, request: async () => { called = true; throw Error('must not reach storage'); } }); + await assert.rejects(storage.withDeletionAccess(['recovery-artifacts/'], () => assert.fail('must not delete dependencies'))); + assert.equal(called, false); +}); diff --git a/core/core/installations/tests/recovery-artifacts.test.js b/core/core/installations/tests/recovery-artifacts.test.js new file mode 100644 index 0000000..1349073 --- /dev/null +++ b/core/core/installations/tests/recovery-artifacts.test.js @@ -0,0 +1,78 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'); +const capsule = require('../src/recovery-capsule'); +const { createArtifactStore, append, hydrate } = require('../src/recovery-artifacts'); +const { createRestic } = require('../src/offsite-backup'); + +test('two independent backups reuse one encrypted release and restore after local code and cache are removed', { skip: !fs.existsSync('/usr/bin/restic') }, t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-shared-artifacts-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const source = path.join(root, 'release'), cacheRoot = path.join(root, 'cache'); + fs.mkdirSync(source, { mode: 0o755 }); fs.writeFileSync(path.join(source, 'runtime'), Buffer.alloc(2 * 1024 * 1024, 71), { mode: 0o444 }); + const target = '/opt/dispatch-runtime/releases/dispatch_fixture', calls = []; + const run = (repository, args, cwd) => { + calls.push([repository, args]); + return createRestic({ PATH: '/usr/bin:/bin', RESTIC_PASSWORD: 'synthetic-test-password', + RESTIC_REPOSITORY: path.join(root, 'remote', path.basename(repository)) })(args, cwd); + }; + fs.mkdirSync(path.join(root, 'remote')); + const store = createArtifactStore({ accountId: 'a'.repeat(32), bucket: 'dispatch-test', environment: {} }, { + cacheRoot, ownerUid: process.geteuid(), runFactory: env => (args, cwd) => run(env.RESTIC_REPOSITORY, args, cwd), + }); + const a = store.prepare(source, target), b = store.prepare(source, target); + assert.equal(a.digest, b.digest); + assert.equal(calls.filter(([, args]) => args.includes('backup')).length, 1); + const bundles = []; + for (const label of ['first', 'second']) { + const data = path.join(root, label + '-data'); fs.mkdirSync(data); fs.writeFileSync(path.join(data, 'value'), label); + const directory = path.join(root, label), proof = capsule.capture(directory, [{ source: data, target: '/home/fixture/data' }], {}, new Set([process.geteuid()])); + const combined = append(directory, proof, [a]); + assert.equal(fs.readdirSync(path.join(directory, 'files')).length, 1); + bundles.push({ directory, proof: combined }); + } + // Ordinary backup deletion and release cleanup have no access to the artifact repository. + fs.rmSync(bundles[0].directory, { recursive: true }); + fs.rmSync(source, { recursive: true }); fs.rmSync(cacheRoot, { recursive: true }); + const selected = bundles[1]; + hydrate(selected.directory, selected.proof.sha256, (repository, args) => run(repository, args)); + const roots = new Set(['/home/fixture/data', target]); + capsule.materialize(selected.directory, path.join(root, 'restored'), selected.proof.sha256, roots); + assert.equal(fs.readFileSync(path.join(root, 'restored/home/fixture/data/value'), 'utf8'), 'second'); + assert.equal(fs.statSync(path.join(root, 'restored', target, 'runtime')).size, 2 * 1024 * 1024); + const manifest = JSON.parse(fs.readFileSync(path.join(selected.directory, 'recovery.json'))); + const entry = manifest.entries.find(e => e.artifact); + fs.unlinkSync(path.join(selected.directory, entry.payload)); + assert.throws(() => hydrate(selected.directory, '0'.repeat(64), () => assert.fail('must not download tampered manifest'))); + assert.throws(() => hydrate(selected.directory, selected.proof.sha256, () => { throw Error('missing shared artifact'); }), /missing shared artifact/); + assert.throws(() => capsule.verify(selected.directory, selected.proof.sha256, roots)); +}); + +test('shared artifact validation rejects altered restored bytes before materialization', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-shared-tamper-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const source = path.join(root, 'source'); fs.mkdirSync(source); fs.writeFileSync(path.join(source, 'runtime'), 'original'); + const artifact = path.join(root, 'artifact'), target = '/opt/dispatch-runtime/releases/dispatch_fixture'; + const a = capsule.capture(artifact, [{ source, target }], {}, new Set([process.geteuid()])); + const data = path.join(root, 'data'); fs.mkdirSync(data); fs.writeFileSync(path.join(data, 'value'), 'tenant'); + const directory = path.join(root, 'bundle'), p = capsule.capture(directory, [{ source: data, target: '/home/fixture/data' }], {}, new Set([process.geteuid()])); + const proof = append(directory, p, [{ directory: artifact, digest: a.sha256, root: target, snapshotId: 'a'.repeat(64) }]); + assert.throws(() => hydrate(directory, proof.sha256, (_, args) => { + const dest = path.join(args[args.indexOf('--target') + 1], 'artifact'); + fs.cpSync(artifact, dest, { recursive: true }); + fs.writeFileSync(path.join(dest, 'files/00000001'), 'corrupted'); + }), /recovery_capsule_invalid/); +}); + +test('local artifact cleanup removes obsolete and interrupted cache entries without touching remote storage', t => { + const cache = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-artifact-cache-')); + t.after(() => fs.rmSync(cache, { recursive: true, force: true })); + const directory = path.join(cache, 'a'.repeat(64)), partial = path.join(cache, 'b'.repeat(64)); + fs.mkdirSync(directory, { mode: 0o700 }); fs.mkdirSync(partial, { mode: 0o700 }); + const release = '/opt/dispatch-runtime/releases/dispatch_absent_test_' + path.basename(cache).toLowerCase(); + assert.equal(fs.existsSync(release), false); + require('../src/release-delivery-files').atomic(path.join(directory, 'receipt.json'), { root: release, digest: 'c'.repeat(64), snapshotId: 'd'.repeat(64) }); + fs.writeFileSync(path.join(cache, 'worker.lock'), ''); + require('../src/recovery-artifacts').pruneLocalCache(cache, process.geteuid()); + assert.deepEqual(fs.readdirSync(cache), ['worker.lock']); +}); diff --git a/core/core/installations/tests/recovery-capsule.test.js b/core/core/installations/tests/recovery-capsule.test.js new file mode 100644 index 0000000..023919e --- /dev/null +++ b/core/core/installations/tests/recovery-capsule.test.js @@ -0,0 +1,85 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'); +const test = require('node:test'), assert = require('node:assert/strict'); +const { capture, verify, materialize } = require('../src/recovery-capsule'); +test('live SQLite WAL pages are captured without transient sidecars', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-capsule-test-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const source = path.join(root, 'source'); fs.mkdirSync(source); + const { DatabaseSync } = require('node:sqlite'), live = new DatabaseSync(path.join(source, 'state.sqlite3')); + t.after(() => live.close()); + live.exec("PRAGMA journal_mode=WAL; CREATE TABLE records(value); INSERT INTO records VALUES('committed in WAL')"); + const directory = path.join(root, 'bundle'), target = '/opt/dispatch-fixture'; + const proof = capture(directory, [{ source, target }], {}, new Set([process.geteuid()])); + const manifest = verify(directory, proof.sha256, new Set([target])); + assert.equal(manifest.entries.some(e => /-(wal|shm)$/.test(e.path)), false); + const file = manifest.entries.find(e => e.path.endsWith('.sqlite3')); + const restored = new DatabaseSync(path.join(directory, file.payload), { readOnly: true }); + try { assert.equal(restored.prepare('SELECT value FROM records').get().value, 'committed in WAL'); } + finally { restored.close(); } +}); +test('recovery capsule preserves code, secrets and configuration with authenticated ownership metadata', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-capsule-test-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const source = path.join(root, 'source'); fs.mkdirSync(source, { mode: 0o700 }); + fs.writeFileSync(path.join(source, 'server'), 'exact executable', { mode: 0o555 }); + fs.writeFileSync(path.join(source, 'vault-key'), 'fixture encryption key', { mode: 0o600 }); + fs.symlinkSync('server', path.join(source, 'current')); + const bundle = path.join(root, 'bundle'), target = '/opt/dispatch-fixture'; + const proof = capture(bundle, [{ source, target }], { kind: 'core', releaseId: 'dispatch_fixture' }, new Set([process.geteuid()])); + const allowed = new Set([target]), inventory = verify(bundle, proof.sha256, allowed); + assert.equal(inventory.entries.find(e => e.path.endsWith('/server')).mode, 0o555); + const staging = path.join(root, 'restored'); materialize(bundle, staging, proof.sha256, allowed); + assert.equal(fs.readFileSync(path.join(staging, target, 'vault-key'), 'utf8'), 'fixture encryption key'); + assert.equal(fs.readFileSync(path.join(staging, target, 'current'), 'utf8'), 'exact executable'); + const payload = inventory.entries.find(e => e.path.endsWith('/vault-key')).payload; + fs.writeFileSync(path.join(bundle, payload), 'changed'); + assert.throws(() => verify(bundle, proof.sha256, allowed), /recovery_capsule_invalid/); +}); +test('recovery refuses links escaping the approved restore roots and forged target roots', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-capsule-test-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const source = path.join(root, 'source'); fs.mkdirSync(source); + fs.symlinkSync('/etc/passwd', path.join(source, 'escape')); + const bundle = path.join(root, 'bundle'), target = '/opt/dispatch-fixture'; + const proof = capture(bundle, [{ source, target }], {}, new Set([process.geteuid()])); + assert.throws(() => verify(bundle, proof.sha256, new Set([target])), /recovery_capsule_invalid/); + assert.throws(() => materialize(bundle, path.join(root, 'restored'), proof.sha256, new Set(['/etc'])), /recovery_capsule_invalid/); + assert.equal(fs.existsSync(path.join(root, 'restored')), false); +}); +test('sealed DSP snapshot overrides retain their exact tree while SQLite is captured', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-capsule-sealed-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const liveRoot = path.join(root, 'live'), snapshot = path.join(root, 'snapshot'); + fs.mkdirSync(liveRoot); fs.mkdirSync(snapshot); + const { DatabaseSync } = require('node:sqlite'); + const original = new DatabaseSync(path.join(liveRoot, 'data.sqlite3')); + original.exec("PRAGMA journal_mode=WAL; CREATE TABLE records(value); INSERT INTO records VALUES('sealed')"); + original.close(); + fs.copyFileSync(path.join(liveRoot, 'data.sqlite3'), path.join(snapshot, 'data.sqlite3')); + const before = fs.readFileSync(path.join(snapshot, 'data.sqlite3')); + const bundle = path.join(root, 'bundle'), target = '/opt/dispatch-fixture'; + const proof = capture(bundle, [{ source: liveRoot, target, overrides: { 'data.sqlite3': path.join(snapshot, 'data.sqlite3') } }], {}, new Set([process.geteuid()])); + assert.deepEqual(fs.readdirSync(snapshot), ['data.sqlite3']); + assert.deepEqual(fs.readFileSync(path.join(snapshot, 'data.sqlite3')), before); + const manifest = verify(bundle, proof.sha256, new Set([target])); + const file = manifest.entries.find(e => e.path.endsWith('.sqlite3')); + const restored = new DatabaseSync(path.join(bundle, file.payload), { readOnly: true }); + try { assert.equal(restored.prepare('SELECT value FROM records').get().value, 'sealed'); } + finally { restored.close(); } +}); + +test('a DSP stopped for backup resumes its captured ready state during disaster recovery',()=>{ + const {servicesForRecovery}=require('../src/host-recovery-bundle'),{opaqueRuntimeSuffix}=require('../../runtime-host-identity'); + const runtime='runtime_fixture',name=`dispatch-dsp-${opaqueRuntimeSuffix(runtime)}.service`,services=[{name,active:false,enabled:true}],installations=[{runtime_key:runtime,status:'ready',organization_status:'active'}]; + assert.equal(servicesForRecovery('dsp',services,installations)[0].active,true);assert.equal(services[0].active,false); + assert.equal(servicesForRecovery('core',services,installations)[0].active,false); + assert.equal(servicesForRecovery('dsp',services,[{...installations[0],status:'suspended'}])[0].active,false); +}); +test('a DSP recovery capsule permits only its own Core-side registration credential',()=>{ + const {recoveryRoots}=require('../src/host-recovery-bundle'),{hostAccountName,opaqueRuntimeSuffix,HOST_TENANT_ROOT}=require('../../runtime-host-identity'); + const key='runtime_fixture',root=HOST_TENANT_ROOT+'/'+opaqueRuntimeSuffix(key),localRoot='/home/core_fixture/local'; + const metadata={kind:'dsp',platform:'ubuntu-24.04-amd64',localRoot,services:[],accounts:[{name:'core_fixture',uid:1001,gid:1001,home:'/home/core_fixture'},{name:hostAccountName(key),uid:20001,gid:20001,home:root+'/home'}],installations:[{organization_id:'org_fixture',runtime_key:key,backend:'native_service_v1',status:'ready'}]}; + const token=localRoot+'/secrets/oci-runtime-agents/'+key+'.token';assert.ok(recoveryRoots(metadata,[token]).has(token)); + assert.throws(()=>recoveryRoots(metadata,[localRoot+'/secrets/oci-runtime-agents/runtime_peer.token']),/host_recovery_unavailable/); +}); diff --git a/core/core/installations/tests/recovery-prewarm.test.js b/core/core/installations/tests/recovery-prewarm.test.js new file mode 100644 index 0000000..62a5898 --- /dev/null +++ b/core/core/installations/tests/recovery-prewarm.test.js @@ -0,0 +1,28 @@ +'use strict'; +const test=require('node:test'),assert=require('node:assert/strict'); +const {prewarm}=require('../src/recovery-prewarm'); +test('background warming prepares immutable roots independently and leaves failures for normal backup verification',()=>{ + const called=[]; + const result=prewarm({roots:['/opt/dispatch-platform/releases/dispatch_1.2.3','/opt/dispatch-runtime/releases/dispatch_1.2.3'],config:{}, + prepare:(_,root)=>{called.push(root);if(root.includes('runtime'))throw Error('unavailable');}}); + assert.equal(called.length,2);assert.deepEqual(result,{status:'recovery_prewarm_incomplete',prepared:1,failed:1}); +}); +test('warming discovers only sealed release roots and excludes installation stages', t=>{ + const fs=require('node:fs'),{candidates}=require('../src/recovery-prewarm'); + const base='/opt/dispatch-platform/releases'; + t.mock.method(fs,'existsSync',()=>true); + t.mock.method(fs,'realpathSync',file=>file); + t.mock.method(fs,'readdirSync',()=>['dispatch_1.2.3','dispatch_1.2.4.pending','dispatch_1.2.5'].map(name=>({name,isDirectory:()=>true}))); + t.mock.method(fs,'lstatSync',file=>({uid:0,isDirectory:()=>true,mode:file===base||file.endsWith('1.2.5')?0o40755:0o40555})); + assert.deepEqual(candidates([base]),[base+'/dispatch_1.2.3']); +}); +test('prewarming reports bounded progress and failures without exposing exception details', () => { + const reports = []; + prewarm({ roots: ['/opt/dispatch-platform/releases/dispatch_1.2.3'], config: {}, + prepare: () => { throw Error('secret payload'); }, report: value => reports.push(JSON.parse(JSON.stringify(value))) }); + assert.equal(reports[0].status, 'running'); + assert.equal(reports.at(-1).status, 'attention'); + assert.equal(reports.at(-1).stages[0].status, 'failed'); + assert.equal(reports.at(-1).failed, 1); + assert(!JSON.stringify(reports).includes('secret')); +}); diff --git a/core/core/installations/tests/release-build.test.js b/core/core/installations/tests/release-build.test.js new file mode 100644 index 0000000..264be5c --- /dev/null +++ b/core/core/installations/tests/release-build.test.js @@ -0,0 +1,45 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const {preflight, checkArchiveResult, withOutput} = require('../src/release-build-space'); +const {removeStage} = require('../src/release-delivery-install'); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'release-build-test-')); + t.after(() => removeStage(root)); + return root; +} +test('disk preflight checks the output filesystem and reports required/available bytes', t => { + const root = fixture(t), output = path.join(root, 'out'); + assert.throws(() => preflight(output, {browserRoot:root, format:'both', statfs(directory) { + assert.equal(directory, root); return {bavail:1, bsize:4096}; + }}), error => error.message === 'release_build_insufficient_space' && error.requiredBytes > 1024**3 && error.availableBytes === 4096); + assert.equal(fs.existsSync(output), false); +}); +test('failed builds clean sealed scratch but preserve preexisting output', async t => { + const root = fixture(t), output = path.join(root, 'out'); + await assert.rejects(withOutput(output, async () => { + const stage = path.join(output, 'stage'); fs.mkdirSync(stage); + fs.writeFileSync(path.join(stage, 'file'), 'partial', {mode:0o444}); fs.chmodSync(stage, 0o555); + throw Error('injected_pack_failure'); + }), /injected_pack_failure/); + assert.equal(fs.existsSync(output), false); + fs.mkdirSync(output); fs.writeFileSync(path.join(output, 'keep'), 'existing'); + await assert.rejects(withOutput(output, async () => assert.fail()), {code:'EEXIST'}); + assert.equal(fs.readFileSync(path.join(output, 'keep'), 'utf8'), 'existing'); +}); +test('concurrent builds cannot claim or remove each other’s output', async t => { + const output = path.join(fixture(t), 'out'); + let release; + const first = withOutput(output, () => new Promise(resolve => { release = resolve; })); + await assert.rejects(withOutput(output, () => assert.fail()), {code:'EEXIST'}); + assert.equal(fs.existsSync(output), true); + release('done'); assert.equal(await first, 'done'); +}); +test('archive exhaustion, timeout and interruption have stable diagnostics', () => { + for (const [result, message] of [ + [{status:1, stderr:'archive_disk_full\n'}, 'release_archive_disk_full'], + [{status:null, error:{code:'ETIMEDOUT'}}, 'release_archive_timeout'], + [{status:null, signal:'SIGTERM'}, 'release_archive_interrupted'], + [{status:1, stderr:'private file path'}, 'release_package_invalid'], + ]) assert.throws(() => checkArchiveResult(result, 'release_package_invalid'), {message}); +}); diff --git a/core/core/installations/tests/release-delivery-files.test.js b/core/core/installations/tests/release-delivery-files.test.js new file mode 100644 index 0000000..638ddab --- /dev/null +++ b/core/core/installations/tests/release-delivery-files.test.js @@ -0,0 +1,23 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { atomic } = require('../src/release-delivery-files'); + +test('atomic public receipts remain readable under the root worker private umask', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-receipt-mode-')); + const previous = process.umask(0o077); + try { + const receipt = path.join(root, 'rollout.cleanup.json'); + atomic(receipt, { status: 'completed' }, 0o644); + assert.equal(fs.statSync(receipt).mode & 0o777, 0o644); + assert.equal(JSON.parse(fs.readFileSync(receipt)).status, 'completed'); + atomic(path.join(root, 'private.json'), { secret: 'private' }); + assert.equal(fs.statSync(path.join(root, 'private.json')).mode & 0o777, 0o600); + } finally { + process.umask(previous); + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/core/core/installations/tests/release-delivery-install.test.js b/core/core/installations/tests/release-delivery-install.test.js new file mode 100644 index 0000000..d9fb24f --- /dev/null +++ b/core/core/installations/tests/release-delivery-install.test.js @@ -0,0 +1,87 @@ +'use strict'; +const test=require('node:test'); +const assert=require('node:assert/strict'); +const fs=require('node:fs'); +const path=require('node:path'); +const os=require('node:os'); +const {spawnSync}=require('node:child_process'); +const {sha,releaseManifest,identity}=require('../src/release-delivery-contract'); +const {prepareRelease,removeStage,tree}=require('../src/release-delivery-install'); +const commit='a'.repeat(40),version='987654.0.1+hotfix.1',id=identity(version,commit); +function entries(root,prefix){return tree(root).map(item=>({path:prefix+'/'+item.path,mode:item.mode.toString(8),sha256:item.hash,data:fs.readFileSync(path.join(root,item.path)).toString('base64')}));} +if(process.geteuid()!==0){ + for (const format of ['legacy', 'split']) test(`immutable ${format} release preparation and restart recovery on a disposable host release`,t=>{ + if(spawnSync('/usr/bin/sudo',['-n','/usr/bin/true']).status!==0)return t.skip('requires noninteractive sudo'); + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-delivery-install-'));t.after(()=>removeStage(root)); + for(const [script,target] of [['create-host-helper-artifact','helper'],['create-bridge-artifact','bridge']]){ + const result=spawnSync(process.execPath,[path.resolve(__dirname,`../src/${script}.js`),path.join(root,target)],{encoding:'utf8'});assert.equal(result.status,0,result.stderr); + } + const helperFiles=entries(path.join(root,'helper'),'host-helper-artifact'); + const coreFile={path:'code/core/installations/src/core-systemd-deployment.js',mode:'444',sha256:sha('module.exports={};'),data:Buffer.from('module.exports={};').toString('base64')}; + fs.writeFileSync(path.join(root,'dispatch-core.json'),JSON.stringify({schemaVersion:1,kind:'core',sourceCommit:commit,files:[coreFile,...helperFiles]})); + fs.writeFileSync(path.join(root,'dispatch-bridge.json'),JSON.stringify({schemaVersion:1,kind:'bridge',sourceCommit:commit,files:entries(path.join(root,'bridge'),'bridge-artifact')})); + fs.writeFileSync(path.join(root,'runtime-image.tar'),'synthetic archive; never executed'); + const assets={};for(const [kind,name]of Object.entries({core:'dispatch-core.json',bridge:'dispatch-bridge.json',runtime:'runtime-image.tar'})){const bytes=fs.readFileSync(path.join(root,name));assets[kind]={name,size:bytes.length,sha256:sha(bytes)};} + const runtime={version:2,backend:'oci_container_v1',releaseId:id,channel:'production',image:'ghcr.io/example-organization/dispatch-runtime@sha256:'+'b'.repeat(64),imageDigest:'sha256:'+'b'.repeat(64),imageId:'c'.repeat(64),sourceCommit:commit,platform:'linux/amd64',runtimeAgentProtocol:1,runtimeGatewayProtocol:1,embeddedManifestSha256:'d'.repeat(64),imageArchiveSha256:assets.runtime.sha256,bridgeManifestSha256:sha(fs.readFileSync(path.join(root,'bridge/manifest.json')))}; + fs.writeFileSync(path.join(root,'manifest.json'),JSON.stringify(releaseManifest({schemaVersion:1,version,releaseId:id,sourceCommit:commit,changelog:[{kind:'fixed',title:'Fixture',description:''}],assets,runtime}))); + if (format === 'split') splitPackages(root); + const result=spawnSync('/usr/bin/sudo',['-n','/usr/bin/env','-i','PATH=/usr/bin:/bin',`FIXTURE_SOURCE=${root}`,`FIXTURE_UID=${process.geteuid()}`,`FIXTURE_GID=${process.getegid()}`,'/usr/bin/node','--no-warnings','--test',__filename],{encoding:'utf8',timeout:60_000}); + assert.equal(result.status,0,result.stdout+result.stderr); + }); +}else{ + test('host preparation seals packages, renders local paths, preserves active services, and rejects replacement',async t=>{ + process.umask(0o022); + const targets=['/opt/dispatch-platform/releases/','/opt/dispatch-control/releases/','/opt/dispatch-runtime/releases/'].map(base=>base+id); + const sudoFile='/etc/sudoers.d/dispatch-release-987654_0_1_hotfix_1'; + for(const file of [...targets,sudoFile])assert.equal(fs.existsSync(file),false,'never adopt an existing fixture path'); + t.after(()=>{for(const root of targets)removeStage(root);fs.rmSync(sudoFile,{force:true});}); + const directory=fs.mkdtempSync('/tmp/dispatch-root-delivery-');t.after(()=>removeStage(directory)); + const manifest=JSON.parse(fs.readFileSync(path.join(process.env.FIXTURE_SOURCE,'manifest.json'))); + for(const {name} of Object.values(manifest.assets))fs.copyFileSync(path.join(process.env.FIXTURE_SOURCE,name),path.join(directory,name)); + const config={uid:Number(process.env.FIXTURE_UID),gid:Number(process.env.FIXTURE_GID),localRoot:'/tmp/dispatch-fixture-platform',unitRoot:'/tmp/dispatch-fixture-units',publicOrigin:'https://dispatch.example.test',port:4319}; + const current=()=>fs.existsSync('/opt/dispatch-control/current')?fs.readlinkSync('/opt/dispatch-control/current'):null; + const before=current(); + const input={config,manifest,directory,configureSandbox:()=>{},publishedAt:'2026-09-05T00:00:00.000Z'}; + const result=await prepareRelease(input);assert.equal(current(),before); + const deployment=JSON.parse(fs.readFileSync(result.core.artifactPath+'/deployment.json')); + assert.equal(deployment.localRoot,config.localRoot);assert.equal(deployment.sourceCommit,commit); + assert.equal(fs.statSync(result.core.artifactPath).mode&0o777,0o555); + assert.equal((await prepareRelease(input)).core.manifestSha256,result.core.manifestSha256); + if (manifest.schemaVersion === 2) { + const invalid = structuredClone(manifest); invalid.assets.dependencies.unpackedSize++; + await assert.rejects(prepareRelease({ ...input, manifest: invalid }), { code: 'release_package_invalid' }); + assert.equal(fs.existsSync(path.join(directory, 'packages')), false, 'failed extraction must not consume retry disk space'); + assert.equal(fs.existsSync(path.join(directory, 'prepared')), false); + } + const file=result.core.artifactPath+'/code/core/installations/src/core-systemd-deployment.js'; + fs.chmodSync(file,0o644);fs.writeFileSync(file,'tampered');fs.chmodSync(file,0o444); + await assert.rejects(prepareRelease(input),{code:'immutable_release_conflict'}); + assert.equal(current(),before); + }); +} + +function splitPackages(root) { + const { writeBundle } = require('../src/release-delivery-install'); + const { pack, runtimeIdentity } = require('../src/release-package'); + const app = path.join(root, 'app'), deps = path.join(root, 'dependencies-package'); + fs.mkdirSync(app); fs.mkdirSync(deps); + for (const kind of ['core', 'bridge']) { + const target = path.join(app, kind); fs.mkdirSync(target); + writeBundle(JSON.parse(fs.readFileSync(path.join(root, `dispatch-${kind}.json`))), target); + } + const files = []; + for (const relative of ['shared/fixture.js', 'dependencies/node/bin/node', 'dependencies/browser/chrome']) { + const base = relative.startsWith('dependencies/') ? deps : path.join(app, 'runtime'); + const file = path.join(base, relative); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, 'fixture'); fs.chmodSync(file, 0o444); + files.push({ path: relative, mode: '444', size: 7, sha256: sha('fixture') }); + } + const bytes = JSON.stringify({ schemaVersion: 1, backend: 'native_service_v1', sourceCommit: commit, platform: 'linux/amd64', files }) + '\n'; + fs.writeFileSync(path.join(app, 'runtime/runtime-release-manifest.json'), bytes); + const assets = { app: pack(app, path.join(root, 'dispatch-app.tar.gz'), 'app', commit), + dependencies: pack(deps, path.join(root, 'dispatch-dependencies.tar.gz'), 'dependencies') }; + const old = JSON.parse(fs.readFileSync(path.join(root, 'manifest.json'))); + const runtime = { version: 1, backend: 'native_service_v1', releaseId: id, channel: 'production', sourceCommit: commit, + platform: 'linux/amd64', runtimeAgentProtocol: 1, runtimeGatewayProtocol: 1, artifactSha256: runtimeIdentity(assets), embeddedManifestSha256: sha(bytes), bridgeManifestSha256: old.runtime.bridgeManifestSha256 }; + fs.writeFileSync(path.join(root, 'manifest.json'), JSON.stringify(releaseManifest({ ...old, schemaVersion: 2, assets, runtime, notes: null, + dependencies: { node: '22.23.2', chrome: '151.0.7922.138' } }))); +} diff --git a/core/core/installations/tests/release-delivery.test.js b/core/core/installations/tests/release-delivery.test.js new file mode 100644 index 0000000..76b80b0 --- /dev/null +++ b/core/core/installations/tests/release-delivery.test.js @@ -0,0 +1,339 @@ +'use strict'; +const assert=require('node:assert/strict'); +const fs=require('node:fs'); +const os=require('node:os'); +const path=require('node:path'); +const test=require('node:test'); +const {createReleaseWatcher}=require('../src/release-delivery-watch'); +const {createGitHubReleaseSource}=require('../src/release-delivery-github'); +const {bundle,releaseManifest,sha,identity}=require('../src/release-delivery-contract'); +const commit='a'.repeat(40); +function fixture(t,version='1.2.3'){ + const id=identity(version,commit); + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-release-watch-'));fs.chmodSync(root,0o700);t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + const payloads={'dispatch-core.json':Buffer.from('core'),'dispatch-bridge.json':Buffer.from('bridge'),'runtime-image.tar':Buffer.from('runtime')}; + const assets={}; + for(const [key,name] of Object.entries({core:'dispatch-core.json',bridge:'dispatch-bridge.json',runtime:'runtime-image.tar'}))assets[key]={name,size:payloads[name].length,sha256:sha(payloads[name])}; + const runtime={version:2,backend:'oci_container_v1',releaseId:id,channel:'production',image:'ghcr.io/example-organization/dispatch-runtime@sha256:'+'b'.repeat(64),imageDigest:'sha256:'+'b'.repeat(64),imageId:'c'.repeat(64),sourceCommit:commit,platform:'linux/amd64',runtimeAgentProtocol:1,runtimeGatewayProtocol:1,embeddedManifestSha256:'d'.repeat(64),imageArchiveSha256:assets.runtime.sha256,bridgeManifestSha256:'e'.repeat(64)}; + const manifest=releaseManifest({schemaVersion:1,version,releaseId:id,sourceCommit:commit,changelog:[{kind:'fixed',title:'Better updates',description:'Updates arrive automatically.'}],assets,runtime}); + payloads['dispatch-release.json']=Buffer.from(JSON.stringify(manifest)); + const release={id:12,tag_name:version,published_at:'2026-09-05T10:00:00Z',draft:false,prerelease:false,assets:Object.entries(payloads).map(([name,bytes],i)=>({id:i+1,name,size:bytes.length,digest:'sha256:'+sha(bytes),state:'uploaded'}))}; + const calls=[],statuses=[],published=[];let now=1000,failDownload=true,retry=null; + const source={list:async()=>[release],verifyCommit:async(v,c)=>{assert.equal(c,commit);},download:async(asset,file)=>{ + calls.push(asset.name); + if(asset.name==='runtime-image.tar'&&failDownload){fs.writeFileSync(file,'partial');throw Error('secret diagnostic');} + fs.writeFileSync(file,payloads[asset.name],{flag:'wx'}); + }}; + const options={root,source,clock:()=>now,prepare:async input=>{assert.equal(input.manifest.version,version);return {version};},publish:async value=>published.push(value),status:async value=>statuses.push(value),retryRequest:()=>retry}; + return {root,source,options,calls,statuses,published,release,payloads,manifest,setNow:value=>now=value,allowDownload:()=>failDownload=false,retry:()=>retry={nonce:'1'.repeat(32)}}; +} +test('discovery resumes after interrupted downloads, waits for verification, and never starts a rollout',async t=>{ + const f=fixture(t);let watcher=createReleaseWatcher(f.options); + assert.equal((await watcher.run()).status,'release_preparation_failed');assert.equal(f.published.length,0); + assert.equal(f.statuses.at(-1).state,'failed');assert.equal(JSON.stringify(f.statuses).includes('secret'),false); + assert.equal((await watcher.run()).status,'backoff'); + f.allowDownload();f.retry();watcher=createReleaseWatcher(f.options); + assert.equal((await watcher.run()).status,'release_ready');assert.equal(f.published.length,1); + assert.equal(f.calls.filter(x=>x==='dispatch-core.json').length,1); + assert.equal(f.calls.filter(x=>x==='runtime-image.tar').length,2); + assert.equal(fs.existsSync(path.join(f.root,'release-12')),false); + assert.equal((await createReleaseWatcher(f.options).run()).status,'idle');assert.equal(f.published.length,1); +}); +test('failed preparation and catalog registration are retried after restart without losing downloaded packages',async t=>{ + const f=fixture(t);f.allowDownload();let attempts=0; + const options={...f.options,publish:async value=>{if(++attempts===1)throw Error('crash before registration');f.published.push(value);}}; + assert.equal((await createReleaseWatcher(options).run()).status,'release_preparation_failed'); + f.setNow(100_000);assert.equal((await createReleaseWatcher(options).run()).status,'release_ready'); + assert.equal(f.calls.length,4);assert.equal(f.published.length,1); +}); +test('drafts, prereleases and legacy releases are ignored, and changed published manifests are never adopted',async t=>{ + const f=fixture(t);f.allowDownload(); + f.source.list=async()=>[{...f.release,draft:true},{...f.release,prerelease:true},{...f.release,assets:[]}]; + assert.equal((await createReleaseWatcher(f.options).run()).status,'idle');assert.equal(f.calls.length,0); + f.source.list=async()=>[f.release];await createReleaseWatcher(f.options).run(); + f.release.assets.find(a=>a.name==='dispatch-release.json').digest='sha256:'+'f'.repeat(64); + await createReleaseWatcher(f.options).run();assert.equal(f.published.length,1);assert.equal(f.statuses.at(-1).retryable,false); +}); +test('corrupt asset bytes and mismatched tags cannot enter the release catalog',async t=>{ + const f=fixture(t);f.allowDownload();f.payloads['runtime-image.tar']=Buffer.from('corrupt'); + assert.equal((await createReleaseWatcher(f.options).run()).status,'release_preparation_failed');assert.equal(f.published.length,0); + f.setNow(100_000);f.source.verifyCommit=async()=>{throw Object.assign(Error(),{code:'release_commit_mismatch'});}; + assert.equal((await createReleaseWatcher(f.options).run()).code,'release_commit_mismatch');assert.equal(f.published.length,0); +}); +test('portable bundles reject traversal, linked-file shapes, unexpected roots, duplicate and ancestor paths',()=>{ + const file={path:'code/shared/test.js',mode:'444',sha256:sha('x'),data:Buffer.from('x').toString('base64')}; + const make=files=>({schemaVersion:1,kind:'core',sourceCommit:commit,files}); + assert.equal(bundle(make([file]),'core',commit).files.length,1); + for(const name of ['../escape','code/../escape','/etc/passwd','code/runtime/x','code/shared//x'])assert.throws(()=>bundle(make([{...file,path:name}]),'core',commit)); + assert.throws(()=>bundle(make([file,file]),'core',commit)); + assert.throws(()=>bundle(make([file,{...file,path:file.path+'/child'}]),'core',commit)); + assert.throws(()=>bundle(make([{...file,symlink:'/etc/passwd'}]),'core',commit)); + assert.throws(()=>bundle(make([{...file,mode:'777'}]),'core',commit)); + assert.throws(()=>bundle(make([{...file,data:'bad'}]),'core',commit)); +}); +test('GitHub download validates metadata and strips credentials from approved redirects',async t=>{ + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-gh-download-'));t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + const calls=[],bytes=Buffer.from('download'); + const source=createGitHubReleaseSource({token:'fixture-token',fetcher:async(url,options)=>{ + calls.push({url,options});return calls.length===1?new Response(null,{status:302,headers:{location:'https://release-assets.githubusercontent.com/file?signature=private'}}):new Response(bytes); + }}); + const asset={id:1,state:'uploaded',size:bytes.length,digest:'sha256:'+sha(bytes)}; + await source.download(asset,path.join(root,'asset'),{size:bytes.length,sha256:sha(bytes)}); + assert.equal(calls[0].options.headers.Authorization,'Bearer fixture-token');assert.equal(calls[1].options.headers.Authorization,undefined); + assert.equal(fs.readFileSync(path.join(root,'asset'),'utf8'),'download'); + await assert.rejects(source.download({...asset,digest:'sha256:'+'0'.repeat(64)},path.join(root,'bad'),{size:bytes.length,sha256:sha(bytes)})); + const unsafe=createGitHubReleaseSource({token:'fixture-token',fetcher:async()=>new Response(null,{status:302,headers:{location:'https://attacker.example/file'}})}); + await assert.rejects(unsafe.download(asset,path.join(root,'unsafe'),{size:bytes.length,sha256:sha(bytes)}),{code:'github_redirect_invalid'}); +}); + +test('GitHub tag and main ancestry must agree with the package commit',async()=>{ + const source=createGitHubReleaseSource({token:'fixture',fetcher:async url=>new Response(JSON.stringify( + url.includes('/git/ref/')?{object:{type:'tag',sha:'b'.repeat(40)}}:url.includes('/git/tags/')?{object:{type:'commit',sha:commit}}:{status:'ahead'}))}); + await source.verifyCommit('1.2.3',commit); + await assert.rejects(source.verifyCommit('1.2.3','c'.repeat(40)),{code:'release_commit_mismatch'}); + const divergent=createGitHubReleaseSource({token:'fixture',fetcher:async url=>new Response(JSON.stringify(url.includes('/git/ref/')?{object:{type:'commit',sha:commit}}:{status:'diverged'}))}); + await assert.rejects(divergent.verifyCommit('1.2.3',commit),{code:'release_commit_mismatch'}); +}); + +for(const version of ['1.2.3','0.0.7+hotfix.1']) test(`publication verifies every asset and preserves existing ${version}`,async t=>{ + const f=fixture(t,version);const {publish}=require('../src/release-delivery-publish-github'); + for(const [name,bytes]of Object.entries(f.payloads))fs.writeFileSync(path.join(f.root,name),bytes); + fs.writeFileSync(path.join(f.root,'SHA256SUMS'),'fixture checksums');fs.writeFileSync(path.join(f.root,'CHANGELOG.md'),'Readable notes'); + let assets=[],existing=[],corrupt=false;const calls=[]; + const run=args=>{ + calls.push(args); + if(args[0]==='api')return JSON.stringify(args[1].includes('matching-refs')?[]:existing); + if(args[1]==='view')return JSON.stringify({assets}); + if(args[1]==='upload'){const file=args[3];assets.push({name:path.basename(file),digest:'sha256:'+sha(fs.readFileSync(file)),state:corrupt?'new':'uploaded'});} + if(args[1]==='edit')assert.equal(assets.length,5); + return ''; + }; + await publish(f.root,run);assert.equal(calls.filter(c=>c[1]==='edit').length,1); + calls.length=0;existing=[{tag_name:version,draft:false,target_commitish:commit}]; + await assert.rejects(publish(f.root,run),/release_exists/);assert.equal(calls.some(c=>c[0]==='release'),false); + existing=[{tag_name:version,draft:true,target_commitish:commit}];assets=[];calls.length=0;corrupt=true; + await assert.rejects(publish(f.root,run),/upload_verification_failed/);assert.equal(calls.some(c=>c[1]==='edit'),false); + calls.length=0;await assert.rejects(publish(f.root,run),/draft_asset_conflict/);assert.equal(calls.some(c=>c[1]==='upload'||c[1]==='edit'),false); + calls.length=0;fs.writeFileSync(path.join(f.root,'runtime-image.tar'),'changed'); + await assert.rejects(publish(f.root,run),/local_asset_mismatch/);assert.equal(calls.length,0); +}); + +for(const version of ['1.2.3','0.0.7+hotfix.1']) test(`delivered ${version} catalogs advance Core then two DSPs through an owner rollout`,async t=>{ + const id=identity(version,commit),f=fixture(t,version);f.allowDownload(); + const {AccessStore,AccessControlService}=require('../../accounts/src'); + const {createPlatformUpdates}=require('../../accounts/src/platform-updates'); + const {createPlatformCoreUpdater}=require('../src/platform-core-update'); + const {publish}=require('../src/release-delivery-publish'); + const {atomic}=require('../src/release-delivery-files'); + const {loadPrivateOciReleaseCatalog}=require('../src/release-catalog'); + const {loadPlatformReleaseCatalog}=require('../src/platform-release-catalog'); + const {createReleaseDelivery}=require('../../../dashboard/server/release-delivery'); + fs.mkdirSync(path.join(f.root,'config'),{mode:0o700}); + for(const name of ['oci-releases.json','platform-releases.json'])atomic(path.join(f.root,'config',name),{schemaVersion:1,releases:{}}); + const config={uid:process.geteuid(),gid:process.getegid(),localRoot:f.root}; + const store=new AccessStore({databaseRoot:path.join(f.root,'access'),database:path.join(f.root,'access/access-control.sqlite3')});t.after(()=>store.close()); + const access=new AccessControlService(store,{installationOperatorEnabled:true,installationBackend:'oci_container_v1'}); + const invite=access.createPlatformBootstrap({email:'platform@example.test'}); + const owner=await access.acceptNewUser({token:invite.token,firstName:'Platform',lastName:'Owner',password:'release fixture password',confirmPassword:'release fixture password'}); + for(let i=0;i<2;i++){ + const dsp=access.createOrganization(owner.session,{ownerEmail:`owner${i}@example.test`,idempotencyKey:`release:fixture:${i}`,name:`DSP ${i}`,stationCode:'TST1',timezone:'UTC'}); + store.db.prepare("UPDATE installations SET status='ready' WHERE organization_id=?").run(dsp.organization.id);store.updateOrganizationStatus(dsp.organization.id,'active',Date.now()); + } + const catalogs=()=>{const releases=loadPrivateOciReleaseCatalog(path.join(f.root,'config/oci-releases.json'));return {releases,platformReleases:loadPlatformReleaseCatalog(path.join(f.root,'config/platform-releases.json'),releases)};}; + const updates=createPlatformUpdates({store,enabled:true,loadCatalogs:catalogs,delivery:createReleaseDelivery(f.root)}); + assert.equal(updates.view().releases.length,0); + let original; + if(version.includes('+hotfix.')) { + const base=fixture(t,'0.0.7'); + original={version:'0.0.7',publishedAt:'2026-09-06T00:00:00.000Z',sourceCommit:commit,runtimeImageDigest:base.manifest.runtime.imageDigest, + changelog:base.manifest.changelog,core:{artifactPath:'/opt/dispatch-platform/releases/dispatch_0.0.7/core-artifact',manifestSha256:'b'.repeat(64)}}; + publish(config,{action:'publish',release:original,runtime:base.manifest.runtime}); + } + const watcher=createReleaseWatcher({...f.options, + prepare:async({manifest,publishedAt})=>({version:manifest.version,publishedAt,sourceCommit:commit,runtimeImageDigest:manifest.runtime.imageDigest,changelog:manifest.changelog,core:{artifactPath:`/opt/dispatch-platform/releases/${id}/core-artifact`,manifestSha256:'a'.repeat(64)}}), + publish:input=>publish(config,{action:'publish',...input}),status:status=>publish(config,{action:'status',status})}); + assert.equal((await watcher.run()).status,'release_ready'); + assert.equal(updates.view().releases[0].version,version);assert.equal(updates.view().delivery.state,'ready'); + assert.equal(store.db.prepare('SELECT count(*) n FROM platform_rollouts').get().n,0); + updates.command(owner.session,{action:'start',releaseId:id,idempotencyKey:'release:rollout:fixture'}); + require('./helpers/rollout-backups').completeRolloutBackups(store); + updates.tick();assert.equal(store.db.prepare("SELECT count(*) n FROM installation_lifecycle_jobs WHERE operation='upgrade'").get().n,0); + const stages=[];const worker=createPlatformCoreUpdater({store,platformReleases:catalogs().platformReleases,execute:async action=>{stages.push(action);updates.tick();assert.equal(store.db.prepare("SELECT count(*) n FROM installation_lifecycle_jobs WHERE operation='upgrade'").get().n,0);}}); + assert.equal((await worker.run()).status,'core_verified');assert.deepEqual(stages,['apply','verify']); + for(let i=0;i<2;i++){ + updates.tick();updates.tick();const jobs=store.db.prepare("SELECT * FROM installation_lifecycle_jobs WHERE operation='upgrade' ORDER BY rowid").all();assert.equal(jobs.length,i+1); + const job=jobs[i];store.db.prepare("UPDATE installation_lifecycle_jobs SET status='succeeded',finished_at=?,result_json='{}' WHERE id=?").run(Date.now(),job.id); + store.db.prepare("UPDATE installations SET status='ready',release_id=? WHERE organization_id=?").run(id,job.organization_id);updates.tick(); + } + updates.tick();assert.equal(updates.view().rollout.status,'completed');assert.equal(updates.view().rollout.updated,2); + assert.equal(updates.view().releases.length,0,'a newer publication date must not offer an older build'); + if(original)assert.deepEqual(catalogs().platformReleases['dispatch_0.0.7'],original); +}); + + +test('hotfix identity is distinct, constrained, and ordered independently of SemVer metadata',()=>{ + const {compareVersions}=require('../../../shared/release-version'); + assert.equal(identity('0.0.7',commit),'dispatch_0.0.7'); + assert.equal(identity('0.0.7+hotfix.1',commit),'dispatch_0.0.7_hotfix.1'); + for(const version of ['0.0.7+hotfix.0','0.0.7+hotfix.01','0.0.7_hotfix.1','0.0.7+hotfix.1.extra','0.0.7+build.1','0.0.7-hotfix.1','0.0.7+hotfix.-1']) + assert.throws(()=>identity(version,commit),{code:'release_invalid'}); + for(const [newer,older] of [['0.0.7+hotfix.1','0.0.7'],['0.0.7+hotfix.10','0.0.7+hotfix.2'],['0.0.8','0.0.7+hotfix.99']]) { + assert.equal(compareVersions(newer,older),1);assert.equal(compareVersions(older,newer),-1); + } +}); + +test('hotfix discovery preserves the original release fingerprint and ignores publication-date downgrades',async t=>{ + const base=fixture(t,'0.0.7'),hotfix=fixture(t,'0.0.7+hotfix.1'); + base.allowDownload();hotfix.allowDownload(); + assert.equal((await createReleaseWatcher(base.options).run()).status,'release_ready'); + const original=JSON.parse(fs.readFileSync(path.join(base.root,'state.json'))).releases['12']; + hotfix.release.id=13; + base.release.published_at='2026-09-06T10:00:00Z'; + hotfix.source.list=async()=>[base.release,hotfix.release]; + assert.equal((await createReleaseWatcher({...hotfix.options,root:base.root}).run()).version,'0.0.7+hotfix.1'); + const state=JSON.parse(fs.readFileSync(path.join(base.root,'state.json'))); + assert.deepEqual(state.releases['12'],original); + assert.equal(state.releases['13'].status,'ready'); + assert.equal(hotfix.published[0].runtime.releaseId,'dispatch_0.0.7_hotfix.1'); +}); + +function addNotes(f, mutate = value => value) { + const { NAME } = require('../src/release-notes'); + const notes = mutate({ schemaVersion: 1, releaseId: f.manifest.releaseId, sourceCommit: commit, + groups: [{ id: 'updates', title: 'Updates', icon: 'refresh-cw' }], + changelog: f.manifest.changelog.map(change => ({ ...change, group: 'updates', icon: 'check-circle', details: 'Expanded explanation.' })), + afterUpdating: [{ title: 'Check settings', description: 'Review your update settings.' }] }); + f.payloads[NAME] = Buffer.from(JSON.stringify(notes)); + f.release.assets.push({ name: NAME, id: 20, state: 'uploaded', size: f.payloads[NAME].length, digest: `sha256:${sha(f.payloads[NAME])}` }); + return notes; +} +test('rich notes are verified, shown during preparation, and registered without changing the v1 manifest', async t => { + const f = fixture(t); const notes = addNotes(f); f.allowDownload(); const saved = []; + const oldManifest = JSON.stringify(f.manifest); + assert.equal((await createReleaseWatcher({ ...f.options, publishNotes: value => saved.push(value) }).run()).status, 'release_ready'); + assert.deepEqual(saved, [notes]); + assert.deepEqual(f.statuses.find(s => s.notes)?.notes, notes); + assert.equal(JSON.stringify(f.manifest), oldManifest); + assert.equal(f.published.length, 1); + const asset = f.release.assets.find(a => a.name === 'dispatch-release-notes.json'); + asset.digest = `sha256:${'f'.repeat(64)}`; + await createReleaseWatcher({ ...f.options, publishNotes: value => saved.push(value) }).run(); + assert.equal(saved.length, 1); assert.equal(f.statuses.at(-1).retryable, false); +}); +test('an upgraded watcher enriches an already prepared release without redownloading packages', async t => { + const f = fixture(t); f.allowDownload(); await createReleaseWatcher(f.options).run(); + const stateFile = path.join(f.root, 'state.json'); + const state = JSON.parse(fs.readFileSync(stateFile)); delete state.releases['12'].notesFingerprint; + fs.writeFileSync(stateFile, JSON.stringify(state)); // State produced by the pre-sidecar watcher. + const notes = addNotes(f), saved = []; f.calls.length = 0; + await createReleaseWatcher({ ...f.options, publishNotes: value => saved.push(value) }).run(); + assert.deepEqual(saved, [notes]); + assert.deepEqual(f.calls, ['dispatch-release.json', 'dispatch-release-notes.json']); + assert.equal(f.published.length, 1); +}); +for (const failure of ['commit', 'text', 'bytes', 'duplicate']) test(`invalid release notes cannot be published (${failure})`, async t => { + const f = fixture(t); addNotes(f, value => { + if (failure === 'commit') value.sourceCommit = 'f'.repeat(40); + if (failure === 'text') value.changelog[0].title = 'Different release'; + return value; + }); + if (failure === 'bytes') f.payloads['dispatch-release-notes.json'] = Buffer.from('changed'); + if (failure === 'duplicate') f.release.assets.push(f.release.assets.at(-1)); + f.allowDownload(); const saved = []; + assert.equal((await createReleaseWatcher({ ...f.options, publishNotes: value => saved.push(value) }).run()).status, 'release_preparation_failed'); + assert.equal(saved.length, 0); assert.equal(f.published.length, 0); +}); +test('GitHub publication uploads and verifies the optional notes attachment', async t => { + const f = fixture(t); addNotes(f); + for (const [name, bytes] of Object.entries(f.payloads)) fs.writeFileSync(path.join(f.root, name), bytes); + fs.writeFileSync(path.join(f.root, 'SHA256SUMS'), 'fixture'); fs.writeFileSync(path.join(f.root, 'CHANGELOG.md'), 'fixture'); + const assets = []; let published = false; + await require('../src/release-delivery-publish-github').publish(f.root, args => { + if (args[0] === 'api') return '[]'; + if (args[1] === 'view') return JSON.stringify({ assets }); + if (args[1] === 'upload') assets.push({ name: path.basename(args[3]), state: 'uploaded', digest: `sha256:${sha(fs.readFileSync(args[3]))}` }); + if (args[1] === 'edit') { assert.equal(assets.length, 6); published = true; } + return ''; + }); + assert.equal(published, true); +}); + +test('history backfill fetches only verified manifests/notes, caches results and never prepares runtimes', async t => { + const f = fixture(t); const notes = addNotes(f); const saved = []; + const { createReleaseHistorySync } = require('../src/release-history-sync'); + const history = createReleaseHistorySync({ root:f.root, source:f.source, publish:input=>saved.push(input) }); + assert.deepEqual(await history.run(), {processed:1,failed:0}); + assert.deepEqual(f.calls,['dispatch-release.json','dispatch-release-notes.json']); + assert.deepEqual(saved[0].notes,notes); assert.equal(f.published.length,0); + assert.deepEqual(await history.run(),{processed:0,failed:0}); + f.release.assets.at(-1).digest=`sha256:${'f'.repeat(64)}`; + assert.equal((await history.run()).failed,1); assert.equal(saved.length,1); +}); +test('corrupt history is retried without changing current update availability', async t => { + const f=fixture(t), saved=[];const { createReleaseHistorySync }=require('../src/release-history-sync'); + f.payloads['dispatch-release.json']=Buffer.from('bad'); + const history=createReleaseHistorySync({root:f.root,source:f.source,publish:input=>saved.push(input)}); + assert.equal((await history.run()).failed,1);assert.equal(saved.length,0); + assert.deepEqual(await history.run(),{processed:0,failed:0}); + assert.equal(f.statuses.length,0); +}); + +function splitFixture(t, version = '1.2.3') { + const f = fixture(t, version), notes = addNotes(f); + const payloads = { 'dispatch-app.tar.gz': Buffer.from('app'), 'dispatch-dependencies.tar.gz': Buffer.from('dependencies') }; + const assets = Object.fromEntries(Object.entries({ app: 'dispatch-app.tar.gz', dependencies: 'dispatch-dependencies.tar.gz' }).map(([key, name]) => + [key, { name, size: payloads[name].length, unpackedSize: 100, sha256: sha(payloads[name]) }])); + const runtime = { version: 1, backend: 'native_service_v1', releaseId: f.manifest.releaseId, channel: 'production', sourceCommit: commit, + platform: 'linux/amd64', runtimeAgentProtocol: 1, runtimeGatewayProtocol: 1, artifactSha256: require('../src/release-package').runtimeIdentity(assets), + embeddedManifestSha256: 'd'.repeat(64), bridgeManifestSha256: 'e'.repeat(64) }; + f.manifest = releaseManifest({ ...f.manifest, schemaVersion: 2, assets, runtime, notes, dependencies: { node: '22.23.2', chrome: '151.0.7922.138' } }); + for (const name of Object.keys(f.payloads)) delete f.payloads[name]; + Object.assign(f.payloads, payloads, { 'dispatch-release.json': Buffer.from(JSON.stringify(f.manifest)) }); + f.release.assets = Object.entries(f.payloads).map(([name, bytes], i) => ({ name, id: i + 1, size: bytes.length, digest: `sha256:${sha(bytes)}`, state: 'uploaded' })); + return f; +} +test('split delivery prepares three assets, preserves embedded notes, and reuses dependencies across releases', async t => { + const legacy = fixture(t); legacy.allowDownload(); + assert.equal((await createReleaseWatcher(legacy.options).run()).status, 'release_ready'); + const first = splitFixture(t, '1.2.4'); first.release.id = 13; + const notes = []; + assert.equal((await createReleaseWatcher({ ...first.options, root: legacy.root, publishNotes: value => notes.push(value) }).run()).status, 'release_ready'); + assert.equal(first.calls.length, 3); assert.deepEqual(notes, [first.manifest.notes]); + const second = splitFixture(t, '1.2.5'); second.release.id = 14; + assert.equal((await createReleaseWatcher({ ...second.options, root: legacy.root }).run()).status, 'release_ready'); + assert.deepEqual(second.calls, ['dispatch-release.json', 'dispatch-app.tar.gz']); + const progress = JSON.parse(fs.readFileSync(path.join(legacy.root, 'preparation-progress.json'))); + assert.equal(progress.stage, 'ready'); assert.equal(progress.assets['dispatch-dependencies.tar.gz'].reused, true); + assert.equal(fs.readdirSync(path.join(legacy.root, 'dependencies')).length, 1); +}); +test('split manifests bind both packages, reject unsupported schemas, and enforce embedded notes identity', t => { + const f = splitFixture(t); + for (const mutate of [m => m.assets.app.sha256 = 'f'.repeat(64), m => m.assets.dependencies.sha256 = 'f'.repeat(64), + m => m.schemaVersion = 3, m => m.notes.sourceCommit = 'f'.repeat(40), m => m.assets.dependencies.unpackedSize = -1]) { + const value = structuredClone(f.manifest); mutate(value); assert.throws(() => releaseManifest(value)); + } +}); +test('split history downloads only the manifest and publishes embedded notes', async t => { + const f = splitFixture(t), saved = []; + const sync = require('../src/release-history-sync').createReleaseHistorySync({ root: f.root, source: f.source, publish: value => saved.push(value) }); + assert.deepEqual(await sync.run(), { processed: 1, failed: 0 }); + assert.deepEqual(f.calls, ['dispatch-release.json']); assert.deepEqual(saved[0].notes, f.manifest.notes); +}); +test('split publication uploads exactly three verified assets', async t => { + const f = splitFixture(t), assets = []; + for (const [name, bytes] of Object.entries(f.payloads)) fs.writeFileSync(path.join(f.root, name), bytes); + fs.writeFileSync(path.join(f.root, 'CHANGELOG.md'), 'fixture'); + await require('../src/release-delivery-publish-github').publish(f.root, args => { + if (args[0] === 'api') return '[]'; + if (args[1] === 'view') return JSON.stringify({ assets }); + if (args[1] === 'upload') assets.push({ name: path.basename(args[3]), state: 'uploaded', digest: `sha256:${sha(fs.readFileSync(args[3]))}` }); + if (args[1] === 'edit') assert.deepEqual(assets.map(a => a.name).sort(), Object.keys(f.payloads).sort()); + return ''; + }); +}); +test('an explicit installation target cannot silently select a newer release', async t => { + const f = splitFixture(t), newer = splitFixture(t, '1.3.0'); newer.release.id = 20; + f.source.list = async () => [newer.release, f.release]; + assert.equal((await createReleaseWatcher({ ...f.options, target: { version: '1.2.3', sourceCommit: commit } }).run()).version, '1.2.3'); + assert.equal((await createReleaseWatcher({ ...f.options, target: { version: '1.2.2', sourceCommit: commit } }).run()).status, 'release_not_found'); +}); diff --git a/core/core/installations/tests/release-download-resume.test.js b/core/core/installations/tests/release-download-resume.test.js new file mode 100644 index 0000000..89226de --- /dev/null +++ b/core/core/installations/tests/release-download-resume.test.js @@ -0,0 +1,42 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const { sha } = require('../src/release-delivery-contract'); +const { createGitHubReleaseSource } = require('../src/release-delivery-github'); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-resume-')); t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const bytes = Buffer.from('complete verified package'); + return { file: path.join(root, 'download.partial'), bytes, expected: { size: bytes.length, sha256: sha(bytes) }, + asset: { id: 10, state: 'uploaded', size: bytes.length, digest: `sha256:${sha(bytes)}` } }; +} +async function interrupt(f) { + let reads = 0; + const source = createGitHubReleaseSource({ token: 'fixture', fetcher: async () => new Response(new ReadableStream({ + pull(controller) { if (reads++ === 0) controller.enqueue(f.bytes.subarray(0, 8)); else controller.error(Error('connection lost')); }, + })) }); + await assert.rejects(source.download(f.asset, f.file, f.expected)); + assert.equal(fs.statSync(f.file).size, 8); +} +for (const range of [true, false]) test(`interrupted download ${range ? 'resumes' : 'restarts if range is ignored'} and verifies the whole file`, async t => { + const f = fixture(t); await interrupt(f); const calls = []; + const source = createGitHubReleaseSource({ token: 'fixture', fetcher: async (url, options) => { + calls.push(options); return new Response(range ? f.bytes.subarray(8) : f.bytes, range ? { status: 206, headers: { 'content-range': `bytes 8-${f.bytes.length - 1}/${f.bytes.length}` } } : {}); + } }); + await source.download(f.asset, f.file, f.expected); + assert.equal(calls[0].headers.Range, 'bytes=8-'); assert.deepEqual(fs.readFileSync(f.file), f.bytes); +}); +test('wrong ranges and corrupted resumed bytes are discarded', async t => { + const f = fixture(t); await interrupt(f); + const wrong = createGitHubReleaseSource({ token: 'fixture', fetcher: async () => new Response(f.bytes.subarray(8), { status: 206, headers: { 'content-range': 'bytes 0-9/10' } }) }); + await assert.rejects(wrong.download(f.asset, f.file, f.expected), { code: 'release_asset_invalid' }); assert.equal(fs.existsSync(f.file), false); + await interrupt(f); fs.writeFileSync(f.file, 'tampered'); + const corrupt = createGitHubReleaseSource({ token: 'fixture', fetcher: async () => new Response(f.bytes.subarray(8), { status: 206, headers: { 'content-range': `bytes 8-${f.bytes.length - 1}/${f.bytes.length}` } }) }); + await assert.rejects(corrupt.download(f.asset, f.file, f.expected), { code: 'release_checksum_failed' }); assert.equal(fs.existsSync(f.file), false); +}); +test('partials from another asset are never appended and symlinks are rejected', async t => { + const f = fixture(t); await interrupt(f); let headers; + const source = createGitHubReleaseSource({ token: 'fixture', fetcher: async (_, options) => { headers = options.headers; return new Response(f.bytes); } }); + await source.download({ ...f.asset, id: 11 }, f.file, f.expected); assert.equal(headers.Range, undefined); + fs.unlinkSync(f.file); fs.symlinkSync('/etc/passwd', f.file); + await assert.rejects(source.download(f.asset, f.file, f.expected), { code: 'unsafe_release_storage' }); +}); diff --git a/core/core/installations/tests/release-frontend.test.js b/core/core/installations/tests/release-frontend.test.js new file mode 100644 index 0000000..019e18d --- /dev/null +++ b/core/core/installations/tests/release-frontend.test.js @@ -0,0 +1,77 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const {spawnSync, execFileSync} = require('node:child_process'); +const {buildFrontend} = require('../src/release-frontend'); +const {sha} = require('../src/release-delivery-contract'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'frontend-package-')); + t.after(() => fs.rmSync(root, {recursive:true, force:true})); + const source = path.join(root, 'source'), output = path.join(root, 'output'); + fs.mkdirSync(source); fs.mkdirSync(output); + const git = args => execFileSync('/usr/bin/git', args, {cwd:source, encoding:'utf8', stdio:['pipe','pipe','pipe']}).trim(); + git(['init']); git(['config','user.name','Test']); git(['config','user.email','test@example.invalid']); + fs.mkdirSync(path.join(source, 'dashboard/public/assets'), {recursive:true}); + fs.writeFileSync(path.join(source, 'dashboard/input.txt'), 'committed source'); + fs.writeFileSync(path.join(source, '.gitignore'), 'dashboard/public/assets/\n'); + fs.mkdirSync(path.join(source, 'plugins/paycom/frontend'), {recursive:true}); + fs.writeFileSync(path.join(source, 'plugins/paycom/frontend/index.tsx'), 'committed plugin'); + git(['add','.']); git(['-c','commit.gpgsign=false','commit','-m','fixture']); + const commit = git(['rev-parse','HEAD']); + fs.writeFileSync(path.join(source, 'dashboard/input.txt'), 'uncommitted source'); + fs.writeFileSync(path.join(source, 'dashboard/public/assets/frontend.js'), 'stale checkout bundle'); + return {source, output, commit}; +} + +test('release frontend uses the selected Git snapshot and hashes generated assets', t => { + const {source, output, commit} = fixture(t), steps = []; + const files = buildFrontend(source, commit, output, {run(executable, args, options) { + if (executable !== 'npm') return spawnSync(executable, args, options); + steps.push(args); + assert.equal(fs.readFileSync(path.resolve(options.cwd, '../plugins/paycom/frontend/index.tsx'), 'utf8'), 'committed plugin'); + assert.equal(fs.readFileSync(path.join(options.cwd, 'input.txt'), 'utf8'), 'committed source'); + const assets = path.join(options.cwd, 'public/assets'); + assert.equal(fs.existsSync(path.join(assets, 'frontend.js')), false); + if (args.includes('vite')) { + fs.mkdirSync(assets, {recursive:true}); + fs.writeFileSync(path.join(assets, 'frontend.js'), 'fresh javascript'); + fs.writeFileSync(path.join(assets, 'styles.css'), 'fresh stylesheet'); + } + return {status:0, stdout:''}; + }}); + assert.deepEqual(steps, [['ci','--no-audit','--no-fund'], ['exec','--','tsc','--noEmit'], ['exec','--','vite','build']]); + assert.deepEqual(files.map(file => Buffer.from(file.data, 'base64').toString()), ['fresh javascript', 'fresh stylesheet']); + for (const file of files) { + assert.equal(file.sha256, sha(Buffer.from(file.data, 'base64'))); + assert.equal(file.mode, '444'); + } + assert.deepEqual(fs.readdirSync(output), []); + assert.equal(fs.readFileSync(path.join(source, 'dashboard/public/assets/frontend.js'), 'utf8'), 'stale checkout bundle'); +}); + +test('failed or interrupted frontend commands stop packaging and clean scratch', t => { + const {source, output, commit} = fixture(t); + for (const failure of [{status:1}, {status:null, signal:'SIGTERM'}, {status:null, error:{code:'ETIMEDOUT'}}]) { + assert.throws(() => buildFrontend(source, commit, output, {run(executable, args, options) { + return executable === 'npm' ? failure : spawnSync(executable, args, options); + }}), /release_frontend_build_failed:npm/); + assert.deepEqual(fs.readdirSync(output), []); + } +}); + +test('missing, empty, or symlinked frontend outputs cannot become release assets', t => { + const {source, output, commit} = fixture(t); + for (const invalid of ['missing', 'empty', 'symlink']) { + assert.throws(() => buildFrontend(source, commit, output, {run(executable, args, options) { + if (executable !== 'npm') return spawnSync(executable, args, options); + if (args.includes('vite')) { + const assets = path.join(options.cwd, 'public/assets'); fs.mkdirSync(assets, {recursive:true}); + if (invalid === 'empty') fs.writeFileSync(path.join(assets, 'frontend.js'), ''); + if (invalid === 'symlink') fs.symlinkSync(path.join(options.cwd, 'input.txt'), path.join(assets, 'frontend.js')); + } + return {status:0, stdout:''}; + }})); + assert.deepEqual(fs.readdirSync(output), []); + } +}); diff --git a/core/core/installations/tests/release-notes.test.js b/core/core/installations/tests/release-notes.test.js new file mode 100644 index 0000000..8eabcf4 --- /dev/null +++ b/core/core/installations/tests/release-notes.test.js @@ -0,0 +1,74 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { authoring, releaseNotes, markdown, saveReleaseNotes, loadReleaseNotes } = require('../src/release-notes'); +const { saveReleaseHistory, loadReleaseHistory } = require('../src/release-history'); +const input = () => ({ groups: [{id:'backups',title:'Backups & recovery',icon:'database'}], + changelog: [{kind:'added',title:'Independent backups',description:'Back up Core separately.',group:'backups',icon:'copy',details:'Longer explanation.\nSecond paragraph.'}], + afterUpdating: [{title:'Enable schedules',description:'Choose the schedules to run.'}] }); +const release = () => ({releaseId:'dispatch_1.2.3',version:'1.2.3',publishedAt:'2026-09-07T00:00:00.000Z',sourceCommit:'a'.repeat(40),changelog:authoring(input()).changelog}); +test('rich authoring produces the legacy changelog and one consistent human-readable GitHub body', () => { + const {changelog,notes} = authoring(input()); + assert.deepEqual(Object.keys(changelog[0]), ['kind','title','description']); + assert.deepEqual(authoring(changelog), {changelog,notes:null}); + const body = markdown('1.2.3',changelog,notes); + for (const text of ['## New','Independent backups','Second paragraph','## After updating','Choose the schedules']) assert.ok(body.includes(text)); + assert.match(markdown('1.2.3',changelog,null), /## New/); +}); +for (const mutate of [n=>n.groups.push(n.groups[0]), n=>n.changelog[0].group='unknown', n=>n.changelog[0].icon='', + n=>n.changelog[0].title='x'.repeat(161), n=>n.changelog[0].details='x'.repeat(4001), n=>n.afterUpdating[0].url='https://example.test', + n=>n.groups.push({id:'empty',title:'Unused',icon:'info'})]) test('authoring rejects unsupported or inconsistent presentation', () => { + const notes=input(); mutate(notes); assert.throws(()=>authoring(notes)); +}); +test('history and rich notes survive removal of installation catalogs, with immutable content and safe fallback', t => { + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-notes-')); t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + const r=release(), id=r.releaseId; + const notes={schemaVersion:1,releaseId:id,sourceCommit:r.sourceCommit,...input()}; + releaseNotes(notes,r);saveReleaseHistory(root,{[id]:r});saveReleaseNotes(root,notes,r); + assert.equal(loadReleaseHistory(root)[id].version,'1.2.3'); + assert.deepEqual(loadReleaseNotes(root,id,r),notes); + assert.throws(()=>saveReleaseHistory(root,{[id]:{...r,version:'1.2.4'}})); + assert.throws(()=>saveReleaseNotes(root,{...notes,afterUpdating:[]},r)); + assert.equal(loadReleaseNotes(root,'../escape',r),null); + const file=path.join(root,'config/release-notes',`${id}.json`);fs.chmodSync(file,0o644); + assert.equal(loadReleaseNotes(root,id,r),null); + assert.equal(loadReleaseHistory(root)[id].changelog[0].title,'Independent backups'); +}); + +test('rich notes leave enough space for the preparation receipt and its legacy compatibility copy', () => { + const notes=input(); notes.changelog=Array.from({length:60},(_,i)=>({...notes.changelog[0],title:`Change ${i}`,description:'x'.repeat(600),details:'x'.repeat(3000)})); + assert.ok(Buffer.byteLength(JSON.stringify(notes))<256*1024); + assert.throws(()=>authoring(notes),/release_notes_invalid/); +}); + +const curatedInput = () => JSON.parse(fs.readFileSync(path.join(__dirname, "../../../dashboard/examples/popup-changelog.json"), 'utf8')); +test('curated authoring keeps v1 sidecar and GitHub notes intact and derives role-scoped popup copy', () => { + const source = curatedInput(); + const { changelog, notes, popup } = authoring(source); + assert.doesNotMatch(JSON.stringify(notes), /"audience"|"popup"/); + assert.doesNotMatch(JSON.stringify(changelog), /"audience"|"popup"/); + assert.equal(popup.changelog.length, changelog.length); + assert.equal(popup.changelog[0].audience, 'platform'); + assert.equal(popup.changelog[0].title, source.changelog[0].popup.title); + assert.equal(popup.changelog[2].title, source.changelog[2].title); + const r = { ...release(), changelog }; + assert.ok(releaseNotes({ schemaVersion: 1, releaseId: r.releaseId, sourceCommit: r.sourceCommit, ...notes }, r)); + assert.match(markdown(r.version, changelog, notes), /Independent backup schedules/); + assert.equal(authoring(input()).popup, undefined); + source.afterUpdating = [{ title: 'Action', description: 'Actual required action.', audience: 'dsp' }]; + assert.equal(authoring(source).popup.afterUpdating[0].audience, 'dsp'); + assert.equal(authoring(source).notes.afterUpdating[0].audience, undefined); +}); +for (const mutate of [ + n => delete n.changelog[1].audience, + n => n.changelog[1].audience = 'all', + n => n.changelog[0].popup = null, + n => n.changelog[0].popup.title = '', + n => n.changelog[0].popup.url = 'https://example.test', + n => n.afterUpdating.push({ title: 'Action', description: 'Action without an audience.' }), +]) test('curated authoring rejects incomplete audience classification and invalid popup copy', () => { + const source = curatedInput(); mutate(source); assert.throws(() => authoring(source)); +}); diff --git a/core/core/installations/tests/release-package.test.py b/core/core/installations/tests/release-package.test.py new file mode 100644 index 0000000..784063b --- /dev/null +++ b/core/core/installations/tests/release-package.test.py @@ -0,0 +1,56 @@ +import gzip +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import tarfile +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location('package', Path(__file__).resolve().parents[1] / 'src/release-package.py') +package = importlib.util.module_from_spec(spec) +spec.loader.exec_module(package) +COMMIT = 'a' * 40 + + +class Packages(unittest.TestCase): + def test_deterministic_archives_and_round_trip(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / 'input' + file = root / 'dependencies/node/bin/node' + file.parent.mkdir(parents=True) + file.write_bytes(b'fixture') + file.chmod(0o555) + a, b = Path(tmp) / 'a.gz', Path(tmp) / 'b.gz' + self.assertEqual(package.pack(root, a, 'dependencies', '-'), 7) + package.pack(root, b, 'dependencies', '-') + self.assertEqual(a.read_bytes(), b.read_bytes()) + output = Path(tmp) / 'output' + package.unpack(a, str(output), 'dependencies', '-', package.digest(a)) + self.assertEqual((output / 'dependencies/node/bin/node').read_bytes(), b'fixture') + self.assertEqual((output / 'dependencies/node/bin/node').stat().st_mode & 0o777, 0o555) + for directory in [output, output / 'dependencies', output / 'dependencies/node', output / 'dependencies/node/bin']: + directory.chmod(0o700) + + def test_rejects_traversal_links_duplicates_oversize_missing_and_wrong_commit(self): + for bad in ['traversal', 'symlink', 'duplicate', 'oversize', 'missing', 'commit', 'unlisted', 'corrupt', 'wrong-root']: + with self.subTest(bad=bad), tempfile.TemporaryDirectory() as tmp: + name = '../escape' if bad == 'traversal' else 'runtime/dependencies/node' if bad == 'wrong-root' else 'core/code/shared/fixture.js' + entry = {'path': name, 'mode': '444', 'size': 128 * 1024 ** 2 + 1 if bad == 'oversize' else 1, 'sha256': hashlib.sha256(b'x').hexdigest()} + data = json.dumps({'schemaVersion': 1, 'kind': 'app', 'sourceCommit': 'b' * 40 if bad == 'commit' else COMMIT, 'files': [entry]}).encode() + archive = Path(tmp) / 'bad.gz' + with tarfile.open(archive, 'w:gz') as target: + m = tarfile.TarInfo(package.MANIFEST); m.size = len(data); target.addfile(m, io.BytesIO(data)) + if bad != 'missing': + m = tarfile.TarInfo('core/code/shared/other.js' if bad == 'unlisted' else name); m.size = 1; m.mode = 0o444 + if bad == 'symlink': m.type = tarfile.SYMTYPE; m.linkname = '/etc/passwd'; m.size = 0 + target.addfile(m, io.BytesIO(b'y' if bad == 'corrupt' else b'x')) + if bad == 'duplicate': target.addfile(m, io.BytesIO(b'x')) + with self.assertRaises(ValueError): + package.unpack(archive, str(Path(tmp) / 'out'), 'app', COMMIT, package.digest(archive)) + self.assertFalse((Path(tmp) / 'escape').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/core/core/installations/tests/release-reuse.test.js b/core/core/installations/tests/release-reuse.test.js new file mode 100644 index 0000000..19a0626 --- /dev/null +++ b/core/core/installations/tests/release-reuse.test.js @@ -0,0 +1,83 @@ +'use strict'; +const fs=require('node:fs'),os=require('node:os'),path=require('node:path'); +const test=require('node:test'),assert=require('node:assert/strict'); +const {sha,releaseManifest}=require('../src/release-delivery-contract'); +const {reuseComponents}=require('../src/release-delivery-build'); +function fixture(t) { + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-reuse-')); t.after(()=>require('../src/release-delivery-install').removeStage(root)); + const source=path.join(root,'source');fs.mkdirSync(source); + const commit='a'.repeat(40), version='987654.0.2',id='dispatch_'+version; + const entry=(name,data)=>({path:name,mode:'444',sha256:sha(data),data:Buffer.from(data).toString('base64')}); + const make=(kind,files)=>JSON.stringify({schemaVersion:1,kind,sourceCommit:commit,files}); + const files={'dispatch-core.json':make('core',[entry('code/shared/fixture.js','fixture'),entry('code/dashboard/release-popup.json','obsolete fixture popup')]), + 'dispatch-bridge.json':make('bridge',[entry('bridge-artifact/manifest.json','bridge fixture')]),'dispatch-runtime.tar.gz':'runtime fixture'}; + const assets={}; + for(const [kind,name] of Object.entries({core:'dispatch-core.json',bridge:'dispatch-bridge.json',runtime:'dispatch-runtime.tar.gz'})) { + fs.writeFileSync(path.join(source,name),files[name]);assets[kind]={name,size:Buffer.byteLength(files[name]),sha256:sha(files[name])}; + } + const notes=require('../examples/fictional-fragment.json'); + const {changelog,popup}=require('../src/release-notes').authoring(notes); + const runtime={version:1,backend:'native_service_v1',releaseId:id,channel:'production',sourceCommit:commit,platform:'linux/amd64',runtimeAgentProtocol:1,runtimeGatewayProtocol:1, + artifactSha256:assets.runtime.sha256,embeddedManifestSha256:'b'.repeat(64),bridgeManifestSha256:sha('bridge fixture')}; + fs.writeFileSync(path.join(source,'dispatch-release.json'),JSON.stringify(releaseManifest({schemaVersion:1,version,releaseId:id,sourceCommit:commit,changelog,assets,runtime}))); + return {root,source,commit,popup,files,out:()=>fs.mkdtempSync(path.join(root,'out-'))}; +} +test('reused components retain exact runtime bytes and replace only release-specific popup',async t=>{ + const f=fixture(t),out=f.out(),popup={schemaVersion:1,releaseId:'dispatch_1.2.3',version:'1.2.3',sourceCommit:f.commit,...f.popup}; + const result=await reuseComponents(f.source,out,f.commit,popup); + assert.equal(fs.readFileSync(path.join(out,'dispatch-runtime.tar.gz'),'utf8'),f.files['dispatch-runtime.tar.gz']); + assert.equal(result.artifact.artifactSha256,sha(f.files['dispatch-runtime.tar.gz'])); + const core=JSON.parse(fs.readFileSync(path.join(out,'dispatch-core.json'))); + assert.equal(core.files.length,2); + assert.deepEqual(JSON.parse(Buffer.from(core.files[1].data,'base64')),popup); +}); +test('mismatched source or tampered component prevents reuse',async t=>{ + const f=fixture(t); + await assert.rejects(reuseComponents(f.source,f.out(),'b'.repeat(40),null),/verified_source_mismatch/); + fs.appendFileSync(path.join(f.source,'dispatch-runtime.tar.gz'),'corrupt'); + await assert.rejects(reuseComponents(f.source,f.out(),f.commit,null),/verified_asset_mismatch/); +}); + +function splitFixture(t) { + const f = fixture(t), { pack, runtimeIdentity } = require('../src/release-package'); + const app = path.join(f.root, 'app'), dependencies = path.join(f.root, 'dependencies'); + const write = (root, name, bytes) => { + const file = path.join(root, name); fs.mkdirSync(path.dirname(file), { recursive: true }); fs.writeFileSync(file, bytes); + }; + write(app, 'core/code/dashboard/release-popup.json', 'obsolete popup'); + write(app, 'runtime/shared/fixture.js', 'unchanged runtime'); + write(dependencies, 'dependencies/node/bin/node', 'dependency fixture'); + const assets = { + app: pack(app, path.join(f.source, 'dispatch-app.tar.gz'), 'app', f.commit), + dependencies: pack(dependencies, path.join(f.source, 'dispatch-dependencies.tar.gz'), 'dependencies'), + }; + const original = JSON.parse(fs.readFileSync(path.join(f.source, 'dispatch-release.json'))); + const manifest = releaseManifest({ ...original, schemaVersion: 2, assets, notes: null, + dependencies: { node: '22.23.2', chrome: '151.0.7922.138' }, runtime: { ...original.runtime, artifactSha256: runtimeIdentity(assets) } }); + fs.writeFileSync(path.join(f.source, 'dispatch-release.json'), JSON.stringify(manifest)); + return { ...f, manifest }; +} +test('split reuse refreshes popup and package identity while preserving runtime and dependency bytes', async t => { + const f = splitFixture(t), out = f.out(); + const popup = { schemaVersion: 1, releaseId: 'dispatch_1.2.3', version: '1.2.3', sourceCommit: f.commit, ...f.popup }; + const result = await reuseComponents(f.source, out, f.commit, popup, 'split'); + assert.deepEqual(result.assets.dependencies, f.manifest.assets.dependencies); + assert.deepEqual(fs.readFileSync(path.join(out, 'dispatch-dependencies.tar.gz')), fs.readFileSync(path.join(f.source, 'dispatch-dependencies.tar.gz'))); + assert.notEqual(result.assets.app.sha256, f.manifest.assets.app.sha256); + assert.equal(result.artifact.artifactSha256, require('../src/release-package').runtimeIdentity(result.assets)); + const app = path.join(f.root, 'check'); + require('../src/release-package').unpack(path.join(out, 'dispatch-app.tar.gz'), app, 'app', f.commit, result.assets.app.sha256, result.assets.app.unpackedSize); + t.after(() => require('../src/release-delivery-install').removeStage(app)); + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(app, 'core/code/dashboard/release-popup.json'))), popup); + assert.equal(fs.readFileSync(path.join(app, 'runtime/shared/fixture.js'), 'utf8'), 'unchanged runtime'); + assert.equal(fs.existsSync(path.join(out, 'reuse-app')), false); +}); +test('split reuse rejects format mismatch, corrupt dependencies and cleans up after invalid popup', async t => { + const f = splitFixture(t); + await assert.rejects(reuseComponents(f.source, f.out(), f.commit, null), /verified_format_mismatch/); + const out = f.out(); + await assert.rejects(reuseComponents(f.source, out, f.commit, { invalid: true }, 'split')); + assert.equal(fs.existsSync(path.join(out, 'reuse-app')), false); + fs.appendFileSync(path.join(f.source, 'dispatch-dependencies.tar.gz'), 'corrupt'); + await assert.rejects(reuseComponents(f.source, f.out(), f.commit, null, 'split'), /verified_asset_mismatch/); +}); diff --git a/core/core/installations/tests/rollout-backups.test.js b/core/core/installations/tests/rollout-backups.test.js new file mode 100644 index 0000000..368324a --- /dev/null +++ b/core/core/installations/tests/rollout-backups.test.js @@ -0,0 +1,274 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'), crypto = require('node:crypto'); +const { AccessStore } = require('../../accounts/src/store'); +const { createPlatformUpdates } = require('../../accounts/src/platform-updates'); +const { createPlatformCoreUpdater } = require('../src/platform-core-update'); +const { createPlatformBackupWorker } = require('../src/platform-backup-worker'); +const { replacePreUpdateBackups, recordCategory } = require('../../accounts/src/backup-categories'); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-rollout-backups-')); + fs.mkdirSync(path.join(root, 'data'), { mode: 0o700 }); + const store = new AccessStore({ databaseRoot: path.join(root, 'data/access-control'), database: path.join(root, 'data/access-control/access-control.sqlite3') }); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + store.insertUser({ id: 'user_platform', email: 'owner@example.test', firstName: 'Owner', lastName: 'Test', passwordHash: 'synthetic', platformRole: 'owner', timestamp: 1000 }); + for (const [id, state] of [['alpha','ready'], ['beta','suspended'], ['gamma','waiting_for_provider_auth']]) { + store.createOrganization({ id: `org_${id}`, name: id, abbreviation: null, timezone: 'UTC', status: state === 'suspended' ? 'suspended' : state === 'ready' ? 'active' : 'setup_required', createdBy: null, timestamp: 1000 }); + store.insertStation(`org_${id}`, 'TST1', true, 1000); + store.createInstallation(`org_${id}`, `runtime_${id}`, state, 1000, 'dispatch_current_1', 'native_service_v1'); + } + let now = 10000; + const remote = { status: 'connected', backups: {} }; + const platformReleases = { dispatch_update_2: { version: '0.0.2', publishedAt: '2026-09-07T00:00:00.000Z', core: {}, changelog: [] } }; + const updates = () => createPlatformUpdates({ store, releases: { dispatch_update_2: { backend: 'native_service_v1' } }, platformReleases, enabled: true, clock: () => now, cleanupReady: () => true }); + const coreCalls = []; + const core = createPlatformCoreUpdater({ store, platformReleases, execute: async stage => coreCalls.push(stage), clock: () => now }); + const worker = () => createPlatformBackupWorker({ store, localRoot: root, archive: () => remote, clock: () => now }); + const session = { user: { id: 'user_platform' } }; + function uploaded(id) { + const row = store.db.prepare('SELECT metadata_json FROM platform_backup_records WHERE id=?').get(id); + remote.backups[id] = { status: 'verified', verification: 'upload', format: 2, metadataDigest: crypto.createHash('sha256').update(row.metadata_json).digest('hex') }; + } + function finishSnapshots() { + for (const job of store.db.prepare("SELECT * FROM installation_lifecycle_jobs WHERE operation='backup' AND status='queued'").all()) { + store.db.prepare("UPDATE installation_lifecycle_jobs SET status='succeeded',finished_at=?,result_json='{}' WHERE id=?").run(now, job.id); + store.db.prepare('UPDATE installations SET status=? WHERE organization_id=?').run(job.starting_state, job.organization_id); + store.db.prepare("UPDATE installation_backups SET status='available',tree_digest=?,file_count=1,total_bytes=10,completed_at=? WHERE id=?").run('a'.repeat(64), now, job.backup_id); + } + } + function start() { updates().command(session, { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'rollout:backup:test' }); } + return { store, remote, updates, worker, core, coreCalls, uploaded, finishSnapshots, start, session, advance: () => { now += 3600001; } }; +} +test('Core and every DSP enqueue together, all uploads gate Core, and upgrades reuse the original snapshot', async t => { + const f = fixture(t); f.start(); f.start(); + assert.equal(f.updates().view().rollout.phase, 'backups'); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM platform_backup_requests').get().n, 4); + assert.equal((await f.core.run()).status, 'waiting_for_backups'); + await f.worker().tick(); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM installation_lifecycle_jobs WHERE operation='backup' AND status='queued'").get().n, 3); + // Core is uploading while all three DSP snapshot jobs are queued. + assert.equal(f.store.db.prepare("SELECT phase FROM platform_backup_requests WHERE kind='core'").get().phase, 'uploading'); + f.finishSnapshots(); await f.worker().tick(); + const requests = f.store.db.prepare('SELECT * FROM platform_backup_requests').all(); + const backupId = r => r.job_id ? f.store.lifecycleJob(r.job_id).backup_id : r.id; + for (const r of requests.slice(0, -1)) f.uploaded(backupId(r)); + await f.worker().tick(); + assert.equal((await f.core.run()).status, 'waiting_for_backups'); + assert.deepEqual(f.coreCalls, []); + f.uploaded(backupId(requests.at(-1))); await f.worker().tick(); + assert.equal((await f.core.run()).status, 'core_verified'); + assert.deepEqual(f.coreCalls, ['apply', 'verify']); + f.updates().tick(); f.updates().tick(); + const job = f.store.db.prepare("SELECT * FROM installation_lifecycle_jobs WHERE operation='upgrade'").get(); + assert.ok(job); + const prior = requests.find(r => r.organization_id === job.organization_id); + assert.equal(job.backup_id, backupId(prior)); + assert.equal(JSON.parse(job.stage_receipts_json).__preUpdateBackup, job.backup_id); + assert.equal(f.store.db.prepare('SELECT count(*) n FROM installation_backups').get().n, 3); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM backup_categories WHERE category='pre_update'").get().n, 4); +}); +test('upload timeout pauses the rollout; resume retries only unfinished uploads across worker restarts', async t => { + const f = fixture(t); f.start(); await f.worker().tick(); f.finishSnapshots(); await f.worker().tick(); + const coreRequest = f.store.db.prepare("SELECT * FROM platform_backup_requests WHERE kind='core'").get(); + f.uploaded(coreRequest.id); await f.worker().tick(); + f.advance(); await f.worker().tick(); f.updates().tick(); + assert.equal(f.updates().view().rollout.status, 'paused'); + assert.equal(f.updates().view().rollout.backups.completed, 1); + assert.equal((await f.core.run()).status, 'idle'); + const ids = f.store.db.prepare('SELECT id FROM installation_lifecycle_jobs').all(); + f.updates().command(f.session, { action: 'resume' }); + for (const row of f.store.db.prepare('SELECT backup_id FROM installation_lifecycle_jobs').all()) f.uploaded(row.backup_id); + await f.worker().tick(); + assert.equal(f.updates().view().rollout.backups.status, 'completed'); + assert.deepEqual(f.store.db.prepare('SELECT id FROM installation_lifecycle_jobs').all(), ids); + assert.equal((await f.core.run()).status, 'core_verified'); +}); +test('snapshot retries replace persisted backup identities before Core and DSP upgrades proceed', async t => { + const f = fixture(t), db = f.store.db; + const { rolloutBackupProgress } = require('../../accounts/src/rollout-backups'); + f.start(); await f.worker().tick(); + const rolloutId = db.prepare('SELECT id FROM platform_rollouts').get().id; + const requestId = db.prepare("SELECT id FROM platform_backup_requests WHERE organization_id='org_alpha'").get().id; + const failedIds = []; + for (let attempt = 0; attempt < 2; attempt++) { + // Persist the current snapshot identity before the executor reports failure. + await f.worker().tick(); + const request = db.prepare('SELECT * FROM platform_backup_requests WHERE id=?').get(requestId); + const job = f.store.lifecycleJob(request.job_id); + failedIds.push(job.backup_id); + db.prepare("UPDATE installation_lifecycle_jobs SET status='failed',failure_code='backup_failed',finished_at=10000 WHERE id=?").run(job.id); + db.prepare("UPDATE installations SET status='ready' WHERE organization_id='org_alpha'").run(); + await f.worker().tick(); f.updates().tick(); + assert.equal(f.updates().view().rollout.status, 'paused'); + f.updates().command(f.session, { action: 'resume' }); + assert.equal(rolloutBackupProgress(db, rolloutId).members.find(m => m.organizationId === 'org_alpha').backupId, null); + await f.worker().tick(); + } + f.finishSnapshots(); await f.worker().tick(); + const current = db.prepare('SELECT j.backup_id FROM platform_backup_requests r JOIN installation_lifecycle_jobs j ON j.id=r.job_id WHERE r.id=?').get(requestId).backup_id; + assert.ok(!failedIds.includes(current)); + // A reader must follow the successful request even before set reconciliation. + const set = db.prepare('SELECT * FROM backup_sets').get(); + const staleMembers = JSON.parse(set.members_json); + staleMembers.find(m => m.organizationId === 'org_alpha').backupId = failedIds[0]; + db.prepare('UPDATE backup_sets SET members_json=? WHERE id=?').run(JSON.stringify(staleMembers), set.id); + assert.equal(rolloutBackupProgress(db, rolloutId).members.find(m => m.organizationId === 'org_alpha').backupId, current); + for (const row of db.prepare('SELECT id FROM platform_backup_records').all()) f.uploaded(row.id); + await f.worker().tick(); await f.worker().tick(); + const saved = db.prepare('SELECT * FROM backup_sets WHERE id=?').get(set.id); + assert.equal(saved.status, 'verified'); + assert.equal(JSON.parse(saved.members_json).find(m => m.organizationId === 'org_alpha').backupId, current); + assert.equal((await f.core.run()).status, 'core_verified'); + f.updates().tick(); f.updates().tick(); + const upgrade = db.prepare("SELECT * FROM installation_lifecycle_jobs WHERE operation='upgrade' AND organization_id='org_alpha'").get(); + assert.ok(upgrade); + assert.equal(upgrade.backup_id, current); + const { rolloutBackupProof } = require('../src/rollout-backup-proof'); + assert.ok(rolloutBackupProof(db, rolloutId, id => { + assert.ok(!failedIds.includes(id)); + const record = db.prepare('SELECT * FROM platform_backup_records WHERE id=?').get(id); + return { ...f.remote.backups[id], id, organizationId: record.organization_id, recoveryDigest: 'a'.repeat(64) }; + })); +}); +test('pre-update replacement is per component, upload-first, idempotent, and preserves manual/scheduled backups', t => { + const f = fixture(t), db = f.store.db; + function record(id, category, at, org = null) { + db.prepare('INSERT INTO platform_backup_records VALUES(?,?,?,?,NULL,?,NULL,NULL)').run(id, org, org ? 'dsp' : 'core', '{}', at); + recordCategory(db, id, category); + } + record('breq_old', 'pre_update', 1); record('breq_new', 'pre_update', 2); + record('breq_manual', 'manual', 1); record('breq_scheduled', 'scheduled', 1); + record('backup_peer', 'pre_update', 1, 'org_beta'); + f.uploaded('breq_old'); f.uploaded('backup_peer'); + replacePreUpdateBackups(f.store, f.remote, 3); + assert.equal(db.prepare('SELECT count(*) n FROM backup_deletions').get().n, 0); + f.uploaded('breq_new'); replacePreUpdateBackups(f.store, f.remote, 4); replacePreUpdateBackups(f.store, f.remote, 5); + assert.deepEqual(db.prepare('SELECT backup_id FROM backup_deletions').all().map(r => r.backup_id), ['breq_old']); +}); +test('independent uploads run concurrently with a fixed limit and report individual failures', async () => { + const { parallelBackupExports } = require('../src/parallel-backup-exports'); + let active = 0, peak = 0; + const results = await parallelBackupExports(Array.from({ length: 7 }, (_, i) => ({ id: String(i) })), { concurrency: 3, execute: async id => { + active++; peak = Math.max(peak, active); + await new Promise(resolve => setTimeout(resolve, 10)); active--; + if (id === '2') throw Error(); + return id; + } }); + assert.equal(peak, 3); assert.equal(results.size, 7); + assert.equal(results.get('2').ok, false); assert.equal(results.get('6').ok, true); +}); + +test('release cleanup accepts the shared set and rejects a missing or mismatched component receipt', async t => { + const f = fixture(t); f.start(); await f.worker().tick(); f.finishSnapshots(); await f.worker().tick(); + for (const row of f.store.db.prepare('SELECT id FROM platform_backup_records').all()) f.uploaded(row.id); + await f.worker().tick(); + const rolloutId = f.store.db.prepare('SELECT id FROM platform_rollouts').get().id; + const { rolloutBackupProof } = require('../src/rollout-backup-proof'); + const read = id => { + const row = f.store.db.prepare('SELECT * FROM platform_backup_records WHERE id=?').get(id); + return { ...f.remote.backups[id], id, organizationId: row.organization_id, recoveryDigest: 'a'.repeat(64) }; + }; + assert.equal(rolloutBackupProof(f.store.db, rolloutId, read).organizationId, null); + assert.throws(() => rolloutBackupProof(f.store.db, rolloutId, id => ({ ...read(id), metadataDigest: 'b'.repeat(64) })), /release_cleanup_unavailable/); +}); + +test('lifecycle reconciliation waits for other active backups when one worker throws', async () => { + const { createInstallationLifecycleReconciler } = require('../src/lifecycle-reconcile'); + let finished = false; + const store = { statusLifecycleMismatches: () => [], lifecycleExhaustedCandidates: () => [], lifecycleOutstandingCount: () => 0, + lifecycleExecutionCandidates: () => [{ id: 'job_one', organization_id: 'org_one', operation: 'backup' }, { id: 'job_two', organization_id: 'org_two', operation: 'backup' }] }; + const worker = createInstallationLifecycleReconciler({ store, backupOnly: true, concurrency: 2, + authorityFactory: organizationId => ({ organizationId }), runtimeFactory: (organizationId, authority) => ({ run: async () => { + assert.equal(authority.organizationId, organizationId); + if (organizationId === 'org_one') throw Error('worker_interrupted'); + await new Promise(resolve => setTimeout(resolve, 15)); finished = true; + return { status: 'succeeded' }; + } }) }); + await assert.rejects(worker.runPending('worker_fixture'), /worker_interrupted/); + assert.equal(finished, true); +}); + +test('legacy Core pre-update archives are replaced only after the whole new set uploads; unrelated snapshots survive', async t => { + const f = fixture(t), db = f.store.db; + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-legacy-retention-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const receiptRoot = path.join(root, 'receipts'); fs.mkdirSync(receiptRoot); + const oldId = 'rollout_' + 'e'.repeat(32); + db.prepare("INSERT INTO platform_rollouts VALUES(?,?,?,?,'completed',1,2)").run(oldId, 'dispatch_current_1', 'user_platform', 'old'); + db.prepare("INSERT INTO platform_rollout_core VALUES(?,'succeeded',?,2,NULL,2)").run(oldId, + JSON.stringify({ version: '0.0.1', publishedAt: '2026-09-06T00:00:00.000Z', core: {}, changelog: [] })); + const directory = path.join(root, 'backups/platform-core', oldId); + fs.mkdirSync(directory, { recursive: true }); + const { receiptKey } = require('../src/offsite-policy'); + const tags = [1, 2].map(attempt => receiptKey(path.join(directory, `attempt-${attempt}`))); + let snapshots = tags.map((tag, i) => ({ id: String(i + 1).repeat(64), hostname: 'dispatch', tags: [tag] })); + const unrelated = { id: 'a'.repeat(64), hostname: 'dispatch', tags: ['manual-backup'] }; + snapshots.push(unrelated); + const calls = []; + const run = args => { + calls.push(args); + if (args[0] === 'snapshots') return [snapshots]; + if (args[0] === 'forget') snapshots = snapshots.filter(s => !args.slice(1).includes(s.id)); + return []; + }; + const read = id => { + const row = db.prepare('SELECT * FROM platform_backup_records WHERE id=?').get(id); + return { ...f.remote.backups[id], id, organizationId: row.organization_id, recoveryDigest: 'a'.repeat(64) }; + }; + const { retireLegacyPreUpdateBackups } = require('../src/legacy-pre-update-backups'); + const retire = () => retireLegacyPreUpdateBackups({ db, config: { localRoot: root, coreUid: process.getuid(), prefix: 'legacy' }, + receiptRoot, ownerUid: process.getuid(), record: read, run, + storage: { withDeletionAccess: async (prefixes, task) => { assert.deepEqual(prefixes, ['legacy/data/', 'legacy/index/', 'legacy/snapshots/']); return task(); } } }); + f.start(); await f.worker().tick(); f.finishSnapshots(); await f.worker().tick(); + await retire(); assert.equal(calls.length, 0); assert.ok(fs.existsSync(directory)); + for (const row of db.prepare('SELECT id FROM platform_backup_records').all()) f.uploaded(row.id); + await f.worker().tick(); + const coreId = db.prepare("SELECT id FROM platform_backup_records WHERE kind='core'").get().id; + const savedProof = f.remote.backups[coreId]; delete f.remote.backups[coreId]; + await assert.rejects(retire(), /release_cleanup_unavailable/); + assert.equal(calls.length, 0); assert.ok(fs.existsSync(directory)); + f.remote.backups[coreId] = savedProof; + await retire(); + assert.deepEqual(snapshots, [unrelated]); assert.equal(fs.existsSync(directory), false); + assert.equal(calls.filter(args => args[0] === 'forget').length, 1); + assert.equal(calls.some(args => ['restore', 'check'].includes(args[0])), false); + const count = calls.length; delete f.remote.backups[coreId]; + await retire(); assert.equal(calls.length, count); +}); + +test('resuming after an upload failure ignores the stale error and reuses the existing snapshots', async t => { + const f = fixture(t); f.start(); await f.worker().tick(); f.finishSnapshots(); await f.worker().tick(); + const ids = f.store.db.prepare('SELECT id FROM platform_backup_records').all().map(row => row.id); + for (const id of ids) f.remote.backups[id] = { status: 'failed', checkedAt: 10000 }; + await f.worker().tick(); f.updates().tick(); + assert.equal(f.updates().view().rollout.status, 'paused'); + f.advance(); f.updates().command(f.session, { action: 'resume' }); + await f.worker().tick(); await f.worker().tick(); f.updates().tick(); + assert.equal(f.updates().view().rollout.status, 'running'); + for (const id of ids) f.uploaded(id); + await f.worker().tick(); + assert.equal(f.updates().view().rollout.backups.status, 'completed'); + assert.deepEqual(f.store.db.prepare('SELECT id FROM platform_backup_records').all().map(row => row.id), ids); +}); + +test('new snapshots use an idle export slot while an earlier snapshot is still uploading', async () => { + const {parallelBackupExports} = require('../src/parallel-backup-exports'); + let releaseFirst, secondStarted = false, discoveries = 0; + const gate = new Promise(resolve => { releaseFirst=resolve; }); + const results = await parallelBackupExports([{id:'first'}], {pollMs:1, + discover: async () => ++discoveries > 1 ? [{id:'first'},{id:'second'}] : [], + execute: async id => { if (id === 'first') await gate; else { secondStarted=true; releaseFirst(); } return id; }, + }); + assert.equal(secondStarted,true); assert.equal(results.size,2); +}); + +test('discovery failure waits for active uploads before releasing the parent lock', async () => { + const {parallelBackupExports} = require('../src/parallel-backup-exports'); + let discoveries = 0, finished = false; + await assert.rejects(parallelBackupExports([{id:'first'}], {pollMs:1, + discover: async () => { if (++discoveries > 1) throw Error('database unavailable'); return []; }, + execute: async () => { await new Promise(resolve => setTimeout(resolve,20)); finished=true; }, + }), /database unavailable/); + assert.equal(finished,true); +}); diff --git a/core/core/installations/tests/runtime-agent-authority-cli.test.js b/core/core/installations/tests/runtime-agent-authority-cli.test.js new file mode 100644 index 0000000..2ca78ff --- /dev/null +++ b/core/core/installations/tests/runtime-agent-authority-cli.test.js @@ -0,0 +1,82 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { AccessStore } = require('../../accounts/src/store'); +const { readPrivateRegistrationToken } = require('../../agents/src'); + +const command = path.resolve(__dirname, "../bin/dispatch-runtime-agent-authority"); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-agent-authority-cli-')); + fs.chmodSync(root, 0o700); + const accessRoot = path.join(root, 'access'); + const installationsRoot = path.join(root, 'installations'); + fs.mkdirSync(accessRoot, { mode: 0o700 }); + fs.mkdirSync(installationsRoot, { mode: 0o700 }); + const paths = { databaseRoot: accessRoot, database: path.join(accessRoot, 'access-control.sqlite3') }; + const store = new AccessStore(paths); + store.transaction(() => { + store.createOrganization({ + id: 'org_agent_cli', name: 'Agent CLI', abbreviation: 'ACL', + timezone: 'America/Los_Angeles', status: 'pending_owner', createdBy: null, timestamp: 1_000, + }); + store.insertStation('org_agent_cli', 'TST1', true, 1_000); + store.createInstallation('org_agent_cli', 'runtime_agent_cli', 'provisioning', 1_000); + }); + store.close(); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return { paths, accessRoot, installationsRoot }; +} + +function run(context, operation) { + const result = spawnSync(command, [operation, 'org_agent_cli'], { + env: { + PATH: process.env.PATH, + DISPATCH_ACCESS_CONTROL_DATABASE_ROOT: context.accessRoot, + DISPATCH_INSTALLATIONS_ROOT: context.installationsRoot, + }, + encoding: 'utf8', + timeout: 10_000, + }); + const output = JSON.parse(result.stdout); + assert.equal(result.status, 0, result.stderr || output.status); + assert.equal(output.ok, true); + return output; +} + +function authority(context) { + const store = new AccessStore(context.paths); + try { return store.runtimeAgentAuthority('runtime_agent_cli'); } + finally { store.close(); } +} + +function tokenFile(context) { + return path.join(context.installationsRoot, 'runtime_agent_cli', 'secrets', 'runtime-agent', 'registration-token'); +} + +test('authority CLI issues, rotates, revokes, and explicitly reissues one generation-fenced token', t => { + const context = fixture(t); + assert.equal(run(context, 'issue').generation, 1); + const firstToken = readPrivateRegistrationToken(tokenFile(context)); + assert.equal(authority(context).generation, 1); + + assert.equal(run(context, 'rotate').generation, 2); + assert.notEqual(readPrivateRegistrationToken(tokenFile(context)), firstToken); + assert.equal(authority(context).status, 'active'); + + assert.equal(run(context, 'revoke').generation, 3); + assert.equal(authority(context).status, 'revoked'); + assert.equal(fs.existsSync(tokenFile(context)), false); + + assert.equal(run(context, 'issue').generation, 4); + const current = authority(context); + assert.equal(current.status, 'active'); + assert.equal(current.token_hash, + crypto.createHash('sha256').update(readPrivateRegistrationToken(tokenFile(context))).digest('hex')); +}); diff --git a/core/core/installations/tests/runtime-agent-credential.test.js b/core/core/installations/tests/runtime-agent-credential.test.js new file mode 100644 index 0000000..ea511c1 --- /dev/null +++ b/core/core/installations/tests/runtime-agent-credential.test.js @@ -0,0 +1,73 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { + createRuntimeAgentCredentialManager, + createInstallationLayoutManager, + INSTALLATION_LAYOUT_TEMPLATE, +} = require('../src'); +const { readPrivateRegistrationToken } = require('../../agents/src'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-agent-credential-')); + fs.chmodSync(root, 0o700); + const installationsRoot = path.join(root, 'installations'); + fs.mkdirSync(installationsRoot, { mode: 0o700 }); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return installationsRoot; +} + +function manifest() { + return { + manifestVersion: 1, + revision: 1, + organization: { id: 'org_agent_credential', stationCode: 'TST1', timezone: 'America/Los_Angeles' }, + runtime: { key: 'runtime_agent_credential', templateId: INSTALLATION_LAYOUT_TEMPLATE, releaseId: 'dispatch_fixture_1' }, + }; +} + +function authority(value) { + return { + revision: value.revision, + organization: { ...value.organization }, + runtime: { ...value.runtime }, + }; +} + +test('runtime-agent credential materialization is private, stable, rotatable, and layout-compatible', t => { + const installationsRoot = fixture(t); + const credentials = createRuntimeAgentCredentialManager({ installationsRoot }); + const first = credentials.issue('runtime_agent_credential'); + assert.equal(first.changed, true); + assert.equal(first.tokenChanged, true); + assert.match(first.tokenHash, /^[a-f0-9]{64}$/); + assert.equal(Object.hasOwn(first, 'token'), false); + const tokenFile = credentials.paths('runtime_agent_credential').tokenFile; + const firstToken = readPrivateRegistrationToken(tokenFile); + assert.equal(fs.lstatSync(tokenFile).mode & 0o7777, 0o600); + const orphan = path.join(path.dirname(tokenFile), `.registration-token.${process.pid}.0123456789abcdef.tmp`); + fs.writeFileSync(orphan, `${firstToken}\n`, { mode: 0o600 }); + const replayed = credentials.issue('runtime_agent_credential'); + assert.equal(replayed.tokenHash, first.tokenHash); + assert.equal(replayed.tokenChanged, false); + assert.equal(fs.existsSync(orphan), false); + + const selectedManifest = manifest(); + const layout = createInstallationLayoutManager({ installationsRoot }); + assert.equal(layout.materialize(selectedManifest, authority(selectedManifest)).status, 'verified'); + assert.equal(readPrivateRegistrationToken(tokenFile), firstToken); + + const rotated = credentials.issue('runtime_agent_credential', { rotate: true }); + assert.equal(rotated.tokenChanged, true); + assert.notEqual(rotated.tokenHash, first.tokenHash); + assert.notEqual(readPrivateRegistrationToken(tokenFile), firstToken); + assert.equal(credentials.revoke('runtime_agent_credential', first.tokenHash), false); + assert.equal(fs.existsSync(tokenFile), true); + assert.equal(credentials.revoke('runtime_agent_credential', rotated.tokenHash), true); + assert.equal(credentials.revoke('runtime_agent_credential'), false); + assert.equal(fs.existsSync(tokenFile), false); +}); diff --git a/core/core/installations/tests/service-jobs.test.js b/core/core/installations/tests/service-jobs.test.js new file mode 100644 index 0000000..818f36d --- /dev/null +++ b/core/core/installations/tests/service-jobs.test.js @@ -0,0 +1,464 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const test = require('node:test'); +const { + INSTALLATION_LAYOUT_TEMPLATE, + PRIVATE_DIRECTORY_MODE, + INSTALLATION_SERVICE_PLAN_VERSION, + INSTALLATION_AGENT_SERVICE_COUNT, + createInstallationLayoutManager, + createInstallationServiceManager, + createDurableInstallationProvisioner, +} = require('../src'); +const { + INSTALLATION_SERVICE_PIPELINE_ID, + INSTALLATION_ROLLBACK_MAX_ATTEMPTS, + createInstallationJobStore, +} = require('../src/job-store'); + +const MANAGE = Object.freeze({ + scope: 'operator_fixture', + permission: 'platform.installations.manage', + operatorEnabled: true, +}); +const READ = Object.freeze({ scope: 'operator_fixture', permission: 'platform.installations.read' }); +const FIXTURE_REGISTRATION = Object.freeze({ + fixture: true, + installationState: 'pending', + retainedData: false, +}); + +function manifest(id) { + return { + manifestVersion: 1, + revision: 1, + organization: { id: `org_${id}`, stationCode: 'TST1', timezone: 'America/Los_Angeles' }, + runtime: { + key: `fixture_${id}`, + templateId: INSTALLATION_LAYOUT_TEMPLATE, + releaseId: 'dispatch_fixture_1', + }, + }; +} + +function authority(value) { + return { + revision: value.revision, + organization: { ...value.organization }, + runtime: { ...value.runtime }, + }; +} + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dj4-')); + fs.chmodSync(root, PRIVATE_DIRECTORY_MODE); + const stateRoot = path.join(root, 'c'); + const installationsRoot = path.join(root, 'i'); + const unitRoot = path.join(root, 'u'); + for (const selected of [stateRoot, installationsRoot, unitRoot]) { + fs.mkdirSync(selected, { mode: PRIVATE_DIRECTORY_MODE }); + } + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return { root, stateRoot, installationsRoot, unitRoot }; +} + +function serviceReceipt(status, changed = false) { + return Object.freeze({ + servicePlanVersion: INSTALLATION_SERVICE_PLAN_VERSION, + status, + serviceCount: INSTALLATION_AGENT_SERVICE_COUNT, + changed, + }); +} + +function fakeSupervisor(options = {}) { + const states = new Map(); + const actions = []; + let failStart = Boolean(options.failStart); + let failHealth = Boolean(options.failHealth); + let afterStart = null; + + function state(unit) { + if (!states.has(unit.name)) { + states.set(unit.name, { enabled: false, active: false, enableMode: 'none' }); + } + return states.get(unit.name); + } + + function mutate(capability, action, operation) { + capability(() => { + actions.push(action); + operation(); + }); + } + + const api = { + states, + actions, + setFailStart(value) { failStart = value; }, + setFailHealth(value) { failHealth = value; }, + setAfterStart(callback) { afterStart = callback; }, + snapshot(plan) { + return plan.units.map(unit => ({ id: unit.id, name: unit.name, ...state(unit) })); + }, + reload(_plan, capability) { + mutate(capability, 'reload', () => {}); + return serviceReceipt('reloaded', true); + }, + enable(plan, capability) { + for (const unit of plan.units) { + mutate(capability, `enable:${unit.id}`, () => { + state(unit).enabled = true; + state(unit).enableMode = 'runtime'; + }); + } + return serviceReceipt('enabled', true); + }, + disable(plan, capability) { + for (const unit of [...plan.units].reverse()) { + mutate(capability, `disable:${unit.id}`, () => { + state(unit).enabled = false; + state(unit).enableMode = 'none'; + }); + } + return serviceReceipt('disabled', true); + }, + start(plan, capability) { + for (let index = 0; index < plan.units.length; index += 1) { + const unit = plan.units[index]; + mutate(capability, `start:${unit.id}`, () => { state(unit).active = true; }); + if (failStart && index === 0) { + throw Object.assign(new Error('private supervisor detail'), { code: 'runtime_health_failed' }); + } + } + if (afterStart) afterStart(); + return serviceReceipt('started', true); + }, + stop(plan, capability) { + for (const unit of [...plan.units].reverse()) { + mutate(capability, `stop:${unit.id}`, () => { state(unit).active = false; }); + } + return serviceReceipt('stopped', true); + }, + resetFailed(_plan, capability) { + mutate(capability, 'reset-failed', () => {}); + return serviceReceipt('failure_state_reset', true); + }, + restoreState(plan, prior, capability) { + for (let index = 0; index < plan.units.length; index += 1) { + const unit = plan.units[index]; + mutate(capability, `restore:${unit.id}`, () => { + state(unit).enabled = prior[index].enabled; + state(unit).active = prior[index].active; + state(unit).enableMode = prior[index].enableMode; + }); + } + return serviceReceipt('state_restored', true); + }, + inspect(plan) { + if (plan.units.some(unit => !state(unit).enabled || !state(unit).active)) { + throw Object.assign(new Error('private inspect detail'), { code: 'runtime_health_failed' }); + } + return serviceReceipt('active'); + }, + health(plan) { + api.inspect(plan); + if (failHealth) throw Object.assign(new Error('private health detail'), { code: 'runtime_health_failed' }); + return serviceReceipt('healthy'); + }, + }; + return api; +} + +function provisioner(paths, supervisor, ids) { + return createDurableInstallationProvisioner({ + stateRoot: paths.stateRoot, + installationsRoot: paths.installationsRoot, + unitRoot: paths.unitRoot, + runtimeAgentHubSocket: path.join(paths.stateRoot, 'central-run', 'runtime-agent-hub.sock'), + supervisor, + idFactory: () => ids.shift(), + }); +} + +function request(operation, idempotencyKey, expectedRevision) { + return { operation, idempotencyKey, expectedRevision }; +} + +function journalPhases(paths) { + return fs.readdirSync(paths.unitRoot) + .filter(name => name.startsWith('.dispatch-service-') && name.endsWith('.json')) + .map(name => JSON.parse(fs.readFileSync(path.join(paths.unitRoot, name), 'utf8')).phase) + .sort(); +} + +function installedWork(paths, runtime, jobId) { + const store = createInstallationJobStore({ + stateRoot: paths.stateRoot, + pipelineId: INSTALLATION_SERVICE_PIPELINE_ID, + }); + const layout = createInstallationLayoutManager({ installationsRoot: paths.installationsRoot }); + const services = createInstallationServiceManager({ + unitRoot: paths.unitRoot, + runtimeAgentHubSocket: path.join(paths.stateRoot, 'central-run', 'runtime-agent-hub.sock'), + }); + const supervisor = fakeSupervisor(); + store.registerFixture(runtime, authority(runtime), FIXTURE_REGISTRATION, 1_000); + store.request( + runtime, + authority(runtime), + request('provision', `${jobId}_request`, 1), + MANAGE, + jobId, + 1_001, + ); + const claim = store.claimNext('worker_initial', 1_002, 100); + const guard = mutation => store.mutateClaim(claim, 1_003, mutation); + let current = store.work(claim, 1_003); + assert.equal(current.stage, 'runtime_layout_materialize'); + store.completeStage(claim, current.stage, layout.materialize(runtime, authority(runtime), guard), 1_004); + current = store.work(claim, 1_005); + store.completeStage(claim, current.stage, layout.inspect(runtime, authority(runtime)), 1_006); + const plan = services.plan(runtime, authority(runtime), layout.derive(runtime, authority(runtime))); + current = store.work(claim, 1_007); + store.completeStage(claim, current.stage, services.render(plan, guard), 1_008); + current = store.work(claim, 1_009); + store.completeStage(claim, current.stage, services.validate(plan), 1_010); + current = store.work(claim, 1_011); + store.completeStage( + claim, + current.stage, + services.install(plan, supervisor.snapshot(plan), guard), + 1_012, + ); + assert.equal(store.work(claim, 1_013).stage, 'runtime_service_start'); + return { store, layout, services, supervisor, plan, claim }; +} + +function restoreInstalledWork(work, claim, at, removeCandidate) { + const guard = mutation => work.store.mutateClaim(claim, at, mutation); + const prior = work.services.rollbackState(work.plan); + assert.ok(prior); + work.supervisor.stop(work.plan, guard); + work.supervisor.disable(work.plan, guard); + work.services.restoreFiles(work.plan, guard); + work.supervisor.reload(work.plan, guard); + work.supervisor.restoreState(work.plan, prior, guard); + if (removeCandidate) work.services.removeCandidate(work.plan, guard); + return prior; +} + +test('the durable service pipeline checkpoints and verifies the server-owned runtime units', t => { + const paths = fixture(t); + const supervisor = fakeSupervisor(); + const runtime = manifest('a'); + const selected = provisioner(paths, supervisor, ['job_service_success']); + t.after(() => selected.close()); + selected.registerFixture(runtime, authority(runtime), FIXTURE_REGISTRATION); + selected.request(runtime, authority(runtime), request('provision', 'service_success_request', 1), MANAGE); + const completed = selected.runNext('worker_service_success'); + assert.equal(completed.status, 'succeeded'); + assert.equal(completed.installationState, 'provisioning'); + assert.equal(completed.failure, null); + assert.equal(selected.inspect(runtime, authority(runtime), READ).status, 'succeeded'); + assert.equal(selected.runNext('worker_idle').status, 'idle'); + assert.equal(fs.readdirSync(paths.unitRoot).filter(name => name.endsWith('.service')).length, INSTALLATION_AGENT_SERVICE_COUNT); + assert.deepEqual(journalPhases(paths), ['verified']); + assert.equal(supervisor.snapshot({ units: [...supervisor.states.keys()].map(name => ({ + id: name.includes('auth-broker') ? 'auth_broker' + : name.includes('collection-manager') ? 'collection_manager' + : name.includes('-agent.') ? 'runtime_agent' : 'runtime_gateway', + name, + })) }).every(value => value.enabled && value.active), true); +}); + +test('every persisted legacy service checkpoint is rejected before supervisor mutation', t => { + const paths = fixture(t); + const supervisor = fakeSupervisor(); + const runtime = manifest('legacy_checkpoint'); + const selected = provisioner(paths, supervisor, ['job_legacy_checkpoint']); + selected.registerFixture(runtime, authority(runtime), FIXTURE_REGISTRATION); + selected.request(runtime, authority(runtime), request('provision', 'legacy_checkpoint_request', 1), MANAGE); + assert.equal(selected.runNext('worker_legacy_checkpoint').status, 'succeeded'); + selected.close(); + + const database = path.join(paths.stateRoot, 'provisioner.sqlite3'); + let db = new DatabaseSync(database); + const checkpoints = db.prepare(`SELECT stage_index,stage,receipt_json FROM job_checkpoints + WHERE job_id=? AND stage LIKE 'runtime_service_%' ORDER BY stage_index`).all('job_legacy_checkpoint'); + db.close(); + assert.equal(checkpoints.length, 5); + const actions = supervisor.actions.length; + + for (const checkpoint of checkpoints) { + const legacy = { ...JSON.parse(checkpoint.receipt_json), servicePlanVersion: 1, serviceCount: 2 }; + db = new DatabaseSync(database); + db.prepare('UPDATE job_checkpoints SET receipt_json=? WHERE job_id=? AND stage_index=?') + .run(JSON.stringify(legacy), 'job_legacy_checkpoint', checkpoint.stage_index); + db.close(); + + assert.throws(() => provisioner(paths, supervisor, ['unused']), + error => error?.code === 'service_installation_failed'); + assert.equal(supervisor.actions.length, actions); + + db = new DatabaseSync(database); + db.prepare('UPDATE job_checkpoints SET receipt_json=? WHERE job_id=? AND stage_index=?') + .run(checkpoint.receipt_json, 'job_legacy_checkpoint', checkpoint.stage_index); + db.close(); + } +}); + +test('a legacy service journal is rejected before resumed service-start mutation', t => { + const paths = fixture(t); + const runtime = manifest('legacy_journal'); + const work = installedWork(paths, runtime, 'job_legacy_journal'); + work.store.close(); + const journalName = fs.readdirSync(paths.unitRoot) + .find(name => name.startsWith('.dispatch-service-') && name.endsWith('.json')); + assert.ok(journalName); + const journalPath = path.join(paths.unitRoot, journalName); + const journal = JSON.parse(fs.readFileSync(journalPath, 'utf8')); + fs.writeFileSync(journalPath, JSON.stringify({ ...journal, servicePlanVersion: 1 }), { mode: 0o600 }); + + const actions = work.supervisor.actions.length; + const selected = provisioner(paths, work.supervisor, ['unused']); + t.after(() => selected.close()); + selected.runNext('worker_legacy_journal'); + assert.equal(work.supervisor.actions.length, actions); +}); + +test('partial service start rolls back before a retry succeeds', t => { + const paths = fixture(t); + const supervisor = fakeSupervisor({ failStart: true }); + const runtime = manifest('b'); + const selected = provisioner(paths, supervisor, ['job_service_failure', 'job_service_retry']); + t.after(() => selected.close()); + selected.registerFixture(runtime, authority(runtime), FIXTURE_REGISTRATION); + selected.request(runtime, authority(runtime), request('provision', 'service_failure_request', 1), MANAGE); + const failed = selected.runNext('worker_service_failure'); + assert.equal(failed.status, 'failed'); + assert.deepEqual(failed.failure, { + code: 'runtime_health_failed', category: 'infrastructure', recoverable: true, + }); + assert.equal(fs.readdirSync(paths.unitRoot).filter(name => name.endsWith('.service')).length, 0); + assert.deepEqual(journalPhases(paths), ['restored']); + assert.equal([...supervisor.states.values()].every(value => !value.enabled && !value.active), true); + + supervisor.setFailStart(false); + selected.request( + runtime, + authority(runtime), + request('retry', 'service_retry_request', failed.revision), + MANAGE, + ); + const retried = selected.runNext('worker_service_retry'); + assert.equal(retried.status, 'succeeded'); + assert.equal(retried.installationState, 'provisioning'); + assert.deepEqual(journalPhases(paths), ['verified']); +}); + +test('cancellation after service start rolls back and removes the inactive candidate', t => { + const paths = fixture(t); + const supervisor = fakeSupervisor(); + const runtime = manifest('c'); + const selected = provisioner(paths, supervisor, ['job_service_cancel']); + t.after(() => selected.close()); + selected.registerFixture(runtime, authority(runtime), FIXTURE_REGISTRATION); + selected.request(runtime, authority(runtime), request('provision', 'service_cancel_provision_request', 1), MANAGE); + supervisor.setAfterStart(() => { + supervisor.setAfterStart(null); + selected.request(runtime, authority(runtime), request('cancel', 'service_cancel_request', 2), MANAGE); + }); + const cancelled = selected.runNext('worker_service_cancel'); + assert.equal(cancelled.status, 'cancelled'); + assert.equal(cancelled.installationState, 'pending'); + assert.equal(fs.readdirSync(paths.unitRoot).filter(name => name.endsWith('.service')).length, 0); + assert.deepEqual(journalPhases(paths), ['restored']); + assert.equal([...supervisor.states.values()].every(value => !value.enabled && !value.active), true); + const configRoot = path.join(paths.installationsRoot, runtime.runtime.key, 'config'); + assert.deepEqual(fs.readdirSync(configRoot), []); +}); + +test('expired service cancellation is reclaimed as durable compensation', t => { + const paths = fixture(t); + const runtime = manifest('d'); + const work = installedWork(paths, runtime, 'job_service_reclaim'); + t.after(() => work.store.close()); + work.store.request( + runtime, + authority(runtime), + request('cancel', 'service_reclaim_cancel_request', 2), + MANAGE, + 'ignored_cancel_job', + 1_020, + ); + const compensationClaim = work.store.claimNext('worker_compensation', 1_103, 100); + const compensation = work.store.work(compensationClaim, 1_104); + assert.equal(compensation.compensating, true); + assert.equal(compensation.compensationIntent, 'cancelled'); + assert.throws( + () => work.store.mutateClaim(work.claim, 1_104, () => {}), + error => error?.code === 'installation_operation_in_progress', + ); + restoreInstalledWork(work, compensationClaim, 1_105, true); + const cancelled = work.store.finishCompensation(compensationClaim, 1_106); + work.services.finishRollback(work.plan); + assert.equal(cancelled.status, 'cancelled'); + assert.equal(cancelled.installationState, 'pending'); + assert.deepEqual(fs.readdirSync(paths.unitRoot), []); +}); + +test('attempt exhaustion compensates installed services before terminal failure', t => { + const paths = fixture(t); + const runtime = manifest('e'); + const work = installedWork(paths, runtime, 'job_service_exhaustion'); + t.after(() => work.store.close()); + let claim = work.claim; + let claimedAt = 1_103; + for (let attempt = 2; attempt <= 8; attempt += 1) { + claim = work.store.claimNext(`worker_attempt_${attempt}`, claimedAt, 100); + assert.equal(work.store.work(claim, claimedAt + 1).compensating, false); + claimedAt += 101; + } + const compensationClaim = work.store.claimNext('worker_exhaustion_rollback', claimedAt, 100); + const compensation = work.store.work(compensationClaim, claimedAt + 1); + assert.equal(compensation.compensating, true); + assert.equal(compensation.compensationIntent, 'failed'); + restoreInstalledWork(work, compensationClaim, claimedAt + 2, false); + const failed = work.store.finishCompensation(compensationClaim, claimedAt + 3); + work.services.finishRollback(work.plan); + assert.equal(failed.status, 'failed'); + assert.equal(failed.failure.code, 'installation_operation_failed'); + assert.equal(fs.readdirSync(paths.unitRoot).filter(name => name.endsWith('.service')).length, 0); +}); + +test('rollback retries are code-owned and bounded', t => { + const paths = fixture(t); + const runtime = manifest('f'); + const work = installedWork(paths, runtime, 'job_service_rollback_bound'); + t.after(() => work.store.close()); + work.store.beginCompensation( + work.claim, + 'failed', + Object.assign(new Error('private'), { code: 'runtime_health_failed' }), + 1_020, + ); + let claim = work.claim; + let terminal = null; + for (let attempt = 1; attempt <= INSTALLATION_ROLLBACK_MAX_ATTEMPTS; attempt += 1) { + terminal = work.store.failCompensation(claim, 1_020 + attempt); + if (attempt < INSTALLATION_ROLLBACK_MAX_ATTEMPTS) { + assert.equal(terminal.status, 'running'); + claim = work.store.claimNext(`worker_rollback_${attempt + 1}`, 1_030 + attempt, 100); + assert.equal(work.store.work(claim, 1_031 + attempt).compensating, true); + } + } + assert.equal(terminal.status, 'failed'); + assert.equal(terminal.failure.code, 'service_installation_failed'); + assert.equal(terminal.installationState, 'failed'); +}); diff --git a/core/core/installations/tests/services.test.js b/core/core/installations/tests/services.test.js new file mode 100644 index 0000000..fde42e0 --- /dev/null +++ b/core/core/installations/tests/services.test.js @@ -0,0 +1,277 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { spawnSync } = require('node:child_process'); +const { + INSTALLATION_LAYOUT_TEMPLATE, + PRIVATE_DIRECTORY_MODE, + createInstallationLayoutManager, + INSTALLATION_SERVICE_PLAN_VERSION, + INSTALLATION_SERVICE_COUNT, + INSTALLATION_AGENT_SERVICE_COUNT, + createInstallationServiceManager, +} = require('../src'); + +function selectedManifest(id) { + return { + manifestVersion: 1, + revision: 1, + organization: { id: `org_${id}`, stationCode: 'TST1', timezone: 'America/Los_Angeles' }, + runtime: { + key: `fixture_${id}`, + templateId: INSTALLATION_LAYOUT_TEMPLATE, + releaseId: 'dispatch_fixture_1', + }, + }; +} + +function authority(manifest) { + return { + revision: manifest.revision, + organization: { ...manifest.organization }, + runtime: { ...manifest.runtime }, + }; +} + +function isCode(code) { + return error => error?.code === code && error.message === code; +} + +function fixture(t) { + const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'ds-')); + fs.chmodSync(temporary, PRIVATE_DIRECTORY_MODE); + const root = path.join(temporary, 'x y'); + const installationsRoot = path.join(root, 'installations'); + const unitRoot = path.join(root, 'units'); + fs.mkdirSync(root, { mode: PRIVATE_DIRECTORY_MODE }); + fs.mkdirSync(installationsRoot, { mode: PRIVATE_DIRECTORY_MODE }); + fs.mkdirSync(unitRoot, { mode: PRIVATE_DIRECTORY_MODE }); + t.after(() => fs.rmSync(temporary, { recursive: true, force: true })); + return { temporary, installationsRoot, unitRoot }; +} + +test('managed service plans include a supervised outbound Runtime Agent when configured', t => { + const { temporary, installationsRoot, unitRoot } = fixture(t); + const layoutManager = createInstallationLayoutManager({ installationsRoot }); + const runtimeAgentHubSocket = path.join(temporary, 'central-run', 'runtime-agent-hub.sock'); + const serviceManager = createInstallationServiceManager({ unitRoot, runtimeAgentHubSocket }); + const alpha = selectedManifest('a'); + const bravo = selectedManifest('b'); + layoutManager.materialize(alpha, authority(alpha)); + layoutManager.materialize(bravo, authority(bravo)); + const alphaPlan = serviceManager.plan(alpha, authority(alpha), layoutManager.derive(alpha, authority(alpha))); + const bravoPlan = serviceManager.plan(bravo, authority(bravo), layoutManager.derive(bravo, authority(bravo))); + + assert.equal(alphaPlan.servicePlanVersion, INSTALLATION_SERVICE_PLAN_VERSION); + assert.equal(alphaPlan.units.length, INSTALLATION_AGENT_SERVICE_COUNT); + assert.deepEqual(alphaPlan.units.map(unit => unit.id), [ + 'auth_broker', 'collection_manager', 'runtime_gateway', 'runtime_agent', + ]); + assert.equal(alphaPlan.units.some(unit => bravoPlan.units.some(other => other.name === unit.name)), false); + assert.notEqual(alphaPlan.candidateRoot, bravoPlan.candidateRoot); + assert.equal(serviceManager.render(alphaPlan).changed, true); + assert.equal(serviceManager.render(alphaPlan).changed, false); + assert.equal(serviceManager.render(bravoPlan).changed, true); + assert.deepEqual(serviceManager.validate(alphaPlan), { + servicePlanVersion: INSTALLATION_SERVICE_PLAN_VERSION, + status: 'validated', + serviceCount: INSTALLATION_AGENT_SERVICE_COUNT, + changed: false, + }); + assert.equal(serviceManager.validate(bravoPlan).status, 'validated'); + + const auth = alphaPlan.units.find(unit => unit.id === 'auth_broker'); + const collection = alphaPlan.units.find(unit => unit.id === 'collection_manager'); + const gateway = alphaPlan.units.find(unit => unit.id === 'runtime_gateway'); + const agent = alphaPlan.units.find(unit => unit.id === 'runtime_agent'); + assert.match(auth.content, /Restart=always/); + assert.match(auth.content, /KillMode=control-group/); + assert.match(auth.content, /StartLimitBurst=5/); + assert.match(collection.content, new RegExp(`After=local-fs.target ${auth.name.replaceAll('.', '\\.')}`)); + assert.match(gateway.content, new RegExp(auth.name.replaceAll('.', '\\.'))); + assert.match(gateway.content, new RegExp(collection.name.replaceAll('.', '\\.'))); + assert.match(agent.content, new RegExp(gateway.name.replaceAll('.', '\\.'))); + for (const unit of alphaPlan.units) { + assert.equal(unit.content.includes('Environment=DISPATCH_LOCAL_ROOT='), false); + assert.equal(unit.content.includes('Environment=DISPATCH_ACCESS_CONTROL'), false); + assert.equal(unit.content.includes('\nEnvironment='), false); + assert.equal(unit.content.includes('--ignore-environment'), true); + assert.equal(unit.content.includes('x\\x20y'), true); + const info = fs.lstatSync(unit.candidate); + assert.equal(info.isFile(), true); + assert.equal(info.isSymbolicLink(), false); + assert.equal(info.nlink, 1); + assert.equal(info.mode & 0o7777, 0o600); + } + assert.equal(auth.content.includes('DISPATCH_PAYCOM_DATA_ROOT'), false); + assert.equal(collection.content.includes('DISPATCH_PAYCOM_DATA_ROOT'), true); + assert.equal(collection.content.includes('DISPATCH_CDF_DATA_ROOT'), true); + assert.equal(collection.environment.DISPATCH_MANAGED_RUNTIME, '1'); + assert.equal(Object.hasOwn(auth.environment, 'DISPATCH_MANAGED_RUNTIME'), false); + assert.equal(Object.hasOwn(gateway.environment, 'DISPATCH_MANAGED_RUNTIME'), false); + assert.equal(gateway.content.includes('DISPATCH_RUNTIME_GATEWAY_SOCKET'), true); + assert.equal(gateway.content.includes('DISPATCH_RUNTIME_KEY'), true); + assert.equal(agent.environment.DISPATCH_RUNTIME_AGENT_HUB_SOCKET, runtimeAgentHubSocket); + assert.equal(agent.environment.DISPATCH_RUNTIME_AGENT_TOKEN_FILE.endsWith('/secrets/runtime-agent/registration-token'), true); + assert.equal(agent.environment.DISPATCH_RUNTIME_AGENT_STATUS_SOCKET.endsWith('/run/runtime-agent-status.sock'), true); + assert.equal(gateway.content.includes('DISPATCH_ACCESS_CONTROL_DATABASE_ROOT='), false); + assert.deepEqual(Object.keys(serviceManager.render(alphaPlan)).sort(), [ + 'changed', 'serviceCount', 'servicePlanVersion', 'status', + ]); +}); + +test('service installation is restart-safe and rollback restores prior units', t => { + const { installationsRoot, unitRoot } = fixture(t); + const layoutManager = createInstallationLayoutManager({ installationsRoot }); + const serviceManager = createInstallationServiceManager({ unitRoot }); + const selected = selectedManifest('t'); + layoutManager.materialize(selected, authority(selected)); + const plan = serviceManager.plan(selected, authority(selected), layoutManager.derive(selected, authority(selected))); + serviceManager.render(plan); + serviceManager.validate(plan); + const priorContents = plan.units.map((unit, index) => `prior ${unit.id} unit ${index}\n`); + const priorState = plan.units.map((unit, index) => { + fs.writeFileSync(unit.installed, priorContents[index], { mode: 0o600 }); + return { + id: unit.id, + name: unit.name, + enabled: true, + active: index === 0, + enableMode: 'persistent', + }; + }); + + assert.equal(serviceManager.install(plan, priorState).status, 'installed'); + assert.equal(serviceManager.install(plan, priorState).changed, false); + assert.deepEqual(serviceManager.rollbackState(plan), priorState); + for (const unit of plan.units) assert.equal(fs.readFileSync(unit.installed, 'utf8'), unit.content); + + assert.equal(serviceManager.restoreFiles(plan).status, 'restored'); + for (let index = 0; index < plan.units.length; index += 1) { + assert.equal(fs.readFileSync(plan.units[index].installed, 'utf8'), priorContents[index]); + } + assert.equal(serviceManager.finishRollback(plan).status, 'rolled_back'); + assert.equal(fs.existsSync(plan.journal), false); + + const inactive = plan.units.map(unit => ({ + id: unit.id, name: unit.name, enabled: false, active: false, enableMode: 'none', + })); + assert.equal(serviceManager.install(plan, inactive).status, 'installed'); + assert.equal(serviceManager.markVerified(plan).status, 'verified'); + assert.equal(serviceManager.finalizeSettled(plan).status, 'committed'); + assert.equal(fs.existsSync(plan.journal), false); + assert.equal(serviceManager.inspectInstalled(plan).status, 'installed'); +}); + +test('rollback refuses externally drifted installed units', t => { + const { installationsRoot, unitRoot } = fixture(t); + const layoutManager = createInstallationLayoutManager({ installationsRoot }); + const serviceManager = createInstallationServiceManager({ unitRoot }); + const selected = selectedManifest('d'); + layoutManager.materialize(selected, authority(selected)); + const plan = serviceManager.plan(selected, authority(selected), layoutManager.derive(selected, authority(selected))); + serviceManager.render(plan); + const inactive = plan.units.map(unit => ({ + id: unit.id, name: unit.name, enabled: false, active: false, enableMode: 'none', + })); + serviceManager.install(plan, inactive); + fs.writeFileSync(plan.units[0].installed, 'external drift\n', { mode: 0o600 }); + assert.throws(() => serviceManager.restoreFiles(plan), isCode('service_installation_failed')); + assert.equal(fs.readFileSync(plan.units[0].installed, 'utf8'), 'external drift\n'); + assert.equal(fs.existsSync(plan.journal), true); +}); + +test('service-plan version drift fails closed before changing installed units', t => { + const { installationsRoot, unitRoot } = fixture(t); + const layoutManager = createInstallationLayoutManager({ installationsRoot }); + const serviceManager = createInstallationServiceManager({ unitRoot }); + const selected = selectedManifest('v'); + layoutManager.materialize(selected, authority(selected)); + const plan = serviceManager.plan(selected, authority(selected), layoutManager.derive(selected, authority(selected))); + serviceManager.render(plan); + const inactive = plan.units.map(unit => ({ + id: unit.id, name: unit.name, enabled: false, active: false, enableMode: 'none', + })); + serviceManager.install(plan, inactive); + const journal = JSON.parse(fs.readFileSync(plan.journal, 'utf8')); + fs.writeFileSync(plan.journal, `${JSON.stringify({ ...journal, servicePlanVersion: 1 })}\n`, { mode: 0o600 }); + const installed = plan.units.map(unit => fs.readFileSync(unit.installed, 'utf8')); + assert.throws(() => serviceManager.install(plan, inactive), isCode('runtime_boundary_violation')); + assert.deepEqual(plan.units.map(unit => fs.readFileSync(unit.installed, 'utf8')), installed); + assert.equal(fs.existsSync(plan.journal), true); +}); + +test('service rendering fails closed before stale or unsafe mutation', t => { + const { installationsRoot, unitRoot } = fixture(t); + const layoutManager = createInstallationLayoutManager({ installationsRoot }); + const serviceManager = createInstallationServiceManager({ unitRoot }); + const selected = selectedManifest('f'); + layoutManager.materialize(selected, authority(selected)); + const layout = layoutManager.derive(selected, authority(selected)); + const plan = serviceManager.plan(selected, authority(selected), layout); + + assert.throws(() => serviceManager.render(plan, () => { + throw Object.assign(new Error('installation_operation_in_progress'), { + code: 'installation_operation_in_progress', + }); + }), isCode('installation_operation_in_progress')); + assert.equal(fs.existsSync(plan.candidateRoot), false); + + assert.equal(serviceManager.render(plan).status, 'rendered'); + fs.writeFileSync(path.join(plan.candidateRoot, 'unknown.service'), 'fixture', { mode: 0o600 }); + assert.throws(() => serviceManager.render(plan), isCode('service_installation_failed')); + fs.rmSync(path.join(plan.candidateRoot, 'unknown.service')); + + const moved = `${unitRoot}.old`; + fs.renameSync(unitRoot, moved); + fs.mkdirSync(unitRoot, { mode: PRIVATE_DIRECTORY_MODE }); + assert.throws(() => serviceManager.inspect(plan), isCode('runtime_boundary_violation')); +}); + +test('service planning rejects mismatched runtime and unsafe unit roots', t => { + const { temporary, installationsRoot, unitRoot } = fixture(t); + const layoutManager = createInstallationLayoutManager({ installationsRoot }); + const serviceManager = createInstallationServiceManager({ unitRoot }); + const alpha = selectedManifest('q'); + const bravo = selectedManifest('r'); + layoutManager.materialize(alpha, authority(alpha)); + layoutManager.materialize(bravo, authority(bravo)); + assert.throws(() => serviceManager.plan( + alpha, + authority(alpha), + layoutManager.derive(bravo, authority(bravo)), + ), isCode('runtime_identity_mismatch')); + + const longSocket = selectedManifest('s'); + longSocket.runtime.key = `fixture_${'x'.repeat(88)}`; + const longAuthority = authority(longSocket); + layoutManager.materialize(longSocket, longAuthority); + assert.throws(() => serviceManager.plan( + longSocket, + longAuthority, + layoutManager.derive(longSocket, longAuthority), + ), isCode('runtime_boundary_violation')); + + const unsafe = path.join(temporary, 'unsafe-units'); + fs.mkdirSync(unsafe, { mode: 0o777 }); + assert.throws(() => createInstallationServiceManager({ unitRoot: unsafe }), + isCode('runtime_boundary_violation')); +}); + + +test('service planning accepts immutable root-owned code without relaxing private unit ownership', t => { + if (process.geteuid() === 0) return t.skip('requires the non-root Core identity'); + const f = fixture(t); + const rootCode = path.join(f.temporary, 'root-code'); + const prepared = spawnSync('/usr/bin/sudo', ['-n', '/usr/bin/install', '-d', '-o', '0', '-g', '0', '-m', '0555', rootCode]); + if (prepared.status !== 0) return t.skip('requires noninteractive sudo for the immutable code fixture'); + assert.equal(fs.lstatSync(rootCode).uid, 0); + assert.doesNotThrow(() => createInstallationServiceManager({ unitRoot: f.unitRoot, projectRoot: rootCode })); + assert.throws(() => createInstallationServiceManager({ unitRoot: rootCode, projectRoot: f.temporary }), isCode('runtime_boundary_violation')); + const writableCode = path.join(f.temporary, 'writable-code'); fs.mkdirSync(writableCode); fs.chmodSync(writableCode, 0o777); + assert.throws(() => createInstallationServiceManager({ unitRoot: f.unitRoot, projectRoot: writableCode }), isCode('runtime_boundary_violation')); +}); diff --git a/core/core/installations/tests/support/directory-sandbox.js b/core/core/installations/tests/support/directory-sandbox.js new file mode 100644 index 0000000..94aa3cf --- /dev/null +++ b/core/core/installations/tests/support/directory-sandbox.js @@ -0,0 +1,46 @@ +'use strict'; + +// Acceptance-only launcher. The application runtime uses the directory service. +const fs = require('node:fs'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); +const { directory, platformPaths } = require('../../../../shared/paths/platform-paths'); +const { inspectDsp } = require('../../../../host/storage/storage'); +const { CODE_ROOT, NODE_ROOT, runtimeEnvironment, storageMounts } = require('../../../../host/services/runtime-layout'); + +function fail() { throw new Error('directory_dsp_invalid'); } + +// A minimal filesystem allowlist plus separate PID, IPC, network, user, mount, +// UTS and cgroup namespaces. Never bind the host root or the entire DSP parent. +// Network is isolated in this acceptance backend; provider access is a later gate. +function sandboxArguments(paths, id, { toolsRoot, script, scriptArguments = [] } = {}) { + paths = platformPaths(paths.platformRoot); + const dsp = inspectDsp(paths, id); + const tools = directory(toolsRoot); + if (!tools.startsWith(paths.local + path.sep)) fail(); + if (typeof script !== 'string' || path.isAbsolute(script) || path.normalize(script) !== script + || script.startsWith('..') || !script.endsWith('.js') || !Array.isArray(scriptArguments) + || scriptArguments.some(value => typeof value !== 'string' || value.includes('\0'))) fail(); + const source = path.join(paths.live, script); + if (fs.realpathSync(source) !== source || !fs.statSync(source).isFile()) fail(); + const environment = runtimeEnvironment(id); + const args = ['--unshare-all', '--die-with-parent', '--new-session', '--clearenv', '--cap-drop', 'ALL', + '--hostname', 'dispatch-dsp', '--ro-bind', '/usr', '/usr', + '--symlink', 'usr/lib', '/lib', '--symlink', 'usr/lib64', '/lib64', + '--symlink', 'usr/bin', '/bin', '--symlink', 'usr/sbin', '/sbin', + '--proc', '/proc', '--dev', '/dev', '--size', '536870912', '--tmpfs', '/tmp', '--chmod', '1777', '/tmp', + '--ro-bind', paths.live, CODE_ROOT, '--ro-bind', tools, NODE_ROOT]; + for (const mount of storageMounts(dsp)) args.push('--bind', mount.source, mount.target); + args.push('--remount-ro', '/', '--chdir', CODE_ROOT); + for (const [name, value] of Object.entries(environment)) args.push('--setenv', name, value); + args.push('--', `${NODE_ROOT}/node`, '--no-warnings', path.join(CODE_ROOT, script), ...scriptArguments); + return args; +} + +function startSandbox(paths, id, options) { + return spawn('/usr/bin/bwrap', sandboxArguments(paths, id, options), { + env: { PATH: '/usr/bin:/bin', LANG: 'C.UTF-8' }, stdio: ['ignore', 'pipe', 'pipe'], shell: false, + }); +} + +module.exports = { sandboxArguments, startSandbox }; diff --git a/core/core/installations/tests/system-backup-manifests.test.js b/core/core/installations/tests/system-backup-manifests.test.js new file mode 100644 index 0000000..efb0a25 --- /dev/null +++ b/core/core/installations/tests/system-backup-manifests.test.js @@ -0,0 +1,101 @@ +'use strict'; +const test = require('node:test'), + assert = require('node:assert/strict'), + fs = require('node:fs'), + path = require('node:path'), + os = require('node:os'); +const { DatabaseSync } = require('node:sqlite'), + { initializeBackupSchema } = require('../../accounts/src/backup-schema'), + { createRestic } = require('../src/offsite-backup'), + { syncSystemManifests } = require('../src/system-backup-manifests'); +test( + 'full-system manifest is encrypted and independently recoverable; deleting it never touches component repositories', + { skip: !fs.existsSync('/usr/bin/restic') }, + async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-system-manifest-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const workRoot = path.join(root, 'work'); + fs.mkdirSync(workRoot, { mode: 0o700 }); + const password = path.join(root, 'password'); + fs.writeFileSync(password, 'only-a-disposable-test-password', { mode: 0o600 }); + const db = new DatabaseSync(':memory:'); + t.after(() => db.close()); + initializeBackupSchema(db); + const core = 'breq_' + 'a'.repeat(32), + dsp = 'backup_' + 'b'.repeat(32), + id = 'breq_' + 'c'.repeat(32), + members = [ + { organizationId: null, backupId: core }, + { organizationId: 'org_dsp', backupId: dsp }, + ]; + db.prepare("INSERT INTO backup_sets VALUES(?,1,?,'verified')").run(id, JSON.stringify(members)); + for (const [bid, org, kind] of [ + [core, null, 'core'], + [dsp, 'org_dsp', 'dsp'], + ]) + db.prepare("INSERT INTO platform_backup_records VALUES(?,?,?,'{}',NULL,1,NULL,NULL)").run( + bid, + org, + kind, + ); + const repositories = [], + runFactory = (env) => { + repositories.push(env.RESTIC_REPOSITORY); + return createRestic({ + ...env, + RESTIC_REPOSITORY: path.join(root, 'set-repository'), + RESTIC_PASSWORD_FILE: password, + }); + }; + const options = { + config: { + accountId: 'a'.repeat(32), + bucket: 'fixture', + environment: { PATH: '/usr/bin:/bin' }, + }, + db, + runFactory, + workRoot, + storage: { + removeSet: async (selected) => { + assert.equal(selected, id); + fs.rmSync(path.join(root, 'set-repository'), { recursive: true }); + }, + }, + record: (bid) => ({ + status: 'verified', + kind: bid === core ? 'core' : 'dsp', + organizationId: bid === core ? null : 'org_dsp', + }), + }; + const result = await syncSystemManifests(options); + assert.equal(result[id].status, 'verified'); + assert.ok(repositories.every((r) => r.endsWith('/sets/' + id))); + const run = createRestic({ + PATH: '/usr/bin:/bin', + RESTIC_REPOSITORY: path.join(root, 'set-repository'), + RESTIC_PASSWORD_FILE: password, + }), + saved = run(['dump', 'latest', 'system.json'])[0]; + assert.equal(saved.components.length, 2); + assert.deepEqual( + saved.components.map((c) => c.id), + [core, dsp], + ); + db.prepare("UPDATE backup_sets SET status='deleted' WHERE id=?").run(id); + const deleted = await syncSystemManifests(options); + assert.equal(deleted[id].status, 'deleted'); + assert.equal(fs.existsSync(path.join(root, 'set-repository')), false); + assert.equal( + db.prepare('SELECT count(*) n FROM platform_backup_records WHERE deleted_at IS NULL').get().n, + 2, + ); + }, +); +test('a failed set with no completed components can still finish deletion',async t=>{ + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-system-empty-'));t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + const db=new DatabaseSync(':memory:');t.after(()=>db.close());initializeBackupSchema(db); + const id='breq_'+'d'.repeat(32);db.prepare("INSERT INTO backup_sets VALUES(?,1,?,'deleting')").run(id,JSON.stringify([{organizationId:'org_dsp',backupId:null}])); + const removed=[];const result=await syncSystemManifests({db,config:{accountId:'a'.repeat(32),bucket:'fixture',environment:{}},workRoot:root,runFactory:()=>()=>{throw Error('must not create repository');},storage:{removeSet:async value=>removed.push(value)}}); + assert.deepEqual(removed,[id]);assert.equal(result[id].status,'deleted'); +}); diff --git a/core/core/installations/tests/update-efficiency.test.js b/core/core/installations/tests/update-efficiency.test.js new file mode 100644 index 0000000..6bf4312 --- /dev/null +++ b/core/core/installations/tests/update-efficiency.test.js @@ -0,0 +1,108 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'); +const { DatabaseSync } = require('node:sqlite'); +const { drain } = require('../src/drain-worker'); +const { AccessStore } = require('../../accounts/src/store'); +const { wake } = require('../../accounts/src/worker-wakeup'); + +test('productive passes drain immediately; waiting, failure, and pass limits stop without busy polling', async () => { + let calls = 0; + const result = await drain(async () => ({ progressed: ++calls < 4, pending: true, failed: 0 })); + assert.equal(calls, 4); assert.equal(result.pending, true); + calls = 0; await drain(async () => ({ progressed: ++calls > 0, failed: 1 })); assert.equal(calls, 1); + calls = 0; await drain(async () => ({ progressed: ++calls > 0 }), { maxPasses: 3 }); assert.equal(calls, 3); +}); + +test('wakeups run only after outer commit and disappear on nested or outer rollback', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-commit-wakeup-')); fs.chmodSync(root, 0o700); + const databaseRoot = path.join(root, 'db'), store = new AccessStore({ databaseRoot, database: path.join(databaseRoot, 'access.sqlite3') }); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const events = []; + store.transaction(() => { + store.afterCommit(() => events.push('outer')); + store.transaction(() => store.afterCommit(() => events.push('inner'))); + assert.throws(() => store.transaction(() => { store.afterCommit(() => events.push('rolled-back')); throw Error('rollback'); })); + assert.deepEqual(events, []); + }); + assert.deepEqual(events, ['outer', 'inner']); + assert.throws(() => store.transaction(() => { store.afterCommit(() => events.push('bad')); throw Error('rollback'); })); + assert.deepEqual(events, ['outer', 'inner']); +}); + +test('wake failure leaves committed work for fallback and never wakes a different installation', () => { + let calls = 0; + const options = { localRoot: '/home/fixture/local', databaseRoot: '/home/fixture/local/data/access-control', send: (_, args) => { + calls++; assert.ok(args.includes('--no-block')); throw Error('systemd unavailable'); + } }; + assert.doesNotThrow(() => wake(['core', 'reconcile'], options)); assert.equal(calls, 1); + wake(['core'], { ...options, databaseRoot: '/tmp/test-db' }); + wake(['unknown'], options); assert.equal(calls, 1); +}); + +test('stage timing keeps interrupted attempts and safe failure codes separately from retry success', () => { + const db = new DatabaseSync(':memory:'); let clock = 100; + try { + const timing = require('../src/operation-timing'); + timing.start(db, { jobId: 'life_fixture', attempt: 1, stage: 'snapshot' }, () => clock); + clock = 200; + const fail = timing.start(db, { jobId: 'life_fixture', attempt: 2, stage: 'snapshot' }, () => clock); + clock = 230; fail(new Error('secret must never be logged')); + const done = timing.start(db, { jobId: 'life_fixture', attempt: 3, stage: 'snapshot' }, () => clock); + clock = 250; done(); + const rows = db.prepare('SELECT * FROM operation_stage_timings ORDER BY started_at').all(); + assert.deepEqual(rows.map(r => r.status), ['running', 'failed', 'succeeded']); + assert.deepEqual(rows.map(r => r.duration_ms), [null, 30, 20]); + assert.equal(JSON.stringify(rows).includes('secret'), false); + } finally { db.close(); } +}); + +test('sealed Core and DSP exports leave the owning worker, dashboard dependencies and tenant services running', () => { + const { captureWriters } = require('../src/host-recovery-bundle'); + const services = ['dispatch-installation-reconcile.service', 'dispatch-dashboard.service', 'dispatch-dsp-fixture.service', + 'dispatch-installation-reconcile.timer', 'dispatch-offsite-backup.timer', 'dispatch-platform-update.timer'].map(name => ({ name, active: true })); + assert.deepEqual(captureWriters({ services, snapshotSource: '/sealed/core', scopedCore: true, kind: 'core' }), []); + assert.deepEqual(captureWriters({ services, snapshotSource: '/sealed/dsp', scopedCore: false, kind: 'dsp' }), []); + const legacy = captureWriters({ services, snapshotSource: '/legacy', scopedCore: false, kind: 'core' }); + assert.ok(legacy.some(s => s.name === 'dispatch-dashboard.service')); + assert.equal(legacy.some(s => ['dispatch-offsite-backup.timer', 'dispatch-platform-update.timer'].includes(s.name)), false); +}); + +test('wait reasons retain one interval across repeated polls and close when the dependency completes', () => { + const db = new DatabaseSync(':memory:'); const { wait } = require('../src/operation-timing'); + try { + wait(db, 'rollout_fixture', 'waiting_for_backups', true, () => 10); + wait(db, 'rollout_fixture', 'waiting_for_backups', true, () => 20); + wait(db, 'rollout_fixture', 'waiting_for_backups', false, () => 40); + const rows = db.prepare('SELECT * FROM operation_stage_timings').all(); + assert.equal(rows.length, 1); assert.equal(rows[0].duration_ms, 30); assert.equal(rows[0].status, 'succeeded'); + } finally { db.close(); } +}); + +test('rendered reconciliation survives dashboard restarts and both idle timers use the fallback cadence', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-worker-units-')); + t.after(() => { fs.chmodSync(root, 0o700); fs.chmodSync(path.join(root, 'units'), 0o700); fs.rmSync(root, { recursive: true, force: true }); }); + require('../src/core-artifact-layout').finishCoreArtifact(root, { releaseId: 'dispatch_fixture', sourceCommit: 'a'.repeat(40), localRoot: '/home/fixture/local', port: 4310, publicOrigin: 'https://fixture.example' }); + const unit = fs.readFileSync(path.join(root, 'units/dispatch-installation-reconcile.service'), 'utf8'); + assert.ok(unit.includes('Wants=dispatch-dashboard.service')); + assert.equal(unit.includes('Requires=dispatch-dashboard.service'), false); + for (const name of ['dispatch-installation-reconcile', 'dispatch-platform-update']) + assert.ok(fs.readFileSync(path.join(root, `units/${name}.timer`), 'utf8').includes('OnUnitInactiveSec=60s')); +}); + +test('export scan detects snapshots arriving mid-pass and wakes user workers only for unfinished work', t => { + const localRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-export-drain-')); + const data = path.join(localRoot, 'data/access-control'); fs.mkdirSync(data, { recursive: true }); + const db = new DatabaseSync(path.join(data, 'access-control.sqlite3')); + t.after(() => { db.close(); fs.rmSync(localRoot, { recursive: true, force: true }); }); + db.exec(`CREATE TABLE platform_rollouts(status TEXT); CREATE TABLE platform_backup_requests(status TEXT); + CREATE TABLE installation_lifecycle_jobs(status TEXT); CREATE TABLE platform_backup_records(id TEXT,deleted_at INTEGER); + CREATE TABLE installation_backups(id TEXT,status TEXT,completed_at INTEGER);`); + const { backupQueueKey, workPending } = require('../src/drain-worker'), config = { localRoot }; + const before = backupQueueKey(config); assert.equal(workPending(config), false); + db.exec("INSERT INTO installation_backups VALUES('backup_fixture','available',100); INSERT INTO installation_lifecycle_jobs VALUES('running')"); + assert.notEqual(backupQueueKey(config), before); assert.equal(workPending(config), true); + db.exec("UPDATE installation_lifecycle_jobs SET status='succeeded'"); + assert.equal(workPending(config), false); + db.exec("INSERT INTO platform_rollouts VALUES('running')"); assert.equal(workPending(config), true); +}); diff --git a/core/core/plugins/README.md b/core/core/plugins/README.md new file mode 100644 index 0000000..d2237cb --- /dev/null +++ b/core/core/plugins/README.md @@ -0,0 +1,25 @@ +# Core plugin services + +`sdk-service.js` binds each transport to a host-authenticated DSP, plugin, +installation revision and job. It validates closed SDK messages, checks authority +before and after asynchronous work, and releases browsers acquired during +revocation. Ordinary SDK requests cannot choose their own identity. + +`access-authority.js` checks the existing Core access database on every request. +Connections require both a manifest declaration and an explicit connection policy +grant. Missing grant policy denies connection access. Individual handlers must +also enforce ownership of job, schedule and published resources. + +`package-catalog.js` reads the host-managed approved package digest allowlist from +`local/config/plugin-packages.json`. Packages reside under +`local/packages/plugins///`. No request supplies a package +path or digest. `installation.js` orders the catalog and host lifecycle ports. +Core accounts retains the durable desired/applied installation registry and now +accepts this coordinator during reconciliation. + +The directory API enables the coordinator with the scoped worker launcher, +initialization hooks and independently built packages. `runtime-services.js` +adapts jobs, schedules, publication and structured logging to existing durable +DSP services. Worker execution remains bounded by global and per-DSP admission. +The dashboard's split mode forwards HTTP requests to `core/api/`; private worker +SDK traffic continues using bound Unix sockets. diff --git a/core/core/plugins/access-authority.js b/core/core/plugins/access-authority.js new file mode 100644 index 0000000..9b5c2aa --- /dev/null +++ b/core/core/plugins/access-authority.js @@ -0,0 +1,37 @@ +'use strict'; +const { catalog: defaultCatalog } = require('../../shared/plugin-sdk/catalog'); + +// Identity comes from a bound worker socket. Installation state is read afresh +// for every request and after asynchronous work; a cached SDK grant is never +// sufficient to keep using a disabled or replaced plugin. +function createAccessPluginAuthority({ store, catalog = defaultCatalog, manifestFor, connectionGrant = () => false }) { + if (!store?.db || typeof catalog !== 'function' || typeof connectionGrant !== 'function') throw new TypeError('plugin_authority_required'); + return function authorize(context, request) { + const definition = manifestFor ? manifestFor(context) : catalog().find(item => item.id === context.pluginId); + if (!definition) return false; + const row = store.db.prepare(`SELECT p.revision,p.applied_revision,p.desired_state,p.applied_state,p.version, + i.organization_id,i.status installation_status,o.status organization_status + FROM installations i JOIN organizations o ON o.id=i.organization_id + JOIN dsp_plugins p ON p.organization_id=i.organization_id AND p.plugin_id=? WHERE i.runtime_key=?`) + .get(context.pluginId, context.dspId); + if (!row || row.installation_status !== 'ready' || row.organization_status !== 'active' + || row.revision !== context.installationRevision || row.applied_revision !== row.revision + || row.desired_state !== 'enabled' || row.applied_state !== 'enabled' || row.version !== definition.version + || store.activeLifecycleJob(row.organization_id) + || store.db.prepare('SELECT 1 FROM dsp_removals WHERE organization_id=?').get(row.organization_id) + || store.db.prepare("SELECT 1 FROM directory_lifecycle_requests WHERE organization_id=? AND status IN ('queued','running')").get(row.organization_id)) return false; + if (request.operation === 'connections.status' || request.operation === 'connections.acquire') { + // Declaring a dependency is not a grant. The broker supplies its current + // approved connection policy; absent policy denies all credential access. + return definition.services.includes(request.input.connection) + && connectionGrant(context, request.input.connection) === true; + } + if (request.operation === 'jobs.enqueue') return (definition.jobs || []).includes(request.input.action); + if (request.operation === 'actions.invoke') { + return definition.actions.some(action => action.id === request.input.action); + } + return ['capabilities.get', 'settings.get', 'connections.renew', 'connections.release', 'jobs.status', 'jobs.cancel', 'jobs.retry', + 'schedules.list', 'schedules.status', 'schedules.run', 'schedules.set', 'schedules.remove', 'published.read', 'progress.report', 'log.write'].includes(request.operation); + }; +} +module.exports = { createAccessPluginAuthority }; diff --git a/core/core/plugins/assets.js b/core/core/plugins/assets.js new file mode 100644 index 0000000..a7e04e9 --- /dev/null +++ b/core/core/plugins/assets.js @@ -0,0 +1,18 @@ +'use strict'; +const path = require('node:path'); +const { read } = require('../../shared/plugin-sdk/package-files'); +const { installedPackage } = require('../../host/plugins/install'); + +function createPluginAssets({ dspRoot }) { + if (typeof dspRoot !== 'function') throw new TypeError('plugin_asset_scope_required'); + return function assets({ runtimeKey, pluginId, revision }) { + const selected = installedPackage({ dspRoot: dspRoot(runtimeKey), pluginId, revision }); + const manifest = selected.manifest.plugin; + if (!manifest.frontend || !manifest.frontend.startsWith('frontend/')) throw new Error('plugin_frontend_unavailable'); + const css = path.posix.join(path.posix.dirname(manifest.frontend), 'styles.css'); + return { id: pluginId, version: manifest.version, revision, + javascript: read(selected.directory, manifest.frontend, 2 * 1024 * 1024).toString('utf8'), + stylesheet: selected.manifest.files.some(file => file.path === css) ? read(selected.directory, css, 1024 * 1024).toString('utf8') : '' }; + }; +} +module.exports = { createPluginAssets }; diff --git a/core/core/plugins/backend.js b/core/core/plugins/backend.js new file mode 100644 index 0000000..8152ef6 --- /dev/null +++ b/core/core/plugins/backend.js @@ -0,0 +1,277 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { BrowserStore } = require('../browser-manager/store'); +const { BrowserManager } = require('../browser-manager/manager'); +const { AuthenticationCoordinator } = require('../auth-broker/coordinator'); +const { ScopedWorkerHost } = require('../../host/services/scoped-worker'); +const { AuthenticationWorkerHost } = require('../../host/services/authentication-worker'); +const { browserRelay } = require('../../host/plugins/browser-relay'); +const { installedPackage, installationReceipt } = require('../../host/plugins/install'); +const { privateDirectory } = require('../../host/controller/operations'); +const { privateJson, atomic } = require('../installations/src/release-delivery-files'); +const { createPluginService } = require('./sdk-service'); +const { createAccessPluginAuthority } = require('./access-authority'); +const { createRuntimeServices } = require('./runtime-services'); +const { hasGrant } = require('./connection-grants'); +const { DispatchError } = require('../../sdk/src/protocol'); +const fail = code => { throw new DispatchError(code, { recoverable: true }); }; +const { settingsStore } = require('./settings-store'); +const { initializeSettings, applySettingsPolicy } = require('./settings-policy'); + +// The independent Core backend holds policy, admission and process references. +// Every provider entrypoint runs in a selected DSP's disposable namespace. +async function openPluginBackend({ paths, installation, store, dspRoot, permitted, timezoneFor, + wake, networkPolicy, assistance, sourceRoot = paths.live, runtimeSourceFor = id => require('../../host/releases/runtime').runtimeSource(paths,id), workerHost, authenticationHost, limits = {} }) { + const stateRoot = privateDirectory(path.join(paths.local, 'state/plugin-backend')); + const jobsRoot = privateDirectory(path.join(stateRoot, 'jobs')); + const namespaceRoot = privateDirectory(path.join(paths.local, 'run/plugin-backend-namespaces')); + const jobs = new Map(), active = new Map(), queue = [], cleanup = new Map(); + let closing = false; + const workers = workerHost || new ScopedWorkerHost({ sourceRoot, sourceRootFor: runtimeSourceFor, nodeRoot: installation.nodeRoot, namespaceRoot }); + function rowFor(dspId, pluginId) { + return store.db.prepare(`SELECT p.* FROM dsp_plugins p JOIN installations i ON i.organization_id=p.organization_id + WHERE i.runtime_key=? AND p.plugin_id=?`).get(dspId, pluginId); + } + function selected(dspId, pluginId) { + if (!permitted(dspId)) fail('permission_denied'); + const row = rowFor(dspId, pluginId); + if (!row || row.desired_state !== 'enabled' || row.applied_state !== 'enabled' || row.revision !== row.applied_revision) fail('plugin_disabled'); + const value = installedPackage({ dspRoot: dspRoot(dspId), pluginId, revision: row.revision }); + if (value.manifest.plugin.version !== row.version) fail('plugin_revision_conflict'); + return value; + } + function packages(dspId) { + return store.db.prepare('SELECT p.plugin_id FROM dsp_plugins p JOIN installations i ON i.organization_id=p.organization_id WHERE i.runtime_key=?').all(dspId).flatMap(row => { + try { const value = selected(dspId, row.plugin_id); return [{ id: row.plugin_id, directory: value.directory, digest: value.receipt.digest }]; } + catch { return []; } + }); + } + const authWorkers = authenticationHost || new AuthenticationWorkerHost({ sourceRoot, sourceRootFor: runtimeSourceFor, ...installation, namespaceRoot, + dspRoot, packages, permitted, networkPolicy, assistance }); + const browsers = new BrowserStore(path.join(stateRoot, 'browsers.sqlite3')); + let manager, auth; + function manifestFor(context) { return jobs.get(context.jobId)?.manifest || null; } + const ordinary = createAccessPluginAuthority({ store, manifestFor, + connectionGrant: (context, service) => hasGrant(dspRoot(context.dspId), context, service) }); + const authorize = (context, request) => { + const job = jobs.get(context.jobId); + if (!job || job.cancelled || closing || !permitted(context.dspId)) return false; + if (job.kind !== 'initialize') return ordinary(context, request); + const row = rowFor(context.dspId, context.pluginId); + return ['capabilities.get', 'settings.get', 'progress.report', 'log.write'].includes(request.operation) + && row?.revision === context.installationRevision && row.version === job.manifest.version && row.desired_state === 'enabled'; + }; + const authorizePlugin = (context, connection) => authorize(context, { operation: 'connections.acquire', input: { connection } }); + function authAllowed(dspId, request) { + if (!permitted(dspId)) return false; + const provider = request.input?.service || (request.profile === 'paycom-main' || request.action === 'enroll-paycom' ? 'paycom' : null); + if (!provider || provider === 'cortex') return true; + try { return packages(dspId).some(item => item.id === provider); } catch { return false; } + } + manager = new BrowserManager({ store: browsers, workers: { start: (...args) => authWorkers.startLease(...args), close: row => authWorkers.closeLease(row) }, + authorize: context => permitted(context.dspId), limits: { sessions: 2, tabs: 12, perDsp: 1, ...limits.browser } }); + try { + await manager.start(); + // A Core crash invalidates every SDK binding. Reap each durable identity + // before accepting new work, even if its result was never acknowledged. + for (const name of fs.readdirSync(jobsRoot)) { + if (!/^job_[a-f0-9]{32}\.json$/.test(name)) fail('plugin_worker_boundary'); + const record = privateJson(path.join(jobsRoot, name), process.geteuid()); + if (Object.keys(record).sort().join(',') !== 'dspId,jobId,schemaVersion' || record.schemaVersion !== 1 || record.jobId + '.json' !== name) fail('plugin_worker_boundary'); + const root = dspRoot(record.dspId); + await workers.stop(record.jobId); + fs.rmSync(path.join(root, 'run/plugin-workers', record.jobId), { recursive: true, force: true }); + fs.unlinkSync(path.join(jobsRoot, name)); + } + auth = new AuthenticationCoordinator({ manager, workers: authWorkers, + generationFor: dspId => JSON.stringify(packages(dspId).map(item => [item.id, item.digest])), + contextFor: dspId => ({ dspId, pluginId: 'core-auth', installationRevision: 1, jobId: 'auth_' + crypto.randomBytes(16).toString('hex') }), + authorizeRequest: authAllowed, authorizePlugin, + manualRetryFor: context => require('./manual-auth-retry').manualAuthRetry(dspRoot(context.dspId), context, jobs.get(context.jobId)), + relay: (context, row, browser) => { + const job = jobs.get(context.jobId); + if (!job || job.cancelled) fail('permission_denied'); + return browserRelay({ dspId: context.dspId, authRunRoot: authWorkers.selected(row).runRoot, + runRoot: job.runRoot, browser }); + } }); + } catch (error) { await manager?.close(); browsers.close(); throw error; } + const handlers = createRuntimeServices({ dspRoot, manifestFor, timezoneFor, wake, + invoke: (context, action, input, options) => nested(context, 'invoke', { action, input }, options), + published: (context, view, query, options) => nested(context, 'read', { view, query }, options) }); + const sdk = createPluginService({ authorize, handlers: { ...handlers, ...auth.handlers(), + 'settings.get': context => { + const snapshot = jobs.get(context.jobId)?.settings; + if (!snapshot) fail('capability_unavailable'); + return snapshot; + }, + } }); + const maximum = limits.workers || 8, perDsp = limits.perDsp || 2; + let reaping = false, nextSettingsCheck = 0; + const cleanupTimer = setInterval(async () => { + if (reaping) return; + reaping = true; + try { + for (const [jobId, release] of cleanup) { + try { await workers.reap(jobId); cleanup.delete(jobId); release(); } catch { /* Keep capacity reserved and retry. */ } + } + if (Date.now() >= nextSettingsCheck) { nextSettingsCheck = Date.now() + 15000; + for (const row of store.db.prepare(`SELECT i.runtime_key,p.plugin_id FROM dsp_plugins p + JOIN installations i ON i.organization_id=p.organization_id WHERE p.desired_state='enabled' + AND p.applied_state='enabled' AND p.revision=p.applied_revision AND i.status='ready'`).all()) { + try { if (!settingsStore(dspRoot(row.runtime_key),row.plugin_id).pending()) continue; + const manifest = selected(row.runtime_key,row.plugin_id).manifest.plugin; + if (!require('../../host/releases/guard').updating(paths, row.runtime_key)) applySettingsPolicy(dspRoot(row.runtime_key),manifest); } catch { /* Pending policy is retried after lifecycle/storage recovery. */ } + } + } + } finally { reaping = false; } + }, 1000); + cleanupTimer.unref(); + function pump() { + for (let index = 0; index < queue.length && active.size < maximum;) { + const item = queue[index]; + if (closing || item.signal?.aborted || Date.now() >= item.deadline) { + queue.splice(index, 1); item.reject(new DispatchError(closing ? 'service_unavailable' : 'cancelled')); continue; + } + if ([...active.values()].filter(job => job.dspId === item.dspId).length >= perDsp + || item.writer && !item.nested && [...active.values()].some(job => job.dspId === item.dspId && job.pluginId === item.pluginId && job.writer)) { index++; continue; } + queue.splice(index, 1); const token = Symbol(); active.set(token, item); item.resolve(token); + } + } + async function admission(dspId, pluginId, writer, signal, timeoutMs, nested = false) { + if (closing || queue.length >= 128) fail('plugin_worker_capacity'); + // A parent never waits while holding the last worker slot. Nested calls + // either use spare bounded capacity immediately or return a retryable error. + if (nested && (active.size >= maximum || [...active.values()].filter(job => job.dspId === dspId).length >= perDsp)) fail('plugin_worker_capacity'); + let token; + const timer = setInterval(pump, 1000); timer.unref(); + const cancel = () => pump(); signal?.addEventListener('abort', cancel, { once: true }); + try { token = await new Promise((resolve, reject) => { const item = { dspId, pluginId, writer, nested, signal, deadline: Date.now() + timeoutMs, resolve, reject }; + if (nested) queue.unshift(item); else queue.push(item); pump(); }); } + finally { clearInterval(timer); signal?.removeEventListener('abort', cancel); } + return () => { active.delete(token); pump(); }; + } + function nested(context, kind, input, options) { + const parent = jobs.get(context.jobId); + if (!parent || parent.nested) fail('nested_action_limit'); + return execute(context.dspId, context.pluginId, kind, input, { ...options, nested: true }); + } + async function execute(dspId, pluginId, kind, input, { signal, initialize, timeoutMs = 300000, nested = false } = {}) { + if (!['initialize', 'invoke', 'collect', 'publish', 'read', 'inspect', 'evidence'].includes(kind)) fail('invalid_request'); + const chosen = initialize || selected(dspId, pluginId); + const manifest = chosen.manifest.plugin; + const revision = initialize?.revision || chosen.receipt.revision; + if (manifest.id !== pluginId) fail('permission_denied'); + if (kind === 'invoke' && !manifest.actions.some(item => item.id === input.action)) fail('permission_denied'); + if (kind === 'collect' && !manifest.collectors.includes(input.source?.collector)) fail('permission_denied'); + if (kind === 'collect') { + const remaining = Date.parse(input.deadline) - Date.now(); + if (!Number.isSafeInteger(remaining) || remaining <= 0) fail('collector_timeout'); + timeoutMs = Math.min(3600000, remaining); + // The six-page auth namespace retains one authenticated handoff page. + // Grant collection concurrency from the remaining five page slots. + const requested = input.source?.config?.maxConcurrency; + if (Number.isInteger(requested) && requested >= 1 && requested <= 6) input = { + ...input, source: { ...input.source, config: { ...input.source.config, maxConcurrency: Math.min(requested, 5) } }, + }; + } + if (kind === 'evidence') { + const { exact } = require('../../sdk/src/protocol'); + exact(input, ['batchId', 'preparationRunId', 'definitionDigest']); + const databaseRoot = path.join(dspRoot(dspId), 'data/collection-manager'); + const collections = new (require('dispatch-runtime-kit/collection-manager/src/store').CollectionStore)( + { databaseRoot, database: path.join(databaseRoot, 'collection-manager.sqlite3') }, { readOnly: true, plugins: [manifest] }); + try { + const batch = collections.batch(input.batchId); + const runs = [input.preparationRunId, ...batch.runs.map(item => item.run.id)].map(id => collections.run(id)); + if (runs.some(run => !manifest.collectors.includes(run.collector))) fail('permission_denied'); + input = { ...input, batch, runs }; + } finally { collections.close(); } + } + const revalidate = () => { + if (!permitted(dspId)) fail('permission_denied'); + if (initialize) { + const row = rowFor(dspId, pluginId); + if (row?.revision !== revision || row.version !== manifest.version || row.desired_state !== 'enabled') fail('permission_denied'); + } else { + const current = selected(dspId, pluginId); + if (current.receipt.revision !== revision || current.receipt.digest !== chosen.receipt.digest) fail('plugin_revision_conflict'); + } + }; + revalidate(); + const deadline = Date.now() + timeoutMs; + const release = await admission(dspId, pluginId, kind !== 'read', signal, timeoutMs, nested); + let ownedJobId; + try { + revalidate(); + timeoutMs = Math.max(1, deadline - Date.now()); + if (kind === 'initialize') initializeSettings(dspRoot(dspId), manifest); + const settings = manifest.settings ? settingsStore(dspRoot(dspId), pluginId).read(manifest.settings) : null; + const result = await workers.run({ dspRoot: dspRoot(dspId), dspId, pluginId, version: manifest.version, + digest: chosen.digest || chosen.receipt.digest, signal, timeoutMs, + task: { kind, action: kind === 'invoke' ? input.action : null, input: kind === 'invoke' ? input.input : input, timezone: timezoneFor(dspId) }, + transport: ({ jobId, runRoot }) => { + ownedJobId = jobId; + const context = Object.freeze({ dspId, pluginId, installationRevision: revision, jobId }); + atomic(path.join(jobsRoot, jobId + '.json'), { schemaVersion: 1, dspId, jobId }); + jobs.set(jobId, { context, manifest, settings, runRoot, kind, nested, cancelled: false, + collectionRunId: kind === 'collect' ? input.runId : null }); + return sdk.bind(context); + }, + onClose: async ({ jobId }) => { + for (const [id, session] of auth.sessions) if (session.context.jobId === jobId) await auth.release(session.context, id); + jobs.delete(jobId); fs.unlinkSync(path.join(jobsRoot, jobId + '.json')); + } }); + revalidate(); + return result; + } finally { + if (ownedJobId && jobs.has(ownedJobId)) cleanup.set(ownedJobId, release); + else release(); + } + } + async function revoke(dspId, pluginId) { + for (const job of jobs.values()) if (job.context.dspId === dspId && (!pluginId || job.context.pluginId === pluginId)) job.cancelled = true; + await workers.revoke(dspId, pluginId); + if (pluginId) await auth.revokePlugin(dspId, pluginId); else await auth.revoke(dspId); + } + return { execute, selected, packages, auth, workers, manager, jobs, revoke, + async settingsRequest(dspId, pluginId, request, options) { + const chosen = selected(dspId,pluginId), manifest = chosen.manifest.plugin; + if (!manifest.settings) fail('capability_unavailable'); + const storage = settingsStore(dspRoot(dspId),pluginId); + if (request.action === 'update') { + storage.update(manifest.settings,request.input,request.actor); + applySettingsPolicy(dspRoot(dspId),manifest); wake(dspId); + } + const current = request.action === 'history' ? storage.history(manifest.settings,request.input) : storage.read(manifest.settings); + let choices = {}; + if (request.action === 'options' && manifest.settings.optionsView) { + choices = await execute(dspId,pluginId,'read',{view:manifest.settings.optionsView,query:{}},options); + } + if (selected(dspId,pluginId).receipt.revision !== chosen.receipt.revision) fail('plugin_revision_conflict'); + return request.action === 'options' ? choices : current; + }, + canStart(dspId) { + return !store.db.prepare(`SELECT 1 FROM dsp_plugins p JOIN installations i ON i.organization_id=p.organization_id + WHERE i.runtime_key=? AND p.revision<>p.applied_revision`).get(dspId); + }, + async authRequest(dspId, request, options) { + if (!authAllowed(dspId, request)) fail('permission_denied'); + if (request.action === 'activity' && !auth.dsps.has(dspId)) return { ok: true, status: 'idle', busy: false }; + // Only SDK jobs may obtain a browser capability. Framework operations + // administer connections and submit work, never raw browser sessions. + if (['acquire-browser', 'renew-browser', 'release-browser'].includes(request.action)) fail('permission_denied'); + return auth.request(dspId, request, options); + }, + async close() { + closing = true; pump(); + await Promise.all([...new Set([...jobs.values()].map(job => job.context.dspId))].map(id => revoke(id))); + const deadline = Date.now() + 35000; + while (active.size && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 50)); + if (active.size) fail('plugin_worker_stop_failed'); + clearInterval(cleanupTimer); + await auth.close(); await manager.close(); browsers.close(); + } }; +} +module.exports = { openPluginBackend }; diff --git a/core/core/plugins/connection-grants.js b/core/core/plugins/connection-grants.js new file mode 100644 index 0000000..761dc05 --- /dev/null +++ b/core/core/plugins/connection-grants.js @@ -0,0 +1,28 @@ +'use strict'; +const path = require('node:path'); +const { privateJson, atomic } = require('../installations/src/release-delivery-files'); +const { privateDirectory } = require('../../host/controller/operations'); +const { installationReceipt } = require('../../host/plugins/install'); + +// Reviewed platform policy is separate from what a package requests. An owner +// installing the approved Paycom package grants only its Paycom connection. +const POLICY = Object.freeze({ paycom: Object.freeze(['paycom']) }); +function writeGrants(dspRoot, manifest, digest, revision) { + const root = privateDirectory(path.join(dspRoot, 'config/plugins/grants')); + const services = manifest.services.filter(id => (POLICY[manifest.id] || []).includes(id)); + atomic(path.join(root, `${manifest.id}.json`), { schemaVersion: 1, pluginId: manifest.id, + version: manifest.version, digest, revision, services }); +} +function hasGrant(dspRoot, context, connection) { + try { + const receipt = installationReceipt(dspRoot, context.pluginId); + const grant = privateJson(path.join(dspRoot, 'config/plugins/grants', `${context.pluginId}.json`), process.geteuid()); + return Object.keys(grant).sort().join(',') === 'digest,pluginId,revision,schemaVersion,services,version' + && receipt.state === 'enabled' && receipt.revision === context.installationRevision + && grant.schemaVersion === 1 && grant.pluginId === context.pluginId && grant.version === receipt.version + && grant.digest === receipt.digest && grant.revision === receipt.revision + && Array.isArray(grant.services) && grant.services.includes(connection) + && (POLICY[context.pluginId] || []).includes(connection); + } catch { return false; } +} +module.exports = { writeGrants, hasGrant }; diff --git a/core/core/plugins/installation.js b/core/core/plugins/installation.js new file mode 100644 index 0000000..165c1e6 --- /dev/null +++ b/core/core/plugins/installation.js @@ -0,0 +1,17 @@ +'use strict'; + +// Core accounts remains the durable desired/applied registry. This coordinator +// owns the ordering of package preparation and runtime acknowledgement. Its +// host port must fence the DSP lifecycle for the entire operation. +function createInstallationCoordinator({ catalog, host, needsMigration = () => false, resume = async () => {}, acknowledge = ({ runtimeKey, request, invoke }) => invoke(runtimeKey, 'plugins.manage', request) }) { + if (typeof catalog?.resolve !== 'function' || typeof host?.apply !== 'function') throw new TypeError('plugin_installation_dependencies_required'); + return Object.freeze({ + needsMigration, resume, latest: (id, runtimeKey) => catalog.latest?.(id, runtimeKey) || null, + async apply({ runtimeKey, request, invoke, authorize }) { + const approved = catalog.resolve(request.pluginId, request.version, runtimeKey); + return host.apply({ runtimeKey, request, approved, authorize, + acknowledge: () => acknowledge({ runtimeKey, request, invoke }) }); + }, + }); +} +module.exports = { createInstallationCoordinator }; diff --git a/core/core/plugins/manual-auth-retry.js b/core/core/plugins/manual-auth-retry.js new file mode 100644 index 0000000..84a23c3 --- /dev/null +++ b/core/core/plugins/manual-auth-retry.js @@ -0,0 +1,16 @@ +'use strict'; +const path = require('node:path'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); + +function manualAuthRetry(dspRoot, context, job) { + if (context.pluginId !== 'paycom' || job?.kind !== 'collect' || !job.collectionRunId || job.cancelled + || ['dspId', 'pluginId', 'installationRevision', 'jobId'].some(key => context[key] !== job.context[key])) return false; + const databaseRoot = path.join(dspRoot, 'data/collection-manager'); + const store = new CollectionStore({ databaseRoot, database: path.join(databaseRoot, 'collection-manager.sqlite3') }, { readOnly: true }); + try { + const run = store.run(job.collectionRunId); + return run.collector === context.pluginId && run.status === 'running' && !run.cancelRequested + && run.trigger === 'sync_manual'; + } finally { store.close(); } +} +module.exports = { manualAuthRetry }; diff --git a/core/core/plugins/package-catalog.js b/core/core/plugins/package-catalog.js new file mode 100644 index 0000000..dac0a31 --- /dev/null +++ b/core/core/plugins/package-catalog.js @@ -0,0 +1,69 @@ +'use strict'; +const path = require('node:path'); +const { privateJson } = require('../installations/src/release-delivery-files'); +const { SHA256 } = require('../../shared/plugin-sdk/package'); +const { verifyPackage } = require('../../shared/plugin-sdk/package-files'); +const ID = /^[a-z][a-z0-9-]{0,63}$/; +const VERSION = /^\d+\.\d+\.\d+$/; +const compare = (a, b) => { const x=a.version.split('.').map(Number),y=b.version.split('.').map(Number); return x[0]-y[0]||x[1]-y[1]||x[2]-y[2]; }; +function normalizeCatalog(value) { + const fail = () => { throw new Error('plugin_catalog_invalid'); }; + if (Buffer.byteLength(JSON.stringify(value)||'')>256*1024)fail(); + if (!value || ![1,2].includes(value.schemaVersion) || Object.keys(value).sort().join(',') !== (value.schemaVersion === 1 ? 'items,schemaVersion' : 'approved,items,schemaVersion') + || !Array.isArray(value.items) || value.items.length > 1024) fail(); + const entries = new Map(); + for (const item of value.items) { + if (!item || Object.keys(item).sort().join(',') !== 'digest,pluginId,version' || !ID.test(item.pluginId) + || !VERSION.test(item.version) || !SHA256.test(item.digest) || entries.has(`${item.pluginId}@${item.version}`)) fail(); + entries.set(`${item.pluginId}@${item.version}`, item); + } + // A legacy catalog was already active. Preserve that decision before staging + // a new candidate; never implicitly approve a newly downloaded version. + if (value.schemaVersion === 1) { + const production = Object.fromEntries([...entries.values()].sort(compare).map(item => [item.pluginId,item.version])); + value = { schemaVersion:2, items:value.items, approved:{production,dsps:{}} }; + } + const approved=value.approved; + if (!approved || Object.keys(approved).sort().join(',') !== 'dsps,production') fail(); + const mapping = map => { + if (!map || typeof map !== 'object' || Array.isArray(map)) fail(); + for (const [id, version] of Object.entries(map)) if (!ID.test(id) || !VERSION.test(version) || !entries.has(`${id}@${version}`)) fail(); + }; + mapping(approved.production); + if (!approved.dsps || typeof approved.dsps !== 'object' || Array.isArray(approved.dsps) || Object.keys(approved.dsps).length > 10000) fail(); + for (const [id,map] of Object.entries(approved.dsps)) { if (!/^[a-zA-Z0-9_-]{1,128}$/.test(id)) fail(); mapping(map); } + return value; +} +function packageCatalog({ local }) { + const raw = privateJson(path.join(local,'config/plugin-packages.json'),process.geteuid(),true); + if (!raw) return null; + const value=normalizeCatalog(raw),entries=new Map(value.items.map(item=>[`${item.pluginId}@${item.version}`,item])); + const latest = (id,runtimeKey) => { + const approved = runtimeKey && Object.hasOwn(value.approved.dsps, runtimeKey) + ? value.approved.dsps[runtimeKey] : value.approved.production; + const version = approved[id]; + return version ? entries.get(`${id}@${version}`) : null; + }; + const resolve = (id,version) => { + const item=entries.get(`${id}@${version}`); + if (!item) throw new Error('plugin_package_unavailable'); + const directory=path.join(local,'packages/plugins',id,version),manifest=verifyPackage(directory,item.digest); + if (manifest.plugin.id!==id || manifest.plugin.version!==version) throw new Error('plugin_catalog_invalid'); + return {directory,digest:item.digest,manifest}; + }; + return Object.freeze({ latest, resolve, + resolveApproved(id,version,runtimeKey) { + if (latest(id,runtimeKey)?.version !== version) throw new Error('plugin_package_not_approved'); + return resolve(id,version); + }, + definitions() { + const selected=new Map(); + for (const map of [value.approved.production,...Object.values(value.approved.dsps)]) for (const [id,version] of Object.entries(map)) { + const prior=selected.get(id); + if (!prior || compare({version},prior)>0) selected.set(id,resolve(id,version).manifest.plugin); + } + return [...selected.values()]; + }, + }); +} +module.exports = { packageCatalog, normalizeCatalog }; diff --git a/core/core/plugins/runtime-events.js b/core/core/plugins/runtime-events.js new file mode 100644 index 0000000..2c0c1d1 --- /dev/null +++ b/core/core/plugins/runtime-events.js @@ -0,0 +1,30 @@ +'use strict'; +const path = require('node:path'); +const { openDatabase, transaction } = require('../../shared/published/database'); +const { privateDirectory } = require('../../host/controller/operations'); +const { DispatchError } = require('../../sdk/src/protocol'); +const fail = () => { throw new DispatchError('invalid_request'); }; +function recordEvent(dspRoot, context, type, value) { + const keys = type === 'progress' ? ['phase', 'completed', 'total'] : ['level', 'code', 'counts', 'durationMs']; + if (Object.keys(value).some(key => !keys.includes(key))) fail(); + if (type === 'progress') { + if (!/^[a-z][a-z0-9_]{0,63}$/.test(value.phase || '') || !Number.isSafeInteger(value.completed) + || !Number.isSafeInteger(value.total) || value.completed < 0 || value.completed > value.total || value.total > 1e9) fail(); + } else { + if (!['debug', 'info', 'warn', 'error'].includes(value.level) || !/^[a-z][a-z0-9_]{0,63}$/.test(value.code || '') + || value.durationMs !== undefined && (!Number.isSafeInteger(value.durationMs) || value.durationMs < 0 || value.durationMs > 86400000)) fail(); + if (value.counts !== undefined && (!value.counts || Object.getPrototypeOf(value.counts) !== Object.prototype || Object.keys(value.counts).length > 16 + || Object.entries(value.counts).some(([key, count]) => !/^[a-z][a-z0-9_]{0,31}$/.test(key) || !Number.isSafeInteger(count) || count < 0 || count > 1e9))) fail(); + } + const root = privateDirectory(path.join(dspRoot, 'state/plugins', context.pluginId)); + const db = openDatabase(path.join(root, 'sdk-events.sqlite3'), { write: true }); + try { + db.exec('CREATE TABLE IF NOT EXISTS events(id INTEGER PRIMARY KEY,job_id TEXT NOT NULL,type TEXT NOT NULL,body TEXT NOT NULL,created_at INTEGER NOT NULL);'); + transaction(db, () => { + db.prepare('INSERT INTO events(job_id,type,body,created_at) VALUES(?,?,?,?)').run(context.jobId, type, JSON.stringify(value), Date.now()); + db.exec('DELETE FROM events WHERE id <= (SELECT max(id)-1000 FROM events)'); + }); + } finally { db.close(); } + return { recorded: true }; +} +module.exports = { recordEvent }; diff --git a/core/core/plugins/runtime-services.js b/core/core/plugins/runtime-services.js new file mode 100644 index 0000000..f2407d2 --- /dev/null +++ b/core/core/plugins/runtime-services.js @@ -0,0 +1,94 @@ +'use strict'; +const path = require('node:path'); +const crypto = require('node:crypto'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { StandardCollectionService } = require('dispatch-runtime-kit/collection-manager/src/standard-collections'); +const { LocalSyncManagerPort } = require('dispatch-runtime-kit/adapters/local/sync-manager-port'); +const { SyncClient } = require('dispatch-runtime-kit/sdk/src/sync-client'); +const { DispatchError, boundedJson } = require('../../sdk/src/protocol'); +const { recordEvent } = require('./runtime-events'); +const fail = code => { throw new DispatchError(code); }; +const hash = value => crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex'); + +function createRuntimeServices({ dspRoot, manifestFor, timezoneFor, wake, invoke, published }) { + if ([dspRoot, manifestFor, timezoneFor, wake, invoke, published].some(fn => typeof fn !== 'function')) throw new TypeError('plugin_service_dependencies_required'); + function paths(context) { + const root = dspRoot(context.dspId), databaseRoot = path.join(root, 'data/collection-manager'); + return { databaseRoot, database: path.join(databaseRoot, 'collection-manager.sqlite3'), stateRoot: path.join(root, 'state/collection-manager') }; + } + function withStore(context, write, work) { + const store = new CollectionStore(paths(context), { readOnly: !write, plugins: [manifestFor(context)] }); + try { return work(store, manifestFor(context)); } finally { store.close(); } + } + function ownRun(store, manifest, id) { + const run = store.run(id); + if (!manifest.collectors.includes(run.collector)) fail('job_not_found'); + return run; + } + function mutate(context, operation, input, work) { + const value = withStore(context, true, (store, manifest) => { + store.db.exec(`CREATE TABLE IF NOT EXISTS plugin_sdk_requests( + plugin_id TEXT NOT NULL,key TEXT NOT NULL,request_hash TEXT NOT NULL,response_json TEXT NOT NULL, + created_at INTEGER NOT NULL,PRIMARY KEY(plugin_id,key)) STRICT;`); + return store.transaction(() => { + const digest = hash({ operation, input }); + const saved = store.db.prepare('SELECT request_hash,response_json FROM plugin_sdk_requests WHERE plugin_id=? AND key=?') + .get(context.pluginId, input.idempotencyKey); + if (saved) { + if (saved.request_hash !== digest) fail('idempotency_conflict'); + return JSON.parse(saved.response_json); + } + if (store.db.prepare('SELECT count(*) n FROM plugin_sdk_requests WHERE plugin_id=?').get(context.pluginId).n >= 10000) fail('plugin_request_capacity'); + const result = boundedJson(work(store, manifest)); + store.db.prepare('INSERT INTO plugin_sdk_requests VALUES(?,?,?,?,?)') + .run(context.pluginId, input.idempotencyKey, digest, JSON.stringify(result), Date.now()); + return result; + }); + }); + wake(context.dspId); return value; + } + return { + 'jobs.enqueue': (context, input) => mutate(context, 'jobs.enqueue', input, (store, manifest) => { + if (!(manifest.jobs || []).includes(input.action)) fail('permission_denied'); + const plan = store.plan(input.action), source = store.source(plan.source); + if (!manifest.collectors.includes(source.collector)) fail('permission_denied'); + return store.enqueuePlan(input.action, { input: input.input, logicalKey: 'sdk:' + hash([context.pluginId, input.idempotencyKey]) }); + }), + 'jobs.status': (context, { id }) => withStore(context, false, (store, manifest) => ownRun(store, manifest, id)), + 'jobs.cancel': (context, input) => mutate(context, 'jobs.cancel', input, (store, manifest) => { + const run = ownRun(store, manifest, input.id); + return ['queued', 'running'].includes(run.status) ? store.cancel(input.id) : run; + }), + 'jobs.retry': (context, input) => mutate(context, 'jobs.retry', input, (store, manifest) => { + ownRun(store, manifest, input.id); return store.retry(input.id); + }), + 'schedules.list': context => withStore(context, false, (store, manifest) => ({ items: store.collectionSchedules() + .filter(item => manifest.collectors.includes(store.source(item.request.source).collector)) })), + 'schedules.status': async (context, { id }) => { + if (!manifestFor(context).syncs.includes(id)) fail('permission_denied'); + return { timezone: timezoneFor(context.dspId), result: await new SyncClient({ port: new LocalSyncManagerPort({ paths: paths(context) }) }).status(id) }; + }, + 'schedules.run': async (context, { id, options }) => { + if (!manifestFor(context).syncs.includes(id)) fail('permission_denied'); + const result = await new SyncClient({ port: new LocalSyncManagerPort({ paths: paths(context) }) }).runNow(id, options); + wake(context.dspId); return result; + }, + 'schedules.set': (context, input) => mutate(context, 'schedules.set', input, (store, manifest) => { + const definition = { ...input.definition, id: input.id }; + if (!definition.request || !manifest.collectors.includes(store.source(definition.request.source).collector)) fail('permission_denied'); + const before = store.db.prepare('SELECT 1 FROM collection_schedules WHERE id=?').get(input.id); + if (before && !manifest.collectors.includes(store.source(store.collectionSchedule(input.id).request.source).collector)) fail('permission_denied'); + return new StandardCollectionService(store).putSchedule(definition); + }), + 'schedules.remove': (context, input) => mutate(context, 'schedules.remove', input, (store, manifest) => { + const before = store.collectionSchedule(input.id); + if (!manifest.collectors.includes(store.source(before.request.source).collector)) fail('permission_denied'); + store.removeCollectionSchedule(input.id); return { removed: true }; + }), + 'actions.invoke': (context, { action, input }, options) => invoke(context, action, input, options), + 'published.read': (context, { view, query }, options) => published(context, view, query, options), + 'progress.report': (context, input) => recordEvent(dspRoot(context.dspId), context, 'progress', input.event), + 'log.write': (context, input) => recordEvent(dspRoot(context.dspId), context, 'log', input.event), + }; +} +module.exports = { createRuntimeServices }; diff --git a/core/core/plugins/sdk-service.js b/core/core/plugins/sdk-service.js new file mode 100644 index 0000000..b485607 --- /dev/null +++ b/core/core/plugins/sdk-service.js @@ -0,0 +1,44 @@ +'use strict'; + +const { validateRequest, result, failure, identifier, key, DispatchError, API_VERSION } = require('../../sdk/src/protocol'); + +// Called by the authenticated host transport, never from a request body. Each +// binding retains immutable DSP/plugin/installation/job identity. Transport +// handlers must not expose the bind function or accept identities from callers. +function createPluginService({ authorize, handlers = {} } = {}) { + if (typeof authorize !== 'function') throw new TypeError('plugin_authority_required'); + function bind(value) { + if (!value || Object.keys(value).sort().join(',') !== 'dspId,installationRevision,jobId,pluginId' + || !/^dsp_[a-f0-9]{32}$/.test(value.dspId) || !Number.isSafeInteger(value.installationRevision) + || value.installationRevision < 1) throw new TypeError('plugin_context_invalid'); + identifier(value.pluginId); key(value.jobId); + const context = Object.freeze({ ...value }); + return Object.freeze({ async request(value, { signal } = {}) { + let request; + try { + request = validateRequest(value); + const handler = Object.hasOwn(handlers, request.operation) ? handlers[request.operation] : null; + if (typeof handler !== 'function' && request.operation !== 'capabilities.get') return failure('capability_unavailable'); + if (signal?.aborted) return failure('cancelled', true); + // Release is authenticated to its owning context by the lease service. + // It must remain possible after uninstall/revocation for cleanup. + const cleanup = request.operation === 'connections.release'; + if (!cleanup && !await authorize(context, request)) return failure('permission_denied'); + const data = request.operation === 'capabilities.get' && !handler + ? { apiVersion: API_VERSION, operations: Object.keys(handlers).filter(operation => operation !== 'connections.release') } + : await handler(context, request.input, { signal }); + if (!cleanup && (!await authorize(context, request) || signal?.aborted)) { + if (request.operation === 'connections.acquire' && data?.leaseId && handlers['connections.release']) { + await handlers['connections.release'](context, { leaseId: data.leaseId }, {}); + } + return failure(signal?.aborted ? 'cancelled' : 'permission_denied'); + } + return result(data); + } catch (error) { + return failure(error instanceof DispatchError ? error.code : 'service_unavailable', error?.recoverable === true); + } + } }); + } + return Object.freeze({ bind }); +} +module.exports = { createPluginService }; diff --git a/core/core/plugins/settings-policy.js b/core/core/plugins/settings-policy.js new file mode 100644 index 0000000..b24b0cb --- /dev/null +++ b/core/core/plugins/settings-policy.js @@ -0,0 +1,87 @@ +"use strict"; +const path = require("node:path"); +const { + CollectionStore, +} = require("dispatch-runtime-kit/collection-manager/src/store"); +const { settingsStore } = require("./settings-store"); + +function applySettingsPolicy(dspRoot, manifest, { start = true } = {}) { + if (!manifest.settings) return null; + const storage = settingsStore(dspRoot, manifest.id), + current = storage.read(manifest.settings); + if (current.appliedRevision === current.revision) return current; + const binding = manifest.settings.schedule; + if (binding) { + const databaseRoot = path.join(dspRoot, "data/collection-manager"); + const collections = new CollectionStore({ + databaseRoot, + database: path.join(databaseRoot, "collection-manager.sqlite3"), + }, { plugins: [manifest] }); + try { + if ( + !collections.db + .prepare("SELECT 1 FROM sync_definitions WHERE id=?") + .get(binding.id) + ) + return current; + collections.transaction(() => { + const before = collections.sync(binding.id), + interval = current.values[binding.interval], + enabled = current.values[binding.enabled]; + if (before.intervalSeconds !== interval) + collections.editSync(binding.id, { + intervalSeconds: interval, + jitterSeconds: Math.min(before.jitterSeconds, interval - 1), + }); + if (!enabled) { + collections.setSyncDesiredState(binding.id, "stopped"); + for (const row of collections.db + .prepare( + `SELECT r.id FROM runs r JOIN sync_runs s ON s.run_id=r.id + WHERE s.sync_id=? AND r.status='queued' AND s.trigger<>'sync_manual'`, + ) + .all(binding.id)) + collections.cancel(row.id); + } else if (start && before.desiredState !== "running") { + collections.setSyncDesiredState(binding.id, "running", Date.now(), { + incrementGeneration: true, + }); + collections.setSyncNextDue(binding.id, Date.now() + interval * 1000); + } + }); + } finally { + collections.close(); + } + } + storage.applied(current.revision); + return storage.read(manifest.settings); +} +function initializeSettings(dspRoot, manifest) { + if (!manifest.settings) return null; + const seed = {}, + binding = manifest.settings.schedule; + if (binding) { + const { openDatabase } = require("../../shared/published/database"); + const db = openDatabase( + path.join(dspRoot, "data/collection-manager/collection-manager.sqlite3"), + ); + try { + const row = db + ?.prepare( + "SELECT desired_state,interval_seconds FROM sync_definitions WHERE id=?", + ) + .get(binding.id); + if (row) { + seed[binding.enabled] = row.desired_state === "running"; + seed[binding.interval] = row.interval_seconds; + } + } finally { + db?.close(); + } + } + return settingsStore(dspRoot, manifest.id).initialize( + manifest.settings, + seed, + ); +} +module.exports = { applySettingsPolicy, initializeSettings }; diff --git a/core/core/plugins/settings-store.js b/core/core/plugins/settings-store.js new file mode 100644 index 0000000..b62a551 --- /dev/null +++ b/core/core/plugins/settings-store.js @@ -0,0 +1,383 @@ +"use strict"; +const path = require("node:path"), + crypto = require("node:crypto"); +const { + openDatabase, + transaction, +} = require("../../shared/published/database"); +const { privateDirectory } = require("../../host/controller/operations"); +const { + identifier, + exact, + key, + DispatchError, +} = require("../../sdk/src/protocol"); +const { + validateSettingsDefinition, + settingsValues, + settingsSources, + migrateSettingsState, +} = require("../../sdk/src/settings"); +const { same } = require("../../sdk/src/settings-behavior"); +const fail = (code) => { + throw new DispatchError(code); +}; +function settingsStore(dspRoot, pluginId) { + identifier(pluginId); + const directory = path.join(dspRoot, "config/plugins", pluginId), + file = path.join(directory, "settings.sqlite3"); + function open(write) { + if (write) privateDirectory(directory); + const db = openDatabase(file, { write, journalMode: "DELETE" }); + if (write) { + db.exec(`CREATE TABLE IF NOT EXISTS settings(id INTEGER PRIMARY KEY CHECK(id=1),version INTEGER NOT NULL,revision INTEGER NOT NULL,values_json TEXT NOT NULL,updated_at INTEGER NOT NULL,actor TEXT,applied_revision INTEGER NOT NULL DEFAULT -1); + CREATE TABLE IF NOT EXISTS history(revision INTEGER PRIMARY KEY,version INTEGER NOT NULL,values_json TEXT NOT NULL,updated_at INTEGER NOT NULL,actor TEXT); + CREATE TABLE IF NOT EXISTS requests(key TEXT PRIMARY KEY,digest TEXT NOT NULL,revision INTEGER NOT NULL);`); + transaction(db, () => { + for (const [table, columns] of [ + ["settings", ["sources_json"]], + ["history", ["sources_json", "fields_json"]], + ]) { + const existing = new Set( + db + .prepare(`PRAGMA table_info(${table})`) + .all() + .map((row) => row.name), + ); + for (const column of columns) + if (!existing.has(column)) + db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} TEXT`); + } + }); + } + return db; + } + function sources(row) { + // Historical complete-value saves contain no per-field intent. Preserve all + // of them as overrides, even values that happen to equal today's defaults. + return row.sources_json + ? JSON.parse(row.sources_json) + : Object.fromEntries( + Object.keys(JSON.parse(row.values_json)).map((key) => [ + key, + "override", + ]), + ); + } + function snapshot(row, definition) { + if (!row || row.version !== definition.version) + fail("settings_migration_required"); + const values = settingsValues(definition, JSON.parse(row.values_json)); + return { + revision: row.revision, + definitionVersion: row.version, + values, + sources: settingsSources(definition, sources(row), values), + appliedRevision: row.applied_revision, + updatedAt: row.updated_at, + updatedBy: row.actor, + definition, + }; + } + function persist(db, definition, state, revision, actor) { + const timestamp = Date.now(), + json = JSON.stringify(state.values), + sourceJson = JSON.stringify(state.sources); + db.prepare( + `INSERT INTO settings(id,version,revision,values_json,updated_at,actor,sources_json) VALUES(1,?,?,?,?,?,?) + ON CONFLICT(id) DO UPDATE SET version=excluded.version,revision=excluded.revision,values_json=excluded.values_json,updated_at=excluded.updated_at,actor=excluded.actor,sources_json=excluded.sources_json`, + ).run(definition.version, revision, json, timestamp, actor, sourceJson); + db.prepare( + "INSERT INTO history(revision,version,values_json,updated_at,actor,sources_json,fields_json) VALUES(?,?,?,?,?,?,?)", + ).run( + revision, + definition.version, + json, + timestamp, + actor, + sourceJson, + JSON.stringify( + definition.fields.map(({ id, label, section }) => ({ + id, + label, + section, + })), + ), + ); + } + return { + pending() { + const db = open(false); + if (!db) return false; + try { + return !!db + .prepare("SELECT 1 FROM settings WHERE revision<>applied_revision") + .get(); + } finally { + db.close(); + } + }, + initialize(input, seed = {}) { + const definition = validateSettingsDefinition(input), + db = open(true); + try { + return transaction(db, () => { + const before = db.prepare("SELECT * FROM settings WHERE id=1").get(); + if (!before) { + const values = settingsValues(definition, seed, { defaults: true }); + persist( + db, + definition, + { + values, + sources: Object.fromEntries( + definition.fields.map((field) => [ + field.id, + Object.hasOwn(seed, field.id) ? "override" : "default", + ]), + ), + }, + 0, + null, + ); + } else if (before.version !== definition.version) { + persist( + db, + definition, + migrateSettingsState( + definition, + JSON.parse(before.values_json), + before.version, + sources(before), + ), + before.revision + 1, + null, + ); + } else if (!before.sources_json) { + // Record conservative provenance without changing values or revision. + const encoded = JSON.stringify(sources(before)); + db.prepare("UPDATE settings SET sources_json=? WHERE id=1").run( + encoded, + ); + db.prepare( + "UPDATE history SET sources_json=? WHERE revision=?", + ).run(encoded, before.revision); + } + return snapshot( + db.prepare("SELECT * FROM settings WHERE id=1").get(), + definition, + ); + }); + } finally { + db.close(); + } + }, + read(input) { + const definition = validateSettingsDefinition(input), + db = open(false); + if (!db) fail("settings_not_initialized"); + try { + return snapshot( + db.prepare("SELECT * FROM settings WHERE id=1").get(), + definition, + ); + } finally { + db.close(); + } + }, + update(input, request, actor) { + const definition = validateSettingsDefinition(input); + exact(request, [ + "values", + "expectedRevision", + "definitionVersion", + "idempotencyKey", + ...(Object.hasOwn(request, "sources") ? ["sources"] : []), + ]); + key(request.idempotencyKey); + key(actor); + if ( + !Number.isSafeInteger(request.expectedRevision) || + request.expectedRevision < 0 || + request.definitionVersion !== definition.version + ) + fail("settings_revision_conflict"); + const values = settingsValues(definition, request.values); + const requestedSources = + request.sources === undefined + ? undefined + : settingsSources(definition, request.sources, values); + // Preserve legacy receipt digests and retries from an already open client. + const digest = crypto + .createHash("sha256") + .update( + JSON.stringify([ + actor, + request.expectedRevision, + request.definitionVersion, + values, + ...(requestedSources ? [requestedSources] : []), + ]), + ) + .digest("hex"); + const db = open(true); + try { + return transaction(db, () => { + const before = db.prepare("SELECT * FROM settings WHERE id=1").get(), + current = snapshot(before, definition); + const existing = db + .prepare("SELECT * FROM requests WHERE key=?") + .get(request.idempotencyKey); + if (existing) { + if (existing.digest !== digest) fail("idempotency_conflict"); + return current; + } + if (before.revision !== request.expectedRevision) + fail("settings_revision_conflict"); + if (db.prepare("SELECT count(*) n FROM requests").get().n >= 10000) + fail("settings_request_capacity"); + const resolvedSources = + requestedSources || + Object.fromEntries( + definition.fields.map((field) => [ + field.id, + same(values[field.id], current.values[field.id]) + ? current.sources[field.id] + : "override", + ]), + ); + persist( + db, + definition, + { values, sources: resolvedSources }, + before.revision + 1, + actor, + ); + db.prepare("INSERT INTO requests VALUES(?,?,?)").run( + request.idempotencyKey, + digest, + before.revision + 1, + ); + return snapshot( + db.prepare("SELECT * FROM settings WHERE id=1").get(), + definition, + ); + }); + } finally { + db.close(); + } + }, + history(input, { beforeRevision = null } = {}) { + const definition = validateSettingsDefinition(input); + if ( + beforeRevision !== null && + (!Number.isSafeInteger(beforeRevision) || beforeRevision < 0) + ) + fail("settings_invalid"); + const db = open(false); + if (!db) fail("settings_not_initialized"); + try { + const current = snapshot( + db.prepare("SELECT * FROM settings WHERE id=1").get(), + definition, + ); + const rows = db + .prepare( + "SELECT * FROM history WHERE revision + value && typeof value === "object" + ? 1 + + Object.values(value).reduce( + (sum, item) => sum + countNodes(item), + 0, + ) + : 1; + const preview = (value) => + Array.isArray(value) && value.length > 12 + ? [...value.slice(0, 12), "…"] + : value; + for (const row of rows.slice(0, 10)) { + const values = JSON.parse(row.values_json), + source = sources(row), + previous = db + .prepare( + "SELECT * FROM history WHERE revision + !same(values[id], prior[id]) || source[id] !== priorSources[id], + ) + .map((id) => ({ + field: id, + label: fields.find((field) => field.id === id)?.label || id, + before: preview(prior[id] ?? null), + after: preview(values[id] ?? null), + beforeSource: priorSources[id] || null, + afterSource: source[id] || null, + })); + const item = { + revision: row.revision, + definitionVersion: row.version, + values, + sources: source, + updatedAt: row.updated_at, + updatedBy: row.actor, + kind: row.actor + ? "owner" + : row.revision === 0 + ? "initial" + : "migration", + changes, + canRestore: row.version === definition.version, + }; + const size = Buffer.byteLength(JSON.stringify(item)); + const nodeCount = countNodes(item); + if ( + items.length && + (bytes + size > 180000 || nodes + nodeCount > 16000) + ) + break; + nodes += nodeCount; + bytes += size; + items.push(item); + } + return { + items, + nextBefore: + rows.length > items.length + ? (items.at(-1)?.revision ?? null) + : null, + currentRevision: current.revision, + }; + } finally { + db.close(); + } + }, + applied(revision) { + const db = open(true); + try { + db.prepare( + "UPDATE settings SET applied_revision=? WHERE id=1 AND revision=?", + ).run(revision, revision); + } finally { + db.close(); + } + }, + }; +} +module.exports = { settingsStore }; diff --git a/core/core/plugins/tests/access-authority.test.js b/core/core/plugins/tests/access-authority.test.js new file mode 100644 index 0000000..5243e2c --- /dev/null +++ b/core/core/plugins/tests/access-authority.test.js @@ -0,0 +1,37 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { DatabaseSync } = require('node:sqlite'); +const { createAccessPluginAuthority } = require('../access-authority'); + +test('SDK authority requires acknowledged installation and declared connections/actions', t => { + const db = new DatabaseSync(':memory:'); t.after(() => db.close()); + db.exec(`CREATE TABLE organizations(id TEXT,status TEXT); + CREATE TABLE installations(runtime_key TEXT,organization_id TEXT,status TEXT); + CREATE TABLE dsp_plugins(organization_id TEXT,plugin_id TEXT,revision INTEGER,applied_revision INTEGER,desired_state TEXT,applied_state TEXT,version TEXT); + CREATE TABLE dsp_removals(organization_id TEXT); + CREATE TABLE directory_lifecycle_requests(organization_id TEXT,status TEXT); + INSERT INTO organizations VALUES('org-a','active'); + INSERT INTO installations VALUES('dsp_a','org-a','ready'); + INSERT INTO dsp_plugins VALUES('org-a','sample',1,1,'enabled','enabled','1.0.0');`); + let lifecycle = false, granted = true; + const authorize = createAccessPluginAuthority({ store: { db, activeLifecycleJob: () => lifecycle }, + connectionGrant: () => granted, + catalog: () => [{ id: 'sample', version: '1.0.0', services: ['sample-connection'], actions: [{ id: 'sample.collect' }] }] }); + const context = { dspId: 'dsp_a', pluginId: 'sample', installationRevision: 1, jobId: 'job-a' }; + const connection = { operation: 'connections.acquire', input: { connection: 'sample-connection' } }; + assert.equal(authorize(context, connection), true); + granted = false; assert.equal(authorize(context, connection), false); granted = true; + assert.equal(authorize(context, { ...connection, input: { connection: 'unrelated' } }), false); + assert.equal(authorize({ ...context, dspId: 'dsp_b' }, connection), false); + assert.equal(authorize({ ...context, installationRevision: 2 }, connection), false); + assert.equal(authorize(context, { operation: 'actions.invoke', input: { action: 'sample.collect' } }), true); + assert.equal(authorize(context, { operation: 'actions.invoke', input: { action: 'admin.delete' } }), false); + lifecycle = true; assert.equal(authorize(context, connection), false); lifecycle = false; + db.exec("UPDATE dsp_plugins SET desired_state='disabled',revision=2"); + assert.equal(authorize(context, connection), false); + db.exec("UPDATE dsp_plugins SET desired_state='enabled',revision=3"); + assert.equal(authorize({ ...context, installationRevision: 3 }, connection), false); + db.exec("UPDATE dsp_plugins SET applied_revision=3"); + assert.equal(authorize({ ...context, installationRevision: 3 }, connection), true); +}); diff --git a/core/core/plugins/tests/manual-auth-retry.test.js b/core/core/plugins/tests/manual-auth-retry.test.js new file mode 100644 index 0000000..0bacf0e --- /dev/null +++ b/core/core/plugins/tests/manual-auth-retry.test.js @@ -0,0 +1,44 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { SyncService } = require('dispatch-runtime-kit/collection-manager/src/syncs'); +const { fixture, spec } = require('dispatch-dsp/runtime/collection-manager/tests/helpers.js'); +const { manualAuthRetry } = require('../manual-auth-retry'); + +test('only an active manual Paycom run in the same DSP and plugin job grants authentication retry', t => { + const { root } = fixture(); + fs.mkdirSync(path.join(root, 'data'), { mode: 0o700 }); + const databaseRoot = path.join(root, 'data/collection-manager'); + const store = new CollectionStore({ databaseRoot, database: path.join(databaseRoot, 'collection-manager.sqlite3') }); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const definition = spec(); + definition.collectors[0].id = 'paycom'; definition.sources[0].collector = 'paycom'; + store.applySpec(definition); + require('dispatch-runtime-kit/collection-manager/src/plugin-state').applyState(store, { + command: 'apply', pluginId: 'paycom', state: 'enabled', revision: 1, version: '0.18.5', + }); + const sync = new SyncService(store); + sync.start('fixture-main-sync', { runNow: false }); + const run = store.enqueueSync('fixture-main-sync', { trigger: 'sync_schedule', windowKey: 'scheduled' }); + store.db.prepare("UPDATE runs SET status='running' WHERE id=?").run(run.id); + const context = { dspId: 'dsp_' + 'a'.repeat(32), pluginId: 'paycom', installationRevision: 1, jobId: 'job_a' }; + const job = { context, kind: 'collect', collectionRunId: run.id, cancelled: false }; + assert.equal(manualAuthRetry(root, context, job), false); + store.db.prepare("UPDATE runs SET status='queued' WHERE id=?").run(run.id); + const requested = sync.runNow('fixture-main-sync'); + assert.equal(requested.run.id, run.id); + assert.equal(manualAuthRetry(root, context, job), false); + store.db.prepare("UPDATE runs SET status='running' WHERE id=?").run(run.id); + assert.equal(manualAuthRetry(root, context, job), true); + assert.equal(manualAuthRetry(root, { ...context, dspId: 'other' }, job), false); + assert.equal(manualAuthRetry(root, { ...context, jobId: 'other' }, job), false); + assert.equal(manualAuthRetry(root, context, { ...job, kind: 'invoke' }), false); + assert.equal(manualAuthRetry(root, context, { ...job, cancelled: true }), false); + store.db.prepare('UPDATE runs SET cancel_requested=1 WHERE id=?').run(run.id); + assert.equal(manualAuthRetry(root, context, job), false); + store.db.prepare("UPDATE runs SET status='succeeded',cancel_requested=0 WHERE id=?").run(run.id); + assert.equal(manualAuthRetry(root, context, job), false); +}); diff --git a/core/core/plugins/tests/runtime-services.test.js b/core/core/plugins/tests/runtime-services.test.js new file mode 100644 index 0000000..77fa96a --- /dev/null +++ b/core/core/plugins/tests/runtime-services.test.js @@ -0,0 +1,43 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { spec } = require('dispatch-dsp/runtime/collection-manager/tests/helpers.js'); +const { createRuntimeServices } = require('../runtime-services'); + +test('SDK jobs use the durable collection queue with scoped, atomic idempotent changes', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-sdk-services-')); + fs.mkdirSync(path.join(root, 'data'), { mode: 0o700 }); + const databaseRoot = path.join(root, 'data/collection-manager'); + const paths = { databaseRoot, database: path.join(databaseRoot, 'collection-manager.sqlite3') }; + const store = new CollectionStore(paths); store.applySpec(spec()); + t.after(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const manifest = { id: 'sample', version: '1.0.0', collectors: ['fixture'], syncs: ['fixture-main-sync'], jobs: ['fixture-snapshot'] }; + let wakes = 0; + const handlers = createRuntimeServices({ dspRoot: () => root, manifestFor: context => context.pluginId === 'sample' ? manifest : { id: 'other', version: '1.0.0', collectors: [], syncs: [], jobs: [] }, + timezoneFor: () => 'UTC', wake: () => wakes++, invoke: async () => ({}), published: async () => ({}) }); + const context = { dspId: 'dsp_' + 'a'.repeat(32), pluginId: 'sample', installationRevision: 1, jobId: 'job-sample' }; + const request = { action: 'fixture-snapshot', input: { label: 'first' }, idempotencyKey: 'enqueue-one' }; + const first = handlers['jobs.enqueue'](context, request); + assert.deepEqual(handlers['jobs.enqueue'](context, request), first); + assert.equal(store.runs().length, 1); + assert.throws(() => handlers['jobs.enqueue'](context, { ...request, input: { label: 'changed' } }), { code: 'idempotency_conflict' }); + assert.throws(() => handlers['jobs.status']({ ...context, pluginId: 'other' }, { id: first.id }), { code: 'job_not_found' }); + const cancel = { id: first.id, idempotencyKey: 'cancel-one' }; + assert.equal(handlers['jobs.cancel'](context, cancel).status, 'cancelled'); + const retry = { id: first.id, idempotencyKey: 'retry-one' }; + assert.equal(handlers['jobs.retry'](context, retry).status, 'queued'); + assert.equal(handlers['jobs.cancel'](context, cancel).status, 'cancelled'); + // The old cancel receipt must not cancel the newly retried attempt. + assert.equal(store.run(first.id).status, 'queued'); + assert.equal(handlers['jobs.retry'](context, retry).status, 'queued'); + assert.equal((await handlers['schedules.status'](context, { id: 'fixture-main-sync' })).result.ok, true); + await assert.rejects(handlers['schedules.status'](context, { id: 'other' }), { code: 'permission_denied' }); + assert.equal(handlers['progress.report'](context, { event: { phase: 'collecting', completed: 1, total: 2 } }).recorded, true); + assert.equal(handlers['log.write'](context, { event: { level: 'info', code: 'collected', counts: { records: 1 } } }).recorded, true); + assert.throws(() => handlers['log.write'](context, { event: { level: 'info', code: 'collected', password: 'forbidden' } }), { code: 'invalid_request' }); + assert.ok(wakes >= 3); +}); diff --git a/core/core/plugins/tests/sdk-service.test.js b/core/core/plugins/tests/sdk-service.test.js new file mode 100644 index 0000000..01a2b37 --- /dev/null +++ b/core/core/plugins/tests/sdk-service.test.js @@ -0,0 +1,39 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { createPluginService } = require('../sdk-service'); +const { createDispatchClient } = require('../../../sdk'); + +const context = Object.freeze({ dspId: 'dsp_' + 'a'.repeat(32), pluginId: 'sample', installationRevision: 1, jobId: 'job_1' }); +test('host binds immutable identity and rejects scope fields in requests', async () => { + let seen; + const mutable = { ...context }; + const service = createPluginService({ authorize: () => true, handlers: { 'actions.invoke': selected => { seen = selected; return {}; } } }); + const transport = service.bind(mutable); mutable.dspId = 'dsp_' + 'b'.repeat(32); + const client = createDispatchClient({ transport }); + await client.actions.invoke('sample.read'); + assert.deepEqual(seen, context); + const invalid = await transport.request({ apiVersion: 1, operation: 'actions.invoke', input: { action: 'sample.read', input: {} }, dspId: mutable.dspId }); + assert.equal(invalid.ok, false); +}); +test('revocation while acquisition is pending closes the browser before responding', async () => { + let allowed = true, released = false; + const service = createPluginService({ authorize: () => allowed, handlers: { + 'connections.acquire': () => { allowed = false; return { leaseId: 'lease_1' }; }, + 'connections.release': (selected, input) => { assert.equal(selected.dspId, context.dspId); assert.equal(input.leaseId, 'lease_1'); released = true; return {}; }, + } }); + const client = createDispatchClient({ transport: service.bind(context) }); + await assert.rejects(client.connections.withSession({ connection: 'paycom' }, async () => assert.fail('revoked browser exposed')), { code: 'permission_denied' }); + assert.equal(released, true); +}); +test('denied operations never enter a handler and asynchronous reads are reauthorized', async () => { + let allowed = false, calls = 0; + const service = createPluginService({ authorize: () => allowed, handlers: { + 'actions.invoke': async () => { calls++; allowed = false; return { private: 'unavailable after revocation' }; }, + } }); + const client = createDispatchClient({ transport: service.bind(context) }); + await assert.rejects(client.actions.invoke('sample.read'), { code: 'permission_denied' }); + assert.equal(calls, 0); allowed = true; + await assert.rejects(client.actions.invoke('sample.read'), { code: 'permission_denied' }); + assert.equal(calls, 1); +}); diff --git a/core/core/plugins/tests/settings-policy.test.js b/core/core/plugins/tests/settings-policy.test.js new file mode 100644 index 0000000..397da31 --- /dev/null +++ b/core/core/plugins/tests/settings-policy.test.js @@ -0,0 +1,130 @@ +"use strict"; +const test = require("node:test"), + assert = require("node:assert/strict"), + fs = require("node:fs"), + os = require("node:os"), + path = require("node:path"); +const { + CollectionStore, +} = require("dispatch-runtime-kit/collection-manager/src/store"); +const { + CollectionManager, +} = require("dispatch-dsp/runtime/collection-manager/src/manager.js"); +const { + SyncService, +} = require("dispatch-runtime-kit/collection-manager/src/syncs"); +const { spec } = require("dispatch-dsp/runtime/collection-manager/tests/helpers.js"); +const { settingsStore } = require("../settings-store"); +const { + initializeSettings, + applySettingsPolicy, +} = require("../settings-policy"); +test("settings pause automatic work, retain queued manual work, and preserve independent DSP schedules", async (t) => { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), "dispatch-settings-policy-"), + ); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const manifest = { + id: "example", version: "1.0.0", collectors: ["fixture"], syncs: ["fixture-main-sync"], + settings: { + version: 1, + sections: [{ id: "sync", label: "Sync" }], + fields: [ + { + id: "enabled", + section: "sync", + label: "Automatic", + type: "boolean", + default: true, + }, + { + id: "interval", + section: "sync", + label: "Interval", + type: "integer", + minimum: 10, + maximum: 100, + default: 10, + }, + ], + schedule: { + id: "fixture-main-sync", + enabled: "enabled", + interval: "interval", + }, + }, + }; + const stores = ["a", "b"].map((id) => { + const databaseRoot = path.join(root, id, "data/collection-manager"); + fs.mkdirSync(path.dirname(databaseRoot), { recursive: true, mode: 0o700 }); + return new CollectionStore({ + databaseRoot, + database: path.join(databaseRoot, "collection-manager.sqlite3"), + stateRoot: path.join(root, id, "state"), + }); + }); + t.after(() => stores.forEach((store) => store.close())); + for (const [index, store] of stores.entries()) { + store.applySpec(spec()); + new SyncService(store).start("fixture-main-sync", { runNow: false }); + initializeSettings(path.join(root, index ? "b" : "a"), manifest); + applySettingsPolicy(path.join(root, index ? "b" : "a"), manifest); + } + const [a, b] = stores, + service = new SyncService(a), + storage = settingsStore(path.join(root, "a"), "example"); + const queued = a.enqueueSync("fixture-main-sync", { + trigger: "sync_schedule", + timestamp: Date.now(), + windowKey: "before-pause", + }); + const save = (values) => { + const before = storage.read(manifest.settings); + storage.update( + manifest.settings, + { + values, + expectedRevision: before.revision, + definitionVersion: 1, + idempotencyKey: "settings:" + before.revision, + }, + "owner_a", + ); + return applySettingsPolicy(path.join(root, "a"), manifest); + }; + save({ enabled: false, interval: 30 }); + assert.equal(a.run(queued.id).status, "cancelled"); + assert.equal(a.sync("fixture-main-sync").desiredState, "stopped"); + assert.equal(a.sync("fixture-main-sync").intervalSeconds, 30); + assert.equal(b.sync("fixture-main-sync").intervalSeconds, 10); + assert.equal(b.sync("fixture-main-sync").desiredState, "running"); + const manual = service.runNow("fixture-main-sync", { + idempotencyKey: "paused-click", + }).run; + assert.equal( + service.runNow("fixture-main-sync", { idempotencyKey: "paused-click" }).run + .id, + manual.id, + ); + save({ enabled: false, interval: 40 }); + assert.equal(a.run(manual.id).status, "queued"); + const manager = new CollectionManager(a, { tickMs: 10 }); + t.after(() => manager.stop()); + await manager.start(); + await manager.runUntilIdle({ timeoutMs: 5000 }); + await manager.stop(); + assert.equal(a.run(manual.id).status, "succeeded"); + assert.equal(a.sync("fixture-main-sync").desiredState, "stopped"); + initializeSettings(path.join(root, "a"), manifest); + assert.deepEqual(storage.read(manifest.settings).values, { + enabled: false, + interval: 40, + }); + save({ enabled: true, interval: 40 }); + assert.equal(a.sync("fixture-main-sync").desiredState, "running"); + assert.ok(a.sync("fixture-main-sync").nextDueAt > Date.now()); + const scheduled = a.enqueueSync("fixture-main-sync", {trigger:"sync_schedule",timestamp:Date.now()+40000,windowKey:"manual-coalesced"}); + assert.equal(service.runNow("fixture-main-sync",{idempotencyKey:"manual-coalesced"}).run.id,scheduled.id); + save({enabled:false,interval:40}); + assert.equal(a.run(scheduled.id).status,"queued"); +}); diff --git a/core/core/plugins/tests/settings.test.js b/core/core/plugins/tests/settings.test.js new file mode 100644 index 0000000..cdf1281 --- /dev/null +++ b/core/core/plugins/tests/settings.test.js @@ -0,0 +1,157 @@ +"use strict"; +const test = require("node:test"), + assert = require("node:assert/strict"), + fs = require("node:fs"), + os = require("node:os"), + path = require("node:path"); +const { settingsStore } = require("../settings-store"); +const { + validateSettingsDefinition, + settingsValues, + migrateSettings, +} = require("../../../sdk/src/settings"); +const definition = { + version: 1, + sections: [{ id: "general", label: "General" }], + fields: [ + { + id: "selected", + label: "Selected departments", + section: "general", + type: "strings", + nullable: true, + default: null, + }, + { + id: "frequency", + label: "Frequency", + section: "general", + type: "integer", + minimum: 10, + maximum: 100, + default: 30, + }, + ], +}; +test("SDK settings validate declarations and values, reject identity/path extras and preserve empty selections", () => { + const schema = validateSettingsDefinition(definition); + assert.deepEqual(settingsValues(schema, {}, { defaults: true }), { + selected: null, + frequency: 30, + }); + assert.deepEqual(settingsValues(schema, { selected: [], frequency: 30 }), { + selected: [], + frequency: 30, + }); + assert.throws( + () => + settingsValues(schema, { selected: [], frequency: 30, dspId: "other" }), + /settings_invalid/, + ); + assert.throws( + () => settingsValues(schema, { selected: ["x", "x"], frequency: 30 }), + /settings_invalid/, + ); + assert.throws( + () => settingsValues(schema, { selected: [], frequency: 1 }), + /settings_invalid/, + ); + assert.throws( + () => + validateSettingsDefinition({ + ...definition, + fields: [{ ...definition.fields[1], default: 999 }], + }), + /settings_invalid/, + ); +}); +test("DSP settings survive reopen, isolate identical plugin ids, reject stale writes and audit idempotent edits", (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "dispatch-settings-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + for (const id of ["a", "b"]) + fs.mkdirSync(path.join(root, id, "config/plugins"), { + recursive: true, + mode: 0o700, + }); + const a = settingsStore(path.join(root, "a"), "example"), + b = settingsStore(path.join(root, "b"), "example"); + a.initialize(definition); + b.initialize(definition); + const input = { + values: { selected: ["D1"], frequency: 50 }, + expectedRevision: 0, + definitionVersion: 1, + idempotencyKey: "save:one", + }; + assert.equal(a.update(definition, input, "owner_a").revision, 1); + assert.equal(a.update(definition, input, "owner_a").revision, 1); + assert.deepEqual(b.read(definition).values, { + selected: null, + frequency: 30, + }); + assert.deepEqual( + settingsStore(path.join(root, "a"), "example").read(definition).values, + input.values, + ); + assert.throws( + () => + a.update(definition, { ...input, idempotencyKey: "save:two" }, "owner_a"), + /settings_revision_conflict/, + ); + assert.throws( + () => + a.update( + definition, + { ...input, values: { selected: [], frequency: 30 } }, + "owner_a", + ), + /idempotency_conflict/, + ); + assert.throws( + () => a.update(definition, input, "owner_b"), + /idempotency_conflict/, + ); + a.applied(0); + assert.equal(a.read(definition).appliedRevision, -1); + a.applied(1); + assert.equal(a.read(definition).appliedRevision, 1); +}); +test("future plugin schema upgrades preserve DSP overrides, add defaults, and support declared renames", () => { + const next = validateSettingsDefinition({ + ...definition, + version: 2, + fields: [ + { ...definition.fields[0], id: "departments" }, + definition.fields[1], + { + id: "enabled", + label: "Enabled", + section: "general", + type: "boolean", + default: true, + }, + ], + migrations: [{ fromVersion: 1, rename: { selected: "departments" } }], + }); + assert.deepEqual(migrateSettings(next, { selected: [], frequency: 70 }, 1), { + departments: [], + frequency: 70, + enabled: true, + }); + assert.throws( + () => migrateSettings(next, { selected: [], frequency: 70 }, 3), + /settings_incompatible/, + ); +}); + +test('initialization migrates stored overrides once and rejects incompatible schema changes without partial writes', t => { + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-settings-upgrade-'));t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + const storage=settingsStore(root,'example');storage.initialize(definition); + storage.update(definition,{values:{selected:[],frequency:70},expectedRevision:0,definitionVersion:1,idempotencyKey:'settings:before-upgrade'},'owner_fixture'); + const next={...definition,version:2,fields:[...definition.fields,{id:'enabled',section:'general',label:'Enabled',type:'boolean',default:true}]}; + const migrated=storage.initialize(next);assert.deepEqual(migrated.values,{selected:[],frequency:70,enabled:true});assert.equal(migrated.revision,2); + assert.equal(storage.initialize(next).revision,2); + const incompatible={...next,version:3,fields:next.fields.map(field=>field.id==='frequency'?{...field,maximum:50}:field)}; + assert.throws(()=>storage.initialize(incompatible),/settings_invalid/); + assert.deepEqual(storage.read(next).values,migrated.values);assert.equal(storage.read(next).revision,2); +}); diff --git a/core/core/plugins/tests/smarter-settings.test.js b/core/core/plugins/tests/smarter-settings.test.js new file mode 100644 index 0000000..ff76074 --- /dev/null +++ b/core/core/plugins/tests/smarter-settings.test.js @@ -0,0 +1,216 @@ +"use strict"; +const test = require("node:test"), + assert = require("node:assert/strict"), + fs = require("node:fs"), + os = require("node:os"), + path = require("node:path"), + { DatabaseSync } = require("node:sqlite"); +const { settingsStore } = require("../settings-store"); +const definition = { + version: 1, + sections: [{ id: "view", label: "View" }], + fields: [ + { + id: "order", + section: "view", + label: "Order", + type: "string", + default: "first", + }, + { + id: "rows", + section: "view", + label: "Rows", + type: "integer", + minimum: 1, + maximum: 100, + default: 25, + }, + ], +}; +function root(t) { + const r = fs.mkdtempSync(path.join(os.tmpdir(), "smarter-settings-")); + t.after(() => fs.rmSync(r, { recursive: true, force: true })); + return r; +} +function update(storage, d, values, sources, id) { + const s = storage.read(d); + return storage.update( + d, + { + values, + ...(sources ? { sources } : {}), + expectedRevision: s.revision, + definitionVersion: d.version, + idempotencyKey: id || "save:" + s.revision, + }, + "owner_fixture", + ); +} +test("owner intent is preserved even when the override equals the default; untouched DSPs follow new defaults", (t) => { + const r = root(t), + a = settingsStore(path.join(r, "a"), "sample"), + b = settingsStore(path.join(r, "b"), "sample"); + a.initialize(definition); + b.initialize(definition); + const saved = update( + a, + definition, + { order: "first", rows: 25 }, + { order: "override", rows: "default" }, + ); + assert.equal(saved.revision, 1); + assert.equal(saved.sources.order, "override"); + const next = { + ...definition, + version: 2, + fields: definition.fields.map((f) => + f.id === "order" ? { ...f, default: "last" } : f, + ), + }; + assert.equal(a.initialize(next).values.order, "first"); + assert.equal(b.initialize(next).values.order, "last"); + assert.equal(a.initialize(next).sources.order, "override"); + assert.throws( + () => + update( + a, + next, + { order: "first", rows: 25 }, + { order: "default", rows: "default" }, + ), + /settings_invalid/, + ); + const reset = update( + a, + next, + { order: "last", rows: 25 }, + { order: "default", rows: "default" }, + ); + assert.equal(reset.sources.order, "default"); + assert.equal( + settingsStore(path.join(r, "a"), "sample").initialize(next).revision, + reset.revision, + ); +}); +test("legacy SQLite rows are conservatively upgraded without guessing intent or changing other values", (t) => { + const r = root(t), + dir = path.join(r, "config/plugins/sample"); + fs.mkdirSync(dir, { recursive: true, mode: 0o700 }); + const db = new DatabaseSync(path.join(dir, "settings.sqlite3")); + db.exec( + "CREATE TABLE settings(id INTEGER PRIMARY KEY,version INTEGER,revision INTEGER,values_json TEXT,updated_at INTEGER,actor TEXT,applied_revision INTEGER);CREATE TABLE history(revision INTEGER PRIMARY KEY,version INTEGER,values_json TEXT,updated_at INTEGER,actor TEXT);", + ); + const values = JSON.stringify({ order: "first", rows: 50 }); + db.prepare("INSERT INTO settings VALUES(1,1,4,?,1,NULL,4)").run(values); + db.prepare("INSERT INTO history VALUES(4,1,?,1,NULL)").run(values); + db.close(); + fs.chmodSync(path.join(dir, "settings.sqlite3"), 0o600); + const storage = settingsStore(r, "sample"), + same = storage.initialize(definition); + assert.deepEqual(same.values, { order: "first", rows: 50 }); + assert.deepEqual(same.sources, { order: "override", rows: "override" }); + assert.equal(same.revision, 4); + const next = { + ...definition, + version: 2, + fields: definition.fields.map((f) => + f.id === "order" ? { ...f, default: "last" } : f, + ), + }; + const migrated = storage.initialize(next); + assert.deepEqual(migrated.values, same.values); + assert.equal(migrated.revision, 5); + const broken = { + ...next, + version: 3, + fields: next.fields.map((f) => + f.id === "rows" ? { ...f, maximum: 30 } : f, + ), + }; + assert.throws(() => storage.initialize(broken), /settings_invalid/); + assert.deepEqual(storage.read(next), migrated); + assert.equal( + storage.history(next).items.find((item) => item.revision === 4).canRestore, + false, + ); +}); +test("history is paginated and scoped; source-only changes and retry receipts are durable", (t) => { + const r = root(t), + a = settingsStore(path.join(r, "a"), "sample"), + b = settingsStore(path.join(r, "b"), "sample"); + a.initialize(definition); + b.initialize(definition); + const initial = a.read(definition); + const request = { + values: initial.values, + sources: { order: "override", rows: "default" }, + expectedRevision: 0, + definitionVersion: 1, + idempotencyKey: "same:value", + }; + a.update(definition, request, "owner_fixture"); + assert.equal(a.update(definition, request, "owner_fixture").revision, 1); + assert.throws( + () => + a.update( + definition, + { ...request, sources: initial.sources }, + "owner_fixture", + ), + /idempotency_conflict/, + ); + for (let rows = 26; rows < 39; rows++) + update( + a, + definition, + { order: "first", rows }, + { order: "override", rows: "override" }, + ); + const first = a.history(definition); + assert.equal(first.items.length, 10); + assert(first.nextBefore !== null); + const second = a.history(definition, { beforeRevision: first.nextBefore }); + assert(second.items.every((item) => item.revision < first.nextBefore)); + assert.equal(second.nextBefore, null); + const intent = second.items.find((item) => item.revision === 1); + assert.equal(intent.changes.length, 1); + assert.equal(intent.changes[0].beforeSource, "default"); + assert.equal(intent.changes[0].afterSource, "override"); + assert.equal(b.history(definition).items.length, 1); + assert.throws( + () => a.history(definition, { beforeRevision: -1 }), + /settings_invalid/, + ); +}); +test("large valid settings histories stay within the SDK response envelope", (t) => { + const r = root(t), + a = settingsStore(r, "sample"); + const d = { + version: 1, + sections: definition.sections, + fields: Array.from({ length: 16 }, (_, i) => ({ + id: "groups_" + i, + section: "view", + label: "Groups " + i, + type: "strings", + default: Array.from({ length: 200 }, (_, n) => "g" + n), + })), + }; + a.initialize(d); + for (let i = 0; i < 8; i++) { + const before = a.read(d); + update( + a, + d, + { ...before.values, groups_0: [...before.values.groups_0].reverse() }, + undefined, + "large:" + i, + ); + } + const history = a.history(d); + assert(history.items.length > 0 && history.nextBefore !== null); + const { result, unwrap } = require("../../../sdk/src/protocol"); + assert.equal(unwrap(result(history)).items.length, history.items.length); + assert.equal(history.items[0].values.groups_0.length, 200); +}); diff --git a/core/core/plugins/transport.js b/core/core/plugins/transport.js new file mode 100644 index 0000000..cdaa98a --- /dev/null +++ b/core/core/plugins/transport.js @@ -0,0 +1,86 @@ +'use strict'; +const path = require('node:path'); +const { PluginSdkSocket } = require('../../host/plugins/sdk-socket'); +const { createPrivateTransport } = require('../../sdk/node'); +const { exact, boundedJson, identifier, result, failure, unwrap } = require('../../sdk/src/protocol'); +const { validateDspId } = require('../../shared/paths/platform-paths'); +const { privateDirectory } = require('../../host/controller/operations'); +const { packageCatalog } = require('./package-catalog'); +const { verifyPackage } = require('../../shared/plugin-sdk/package-files'); +const { pluginRequest } = require('../../shared/plugin-sdk/contract'); +const { installationReceipt } = require('../../shared/plugin-sdk/installed'); +const fileFor = paths => path.join(paths.local, 'run/plugin-backend/control.sock'); +function backendClient(paths) { + const transport = createPrivateTransport({ socketPath: fileFor(paths), timeoutMs: 3660000 }); + return { async request(dspId, operation, input = {}, options) { + return unwrap(await transport.request({ schemaVersion: 1, dspId, operation, input }, options)); + } }; +} +async function serveBackend({ paths, backend, dspRoot, permitted, prepare = async () => {} }) { + const bridges = new Map(), preparing = new Map(); + async function ensure(dspId) { + if (bridges.has(dspId)) return; + if (preparing.has(dspId)) return preparing.get(dspId); + const work = (async () => { + if (!permitted(dspId)) throw new Error('permission_denied'); + await prepare(dspId); + const root = privateDirectory(path.join(dspRoot(dspId), '.control')); + const socket = new PluginSdkSocket({ file: path.join(root, 'backend.sock'), transport: bind(dspId), timeoutMs: 3660000 }); + await socket.start(); bridges.set(dspId, socket); + })().finally(() => preparing.delete(dspId)); + preparing.set(dspId, work); return work; + } + function bind(boundDspId) { + return { async request(raw, options) { + try { + const value = boundedJson(raw); + exact(value, boundDspId ? ['schemaVersion', 'operation', 'input'] : ['schemaVersion', 'dspId', 'operation', 'input']); + if (value.schemaVersion !== 1) throw new Error('invalid_request'); + const dspId = validateDspId(boundDspId || value.dspId), input = value.input; + const cleanup = !boundDspId && value.operation === 'plugin.revoke'; + if (!(value.operation === 'auth.request' && input?.action === 'health' && require('../../host/releases/guard').healthAllowed(paths)) && (boundDspId || !['plugin.revoke', 'plugin.initialize', 'dsp.prepare'].includes(value.operation))) require('../../host/releases/guard').assertAvailable(paths, dspId); + if (!cleanup && !permitted(dspId)) throw new Error('permission_denied'); + let response; + if (!boundDspId && value.operation === 'dsp.prepare') { + exact(input, []); + if (!backend.canStart(dspId)) throw new Error('plugin_operation_pending'); + await ensure(dspId); response = { ready: true }; + } + else if (!boundDspId && value.operation === 'plugin.revoke') { + exact(input, ['pluginId']); if (input.pluginId !== null) identifier(input.pluginId); + await backend.revoke(dspId, input.pluginId); response = { revoked: true }; + } else if (!boundDspId && value.operation === 'plugin.initialize') { + const request = pluginRequest(input); + const previous = installationReceipt(dspRoot(dspId), request.pluginId, true); + const approved = previous?.version === request.version ? previous : packageCatalog(paths)?.resolve(request.pluginId, request.version); + if (!approved) throw new Error('plugin_package_unavailable'); + const directory = path.join(dspRoot(dspId), 'plugins', request.pluginId, 'versions', request.version); + const manifest = verifyPackage(directory, approved.digest); + response = await backend.execute(dspId, request.pluginId, 'initialize', {}, { ...options, + initialize: { directory, digest: approved.digest, manifest, revision: request.revision } }); + } else if (value.operation === 'plugin.settings') { + exact(input,['pluginId','request']);identifier(input.pluginId); + const request = input.request; + if (request?.action === 'update') { if(boundDspId)throw new Error('permission_denied');exact(request,['action','input','actor']); } + else if (request?.action === 'history') { if(boundDspId)throw new Error('permission_denied');exact(request,['action','input']);exact(request.input,['beforeRevision']); } + else { exact(request,['action']);if(!['get','options'].includes(request.action))throw new Error('invalid_request'); } + response = await backend.settingsRequest(dspId,input.pluginId,request,options); + } else if (value.operation === 'auth.request') response = await backend.authRequest(dspId, input, options); + else if (['plugin.invoke', 'plugin.collect', 'plugin.publish', 'plugin.read', 'plugin.inspect', 'plugin.evidence'].includes(value.operation)) { + exact(input, ['pluginId', 'request']); identifier(input.pluginId); + response = await backend.execute(dspId, input.pluginId, value.operation.slice(7), input.request, options); + } else throw new Error('capability_unavailable'); + if (!cleanup && !permitted(dspId)) throw new Error('permission_denied'); + return result(response); + } catch (error) { return failure(error.code || error.message, true); } + } }; + } + privateDirectory(path.dirname(fileFor(paths))); + const socket = new PluginSdkSocket({ file: fileFor(paths), transport: bind(null), maximum: 64, timeoutMs: 3660000 }); + await socket.start(); + return { ensure, async close() { + await Promise.allSettled([...preparing.values()]); + await Promise.all([...bridges.values(), socket].map(item => item.close())); + } }; +} +module.exports = { serveBackend, backendClient, fileFor }; diff --git a/core/core/runtime-deployment.js b/core/core/runtime-deployment.js new file mode 100644 index 0000000..e550083 --- /dev/null +++ b/core/core/runtime-deployment.js @@ -0,0 +1,18 @@ +'use strict'; + +const RUNTIME_BACKENDS = Object.freeze([ + 'local_reference', + 'systemd_user', + 'oci_container_v1', + 'native_service_v1', + 'directory_service_v1', +]); + +function runtimeBackend(value) { + if (!RUNTIME_BACKENDS.includes(value)) { + throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); + } + return value; +} + +module.exports = { RUNTIME_BACKENDS, runtimeBackend }; diff --git a/core/core/runtime-host-identity.js b/core/core/runtime-host-identity.js new file mode 100644 index 0000000..44c285c --- /dev/null +++ b/core/core/runtime-host-identity.js @@ -0,0 +1,32 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { INSTALLATION_IDENTIFIER_RE } = require('../shared/contracts/src/installation'); + +const HOST_TENANT_ROOT = '/var/lib/dispatch/tenants'; +const HOST_BRIDGE_ROOT = '/run/dispatch-runtime-agents'; + +function fail() { + throw Object.assign(new Error('runtime_boundary_violation'), { code: 'runtime_boundary_violation' }); +} + +function runtimeKey(value) { + if (typeof value !== 'string' || value === 'local' || !INSTALLATION_IDENTIFIER_RE.test(value)) fail(); + return value; +} + +function opaqueRuntimeSuffix(value) { + return crypto.createHash('sha256').update(runtimeKey(value), 'utf8').digest('hex').slice(0, 20); +} + +function hostAccountName(value) { + return `dsp-${opaqueRuntimeSuffix(value)}`; +} + +module.exports = { + HOST_TENANT_ROOT, + HOST_BRIDGE_ROOT, + runtimeKey, + opaqueRuntimeSuffix, + hostAccountName, +}; diff --git a/core/core/scripts/render-systemd-units b/core/core/scripts/render-systemd-units new file mode 100755 index 0000000..48f3368 --- /dev/null +++ b/core/core/scripts/render-systemd-units @@ -0,0 +1,114 @@ +#!/usr/bin/env node +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { assertExternalRuntimePaths } = require('../../shared/paths/runtime-paths'); +const { trustedCommandPath } = require('../../shared/trusted-command-path'); + +function fail(message) { + process.stderr.write(`render-systemd-units: ${message}\n`); + process.exit(2); +} + +function option(name) { + const index = process.argv.indexOf(name); + if (index < 0 || index + 1 >= process.argv.length) return null; + return process.argv[index + 1]; +} + +function absolute(value, name) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value || /[\0-\x1f\x7f%"'\\]/.test(value)) fail(`${name} must be an absolute canonical path without systemd-reserved characters`); + return value; +} + +function systemdEscape(value) { + let escaped = ''; + for (const character of value) { + if (/^[A-Za-z0-9_./:@+-]$/.test(character)) { + escaped += character; + } else { + for (const byte of Buffer.from(character)) escaped += `\\x${byte.toString(16).padStart(2, '0')}`; + } + } + return escaped; +} + +function canonicalAncestor(value, name) { + let current = value; + while (!fs.existsSync(current)) { + const parent = path.dirname(current); + if (parent === current) fail(`${name} has no existing parent`); + current = parent; + } + let info; + try { info = fs.lstatSync(current); } catch { fail(`${name} is unavailable`); } + if (!info.isDirectory() || info.isSymbolicLink() || fs.realpathSync(current) !== current) fail(`${name} cannot traverse symlinks`); +} + +function ownedDirectory(value, name, { privateMode = false } = {}) { + if (value === path.parse(value).root) fail(`${name} cannot be a filesystem root`); + canonicalAncestor(value, name); + const existed = fs.existsSync(value); + fs.mkdirSync(value, { recursive: true, mode: 0o700 }); + let info; + try { info = fs.lstatSync(value); } catch { fail(`${name} is unavailable`); } + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== process.geteuid() || fs.realpathSync(value) !== value + || (info.mode & 0o022) !== 0 || privateMode && (info.mode & 0o077) !== 0) fail(`${name} must be a private owner-controlled directory`); + if (!existed && privateMode) fs.chmodSync(value, 0o700); + return value; +} + +function safeDestination(directory, name) { + const destination = path.join(directory, name); + if (!fs.existsSync(destination)) return destination; + let info; + try { info = fs.lstatSync(destination); } catch { fail(`${name} is unavailable`); } + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() || info.nlink !== 1 || (info.mode & 0o022) !== 0 + || fs.realpathSync(destination) !== destination) fail(`${name} must be an owner-controlled regular file`); + return destination; +} + +function writeUnit(directory, name, content) { + const destination = safeDestination(directory, name); + const temporary = path.join(directory, `.${name}.${process.pid}.${crypto.randomBytes(8).toString('hex')}.tmp`); + try { + fs.writeFileSync(temporary, content, { mode: 0o600, flag: 'wx' }); + fs.renameSync(temporary, destination); + fs.chmodSync(destination, 0o600); + } finally { + try { fs.rmSync(temporary, { force: true }); } catch {} + } +} + +const projectRoot = absolute(option('--project-root') || path.resolve(__dirname, "../.."), 'project root'); +const localRoot = absolute(option('--local-root'), 'local root'); +const outputRoot = absolute(option('--output-dir'), 'output directory'); +let commandPath; +try { commandPath = trustedCommandPath(); } catch { fail('no trusted command PATH is available'); } +if (process.argv.length !== (process.argv.includes('--project-root') ? 8 : 6)) fail('usage: render-systemd-units [--project-root PATH] --local-root PATH --output-dir PATH'); + +try { assertExternalRuntimePaths(projectRoot, [localRoot]); } catch { fail('local root must be outside the project worktree'); } +ownedDirectory(localRoot, 'local root', { privateMode: true }); +for (const relative of ['config', 'data', 'secrets', 'state', 'run', 'staging', 'staging/paycom', 'logs', 'tmp']) { + ownedDirectory(path.join(localRoot, relative), `local ${relative}`, { privateMode: true }); +} +ownedDirectory(outputRoot, 'output directory'); + +const templates = [ + ['runtime/auth-broker/integration/systemd/dispatch-auth-broker.service.in', 'dispatch-auth-broker.service'], + ['runtime/collection-manager/integration/systemd/dispatch-collection-manager.service.in', 'dispatch-collection-manager.service'], + ['dashboard/integration/systemd/dispatch-dashboard.service.in', 'dispatch-dashboard.service'], + ['dashboard/integration/systemd/dispatch-dashboard-tunnel.service.in', 'dispatch-dashboard-tunnel.service'], +]; +for (const [relative, outputName] of templates) { + const template = fs.readFileSync(path.join(projectRoot, relative), 'utf8'); + const rendered = template + .replaceAll('@PROJECT_ROOT@', systemdEscape(projectRoot)) + .replaceAll('@LOCAL_ROOT@', systemdEscape(localRoot)) + .replaceAll('@COMMAND_PATH@', systemdEscape(commandPath)); + if (/@[A-Z_]+@/.test(rendered)) fail(`unresolved placeholder in ${relative}`); + writeUnit(outputRoot, outputName, rendered); +} +process.stdout.write(`${JSON.stringify({ ok: true, status: 'rendered', outputRoot })}\n`); diff --git a/core/core/scripts/verify-plugin-boundary b/core/core/scripts/verify-plugin-boundary new file mode 100755 index 0000000..7fb8d60 --- /dev/null +++ b/core/core/scripts/verify-plugin-boundary @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +node --no-warnings - "$ROOT" <<'NODE' +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); + +const root = process.argv[2]; +const ignoredDirectories = new Set(['node_modules', 'scripts', 'state', 'tests']); +const legacyDebt = new Map(); + +function walk(directory, files = []) { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + if (entry.isDirectory() && ignoredDirectories.has(entry.name)) continue; + const target = path.join(directory, entry.name); + if (entry.isDirectory()) walk(target, files); + else if (entry.isFile() && target.endsWith('.js')) files.push(target); + } + return files; +} + +function lineNumber(text, index) { + return text.slice(0, index).split('\n').length; +} + +function normalizeReference(value) { + const slash = value.replaceAll('\\', '/'); + const plugin = slash.match(/(?:^|\/)plugins\/([^/]+)\/(backend|dashboard)(?:\/|$)/); + if (plugin) return slash.slice(slash.indexOf('plugins/')); + const marker = slash.indexOf('compatibility/providers/'); + if (marker >= 0) return slash.slice(marker); + if (slash === 'runtime/providers') return 'compatibility/providers/'; + if (/^@dispatch\/plugin-|^dispatch-plugin-/.test(slash)) return `package:${slash}`; + return null; +} + +function references(text) { + const found = []; + const callPattern = /\b(?:require|import)\s*\(([\s\S]{0,500}?)\)/g; + const fromPattern = /\b(?:import|export)\b[\s\S]{0,300}?\bfrom\s*(['"])([^'"]+)\1/g; + + for (const match of text.matchAll(callPattern)) { + const expression = match[1]; + const literals = [...expression.matchAll(/(['"`])([^'"`]+)\1/g)].map(item => item[2]); + const normalized = literals.map(normalizeReference).filter(Boolean); + if (!normalized.length && literals.includes('runtime/providers')) normalized.push('compatibility/providers/'); + for (const reference of normalized) { + found.push({ reference, line: lineNumber(text, match.index) }); + } + } + + for (const match of text.matchAll(fromPattern)) { + const reference = normalizeReference(match[2]); + if (reference) found.push({ reference, line: lineNumber(text, match.index) }); + } + + return found; +} + +const files = walk(root); +const errors = []; +const observedDebt = new Map(); +let referencesChecked = 0; + +for (const file of files) { + const relative = path.relative(root, file).replaceAll('\\', '/'); + const text = fs.readFileSync(file, 'utf8'); + for (const item of references(text)) { + referencesChecked += 1; + const allowed = legacyDebt.get(relative); + if (!allowed || !allowed.has(item.reference)) { + errors.push(`${relative}:${item.line} imports plugin implementation ${item.reference}`); + continue; + } + if (!observedDebt.has(relative)) observedDebt.set(relative, new Set()); + observedDebt.get(relative).add(item.reference); + } +} + +for (const [file, expected] of legacyDebt) { + const observed = observedDebt.get(file) || new Set(); + for (const reference of expected) { + if (!observed.has(reference)) errors.push(`legacy debt allowlist is stale: ${file} no longer imports ${reference}`); + } +} + +const debt = [...observedDebt.entries()].flatMap(([file, values]) => + [...values].sort().map(reference => ({ file, reference }))); + +if (errors.length) { + process.stderr.write(`${JSON.stringify({ + ok: false, + status: 'plugin_boundary_failed', + filesChecked: files.length, + referencesChecked, + legacyDebt: debt, + errors, + })}\n`); + process.exit(1); +} + +process.stdout.write(`${JSON.stringify({ + ok: true, + status: 'plugin_boundary_verified', + filesChecked: files.length, + referencesChecked, + legacyDebt: debt, +})}\n`); +NODE diff --git a/core/core/updates/README.md b/core/core/updates/README.md new file mode 100644 index 0000000..8e4e8f5 --- /dev/null +++ b/core/core/updates/README.md @@ -0,0 +1,169 @@ +# Independent updates + +The Platform Owner Updates page uses separate Core and DSP release histories. +Only the active platform owner outside a DSP view can issue update commands. +Requests require the normal session, CSRF protection and an idempotency key. The +worker rechecks owner authority before executing a command, including after a +release download. Commands never accept an arbitrary repository or executable. + +## Code and private storage + +| Location | Purpose | +| --- | --- | +| `core/updates/github.js`, `extract.py` | Read published releases from the two Dispatch repositories; verify tag, main commit, hosted release-workflow provenance, archive, inventory and plugin packages. | +| `core/updates/local-releases.js` | Durable release identities, selected versions, Dev health proof, rollout and interrupted-operation journal. | +| `core/updates/commands.js`, `worker.js` | Persistent owner commands, one external worker and periodic release discovery. | +| `core/updates/directory.js`, `transport.js` | Private worker-to-API socket and DSP activation under the directory controller. | +| `host/releases/core.js`, `core-state.js` | Stop Core services, snapshot Core state, swap code, verify API/database health and recover. | +| `host/releases/dsp.js`, `runtime.js`, `runtime-package.js` | Drain one DSP, snapshot its state, copy and verify runtime files, select its installed plugins and restore that DSP on failure. | +| `host/releases/provisioning.js` | Give new DSPs the last completely rolled-out release. | +| `host/releases/setup.js`, `bin/dispatch-updates` | Configure permanent Dev, register an existing split baseline, prepare the independent worker and queue offline recovery. | +| `dashboard/frontend/src/pages/Updates.tsx` | Owner controls, changelogs, progress and recovery UI. | +| `local/config/updates.json` | Private permanent Dev identity and API loopback port. | +| `local/state/updates/` | Verified packages, command history, worker heartbeat and release journal. | +| `local/state/dsp-releases/` | Each DSP's selected release receipt. | +| `local/backups/updates/` | Private Core and per-DSP rollback snapshots. | +| `local/tools/update-worker/` | Retained bootstrap code that survives a Core directory swap. | +| `dsps//runtime/releases/` | DSP-owned runtime and catalog metadata; newly prepared copies omit optional plugin payloads. | +| `local/packages/plugins/` | Shared verified plugin package cache, used for installation and release approval. | +| `dsps//plugins//versions/` | That DSP's installed plugin code and retained rollback versions. | + +The worker checks both feeds when idle, about every five minutes. **Check for +updates** refreshes the selected product. Every install/start-rollout command +refreshes its product again; a stale button cannot install an superseded release. +An active rollout continues using its pinned digest when a newer release appears. +Release downloads only stage verified files. They do not select code or start +services. Shared dependencies remain versioned copies inside each DSP release. + +DSP code and optional plugin packages continue to ship in one DSP release. The +host keeps that full release, but each DSP receives only the runtime and catalog +metadata. The original release manifest/digest authenticates the exact copied +subset. Install copies a sealed plugin into only the requesting DSP. Rollout +updates enabled and disabled installed plugins to the selected release versions; +uninstalled plugins are not copied. New DSPs have only the built-in Cortex +connection until plugins are installed, and use the last completed fleet release. +New plugins are visible only to DSPs on a release that approves them. Update Dev +exposes them to Dev first; rollout exposes them to each DSP as it updates. Both +session bootstrap and the Plugins endpoint apply this same catalog filter, so +signing in or opening a platform-owner DSP view cannot reveal a Dev-only plugin. + +Existing full runtime copies remain readable for rollback. Installing Core does +not rewrite those copies. New DSP creation and preparation of a new DSP release +use the smaller layout. Previously installed or historical release code remains +retained for recovery; this change does not delete credentials, settings, business +data or rollback packages. Runtime-only copies are verified by the host's +`verifyRuntime`; `verifyRelease` still requires the complete publication archive. + +## Initial deployment and configuration + +These steps belong to a separately reviewed installation procedure. They are not +performed by publication or by opening the Updates page. + +1. Publish a reviewed Core release containing these controls. Identify the exact + Core and DSP manifests/digests for the initial split deployment. Make private + offline backups and rehearse the cutover and rollback in an isolated platform. +2. Provision or select the permanent testing DSP. It must be an existing, ready + directory installation. Validate Core with synthetic data and a separate API, + dashboard, state, ports and service names before the production cutover. +3. During the installation maintenance window, install the verified Core code at + `live/`. Install and pin a verified DSP runtime for **every** retained DSP; + each must have its own `runtime/releases/` and selected receipt. + Migrate existing plugin receipts/approvals as needed without replacing DSP + credentials, settings or databases. All initial DSPs must share one approved + release. No retained DSP may depend on the old `live/runtime` fallback. +4. Using the installed code and private `DISPATCH_PLATFORM_CONFIG`, run + `bin/dispatch-updates configure DSP_ID API_PORT`. It validates Dev and writes + the private configuration. The Dev identity cannot subsequently be replaced. + Normal decommission and deletion paths protect that DSP. +5. Stage the published baselines through `GitHubReleases.refresh('core')` and + `.refresh('dsp')`, using `LocalReleases` rooted at `local/state/updates` with + the configured Dev identity. Staging has no activation hooks. The host needs + GitHub CLI with `gh attestation verify`, Python 3 and outbound access to GitHub + and its release asset hosts. CLI authentication stays private; no GitHub write + permission is needed for discovery or verification. +6. With Core services stopped, run `bin/dispatch-updates adopt CORE_DIGEST`. + This only records an **already installed** baseline. It verifies live files, + each DSP receipt, protocol compatibility and permanent Dev. It does not perform + the monolithic-to-split migration and will reject a mismatched live tree. +7. Prepare service units with the existing startup tooling. When configured, + `prepareStartup` also renders `dispatch-updates.service`. Review and install + the generated unit alongside the API/dashboard units, then enable the worker. + Its API port must match `updates.json`. Confirm installed versions, worker + health and owner access before leaving maintenance. + +A missing configuration leaves the page disabled. No guessed Dev identity, +version, baseline or production service activation is supplied automatically. +Core `0.0.1` predates the worker, so the initial installation cannot be bootstrapped +by clicking Update Core on that release. The source implementation and its +adoption command are not a completed production migration. + +## Activation and recovery + +Core and DSP updates require matching protocol versions; incompatible updates are +rejected before activation. Releases retain their own SDK/package versions. +Core changes can temporarily interrupt the shared dashboard/API even though DSP +code does not change. The browser reconnects and reloads after the Core digest +changes. A retained bootstrap worker runs outside the replaceable live tree and +hands off to the active Core release's verified worker after a successful update. + +Core snapshots include configuration, secrets and private Core state, excluding +the updater's own journal. A failed health check restores the previous Core code +and compatible state, including database schema. Bootstrap configuration remains +readable during recovery. DSP snapshots include that DSP's credentials, settings, +databases, sessions and plugin installation state. DSP rollback restores only +that DSP's files and associated plugin authority; it does not replace the entire +Core database or another DSP's data. + +Idle DSPs with a recorded scheduler sleep request can wake for installation and +live health verification. Their normal idle policy resumes afterward. A failed +update returns an originally sleeping DSP to sleep. An owner stop, suspension, +uncompleted setup or newer lifecycle request is never silently overridden; the +rollout pauses until the owner resolves it. DSPs already asleep before the sleep +receipt support was installed require one normal wake/sleep cycle before the +updater can distinguish them from deliberate stops. + +**Pause rollout** takes effect between DSPs. **Resume rollout** retries the same +pinned version. Worker restarts mark in-flight commands interrupted and pause +rollouts; an interrupted activation must recover before new installs. Use +**Recover update** when the API is available. If Core is offline, the platform +operator can queue recovery with the retained worker code: + +```sh +DISPATCH_PLATFORM_CONFIG=/absolute/private/config/platform.json \ + /absolute/retained/bin/dispatch-updates recover OWNER_USER_ID +``` + +The user ID must still identify an active platform owner. The external worker +executes recovery. Do not clear the operation journal, swap code manually or +replace a database to bypass an interrupted update. Existing release packages, +command records and snapshots are retained; automatic retention/pruning is not +implemented. Monitor private disk usage and archive only after reviewing which +receipts and rollback operations still reference those files. + +## Verification + +`npm test` covers command authorization/idempotency, release verification, latest +release ordering, lifecycle rollback, interruption recovery, scheduler sleep and +per-DSP isolation. `npm run test:integration` exercises the wider API/dashboard. +After building the dashboard, run these browser checks: + +```sh +cd dashboard +npx playwright test --config playwright.updates.config.cjs +``` + +The explicit `dashboard/examples/independent-updates-preview.js` fixture uses +synthetic owners/DSPs and simulated host actions. It is not a production updater. +The browser checks cover independent controls, newer-release reset, paused +rollout, sequential resume and mobile layout. CI runs the same browser checks. + +`tests/architecture/release-update.acceptance.js` additionally installs a real +Paycom package in a private Linux/systemd namespace lab. It verifies Cortex-only +startup without optional plugin payloads, installs Paycom, switches DSP and +Paycom versions, wakes a sleeping DSP and checks rollback of its private state. Provide sealed +Node/tini and browser tool roots plus explicit Core/DSP release directories via +`DISPATCH_WORKER_TEST_TOOLS`, `DISPATCH_WORKER_TEST_BROWSER`, +`DISPATCH_UPDATE_TEST_CORE` and `DISPATCH_UPDATE_TEST_DSP`. It uses synthetic data +and no provider credentials. Native Core service swapping still requires a +separate deployment rehearsal with isolated service names before live cutover; +unit lifecycle tests inject service control while exercising real files/SQLite. diff --git a/core/core/updates/commands.js b/core/core/updates/commands.js new file mode 100644 index 0000000..762dc05 --- /dev/null +++ b/core/core/updates/commands.js @@ -0,0 +1,46 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto'); +const { atomic, privateJson } = require('../installations/src/release-delivery-files'); +const { privateDirectory, acquireLock } = require('../../host/controller/operations'); +const { AccessError, exact, idempotencyKey } = require('../accounts/src/validation'); +const ACTIONS = ['refresh', 'update_core', 'update_dev', 'rollout', 'pause', 'resume', 'recover']; +class UpdateCommands { + constructor(directory, clock = Date.now) { + this.root = privateDirectory(directory); this.clock = clock; + this.jobs = privateDirectory(path.join(directory, 'commands')); + } + list() { + return fs.readdirSync(this.jobs).filter(name => /^[a-f0-9]{64}\.json$/.test(name)) + .map(name => privateJson(path.join(this.jobs, name), process.geteuid())).sort((a, b) => (a.sequence || 0) - (b.sequence || 0) || a.createdAt - b.createdAt || a.id.localeCompare(b.id)); + } + save(job) { atomic(path.join(this.jobs, `${job.id}.json`), job); } + request(actor, input, targets = []) { + exact(input, ['action', 'product', 'digest', 'idempotencyKey']); idempotencyKey(input.idempotencyKey); + if (!ACTIONS.includes(input.action) || !['core', 'dsp'].includes(input.product) + || !([null, undefined].includes(input.digest) || typeof input.digest === 'string' && /^[a-f0-9]{64}$/.test(input.digest)) + || ['update_core', 'update_dev', 'rollout'].includes(input.action) && !input.digest + || input.action === 'update_core' && input.product !== 'core' + || ['update_dev', 'rollout', 'pause', 'resume'].includes(input.action) && input.product !== 'dsp') throw new AccessError('invalid_input', 400); + const id = crypto.createHash('sha256').update(`${actor}:${input.idempotencyKey}`).digest('hex'); + const intent = JSON.stringify({ action: input.action, product: input.product, digest: input.digest || null }); + const fd = acquireLock({ local: path.join(this.root, 'queue-lock') }); + try { + const jobs = this.list(), prior = jobs.find(job => job.id === id); + if (prior) { + if (prior.intent !== intent) throw new AccessError('idempotency_conflict', 409); + return prior; + } + if (jobs.length >= 10000) throw new AccessError('release_history_capacity', 409); + if (jobs.some(job => ['queued', 'running'].includes(job.status)) && input.action !== 'pause') throw new AccessError('release_busy', 409); + const job = { id, actor, intent, sequence: Math.max(0, ...jobs.map(job => job.sequence || 0)) + 1, ...JSON.parse(intent), targets: input.action === 'rollout' ? targets : [], + status: 'queued', failure: null, createdAt: this.clock(), completedAt: null }; + this.save(job); return job; + } finally { fs.closeSync(fd); } + } + heartbeat(status = 'ready') { atomic(path.join(this.root, 'worker.json'), { status, at: this.clock() }); } + worker() { + const value = privateJson(path.join(this.root, 'worker.json'), process.geteuid(), true); + return { available: Boolean(value && this.clock() - value.at < 45000 && value.status !== 'stopped'), status: value?.status || 'offline' }; + } +} +module.exports = { UpdateCommands }; diff --git a/core/core/updates/configuration.js b/core/core/updates/configuration.js new file mode 100644 index 0000000..73a971e --- /dev/null +++ b/core/core/updates/configuration.js @@ -0,0 +1,14 @@ +'use strict'; +const path = require('node:path'); +const { privateJson } = require('../installations/src/release-delivery-files'); +const { validateDspId } = require('../../shared/paths/platform-paths'); +const rootFor = paths => path.join(paths.local, 'state/updates'); +function loadConfiguration(paths) { + const value = privateJson(path.join(paths.local, 'config/updates.json'), process.geteuid(), true); + if (!value) return null; + if (Object.keys(value).sort().join(',') !== 'apiPort,devDspId,schemaVersion' || value.schemaVersion !== 1 + || !Number.isInteger(value.apiPort) || value.apiPort < 1024 || value.apiPort > 65535) throw new Error('release_configuration_invalid'); + validateDspId(value.devDspId); + return value; +} +module.exports = { loadConfiguration, rootFor }; diff --git a/core/core/updates/dashboard.js b/core/core/updates/dashboard.js new file mode 100644 index 0000000..164e05f --- /dev/null +++ b/core/core/updates/dashboard.js @@ -0,0 +1,51 @@ +'use strict'; +const fs=require('node:fs'),path=require('node:path'); +const {hash,inventory}=require('../../shared/releases/package'); +const {privateJson}=require('../installations/src/release-delivery-files'); +const {AccessError}=require('../accounts/src/validation'); +function readDashboard(directory, product, digest) { + const read=name=>{const file=path.join(directory,'assets',name),stat=fs.lstatSync(file); + if(!stat.isFile()||stat.isSymbolicLink()||stat.size>12*1024*1024)throw Error('release_dashboard_invalid'); + return fs.readFileSync(file,'utf8');}; + let stylesheet=read('styles.css'); + const font=path.join(directory,'assets/inter.woff2'); + if(fs.existsSync(font))stylesheet=stylesheet.replaceAll('/assets/inter.woff2','data:font/woff2;base64,'+fs.readFileSync(font).toString('base64')); + const javascript=read('frontend.js'); + return {product,digest: digest||hash(javascript+stylesheet),javascript,stylesheet}; +} +function dashboardProvider({paths,store,corePublic=path.resolve(__dirname,'../../dashboard/public')}) { + const core=readDashboard(corePublic,'core'),cache=new Map(); + return (session,identityOnly=false)=>{ + let value=core; + if(session && (session.dspView || session.user.platformRole!=='owner') && session.activeOrganizationId) { + if(!session.memberships.some(row=>row.organizationId===session.activeOrganizationId))throw new AccessError('organization_forbidden',403); + const installation=store.installation(session.activeOrganizationId); + if(!installation?.runtimeKey)throw new AccessError('installation_not_ready',409); + const receipt=privateJson(require('../../host/releases/runtime').fileFor(paths,installation.runtimeKey),process.geteuid(),true); + if(!receipt || !/^[a-f0-9]{64}$/.test(receipt.digest))throw new AccessError('release_dashboard_unavailable',503); + const digest=receipt.digest; + if(!cache.has(digest)) { + const state=privateJson(path.join(paths.local,'state/updates/releases.json'),process.geteuid()); + const release=state.releases.dsp[digest]; + if(!release)throw new AccessError('release_dashboard_unavailable',503); + const directory=path.join(release.directory,'dashboard'); + if(fs.existsSync(directory)) { + const manifest=require('../../shared/releases/package').verifyRelease(release.directory,digest); + if(!manifest.files.some(row=>row.path==='dashboard/assets/frontend.js'))throw Error('release_dashboard_invalid'); + cache.set(digest,readDashboard(directory,'dsp',digest)); + } else { + // Explicit migration snapshots keep the pre-monorepo dashboard frozen for old DSP releases. + const baseline=path.join(paths.local,'state/updates/dashboard-baselines',digest); + const checked=privateJson(path.join(baseline,'snapshot.json'),process.geteuid()); + const publicRoot=path.join(baseline,'public'); + if(checked.dspDigest!==digest || checked.digest!==hash(JSON.stringify(inventory(publicRoot))))throw Error('release_dashboard_invalid'); + cache.set(digest,readDashboard(publicRoot,'dsp',digest)); + } + } + value=cache.get(digest); + if(cache.size>8)cache.delete(cache.keys().next().value); + } + return identityOnly?{product:value.product,digest:value.digest}:value; + }; +} +module.exports={dashboardProvider,readDashboard}; diff --git a/core/core/updates/directory.js b/core/core/updates/directory.js new file mode 100644 index 0000000..089ece8 --- /dev/null +++ b/core/core/updates/directory.js @@ -0,0 +1,45 @@ +'use strict'; +const { LocalReleases } = require('./local-releases'); +const { UpdateCommands } = require('./commands'); +const { createUpdatesService } = require('./service'); +const { loadConfiguration, rootFor } = require('./configuration'); +const { dspHooks } = require('../../host/releases/dsp'); +const { serve } = require('./transport'); +const { exact } = require('../../sdk/src/protocol'); +async function directoryUpdates({ paths, store, manager, execution }) { + const configuration = loadConfiguration(paths); + if (!configuration) return { service: createUpdatesService({ enabled: false }), close: async () => {} }; + const directory = rootFor(paths), releases = new LocalReleases({ directory, devDspId: configuration.devDspId, + hooks: dspHooks({ paths, store, manager, execution }) }); + const commands = new UpdateCommands(directory); + store.permanentDevId = configuration.devDspId; + store.releaseBlocked = organizationId => { + const key = store.installationControl(organizationId)?.runtimeKey; + return key && require('../../host/releases/guard').updating(paths, key); + }; + const authorize = actorId => { + const actor = store.userById(actorId); + if (actor?.status !== 'active' || actor.platform_role !== 'owner') throw new Error('release_actor_forbidden'); + }; + const socket = await serve({ paths, execute: async (action, input) => { + if (action === 'authorize') { exact(input, ['actor']); authorize(input.actor); return true; } + if (action === 'update_dev') { exact(input, ['actor', 'digest']); authorize(input.actor); await releases.updateDev(input.digest); } + else if (action === 'rollout') { + exact(input, ['actor', 'digest', 'targets']); authorize(input.actor); + const targets = store.db.prepare("SELECT runtime_key FROM installations WHERE backend='directory_service_v1' AND status<>'decommissioned' ORDER BY runtime_key").all().map(row => row.runtime_key); + // A changed fleet requires a new owner action; never silently expand it. + if (JSON.stringify(targets) !== JSON.stringify(input.targets)) throw new Error('release_fleet_changed'); + await releases.beginRollout(input.digest, targets, input.actor); + } else if (action === 'step') { exact(input, ['actor']); authorize(input.actor); await releases.step(); } + else if (action === 'resume') { exact(input, ['actor']); authorize(input.actor); await releases.resume(); } + else if (action === 'pause') { exact(input, ['actor']); authorize(input.actor); await releases.pause(); } + else if (action === 'recover') { exact(input, ['actor']); authorize(input.actor); + if (releases.state().operation?.product === 'core') throw new Error('release_product_invalid'); + await releases.recover(); + } else throw new Error('release_command_invalid'); + return { completed: true }; + } }); + return { service: createUpdatesService({ releases, commands, store, devDspId: configuration.devDspId }), + close: () => socket.close() }; +} +module.exports = { directoryUpdates }; diff --git a/core/core/updates/extract.py b/core/core/updates/extract.py new file mode 100644 index 0000000..f0bca8a --- /dev/null +++ b/core/core/updates/extract.py @@ -0,0 +1,49 @@ +"""Extract only bounded regular release files. Never extract links or special files.""" +import os +from pathlib import Path, PurePosixPath +import sys +import tarfile + + +def extract(archive, destination): + target = Path(destination) + target.mkdir(mode=0o700) + seen, total = set(), 0 + with tarfile.open(archive, 'r:gz') as bundle: + for index, member in enumerate(bundle): + name = member.name + if name == '.' and member.isdir(): + continue + if name.startswith('./'): + name = name[2:] + name = name.rstrip('/') if member.isdir() else name + parts = PurePosixPath(name).parts + if (index > 100000 or not parts or name.startswith('/') or '\\' in name + or any(c in name for c in '\x00\r\n') or '..' in parts + or str(PurePosixPath(name)) != name or name in seen + or not (member.isdir() or member.isreg()) or member.size < 0): + raise ValueError('release_archive_invalid') + seen.add(name) + total += member.size + if total > 2 * 1024 ** 3: + raise ValueError('release_archive_capacity') + output = target.joinpath(*parts) + if member.isdir(): + output.mkdir(mode=0o755, parents=True, exist_ok=True) + else: + output.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + with bundle.extractfile(member) as source, output.open('xb') as stream: + remaining = member.size + while remaining: + data = source.read(min(1024 * 1024, remaining)) + if not data: + raise ValueError('release_archive_truncated') + stream.write(data) + remaining -= len(data) + stream.flush() + os.fsync(stream.fileno()) + output.chmod(0o755 if member.mode & 0o111 else 0o644) + + +if __name__ == '__main__': + extract(*sys.argv[1:]) diff --git a/core/core/updates/github.js b/core/core/updates/github.js new file mode 100644 index 0000000..3a67990 --- /dev/null +++ b/core/core/updates/github.js @@ -0,0 +1,109 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { execFile } = require('node:child_process'); +const { promisify } = require('node:util'); +const execute = promisify(execFile); +const { hash, verifyRelease } = require('../../shared/releases/package'); +const { privateDirectory } = require('../../host/controller/operations'); +const REPOSITORIES = Object.freeze({ core: 'dillonlille/dispatch-core', dsp: 'dillonlille/dispatch-dsp' }); +const VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/; +const compareVersions = (a, b) => { + const x = a.split('.').map(BigInt), y = b.split('.').map(BigInt); + for (let i = 0; i < 3; i++) if (x[i] !== y[i]) return x[i] > y[i] ? 1 : -1; + return 0; +}; +async function run(executable, args) { + try { return (await execute(executable, args, { timeout: 180000, maxBuffer: 20 * 1024 * 1024 })).stdout; } + catch { throw new Error('release_verification_failed'); } +} +async function download(url, file, maximum, fetchImpl = fetch) { + const response = await fetchImpl(url, { signal: AbortSignal.timeout(180000) }); + if (!response.ok || !response.body) throw new Error('release_download_failed'); + if (Number(response.headers.get('content-length')) > maximum) throw new Error('release_download_capacity'); + const fd = fs.openSync(file, 'wx', 0o600); + let size = 0; + try { + for await (const chunk of response.body) { + size += chunk.length; + if (size > maximum) throw new Error('release_download_capacity'); + let offset = 0; + while (offset < chunk.length) offset += fs.writeSync(fd, chunk, offset, chunk.length - offset); + } + fs.fsyncSync(fd); + } finally { fs.closeSync(fd); } +} +class GitHubReleases { + constructor({ directory, releases, execute = run, fetchImpl = fetch, repositories = REPOSITORIES }) { + this.repositories = repositories; this.root = privateDirectory(directory); this.releases = releases; this.execute = execute; this.fetch = fetchImpl; + } + async catalog(product) { + const repository = this.repositories[product]; + if (!repository) throw new Error('release_product_invalid'); + const pages = JSON.parse(await this.execute('gh', ['api', '--paginate', '--slurp', `repos/${repository}/releases?per_page=100`])); + if (!Array.isArray(pages) || pages.length > 20) throw new Error('release_history_capacity'); + return pages.flat().filter(item => !item.draft && !item.prerelease && typeof item.tag_name === 'string' + && VERSION.test(item.tag_name.slice(1)) && item.tag_name.startsWith('v')) + .sort((a, b) => compareVersions(a.tag_name.slice(1), b.tag_name.slice(1))); + } + async import(product, release, manifestName = 'release.json', expectedDigest = null) { + const repository = this.repositories[product], version = release.tag_name.slice(1); + if (!repository || !VERSION.test(version) || !Number.isSafeInteger(release.id)) throw new Error('release_identity_invalid'); + const archive = `dispatch-${product}-${version}.tar.gz`; + const folder = fs.mkdtempSync(path.join(this.root, '.download-')); + const asset = async (name, max) => { + const matches = release.assets.filter(item => item.name === name); + const url = `https://github.com/${repository}/releases/download/v${version}/${name}`; + if (matches.length !== 1 || matches[0].browser_download_url !== url || matches[0].size > max || matches[0].size < 1) throw new Error('release_asset_invalid'); + const target = path.join(folder, name); + await download(url, target, max, this.fetch); + if (fs.statSync(target).size !== matches[0].size) throw new Error('release_asset_changed'); + return target; + }; + try { + const manifestFile = await asset(manifestName, 16 * 1024 * 1024); + const manifest = JSON.parse(fs.readFileSync(manifestFile)); + if (manifest.product !== product || manifest.channel !== 'release' || manifest.version !== version + || manifest.source?.repository !== repository || manifest.source?.ref !== 'refs/heads/main' + || !/^[a-f0-9]{40}$/.test(manifest.source?.commit)) throw new Error('release_identity_invalid'); + const attest = file => this.execute('gh', ['attestation', 'verify', file, '--repo', repository, + '--signer-workflow', `${repository}/.github/workflows/release.yml`, '--source-ref', 'refs/heads/main', + '--source-digest', manifest.source.commit, '--deny-self-hosted-runners']); + await attest(manifestFile); + const tag = JSON.parse(await this.execute('gh', ['api', `repos/${repository}/git/ref/tags/v${version}`])); + if (tag.object?.type !== 'commit' || tag.object.sha !== manifest.source.commit) throw new Error('release_tag_changed'); + const packed = await asset(archive, 512 * 1024 * 1024); + await attest(packed); + const extracted = path.join(folder, 'extracted'); + await this.execute('/usr/bin/python3', [path.join(__dirname, 'extract.py'), packed, extracted]); + const digest = hash(JSON.stringify(manifest)); + if(expectedDigest && digest!==expectedDigest)throw new Error('release_asset_changed'); + const checked = verifyRelease(extracted, digest); + if (JSON.stringify(checked) !== JSON.stringify(manifest)) throw new Error('release_asset_changed'); + for (const item of manifest.plugins) { + if (!/^[a-z][a-z0-9-]{0,63}$/.test(item.pluginId)) throw new Error('release_plugins_invalid'); + const plugin = require('../../shared/plugin-sdk/package-files').verifyPackage(path.join(extracted, 'plugins', item.pluginId), item.digest).plugin; + if (plugin.id !== item.pluginId || plugin.version !== item.version) throw new Error('release_plugins_invalid'); + } + const notes = fs.readFileSync(path.join(extracted, 'release-notes.md'), 'utf8'); + if (Buffer.byteLength(notes) > 100000 || !notes.trim()) throw new Error('release_notes_invalid'); + return await this.releases.stage(extracted, digest, { notes, source: manifest.source, + publishedAt: release.published_at, url: `https://github.com/${repository}/releases/tag/v${version}` }); + } finally { fs.rmSync(folder, { recursive: true, force: true }); } + } + async refresh(product) { + const items = await this.catalog(product); + // Import ascending versions so history is readable and newest wins. Always + // reverify the newest: replacing a published version must fail closed. + for (const item of items) { + const known = Object.values(this.releases.state().releases[product]).some(row => row.version === item.tag_name.slice(1)); + if (!known || item === items.at(-1)) await this.import(product, item); + } + if (!items.length) throw new Error('release_feed_empty'); + const latest = this.releases.state().releases[product][this.releases.state().latest[product]]; + if (latest.version !== items.at(-1).tag_name.slice(1)) throw new Error('release_feed_regressed'); + return latest.digest; + } +} +module.exports = { GitHubReleases, REPOSITORIES, compareVersions, download }; diff --git a/core/core/updates/local-releases.js b/core/core/updates/local-releases.js new file mode 100644 index 0000000..1ad891a --- /dev/null +++ b/core/core/updates/local-releases.js @@ -0,0 +1,123 @@ +'use strict'; +// Durable independent release orchestration. Callers supply +// lifecycle hooks; constructing this class never starts or modifies a service. +const fs=require('node:fs'),path=require('node:path'); +const {verifyRelease,secureCopy}=require('../../shared/releases/package'); +const {atomic,privateJson}=require('../installations/src/release-delivery-files'); +const {privateDirectory,acquireLock}=require('../../host/controller/operations'); +const {compareVersions}=require('./github'); +const id=value=>typeof value==='string'&&/^[A-Za-z0-9_-]{1,128}$/.test(value); +class LocalReleases { + constructor({directory,devDspId,hooks,allowDevelopment=false}) { + if(!id(devDspId)||!hooks||['drain','snapshot','start','verify','restore'].some(key=>typeof hooks[key]!=='function'))throw new Error('release_configuration_invalid'); + this.root=privateDirectory(directory);this.file=path.join(directory,'releases.json'); + Object.assign(this,{devDspId,hooks,allowDevelopment}); + } + state() { + return privateJson(this.file,process.geteuid(),true)||{schemaVersion:1,devDspId:this.devDspId,releases:{core:{},dsp:{}},latest:{core:null,dsp:null},active:{core:null,dsps:{}},tested:null,defaultDsp:null,rollout:null,operation:null}; + } + async locked(work) { + const fd=acquireLock({local:this.root}); + try{const state=this.state();if(state.devDspId!==this.devDspId)throw new Error('release_dev_identity_changed');return await work(state);} + finally{fs.closeSync(fd);} + } + + save(state){if(Buffer.byteLength(JSON.stringify(state))>256*1024)throw new Error('release_state_capacity');atomic(this.file,state);} + async stage(directory,digest,metadata={}) { + const manifest=verifyRelease(directory,digest); + if(manifest.plugins.some(item=>!item||Object.keys(item).sort().join(',')!=='digest,pluginId,version'||!/^[a-z][a-z0-9-]{0,63}$/.test(item.pluginId)||!/^\d+\.\d+\.\d+$/.test(item.version)||!/^[a-f0-9]{64}$/.test(item.digest))||new Set(manifest.plugins.map(item=>item.pluginId)).size!==manifest.plugins.length)throw new Error('release_plugins_invalid'); + if(manifest.channel==='development'&&!this.allowDevelopment)throw new Error('release_not_published'); + return this.locked(state=>{ + if(state.operation)throw new Error('release_recovery_required'); + const releases=state.releases[manifest.product]; + if(Object.values(releases).some(item=>item.version===manifest.version&&item.digest!==digest&&(item.source?.repository||null)===(metadata.source?.repository||null)))throw new Error('release_version_immutable'); + if(releases[digest])return {digest,staged:true}; + const target=path.join(privateDirectory(path.join(this.root,'packages',manifest.product)),digest); + if(fs.existsSync(target))verifyRelease(target,digest); + else { + const temporary=target+'.stage-'+require('node:crypto').randomBytes(12).toString('hex'); + try{secureCopy(directory,temporary);verifyRelease(temporary,digest);fs.renameSync(temporary,target);require('../../host/controller/operations').syncDirectory(path.dirname(target));} + finally{fs.rmSync(temporary,{recursive:true,force:true});} + } + verifyRelease(target,digest); + releases[digest]={digest,version:manifest.version,protocol:manifest.protocol,directory:target,source:metadata.source||null,publishedAt:metadata.publishedAt||null,url:metadata.url||null}; + const latest=state.latest[manifest.product]; + if(!latest||(metadata.source?.repository==='dillonlille/dispatch-platform' && releases[latest].source?.repository!=='dillonlille/dispatch-platform')||(metadata.source?.repository===releases[latest].source?.repository && compareVersions(manifest.version,releases[latest].version)>0)){ + state.latest[manifest.product]=digest; + if(manifest.product==='dsp')state.tested=null; + } + this.save(state);return {digest,staged:true}; + }); + } + async activate(state,product,digest,dspId=null) { + const release=state.releases[product][digest];if(!release)throw new Error('release_unavailable'); + const manifest=verifyRelease(release.directory,digest); + if(product==='dsp') { + const core=state.active.core&&state.releases.core[state.active.core]; + if(!core||core.protocol!==manifest.protocol)throw new Error('release_incompatible'); + } else for(const selected of Object.values(state.active.dsps))if(state.releases.dsp[selected].protocol!==manifest.protocol)throw new Error('release_incompatible'); + const prior=product==='core'?state.active.core:state.active.dsps[dspId]||null; + const context={product,dspId,digest,previousDigest:prior,directory:release.directory,manifest}; + const work=async()=>{ + if(prior===digest&&!await this.hooks.requiresActivation?.(context)){if(await this.hooks.verify(context)!==true)throw new Error('release_health_failed');return;} + state.operation={product,dspId,digest,prior,phase:'preparing'};this.save(state); + let snapshot; + try { + if(this.hooks.prepare){state.operation.preparation=await this.hooks.prepare(context);context.preparation=state.operation.preparation;} + state.operation.phase='draining';this.save(state); + await this.hooks.drain(context);snapshot=await this.hooks.snapshot(context); + if(snapshot===undefined)throw new Error('release_snapshot_required'); + // The snapshot token must be durable and JSON serializable for recovery. + state.operation.snapshot=JSON.parse(JSON.stringify(snapshot));state.operation.phase='starting';this.save(state); + await this.hooks.start(context); + if(await this.hooks.verify(context)!==true)throw new Error('release_health_failed'); + if(product==='core')state.active.core=digest;else state.active.dsps[dspId]=digest; + state.operation=null;this.save(state); + }catch(error){ + state.operation.phase='failed';this.save(state); + try{state.operation.phase='restoring';this.save(state);await this.hooks.restore({...context,snapshot});state.operation=null;this.save(state);}catch{state.operation.phase='failed';this.save(state);throw new Error('release_recovery_required');} + throw error; + } + }; + return this.hooks.withActivation?this.hooks.withActivation(context,work):work(); + } + updateCore(digest){return this.locked(async state=>{if(state.operation)throw new Error('release_recovery_required');if(digest!==state.latest.core)throw new Error('release_changed');await this.activate(state,'core',digest);});} + updateDev(digest){return this.locked(async state=>{ + if(state.operation||state.rollout&&state.rollout.status!=='completed')throw new Error('release_busy'); + if(digest!==state.latest.dsp)throw new Error('release_changed'); + state.tested=null;this.save(state); + await this.activate(state,'dsp',digest,this.devDspId);state.tested=digest;this.save(state); + });} + beginRollout(digest,dspIds,actor=null){return this.locked(async state=>{ + if(state.operation||state.rollout&&state.rollout.status!=='completed')throw new Error('release_busy'); + if(digest!==state.latest.dsp||digest!==state.tested)throw new Error('release_dev_required'); + if(!Array.isArray(dspIds)||dspIds.some(value=>!id(value))||new Set(dspIds).size!==dspIds.length)throw new Error('release_targets_invalid'); + if(state.active.dsps[this.devDspId]!==digest)throw new Error('release_dev_required'); + state.tested=null;this.save(state); + await this.activate(state,'dsp',digest,this.devDspId);state.tested=digest; + state.rollout={digest,actor,targets:dspIds.filter(value=>value!==this.devDspId),next:0,status:'running',failure:null};this.save(state); + });} + step(){return this.locked(async state=>{ + const rollout=state.rollout; + if(state.operation)throw new Error('release_recovery_required'); + if(!rollout||rollout.status!=='running')throw new Error('release_rollout_not_running'); + if(rollout.next===rollout.targets.length){rollout.status='completed';state.defaultDsp=rollout.digest;this.save(state);return {completed:true};} + const dspId=rollout.targets[rollout.next]; + try{await this.activate(state,'dsp',rollout.digest,dspId);rollout.next++;if(rollout.next===rollout.targets.length){rollout.status='completed';state.defaultDsp=rollout.digest;}} + catch(error){rollout.status='paused';rollout.failure=/^release_[a-z_]+$/.test(error.message)?error.message:'activation_failed';this.save(state);throw error;} + this.save(state);return {dspId,digest:rollout.digest,completed:rollout.status==='completed'}; + });} + resume(){return this.locked(state=>{if(state.operation)throw new Error('release_recovery_required');if(state.rollout?.status!=='paused')throw new Error('release_rollout_not_paused');state.rollout.status='running';state.rollout.failure=null;this.save(state);});} + pause(){return this.locked(state=>{if(state.rollout?.status!=='running')throw new Error('release_rollout_not_running');state.rollout.status='paused';state.rollout.failure='owner_paused';this.save(state);});} + recover(){return this.locked(async state=>{ + const operation=state.operation;if(!operation)return; + const release=state.releases[operation.product][operation.digest]; + const context={...operation,previousDigest:operation.prior,directory:release.directory,manifest:verifyRelease(release.directory,operation.digest)}; + state.operation.phase='restoring';this.save(state); + const restore=()=>this.hooks.restore(context); + try{if(this.hooks.withActivation)await this.hooks.withActivation(context,restore);else await restore();} + catch(error){state.operation.phase='failed';this.save(state);throw error;} + state.operation=null;if(state.rollout?.status==='running'){state.rollout.status='paused';state.rollout.failure='interrupted';}this.save(state); + });} +} +module.exports={LocalReleases}; diff --git a/core/core/updates/migration.js b/core/core/updates/migration.js new file mode 100644 index 0000000..4c9901b --- /dev/null +++ b/core/core/updates/migration.js @@ -0,0 +1,32 @@ +'use strict'; +const fs=require('node:fs'),path=require('node:path'); +const {verifyRelease,inventory,hash,secureCopy}=require('../../shared/releases/package'); +const {privateDirectory}=require('../../host/controller/operations'); +const {atomic,privateJson}=require('../installations/src/release-delivery-files'); +// Caller holds the update lock. Freeze from the verified installed Core artifact, +// never from a working tree. This prepares compatibility; it does not activate code. +function freezeLegacyDashboards(paths,state){ + if(state.operation||['running','paused'].includes(state.rollout?.status))throw Error('release_busy'); + const core=state.releases.core[state.active.core];if(!core)throw Error('release_baseline_required'); + verifyRelease(core.directory,core.digest); + const publicRoot=path.join(core.directory,'code/dashboard/public'); + const snapshotDigest=hash(JSON.stringify(inventory(publicRoot))); + const results=[]; + for(const release of Object.values(state.releases.dsp)){ + verifyRelease(release.directory,release.digest); + if(fs.existsSync(path.join(release.directory,'dashboard')))continue; + const destination=path.join(privateDirectory(path.join(paths.local,'state/updates/dashboard-baselines')),release.digest); + if(!fs.existsSync(destination)) { + const temporary=destination+'.stage'; + if(fs.existsSync(temporary))fs.rmSync(temporary,{recursive:true}); + privateDirectory(temporary);secureCopy(publicRoot,path.join(temporary,'public')); + atomic(path.join(temporary,'snapshot.json'),{dspDigest:release.digest,coreDigest:core.digest,digest:snapshotDigest}); + fs.renameSync(temporary,destination);require('../../host/controller/operations').syncDirectory(path.dirname(destination)); + } + const snapshot=privateJson(path.join(destination,'snapshot.json'),process.geteuid()); + if(snapshot.dspDigest!==release.digest || snapshot.digest!==hash(JSON.stringify(inventory(path.join(destination,'public')))))throw Error('release_dashboard_invalid'); + results.push(release.digest); + } + return {prepared:true,dashboards:results.length,activation:false}; +} +module.exports={freezeLegacyDashboards}; diff --git a/core/core/updates/platform-github.js b/core/core/updates/platform-github.js new file mode 100644 index 0000000..3fba5a2 --- /dev/null +++ b/core/core/updates/platform-github.js @@ -0,0 +1,55 @@ +'use strict'; +const fs=require('node:fs'),path=require('node:path'); +const {GitHubReleases,download,compareVersions}=require('./github'); +const {hash}=require('../../shared/releases/package'); +const REPOSITORY='dillonlille/dispatch-platform'; +class PlatformGitHubReleases extends GitHubReleases { + constructor(options){super({...options,repositories:{core:REPOSITORY,dsp:REPOSITORY}});this.isUnified=true;} + async descriptor(release) { + const asset=release.assets.filter(a=>a.name==='platform-release.json'); + const url=`https://github.com/${REPOSITORY}/releases/download/${release.tag_name}/platform-release.json`; + if(asset.length!==1||asset[0].browser_download_url!==url||asset[0].size>100000)throw Error('release_asset_invalid'); + const folder=fs.mkdtempSync(path.join(this.root,'.platform-')),file=path.join(folder,'platform-release.json'); + try { + await download(url,file,100000,this.fetch); + const value=JSON.parse(fs.readFileSync(file)); + if(value.schemaVersion!==1||value.version!==release.tag_name.slice(1)||value.repository!==REPOSITORY||!/^[a-f0-9]{40}$/.test(value.commit))throw Error('release_identity_invalid'); + await this.execute('gh',['attestation','verify',file,'--repo',REPOSITORY,'--signer-workflow',`${REPOSITORY}/.github/workflows/release.yml`,'--source-ref','refs/heads/main','--source-digest',value.commit,'--deny-self-hosted-runners']); + const tag=JSON.parse(await this.execute('gh',['api',`repos/${REPOSITORY}/git/ref/tags/${release.tag_name}`])); + if(tag.object?.type!=='commit'||tag.object.sha!==value.commit)throw Error('release_tag_changed'); + if(!value.components || !value.changes || ['core','dsp','plugins'].some(key=>typeof value.changes[key]!=='string'))throw Error('release_identity_invalid'); + for(const key of ['core','dsp']){ + const c=value.components[key]; + if(!c||!/^\d+\.\d+\.\d+$/.test(c.version)||compareVersions(c.version,value.version)>0||!/^[a-f0-9]{64}$/.test(c.digest))throw Error('release_identity_invalid'); + } + return value; + }finally{fs.rmSync(folder,{recursive:true,force:true});} + } + async refresh(product) { + if(!['core','dsp'].includes(product))throw Error('release_product_invalid'); + const items=await this.catalog(product); + if(!items.length)throw Error('release_feed_empty'); + const descriptors=[]; + for(const item of items) { + const known=this.releases.state().platform?.history.find(row=>row.version===item.tag_name.slice(1)); + const value=known && item!==items.at(-1) ? known : await this.descriptor(item); + for(const track of ['core','dsp']) { + const component=value.components[track]; + const source=items.find(row=>row.tag_name===`v${component.version}`); + if(!source)throw Error('release_component_missing'); + if(!this.releases.state().releases[track][component.digest] || item===items.at(-1)) + await this.import(track,source,`${track}-release.json`,component.digest); + } + descriptors.push({...value,url:`https://github.com/${REPOSITORY}/releases/tag/v${value.version}`,publishedAt:item.published_at}); + } + await this.releases.locked(state=>{ + const previous=state.platform?.history||[]; + for(const value of descriptors){const prior=previous.find(row=>row.version===value.version); + if(prior&&hash(JSON.stringify(prior))!==hash(JSON.stringify(value)))throw Error('release_version_immutable');} + if(state.platform && compareVersions(descriptors.at(-1).version,state.platform.latest)<0)throw Error('release_feed_regressed'); + state.platform={latest:descriptors.at(-1).version,history:descriptors};this.releases.save(state); + }); + return this.releases.state().latest[product]; + } +} +module.exports={PlatformGitHubReleases,REPOSITORY}; diff --git a/core/core/updates/service.js b/core/core/updates/service.js new file mode 100644 index 0000000..8280b67 --- /dev/null +++ b/core/core/updates/service.js @@ -0,0 +1,71 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +const { AccessError } = require('../accounts/src/validation'); +const { compareVersions } = require('./github'); +function notes(directory) { + try { + const fd = fs.openSync(path.join(directory, 'release-notes.md'), fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { if (fs.fstatSync(fd).size > 100000) return 'Release notes unavailable.'; return fs.readFileSync(fd, 'utf8'); } finally { fs.closeSync(fd); } + } catch { return 'Release notes unavailable.'; } +} +function createUpdatesService({ releases, commands, store, devDspId, enabled = true }) { + const fleet = () => store.db.prepare(`SELECT i.runtime_key id,o.name,i.status FROM installations i + JOIN organizations o ON o.id=i.organization_id WHERE i.backend='directory_service_v1' + AND i.status<>'decommissioned' ORDER BY i.runtime_key`).all(); + return { + ownerOnly: true, + command(session, input) { + if (session.user.platformRole !== 'owner' || session.dspView) throw new AccessError('platform_forbidden', 403); + if (!enabled || !commands.worker().available) throw new AccessError('release_worker_unavailable', 503); + const actor = store.userById(session.user.id); + if (actor?.platform_role !== 'owner' || actor.status !== 'active') throw new AccessError('platform_forbidden', 403); + const targets = fleet().map(item => item.id); + if (['update_dev', 'rollout', 'resume'].includes(input.action) && !targets.includes(devDspId)) throw new AccessError('release_dev_unavailable', 409); + const job = commands.request(session.user.id, input, targets); + if (!store.db.prepare('SELECT 1 FROM audit_events WHERE id=?').get(`aud_update_${job.id}`)) store.createAudit({ id: `aud_update_${job.id}`, actorUserId: session.user.id, organizationId: null, + action: `platform.update.${job.action}`, targetType: 'release_update', targetId: job.id, result: 'succeeded', timestamp: job.createdAt }); + return { id: job.id, status: job.status }; + }, + view(selectedId = null) { + const state = releases?.state(), jobs = commands?.list() || [], worker = commands?.worker() || { available: false, status: 'offline' }; + const rows = enabled ? fleet() : [], labels = new Map(rows.map(item => [item.id, item.name])); + const platformVersion=selectedId ? (selectedId.startsWith('platform_') ? selectedId.slice(9) : null) : state?.platform?.latest; + const platformRelease=state?.platform?.history.find(row=>row.version===platformVersion); + if(selectedId?.startsWith('platform_') && !platformRelease)throw new AccessError('release_not_found',404); + const selected = !selectedId?.startsWith('platform_') && selectedId && /^(core|dsp)_([a-f0-9]{64})$/.exec(selectedId); + if (selectedId && !selectedId.startsWith('platform_') && (!selected || !state?.releases[selected[1]][selected[2]])) throw new AccessError('release_not_found', 404); + const busy = Boolean(state?.operation || jobs.some(job => ['queued', 'running'].includes(job.status))); + const tracks = Object.fromEntries(['core', 'dsp'].map(product => { + const releasesFor = state?.releases[product] || {}, latest = state?.latest[product]; + const digest = platformRelease?.components[product]?.digest || (selected?.[1] === product ? selected[2] : latest); + const item = releasesFor[digest], active = product === 'core' ? state?.active.core : state?.active.dsps[devDspId]; + const history = Object.values(releasesFor).sort((a, b) => compareVersions(b.version, a.version)).map(row => ({ + id: `${product}_${row.digest}`, digest: row.digest, version: row.version, publishedAt: row.publishedAt, + })); + const current = releasesFor[active]; + return [product, { latest, installedLegacy: Boolean(current?.source?.repository && current.source.repository!=='dillonlille/dispatch-platform'), installedVersion: current?.version || null, installedDigest: active || null, + release: item ? { id: `${product}_${item.digest}`, digest: item.digest, version: item.version, + notes: item.notes || notes(item.directory), source: item.source, publishedAt: item.publishedAt, url: item.url } : null, + history, tested: product === 'dsp' && Boolean(latest && state.tested === latest), + canUpdate: enabled && worker.available && !busy && Boolean(latest) && (product === 'core' + ? active !== latest && !['running', 'paused'].includes(state.rollout?.status) + : !['running', 'paused'].includes(state.rollout?.status) && (!state.tested || state.tested !== latest || state.defaultDsp !== latest || rows.some(row => state.active.dsps[row.id] !== latest))), + }]; + })); + const rollout = state?.rollout; + return { mode: 'independent', platformRelease:platformRelease||null, platformHistory:state?.platform?.history.map(row=>({id:'platform_'+row.version,version:row.version}))||[], latestPlatform:state?.platform?.latest||null, enabled, worker, busy, tracks, + dev: { id: devDspId || null, name: labels.get(devDspId) || 'Dev DSP', available: labels.has(devDspId) }, + recoveryRequired: Boolean(state?.operation), + operation: state?.operation ? { product: state.operation.product, phase: state.operation.phase, + dspName: labels.get(state.operation.dspId) || null } : null, + rollout: rollout ? { digest: rollout.digest, version: state.releases.dsp[rollout.digest]?.version, + status: rollout.status, failure: rollout.failure, updated: rollout.next, total: rollout.targets.length, + members: rollout.targets.map((id, index) => ({ name: labels.get(id) || 'Removed DSP', + status: state.operation?.dspId === id ? 'updating' : index < rollout.next ? 'updated' : index === rollout.next && rollout.status === 'paused' ? 'paused' : 'queued' })) } : null, + jobs: jobs.slice(-10).reverse().map(job => ({ id: job.id, action: job.action, product: job.product, + status: job.status, failure: job.failure, createdAt: job.createdAt })), + }; + }, + }; +} +module.exports = { createUpdatesService }; diff --git a/core/core/updates/tests/commands-worker.test.js b/core/core/updates/tests/commands-worker.test.js new file mode 100644 index 0000000..44312d2 --- /dev/null +++ b/core/core/updates/tests/commands-worker.test.js @@ -0,0 +1,51 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'); +const { UpdateCommands } = require('../commands'); +const { UpdateWorker } = require('../worker'); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-commands-')); t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const commands = new UpdateCommands(root), events = []; + let state = { operation: null, rollout: null }; + const releases = { state: () => state, updateCore: async digest => events.push(['core', digest]), + pause: async () => { state.rollout.status = 'paused'; }, recover: async () => { state.operation = null; events.push(['recover']); } }; + const options = { commands, releases, feed: { refresh: async product => events.push(['refresh', product]) }, + invoke: async (action, input) => { events.push([action, input]); }, authorize: async actor => { if (actor !== 'owner') throw new Error('release_actor_forbidden'); } }; + const worker = new UpdateWorker(options); + const request = (action, product = 'dsp', digest = 'a'.repeat(64), key = action) => commands.request('owner', { + action, product, digest, idempotencyKey: `synthetic:command:${key}` }, ['dev', 'a', 'b']); + return { commands, events, releases, options, worker, state, request }; +} +test('queued commands survive reconstruction and duplicate clicks cannot duplicate an update', async t => { + const f = fixture(t), first = f.request('update_core', 'core'); + assert.equal(f.request('update_core', 'core').id, first.id); + assert.equal(f.commands.list().length, 1); + assert.throws(() => f.request('update_core', 'core', 'b'.repeat(64)), /idempotency_conflict/); + await new UpdateWorker(f.options).tick(); + assert.deepEqual(f.events, [['refresh', 'core'], ['core', 'a'.repeat(64)]]); + assert.equal(f.commands.list()[0].status, 'completed'); +}); +test('rollout submission captures the fleet and rechecks the release before activation', async t => { + const f = fixture(t); f.request('rollout'); await f.worker.tick(); + assert.deepEqual(f.events[0], ['refresh', 'dsp']); + assert.deepEqual(f.events[1], ['rollout', { actor: 'owner', digest: 'a'.repeat(64), targets: ['dev', 'a', 'b'] }]); + f.state.rollout = { status: 'running', actor: 'owner', digest: 'a'.repeat(64) }; + await f.worker.tick(); assert.equal(f.events.at(-1)[0], 'step'); +}); +test('revoked owners cannot execute queued updates or advance a fleet', async t => { + const f = fixture(t), job = f.request('update_core', 'core'); job.actor = 'revoked'; f.commands.save(job); + await f.worker.tick(); assert.equal(f.commands.list()[0].failure, 'release_actor_forbidden'); assert.deepEqual(f.events, []); + f.state.rollout = { status: 'running', actor: 'revoked' }; await f.worker.tick(); assert.equal(f.state.rollout.status, 'paused'); +}); +test('worker restart pauses an active fleet and marks an interrupted command without retrying it', async t => { + const f = fixture(t), job = f.request('update_dev'); job.status = 'running'; f.commands.save(job); + f.state.rollout = { status: 'running', actor: 'owner' }; await f.worker.initialize(); + assert.equal(f.commands.list()[0].failure, 'release_interrupted'); assert.equal(f.state.rollout.status, 'paused'); assert.deepEqual(f.events, []); +}); +test('pause can be queued during a step, Core is blocked by an unfinished rollout, and recovery is explicit', async t => { + const f = fixture(t); f.state.rollout = { status: 'running', actor: 'owner' }; + f.request('update_core', 'core'); f.request('pause'); await f.worker.tick(); + assert.equal(f.commands.list().find(job => job.action === 'update_core').failure, 'release_busy'); + await f.worker.tick(); assert.equal(f.events.at(-1)[0], 'pause'); + f.state.operation = { product: 'core' }; f.request('recover', 'core'); await f.worker.tick(); assert.equal(f.state.operation, null); +}); diff --git a/core/core/updates/tests/github.test.js b/core/core/updates/tests/github.test.js new file mode 100644 index 0000000..a9ea85a --- /dev/null +++ b/core/core/updates/tests/github.test.js @@ -0,0 +1,81 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'); +const { execFileSync } = require('node:child_process'); +const { GitHubReleases, compareVersions, download } = require('../github'); +const { LocalReleases } = require('../local-releases'); +const { inventory, hash } = require('../../../shared/releases/package'); +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-feed-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const hooks = Object.fromEntries(['drain', 'snapshot', 'start', 'verify', 'restore'].map(key => [key, async () => true])); + const releases = new LocalReleases({ directory: path.join(root, 'state'), devDspId: 'dev', hooks }); + const items = [], assets = new Map(), attestations = []; + let denied = false, wrongTag = false; + const execute = async (command, args) => { + if (command === '/usr/bin/python3') return execFileSync(command, args, { encoding: 'utf8' }); + if (args[0] === 'attestation') { attestations.push(args); if (denied) throw new Error('untrusted'); return ''; } + if (args.at(-1).includes('/git/ref/')) return JSON.stringify({ object: { type: 'commit', sha: (wrongTag ? 'b' : 'a').repeat(40) } }); + return JSON.stringify([items]); + }; + const feed = new GitHubReleases({ directory: path.join(root, 'downloads'), releases, execute, + fetchImpl: async url => new Response(assets.get(url)) }); + function add(version, content = version) { + const folder = fs.mkdtempSync(path.join(root, 'source-')); + fs.mkdirSync(path.join(folder, 'code')); fs.writeFileSync(path.join(folder, 'code/value'), content); + fs.writeFileSync(path.join(folder, 'release-notes.md'), `Release ${version}\n\nNew synthetic functionality.`); + const manifest = { schemaVersion: 1, product: 'dsp', version, channel: 'release', protocol: 1, minimumProtocol: 1, + sourceDigest: 'a'.repeat(64), source: { repository: 'dillonlille/dispatch-dsp', commit: 'a'.repeat(40), ref: 'refs/heads/main' }, plugins: [], files: inventory(folder) }; + fs.writeFileSync(path.join(folder, 'release.json'), JSON.stringify(manifest)); + const archive = `dispatch-dsp-${version}.tar.gz`, output = path.join(root, `archive-${items.length}.tgz`); + execFileSync('tar', ['-czf', output, '-C', folder, '.']); + const release = { id: items.length + 1, tag_name: `v${version}`, draft: false, prerelease: false, published_at: '2026-01-01T00:00:00Z', assets: [] }; + for (const [name, buffer] of [['release.json', fs.readFileSync(path.join(folder, 'release.json'))], [archive, fs.readFileSync(output)]]) { + const url = `https://github.com/dillonlille/dispatch-dsp/releases/download/v${version}/${name}`; + assets.set(url, buffer); release.assets.push({ name, size: buffer.length, browser_download_url: url }); + } + items.push(release); return { release, digest: hash(JSON.stringify(manifest)) }; + } + return { feed, releases, add, attestations, items, root, deny: () => { denied = true; }, wrongTag: () => { wrongTag = true; } }; +} +test('verified feed stages history in version order and preserves a completed Dev test', async t => { + const f = fixture(t), newest = f.add('1.10.0'); f.add('1.2.0'); + assert.equal(await f.feed.refresh('dsp'), newest.digest); + assert.equal(f.releases.state().active.dsps.dev, undefined); + const state = f.releases.state(); state.tested = newest.digest; f.releases.save(state); + await f.feed.refresh('dsp'); assert.equal(f.releases.state().tested, newest.digest); + assert.equal(f.releases.state().releases.dsp[newest.digest].source.repository, 'dillonlille/dispatch-dsp'); + assert(f.attestations.every(args => args.includes('--deny-self-hosted-runners') && args.includes('--source-digest'))); + assert.equal(compareVersions('9007199254740993.0.0', '9007199254740992.0.0'), 1); +}); +test('changed publication bytes cannot replace an immutable staged version', async t => { + const f = fixture(t), one = f.add('1.0.0'); await f.feed.refresh('dsp'); + f.items.length = 0; f.add('1.0.0', 'replacement'); + await assert.rejects(f.feed.refresh('dsp'), /release_version_immutable/); + assert.equal(f.releases.state().latest.dsp, one.digest); +}); +test('untrusted provenance, changed tag and foreign asset URLs never stage code', async t => { + for (const defect of ['deny', 'wrongTag', 'url']) { + await t.test(defect, async t => { + const f = fixture(t), { release } = f.add('1.0.0'); + if (defect === 'url') release.assets[0].browser_download_url = 'http://127.0.0.1/private'; else f[defect](); + await assert.rejects(f.feed.refresh('dsp')); + assert.equal(f.releases.state().latest.dsp, null); + }); + } +}); +test('downloads enforce streaming capacity when Content-Length is absent', async t => { + const f = fixture(t); + await assert.rejects(download('https://example.test', path.join(f.root, 'large'), 2, + async () => new Response('overflow')), /release_download_capacity/); +}); +test('extractor rejects traversal and symbolic links before writing outside its root', t => { + const f = fixture(t); + for (const name of ['../escape', '/tmp/escape', 'link']) { + const archive = path.join(f.root, `${name === 'link' ? 'link' : 'path'}.tgz`); + execFileSync('/usr/bin/python3', ['-c', "import tarfile,sys,io\nwith tarfile.open(sys.argv[1],'w:gz') as t:\n m=tarfile.TarInfo(sys.argv[2]);m.type=tarfile.SYMTYPE if sys.argv[2]=='link' else tarfile.REGTYPE;m.linkname='/tmp';t.addfile(m,io.BytesIO())", archive, name]); + assert.throws(() => execFileSync('/usr/bin/python3', [path.join(__dirname, '../extract.py'), archive, + path.join(f.root, `extract-${Math.random()}`)], { stdio: 'pipe' })); + } + assert.equal(fs.existsSync(path.join(f.root, 'escape')), false); +}); diff --git a/core/core/updates/tests/health.test.js b/core/core/updates/tests/health.test.js new file mode 100644 index 0000000..572122e --- /dev/null +++ b/core/core/updates/tests/health.test.js @@ -0,0 +1,35 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'), http = require('node:http'); +const { coreHooks } = require('../../../host/releases/core'); +const { requestHealth } = require('../../../host/releases/health'); +const { atomic } = require('../../installations/src/release-delivery-files'); +async function server(t, handler) { + const server = http.createServer(handler); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => { server.closeAllConnections(); return new Promise(resolve => server.close(resolve)); }); + return server.address().port; +} +test('Core loopback health preserves public Host, HTTPS forwarding and recovery nonce', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-core-health-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + fs.mkdirSync(path.join(root, 'config'), { mode: 0o700 }); + atomic(path.join(root, 'config/dashboard.json'), { version: 1, port: 4310, publicOrigin: 'https://dispatch.example.test' }); + const digest = 'a'.repeat(64), nonce = 'b'.repeat(64); let requests = 0; + const port = await server(t, (request, response) => { + requests++; + const allowed = request.headers.host === 'dispatch.example.test' && request.headers['cf-visitor'] === '{"scheme":"https"}' + && request.headers['x-dispatch-recovery-probe'] === nonce; + response.writeHead(allowed ? 200 : 403, { 'Content-Type': 'application/json' }); + response.end(JSON.stringify({ ok: allowed, data: { digest, version: '0.0.2', recoveryProbe: 'passed' } })); + }); + const hooks = coreHooks({ paths: { local: root, platformRoot: root }, configuration: { apiPort: port }, releases: () => ({}), healthTimeoutMs: 1000 }); + assert.equal(await hooks.verify({ digest, manifest: { version: '0.0.2' }, preparation: { nonce } }), true); + assert.equal(requests, 1); +}); +test('Core health rejects an oversized response and respects cancellation', async t => { + const oversized = await server(t, (_request, response) => response.end('x'.repeat(4097))); + await assert.rejects(requestHealth(`http://127.0.0.1:${oversized}/`, { signal: AbortSignal.timeout(1000) }), /release_health_response_invalid/); + const hung = await server(t, () => {}); + await assert.rejects(requestHealth(`http://127.0.0.1:${hung}/`, { signal: AbortSignal.timeout(25) }), { name: 'AbortError' }); +}); diff --git a/core/core/updates/tests/http.test.js b/core/core/updates/tests/http.test.js new file mode 100644 index 0000000..b583348 --- /dev/null +++ b/core/core/updates/tests/http.test.js @@ -0,0 +1,40 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const { createPreview } = require('../../../dashboard/examples/independent-updates-preview'); +async function fixture(t) { + const app = await createPreview({ automatic: false }); t.after(() => app.close()); + const headers = user => ({ Cookie: `dispatch_session=${user.token}`, 'Content-Type': 'application/json', 'X-Dispatch-CSRF': user.session.csrfToken }); + const get = user => fetch(`${app.url}/api/platform/updates`, { headers: headers(user) }); + const post = (user, body, extra = {}) => fetch(`${app.url}/api/platform/updates`, { method: 'POST', headers: { ...headers(user), ...extra }, body: JSON.stringify(body) }); + return { ...app, headers, get, post }; +} +test('only platform owner sessions can read and queue independent update commands', async t => { + const f = await fixture(t); + assert.equal((await fetch(`${f.url}/api/platform/updates`)).status, 401); + assert.equal((await f.get(f.owners[0])).status, 403); + const view = await (await f.get(f.owner)).json(); assert.equal(view.data.mode, 'independent'); + assert.equal(view.data.tracks.core.installedVersion, '0.0.1'); assert.equal(view.data.tracks.dsp.installedVersion, '0.0.1'); + const body = { action: 'update_dev', product: 'dsp', digest: view.data.tracks.dsp.latest, idempotencyKey: 'synthetic:http:update-dev' }; + assert.equal((await f.post(f.owners[0], body)).status, 403); + assert.equal((await f.post(f.owner, body, { 'X-Dispatch-CSRF': 'wrong' })).status, 403); + assert.equal(f.commands.list().length, 0); + assert.equal((await f.post(f.owner, body)).status, 200); + assert.equal((await f.post(f.owner, body)).status, 200); + assert.equal(f.commands.list().length, 1); + assert.equal(f.store.db.prepare("SELECT count(*) n FROM audit_events WHERE action='platform.update.update_dev'").get().n, 1); + await f.worker.tick(); assert.equal(f.releases.state().tested, body.digest); + assert.equal(f.releases.state().active.dsps[f.dsps[1]] === body.digest, false); +}); +test('a new release between owner review and rollout resets Dev and refuses the stale command', async t => { + const f = await fixture(t), digest = f.releases.state().latest.dsp; + await f.releases.updateDev(digest); + const body = { action: 'rollout', product: 'dsp', digest, idempotencyKey: 'synthetic:http:rollout' }; + assert.equal((await f.post(f.owner, body)).status, 200); + await f.publish('dsp', '0.0.3'); await f.worker.tick(); + assert.equal(f.commands.list()[0].failure, 'release_dev_required'); + assert.equal(f.releases.state().rollout, null); + const view = await (await f.get(f.owner)).json(); assert.equal(view.data.tracks.dsp.tested, false); + const historical = await fetch(`${f.url}/api/platform/updates?releaseId=dsp_${digest}`, { headers: f.headers(f.owner) }); + const selected = await historical.json(); assert.equal(selected.data.tracks.dsp.release.version, '0.0.2'); + assert.equal(JSON.stringify(view).includes(f.root), false); +}); diff --git a/core/core/updates/tests/lifecycle.test.js b/core/core/updates/tests/lifecycle.test.js new file mode 100644 index 0000000..5cdba9a --- /dev/null +++ b/core/core/updates/tests/lifecycle.test.js @@ -0,0 +1,245 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'); +const { DatabaseSync } = require('node:sqlite'); +const { LocalReleases } = require('../local-releases'); +const { coreHooks, receiptFile } = require('../../../host/releases/core'); +const { dspHooks } = require('../../../host/releases/dsp'); +const { atomic } = require('../../installations/src/release-delivery-files'); +const { privateDirectory } = require('../../../host/controller/operations'); +const { ensureDsp } = require('../../../host/storage/storage'); +const { fileFor, prepareDspRelease, selectDspRelease } = require('../../../host/releases/runtime'); +const { hash, inventory, secureCopy } = require('../../../shared/releases/package'); +const DEV = `dsp_${'a'.repeat(32)}`, OTHER = `dsp_${'b'.repeat(32)}`; +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-update-lifecycle-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const paths = { platformRoot: root }; + for (const name of ['local', 'live', 'dsps', 'dev', 'worktrees']) { paths[name] = path.join(root, name); fs.mkdirSync(paths[name], { mode: 0o700 }); } + privateDirectory(path.join(paths.local, 'config')); privateDirectory(path.join(paths.local, 'state/access-control')); + atomic(path.join(paths.local, 'config/platform.json'), { version: 1, platformRoot: root }); + const database = path.join(paths.local, 'state/access-control/access-control.sqlite3'); + const db = new DatabaseSync(database); fs.chmodSync(database, 0o600); + db.exec("CREATE TABLE users(id TEXT,platform_role TEXT,status TEXT); INSERT INTO users VALUES('owner','owner','active'); CREATE TABLE marker(value TEXT); INSERT INTO marker VALUES('before');"); + db.close(); + function artifact(product, version) { + const directory = path.join(root, `${product}-${version}`); fs.mkdirSync(directory, { mode: 0o700 }); + fs.mkdirSync(path.join(directory, 'code'), { mode: 0o755 }); + fs.writeFileSync(path.join(directory, 'code/value.js'), `module.exports='${version}';`, { mode: 0o644 }); + const manifest = { schemaVersion: 1, product, version, channel: 'development', protocol: 1, minimumProtocol: 1, sourceDigest: 'a'.repeat(64), plugins: [], files: inventory(directory) }; + fs.writeFileSync(path.join(directory, 'release.json'), JSON.stringify(manifest), { mode: 0o600 }); + return { directory, digest: hash(JSON.stringify(manifest)), manifest }; + } + return { root, paths, database, artifact }; +} +async function coreFixture(t, { missingBackend = false, stopFailure = false } = {}) { + const f = fixture(t), before = f.artifact('core', '1.0.0'), next = f.artifact('core', '1.1.0'); + fs.rmdirSync(f.paths.live); secureCopy(path.join(before.directory, 'code'), f.paths.live); + let releases, fail = false; + const events = []; + const hooks = coreHooks({ paths: f.paths, configuration: { apiPort: 4999 }, releases: () => releases.state(), healthTimeoutMs: 5, + systemctl: async args => { + events.push(args); + if (args[0] === 'stop' && stopFailure) throw new Error('service_stop_failed'); + if (missingBackend && args[1].startsWith('dispatch-backend-')) { + if (args[0] === 'stop') throw new Error('unit_not_loaded'); + if (args.includes('LoadState')) return 'LoadState=not-found\n'; + } + if (args.includes('LoadState')) return 'LoadState=loaded\n'; + if (args[0] === 'show') return 'ActiveState=inactive\nMainPID=0\nControlPID=0\n'; + if (args[0] === 'start' && args[1] === 'dispatch-api.service') { + const current = JSON.parse(fs.readFileSync(receiptFile(f.paths))); + if (current.digest === next.digest) { + const db = new DatabaseSync(f.database); db.exec("ALTER TABLE marker ADD COLUMN migrated INTEGER; UPDATE marker SET value='after';"); db.close(); + privateDirectory(path.join(f.paths.local, 'state/new-migration')); + fs.writeFileSync(path.join(f.paths.local, 'state/new-file.json'), '{}', { mode: 0o600 }); + } + } + return ''; + }, fetchImpl: async () => { + const current = JSON.parse(fs.readFileSync(receiptFile(f.paths))); + return new Response(JSON.stringify({ ok: true, data: { ...current, recoveryProbe: 'passed', digest: fail && current.digest === next.digest ? 'wrong' : current.digest } })); + } }); + releases = new LocalReleases({ directory: path.join(f.paths.local, 'state/updates'), devDspId: DEV, allowDevelopment: true, hooks }); + await releases.stage(before.directory, before.digest); await releases.stage(next.directory, next.digest); + const state = releases.state(); state.active.core = before.digest; releases.save(state); + return { ...f, before, next, releases, hooks, events, fail: () => { fail = true; } }; +} +test('Core activation swaps only Core code and verifies the new service identity', async t => { + const f = await coreFixture(t); fs.writeFileSync(path.join(f.paths.dsps, 'unchanged'), 'DSP state', { mode: 0o600 }); + await f.releases.updateCore(f.next.digest); + assert.match(fs.readFileSync(path.join(f.paths.live, 'value.js'), 'utf8'), /1.1.0/); + assert.equal(fs.readFileSync(path.join(f.paths.dsps, 'unchanged'), 'utf8'), 'DSP state'); + assert.equal(f.releases.state().active.core, f.next.digest); + assert.equal(f.events.filter(args => args[0] === 'stop').length, 3); +}); +test('Core failed health restores the previous code and compatible database schema', async t => { + const f = await coreFixture(t); f.fail(); + await assert.rejects(f.releases.updateCore(f.next.digest), /release_health_failed/); + assert.match(fs.readFileSync(path.join(f.paths.live, 'value.js'), 'utf8'), /1.0.0/); + const db = new DatabaseSync(f.database); + try { assert.deepEqual(db.prepare('SELECT * FROM marker').get(), Object.assign(Object.create(null), { value: 'before' })); } + finally { db.close(); } + assert.equal(fs.existsSync(path.join(f.paths.local, 'state/new-migration')), false); + assert.equal(fs.existsSync(path.join(f.paths.local, 'state/new-file.json')), false); + assert.equal(f.releases.state().active.core, f.before.digest); assert.equal(f.releases.state().operation, null); +}); +test('Core rollback accepts an already collected backend service', async t => { + const f = await coreFixture(t, { missingBackend: true }); f.fail(); + await assert.rejects(f.releases.updateCore(f.next.digest), /release_health_failed/); + assert.equal(f.releases.state().active.core, f.before.digest); + assert.equal(f.releases.state().operation, null); + assert.match(fs.readFileSync(path.join(f.paths.live, 'value.js'), 'utf8'), /1.0.0/); +}); +test('Core stop failures for a loaded service still prevent the swap', async t => { + const f = await coreFixture(t, { stopFailure: true }); + await assert.rejects(f.releases.updateCore(f.next.digest), /release_recovery_required/); + assert.match(fs.readFileSync(path.join(f.paths.live, 'value.js'), 'utf8'), /1.0.0/); + assert.equal(f.releases.state().operation.phase, 'failed'); +}); +test('Core recovery repairs a crash between the two live-directory renames', async t => { + const f = await coreFixture(t); + const c = { product: 'core', digest: f.next.digest, previousDigest: f.before.digest, directory: f.next.directory, manifest: f.next.manifest }; + c.preparation = await f.hooks.prepare(c); await f.hooks.drain(c); c.snapshot = await f.hooks.snapshot(c); + fs.renameSync(f.paths.live, path.join(f.paths.local, 'backups/updates/core', c.preparation.id, 'previous')); + const state = f.releases.state(); state.operation = { ...c, prior: c.previousDigest, phase: 'starting' }; f.releases.save(state); + const recoveredPaths = require('../../../host/releases/setup').loadWorkerPaths(path.join(f.paths.local, 'config/platform.json')); + assert.equal(recoveredPaths.live, f.paths.live); + await f.releases.recover(); assert.match(fs.readFileSync(path.join(f.paths.live, 'value.js'), 'utf8'), /1.0.0/); + assert.equal(f.releases.state().operation, null); +}); +async function dspFixture(t) { + const f = fixture(t), before = f.artifact('dsp', '1.0.0'), next = f.artifact('dsp', '1.1.0'), core = f.artifact('core', '1.0.0'); + const db = new DatabaseSync(f.database); t.after(() => db.close()); + db.exec('CREATE TABLE installations(runtime_key TEXT,organization_id TEXT,backend TEXT,status TEXT,revision INTEGER); CREATE TABLE dsp_plugins(organization_id TEXT,plugin_id TEXT,version TEXT,desired_state TEXT,revision INTEGER,applied_revision INTEGER,failure_code TEXT); CREATE TABLE directory_lifecycle_requests(organization_id TEXT,status TEXT); CREATE TABLE installation_onboarding_requests(organization_id TEXT,status TEXT);'); + const records = new Map(), roots = new Map(); let failed = false; + for (const id of [DEV, OTHER]) { + const creationId = `create_${id.slice(4)}`; + roots.set(id, ensureDsp(f.paths, id, creationId).root); + records.set(id, { id, creationId, latestRequest: 'a'.repeat(64), desiredState: 'running' }); + db.prepare('INSERT INTO installations VALUES(?,?,?,?,?)').run(id, id, 'directory_service_v1', 'ready', 1); + fs.writeFileSync(path.join(roots.get(id), 'data/value'), 'private before', { mode: 0o600 }); + fs.writeFileSync(path.join(roots.get(id), 'secrets/value'), `secret ${id}`, { mode: 0o600 }); + prepareDspRelease(f.paths, id, before.directory, before.digest); selectDspRelease(f.paths, id, before.digest, null); + } + const manager = { journal: { record: id => records.get(id), saveRecord: value => records.set(value.id, value) }, checkedDsp: record => ({ root: roots.get(record.id) }), + credentials: () => {}, bridge: async () => {}, ready: async id => { + if (failed && JSON.parse(fs.readFileSync(fileFor(f.paths, id))).digest === next.digest) throw new Error('release_health_failed'); + }, host: { stop: async () => {}, prepare: async () => {}, start: async id => { + if (JSON.parse(fs.readFileSync(fileFor(f.paths, id))).digest === next.digest) fs.writeFileSync(path.join(roots.get(id), 'data/value'), 'migrated', { mode: 0o600 }); + } } }; + const sleepingRows = new Map(); + const execution = { eligible: id => sleepingRows.has(id), store: { get: id => sleepingRows.get(id), + update: (id, values) => sleepingRows.set(id, { ...sleepingRows.get(id), ...values }) }, + checkpoint: async () => ({ nextWakeAt: null }), locked: (_id, work) => work() }; + const hooks = dspHooks({ paths: f.paths, store: { db }, manager, execution }); + const releases = new LocalReleases({ directory: path.join(f.paths.local, 'state/updates'), devDspId: DEV, allowDevelopment: true, hooks }); + for (const item of [core, before, next]) await releases.stage(item.directory, item.digest); + const state = releases.state(); state.active = { core: core.digest, dsps: { [DEV]: before.digest, [OTHER]: before.digest } }; releases.save(state); + return { ...f, before, next, releases, roots, records, sleepingRows, sleep(id) { + const operation = 'sleep_' + 'e'.repeat(32), current = records.get(id); + records.set(id, { ...current, desiredState: 'stopped', latestRequest: require('node:crypto').createHash('sha256').update(operation).digest('hex') }); + sleepingRows.set(id, { mode: 'on_demand', state: 'sleeping', operation_id: operation, next_wake_at: null, check_at: null, last_activity: 1000, snapshot_ready: 1, failure_code: null }); + }, fail: () => { failed = true; } }; +} +test('Dev activation changes only Dev runtime and leaves other DSP credentials and data intact', async t => { + const f = await dspFixture(t); await f.releases.updateDev(f.next.digest); + assert.equal(f.releases.state().active.dsps[DEV], f.next.digest); + assert.equal(f.releases.state().active.dsps[OTHER], f.before.digest); + assert.equal(fs.readFileSync(path.join(f.roots.get(OTHER), 'data/value'), 'utf8'), 'private before'); + assert.equal(fs.readFileSync(path.join(f.roots.get(DEV), 'secrets/value'), 'utf8'), `secret ${DEV}`); +}); +test('failed DSP activation restores private state and runtime receipt without touching another DSP', async t => { + const f = await dspFixture(t); f.fail(); await assert.rejects(f.releases.updateDev(f.next.digest), /release_health_failed/); + assert.equal(fs.readFileSync(path.join(f.roots.get(DEV), 'data/value'), 'utf8'), 'private before'); + assert.equal(JSON.parse(fs.readFileSync(fileFor(f.paths, DEV))).digest, f.before.digest); + assert.equal(f.releases.state().tested, null); +}); +test('stopped DSPs cannot be revived by an update', async t => { + const f = await dspFixture(t); f.records.get(DEV).desiredState = 'stopped'; + await assert.rejects(f.releases.updateDev(f.next.digest), /release_dsp_not_ready/); + assert.equal(f.records.get(DEV).desiredState, 'stopped'); + assert.equal(JSON.parse(fs.readFileSync(fileFor(f.paths, DEV))).digest, f.before.digest); +}); +test('new DSP provisioning uses the completed fleet release while Dev has a newer candidate', async t => { + const f = await dspFixture(t), id = `dsp_${'c'.repeat(32)}`; + const state = f.releases.state(); state.defaultDsp = f.before.digest; f.releases.save(state); + atomic(path.join(f.paths.local, 'config/updates.json'), { schemaVersion: 1, devDspId: DEV, apiPort: 4999 }); + await f.releases.updateDev(f.next.digest); + await require('../../../host/releases/provisioning').withCreation(f.paths, 'create', assign => + require('../../../host/controller/operations').withLock(f.paths, async fd => { + ensureDsp(f.paths, id, `create_${'c'.repeat(32)}`); await assign(id, fd); + })); + assert.equal(JSON.parse(fs.readFileSync(fileFor(f.paths, id))).digest, f.before.digest); + assert.equal(f.releases.state().active.dsps[id], f.before.digest); + assert.equal(f.releases.state().active.dsps[DEV], f.next.digest); +}); + +test('Core state recovery can restart after clearing configuration without losing updater bootstrap files', t => { + const f = fixture(t), backup = privateDirectory(path.join(f.root, 'backup')); + const configuration = { schemaVersion: 1, devDspId: DEV, apiPort: 4999 }; + atomic(path.join(f.paths.local, 'config/updates.json'), configuration); + atomic(path.join(f.paths.local, 'config/extra.json'), { before: true }); + const state = require('../../../host/releases/core-state'); + const roots = state.capture(f.paths, backup); + const files = require('../../../host/storage/backup-files'), original = files.copyContents; + files.copyContents = () => { throw new Error('simulated_power_loss'); }; + try { assert.throws(() => state.restore(f.paths, backup, roots), /simulated_power_loss/); } + finally { files.copyContents = original; } + assert.equal(require('../../../host/releases/setup').loadWorkerPaths(path.join(f.paths.local, 'config/platform.json')).platformRoot, f.root); + assert.deepEqual(require('../configuration').loadConfiguration(f.paths), configuration); + state.restore(f.paths, backup, roots); + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(f.paths.local, 'config/extra.json'))), { before: true }); +}); +test('updater bootstrap follows the active verified Core and rejects changed worker bytes', async t => { + const f = fixture(t), candidate = f.artifact('core', '1.2.0'); + fs.mkdirSync(path.join(candidate.directory, 'code/bin'), { mode: 0o755 }); + fs.writeFileSync(path.join(candidate.directory, 'code/bin/dispatch-updates'), '#!/usr/bin/env node\n', { mode: 0o755 }); + fs.unlinkSync(path.join(candidate.directory, 'release.json')); + candidate.manifest.channel = 'release'; candidate.manifest.files = inventory(candidate.directory); + fs.writeFileSync(path.join(candidate.directory, 'release.json'), JSON.stringify(candidate.manifest), { mode: 0o600 }); + candidate.digest = hash(JSON.stringify(candidate.manifest)); + const hooks = Object.fromEntries(['drain', 'snapshot', 'start', 'verify', 'restore'].map(name => [name, async () => {}])); + const releases = new LocalReleases({ directory: path.join(f.paths.local, 'state/updates'), devDspId: DEV, hooks }); + const { workerEntrypoint } = require('../../../host/releases/setup'); + assert.equal(workerEntrypoint(f.paths, __filename), null); + await releases.stage(candidate.directory, candidate.digest); + const state = releases.state(); state.active.core = candidate.digest; releases.save(state); + const entry = path.join(state.releases.core[candidate.digest].directory, 'code/bin/dispatch-updates'); + assert.equal(workerEntrypoint(f.paths, __filename), entry); + assert.equal(workerEntrypoint(f.paths, entry), null); + fs.writeFileSync(entry, 'changed'); + assert.throws(() => workerEntrypoint(f.paths, __filename), /release_digest_mismatch/); +}); + +test('idle DSPs wake for an update and for a later Dev health check, while owner stops remain protected', async t => { + const f = await dspFixture(t); f.sleep(DEV); + await f.releases.updateDev(f.next.digest); + assert.equal(f.records.get(DEV).desiredState, 'running'); + assert.equal(f.sleepingRows.get(DEV).state, 'starting'); + f.sleep(DEV); + await f.releases.beginRollout(f.next.digest, [DEV, OTHER], 'owner'); + assert.equal(f.records.get(DEV).desiredState, 'running'); + assert.equal(f.releases.state().rollout.status, 'running'); +}); +test('a failed update returns an idle DSP to its previous release and sleeping state', async t => { + const f = await dspFixture(t); f.sleep(DEV); const before = { ...f.sleepingRows.get(DEV) }; f.fail(); + await assert.rejects(f.releases.updateDev(f.next.digest), /release_health_failed/); + assert.equal(f.records.get(DEV).desiredState, 'stopped'); + assert.deepEqual(f.sleepingRows.get(DEV), before); + assert.equal(JSON.parse(fs.readFileSync(fileFor(f.paths, DEV))).digest, f.before.digest); +}); +test('a newer owner stop supersedes the scheduler sleep permission', async t => { + const f = await dspFixture(t); f.sleep(DEV); f.records.get(DEV).latestRequest = 'f'.repeat(64); + await assert.rejects(f.releases.updateDev(f.next.digest), /release_dsp_not_ready/); + assert.equal(f.records.get(DEV).desiredState, 'stopped'); +}); + +test('permanent Dev configuration rejects replacement with another ready DSP', async t => { + const f = await dspFixture(t), journal = new (require('../../../host/controller/journal').DirectoryJournal)(f.paths); + for (const record of f.records.values()) journal.saveRecord({ version: 1, tokenHash: null, ...record }); + const { configure } = require('../../../host/releases/setup'); + assert.equal(configure(f.paths, DEV, 4999).configured, true); + assert.throws(() => configure(f.paths, OTHER, 4999), /release_dev_identity_changed/); + assert.equal(require('../configuration').loadConfiguration(f.paths).devDspId, DEV); +}); diff --git a/core/core/updates/tests/local-releases.test.js b/core/core/updates/tests/local-releases.test.js new file mode 100644 index 0000000..58c78a7 --- /dev/null +++ b/core/core/updates/tests/local-releases.test.js @@ -0,0 +1,76 @@ +'use strict'; +const test=require('node:test'),assert=require('node:assert/strict'),fs=require('node:fs'),os=require('node:os'),path=require('node:path'); +const {LocalReleases}=require('../local-releases'); +const {hash,inventory}=require('../../../shared/releases/package'); +const {prepareDspRelease,selectDspRelease,runtimeSource}=require('../../../host/releases/runtime'); +function fixture(t){ + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-releases-'));t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + let failure=null;const events=[],data={}; + const hooks={drain:async c=>events.push(['drain',c.dspId]),snapshot:async c=>({value:data[c.dspId]||0}), + start:async c=>{events.push(['start',c.dspId,c.digest]);data[c.dspId]=42;},verify:async c=>!failure||c.dspId!==failure, + restore:async c=>{events.push(['restore',c.dspId]);data[c.dspId]=c.snapshot?.value||0;}}; + const options={directory:path.join(root,'state'),devDspId:'dev',hooks,allowDevelopment:true}; + const app=new LocalReleases(options); + function artifact(product,version,protocol=1){ + const directory=path.join(root,`${product}-${version}`);fs.mkdirSync(directory); + fs.mkdirSync(path.join(directory,'code'));fs.writeFileSync(path.join(directory,'code/value.js'),`module.exports=${JSON.stringify(version)};`); + const manifest={schemaVersion:1,product,version,channel:'development',protocol,minimumProtocol:protocol,sourceDigest:'a'.repeat(64),plugins:[],files:inventory(directory)}; + const digest=hash(JSON.stringify(manifest));fs.writeFileSync(path.join(directory,'release.json'),JSON.stringify(manifest));return {directory,digest}; + } + return {app,options,root,events,data,artifact,fail:id=>{failure=id;}}; +} +test('staging is inert; Dev approval resets on a new release and sequential rollout stays pinned',async t=>{ + const f=fixture(t),core=f.artifact('core','1.0.0'),one=f.artifact('dsp','1.0.0'),two=f.artifact('dsp','1.1.0'),three=f.artifact('dsp','1.2.0'); + await f.app.stage(core.directory,core.digest);await f.app.stage(one.directory,one.digest); + assert.deepEqual(f.app.state().active,{core:null,dsps:{}});assert.equal(f.events.length,0); + await assert.rejects(f.app.updateDev(one.digest),/release_incompatible/); + await f.app.updateCore(core.digest);await f.app.updateDev(one.digest); + assert.deepEqual(f.app.state().active.dsps,{dev:one.digest}); + await f.app.stage(two.directory,two.digest);assert.equal(f.app.state().tested,null); + await assert.rejects(f.app.beginRollout(one.digest,['a','b']),/release_dev_required/); + await f.app.updateDev(two.digest);await f.app.beginRollout(two.digest,['dev','a','b']); + await f.app.step();assert.equal(f.app.state().active.dsps.a,two.digest);assert.equal(f.app.state().active.dsps.b,undefined); + await f.app.stage(three.directory,three.digest);await f.app.step(); + assert.equal(f.app.state().active.dsps.b,two.digest);assert.equal(f.app.state().latest.dsp,three.digest);assert.equal(f.app.state().tested,null); + assert.equal(new LocalReleases(f.options).state().rollout.status,'completed'); + assert.equal(f.app.state().defaultDsp,two.digest); // A newly published candidate is not the provisioning default. +}); +test('failed health restores the selected DSP, pauses rollout and resumes without repeating completed targets',async t=>{ + const f=fixture(t),core=f.artifact('core','1.0.0'),dsp=f.artifact('dsp','1.0.0'); + await f.app.stage(core.directory,core.digest);await f.app.updateCore(core.digest); + await f.app.stage(dsp.directory,dsp.digest);await f.app.updateDev(dsp.digest);await f.app.beginRollout(dsp.digest,['a','b']); + await f.app.step();f.fail('b');await assert.rejects(f.app.step(),/release_health_failed/); + assert.equal(f.app.state().rollout.status,'paused');assert.equal(f.app.state().active.dsps.b,undefined);assert.equal(f.data.b,0); + f.fail(null);await f.app.resume();await f.app.step();assert.equal(f.events.filter(x=>x[0]==='start'&&x[1]==='a').length,1); + const bad=f.artifact('core','2.0.0',2);await f.app.stage(bad.directory,bad.digest); + await assert.rejects(f.app.updateCore(bad.digest),/release_incompatible/);assert.equal(f.app.state().active.core,core.digest); +}); +test('packages are copied, verified and selected per DSP; private state is preserved',t=>{ + const f=fixture(t),candidate=f.artifact('dsp','1.0.0'); + const ids=['dsp_'+ 'a'.repeat(32),'dsp_'+'b'.repeat(32)]; + const paths={local:path.join(f.root,'local'),dsps:path.join(f.root,'dsps'),live:path.join(f.root,'live')}; + fs.mkdirSync(paths.live);fs.mkdirSync(path.join(paths.live,'runtime')); + for(const id of ids){fs.mkdirSync(path.join(paths.dsps,id,'data'),{recursive:true,mode:0o700});fs.writeFileSync(path.join(paths.dsps,id,'data/keep'),'private');} + const copy=prepareDspRelease(paths,ids[0],candidate.directory,candidate.digest); + assert.equal(runtimeSource(paths,ids[0]),paths.live); + selectDspRelease(paths,ids[0],candidate.digest,null); + assert.equal(runtimeSource(paths,ids[0]),path.join(copy.directory,'code'));assert.equal(runtimeSource(paths,ids[1]),paths.live); + assert.equal(fs.readFileSync(path.join(paths.dsps,ids[0],'data/keep'),'utf8'),'private'); + assert.notEqual(fs.statSync(path.join(candidate.directory,'code/value.js')).ino,fs.statSync(path.join(copy.directory,'code/value.js')).ino); + fs.appendFileSync(path.join(copy.directory,'code/value.js'),'// tampered'); + assert.throws(()=>runtimeSource(paths,ids[0]),/release_digest_mismatch/); +}); +test('development artifacts are refused by publication mode and interrupted activation requires recovery',async t=>{ + const f=fixture(t),core=f.artifact('core','1.0.0'); + await assert.rejects(new LocalReleases({...f.options,allowDevelopment:false}).stage(core.directory,core.digest),/release_not_published/); + await f.app.stage(core.directory,core.digest); + const state=f.app.state();state.operation={product:'core',digest:core.digest,prior:null,dspId:null,snapshot:{value:7},phase:'starting'};f.app.save(state); + await assert.rejects(f.app.updateCore(core.digest),/release_recovery_required/);await f.app.recover();assert.equal(f.app.state().operation,null);assert.equal(f.data.null,7); +}); +test('a failed fresh Dev health check revokes rollout eligibility',async t=>{ + const f=fixture(t),core=f.artifact('core','1.0.0'),dsp=f.artifact('dsp','1.0.0'); + await f.app.stage(core.directory,core.digest);await f.app.updateCore(core.digest); + await f.app.stage(dsp.directory,dsp.digest);await f.app.updateDev(dsp.digest); + f.fail('dev');await assert.rejects(f.app.beginRollout(dsp.digest,['a']),/release_health_failed/); + assert.equal(f.app.state().tested,null);assert.equal(f.app.state().rollout,null); +}); diff --git a/core/core/updates/tests/metadata-scope.test.js b/core/core/updates/tests/metadata-scope.test.js new file mode 100644 index 0000000..1068c39 --- /dev/null +++ b/core/core/updates/tests/metadata-scope.test.js @@ -0,0 +1,16 @@ +'use strict'; +const test=require('node:test'),assert=require('node:assert/strict'),fs=require('node:fs'),os=require('node:os'),path=require('node:path'); +const {CollectionStore}=require('dispatch-runtime-kit/collection-manager/src/store'); +const {collectorEnabled,applyState}=require('dispatch-runtime-kit/collection-manager/src/plugin-state'); +test('concurrent DSP databases use their own installed declarations during a mixed-version rollout',t=>{ + const root=fs.mkdtempSync(path.join(os.tmpdir(),'dispatch-metadata-'));t.after(()=>fs.rmSync(root,{recursive:true,force:true})); + const stores=['old','new'].map((collector,index)=>{ + const manifest={...require('../../../tests/fixtures/paycom-plugin.json'),version:`1.${index}.0`,collectors:[collector],syncs:[]}; + const databaseRoot=path.join(root,collector),store=new CollectionStore({databaseRoot,database:path.join(databaseRoot,'collections.sqlite3')},{plugins:[manifest]}); + t.after(()=>store.close());return store; + }); + assert.equal(collectorEnabled(stores[0].db,'old'),false);assert.equal(collectorEnabled(stores[1].db,'new'),false); + applyState(stores[0],{command:'apply',pluginId:'paycom',version:'1.0.0',revision:1,state:'enabled'}); + assert.equal(collectorEnabled(stores[0].db,'old'),true);assert.equal(collectorEnabled(stores[1].db,'new'),false); + assert.equal(stores[1].db.prepare('SELECT version FROM plugin_installations').get().version,'1.1.0'); +}); diff --git a/core/core/updates/tests/runtime-package.test.js b/core/core/updates/tests/runtime-package.test.js new file mode 100644 index 0000000..c15eaf8 --- /dev/null +++ b/core/core/updates/tests/runtime-package.test.js @@ -0,0 +1,79 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), os = require('node:os'), path = require('node:path'); +const { hash, inventory, secureCopy, verifyRelease } = require('../../../shared/releases/package'); +const { prepareDspRelease, selectDspRelease, runtimeSource } = require('../../../host/releases/runtime'); +const { verifyRuntime } = require('../../../host/releases/runtime-package'); +const ID = 'dsp_' + 'a'.repeat(32); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-runtime-package-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const paths = { local: path.join(root, 'local'), dsps: path.join(root, 'dsps') }; + const source = path.join(root, 'release'); + for (const [name, bytes] of Object.entries({ + 'code/runtime/supervisor.js': 'runtime', 'code/compatibility/cortex.js': 'built-in connection', + 'code/plugins/paycom/dispatch-plugin.json': '{}', 'code/node_modules/dispatch-sdk/index.js': 'dependency', + 'plugins/paycom/backend.js': 'optional Paycom code', 'plugins/paycom/frontend.js': 'optional Paycom frontend', + 'plugins/sample/backend.js': 'another optional plugin', 'release-notes.md': 'notes', + })) { + fs.mkdirSync(path.dirname(path.join(source, name)), { recursive: true, mode: 0o700 }); + fs.writeFileSync(path.join(source, name), bytes, { mode: 0o600 }); + } + const manifest = { schemaVersion: 1, product: 'dsp', version: '1.0.0', channel: 'release', protocol: 1, + minimumProtocol: 1, sourceDigest: 'a'.repeat(64), plugins: [], files: inventory(source) }; + fs.writeFileSync(path.join(source, 'release.json'), JSON.stringify(manifest), { mode: 0o600 }); + const digest = hash(JSON.stringify(manifest)); + const prepare = () => prepareDspRelease(paths, ID, source, digest); + return { root, paths, source, manifest, digest, prepare }; +} + +test('DSP runtime copies omit every optional plugin payload and retain the original authenticated manifest', t => { + const f = fixture(t), installed = f.prepare(); + assert.equal(fs.existsSync(path.join(installed.directory, 'plugins')), false); + assert.equal(fs.existsSync(path.join(installed.directory, 'code/compatibility/cortex.js')), true); + assert.equal(fs.existsSync(path.join(installed.directory, 'code/plugins/paycom/dispatch-plugin.json')), true); + assert.equal(fs.existsSync(path.join(f.paths.dsps, ID, 'plugins/paycom')), false); + assert.deepEqual(verifyRuntime(installed.directory, f.digest), f.manifest); + assert.deepEqual(verifyRelease(f.source, f.digest), f.manifest); + assert.equal(fs.readFileSync(path.join(installed.directory, 'release.json'), 'utf8'), fs.readFileSync(path.join(f.source, 'release.json'), 'utf8')); + assert.notEqual(fs.statSync(path.join(installed.directory, 'code/runtime/supervisor.js')).ino, fs.statSync(path.join(f.source, 'code/runtime/supervisor.js')).ino); + selectDspRelease(f.paths, ID, f.digest, null); + assert.equal(runtimeSource(f.paths, ID), path.join(installed.directory, 'code')); + assert.deepEqual(f.prepare(), installed); +}); + +for (const change of ['runtime bytes', 'missing runtime', 'extra file', 'partial plugin', 'symlink', 'manifest']) { + test(`runtime verification rejects ${change}`, t => { + const f = fixture(t), installed = f.prepare(), runtime = path.join(installed.directory, 'code/runtime/supervisor.js'); + selectDspRelease(f.paths, ID, f.digest, null); + if (change === 'runtime bytes') fs.writeFileSync(runtime, 'changed'); + if (change === 'missing runtime') fs.unlinkSync(runtime); + if (change === 'extra file') fs.writeFileSync(path.join(installed.directory, 'code/extra.js'), 'extra'); + if (change === 'partial plugin') { + fs.mkdirSync(path.join(installed.directory, 'plugins/paycom'), { recursive: true }); + fs.copyFileSync(path.join(f.source, 'plugins/paycom/backend.js'), path.join(installed.directory, 'plugins/paycom/backend.js')); + } + if (change === 'symlink') { fs.unlinkSync(runtime); fs.symlinkSync(path.join(f.source, 'code/runtime/supervisor.js'), runtime); } + if (change === 'manifest') fs.writeFileSync(path.join(installed.directory, 'release.json'), JSON.stringify({ ...f.manifest, version: '2.0.0' })); + assert.throws(() => runtimeSource(f.paths, ID), /release_(digest_mismatch|entry_invalid|manifest_invalid)/); + }); +} + +test('invalid plugin bytes in the downloaded release are rejected even though they will not be copied to the DSP', t => { + const f = fixture(t); + fs.writeFileSync(path.join(f.source, 'plugins/paycom/backend.js'), 'tampered'); + assert.throws(f.prepare, /release_digest_mismatch/); + assert.equal(fs.existsSync(path.join(f.paths.dsps, ID, 'runtime/releases', f.digest)), false); +}); + +test('legacy full DSP releases remain verified and selectable for rollback', t => { + const f = fixture(t), target = path.join(f.paths.dsps, ID, 'runtime/releases', f.digest); + fs.mkdirSync(path.dirname(target), { recursive: true, mode: 0o700 }); + secureCopy(f.source, target); + selectDspRelease(f.paths, ID, f.digest, null); + assert.equal(runtimeSource(f.paths, ID), path.join(target, 'code')); + assert.equal(f.prepare().directory, target); + fs.writeFileSync(path.join(target, 'plugins/paycom/backend.js'), 'tampered'); + assert.throws(() => runtimeSource(f.paths, ID), /release_digest_mismatch/); +}); diff --git a/core/core/updates/tests/selective-delivery-fixture.js b/core/core/updates/tests/selective-delivery-fixture.js new file mode 100644 index 0000000..2f51e5f --- /dev/null +++ b/core/core/updates/tests/selective-delivery-fixture.js @@ -0,0 +1,97 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +const { fixture: accounts } = require('../../accounts/tests/plugin-fixture'); +const { privateDirectory, withLock } = require('../../../host/controller/operations'); +const { ensureDsp } = require('../../../host/storage/storage'); +const { atomic } = require('../../installations/src/release-delivery-files'); +const { sealPackage } = require('../../../tooling/build-plugin-package'); +const { hash, inventory } = require('../../../shared/releases/package'); +const { packageCatalog } = require('../../plugins/package-catalog'); +const { LocalReleases } = require('../local-releases'); +const { dspHooks } = require('../../../host/releases/dsp'); +const { withCreation } = require('../../../host/releases/provisioning'); +const { fileFor } = require('../../../host/releases/runtime'); +const { createDirectoryInstallation } = require('../../../host/plugins/directory-lifecycle'); +const { createPluginService } = require('../../accounts/src/plugins'); + +async function fixture(t) { + const f = await accounts(t, { dspCount: 4 }); + const paths = { platformRoot: f.root }; + for (const name of ['local', 'live', 'dsps', 'dev', 'worktrees']) paths[name] = privateDirectory(path.join(f.root, name)); + const records = new Map(), roots = new Map(), events = []; + let failedDigest = null; + for (const dsp of f.dsps) { + const creationId = 'create_' + dsp.runtimeKey.slice(4); + roots.set(dsp.runtimeKey, ensureDsp(paths, dsp.runtimeKey, creationId).root); + records.set(dsp.runtimeKey, { id: dsp.runtimeKey, creationId, latestRequest: 'a'.repeat(64), desiredState: 'running' }); + f.store.db.prepare('INSERT INTO plugin_migration_checks(organization_id) VALUES(?)').run(dsp.id); + } + const manager = { + journal: { record: id => records.get(id), saveRecord: value => records.set(value.id, value) }, + checkedDsp: record => ({ root: roots.get(record.id) }), credentials: () => {}, bridge: async () => {}, + ready: async id => { if (JSON.parse(fs.readFileSync(fileFor(paths, id))).digest === failedDigest) throw new Error('release_health_failed'); }, + apply: async () => {}, + host: { stop: async id => events.push(['stop', id]), prepare: async () => {}, start: async id => events.push(['start', id]) }, + pluginBackend: { request: async (id, action, input) => { + events.push([action, id, input.version]); + if (action === 'plugin.initialize') return true; + if (action !== 'plugin.revoke') throw new Error('unexpected_backend_request'); + } }, + }; + const execution = { eligible: () => false, locked: (_id, work) => work() }; + const hooks = dspHooks({ paths, store: f.store, manager, execution }); + const releases = new LocalReleases({ directory: path.join(paths.local, 'state/updates'), devDspId: f.dsps[0].runtimeKey, hooks, allowDevelopment: true }); + const coordinator = createDirectoryInstallation({ paths, store: f.store, manager, execution }); + const plugins = createPluginService({ store: f.store, access: f.access, installationCoordinator: coordinator, invoke: async () => { throw new Error('unexpected_runtime_request'); } }); + const definitions = () => packageCatalog(paths)?.definitions() || []; + require('../../../shared/plugin-sdk/catalog').configureCatalog(definitions); + require('dispatch-protocol/plugin-sdk/catalog').configureCatalog(definitions); + t.after(() => { + const defaults = () => [require('../../../tests/fixtures/paycom-plugin.json')]; + require('../../../shared/plugin-sdk/catalog').configureCatalog(defaults); + require('dispatch-protocol/plugin-sdk/catalog').configureCatalog(defaults); + }); + function artifact(product, version, pluginVersion = null, pluginIds = ['paycom', 'sample']) { + const directory = privateDirectory(path.join(paths.dev, product + '-' + version)); + privateDirectory(path.join(directory, 'code/runtime')); + fs.writeFileSync(path.join(directory, 'code/runtime/index.js'), `module.exports='${version}';`, { mode: 0o600 }); + const packaged = []; + if (pluginVersion) for (const id of pluginIds) { + const root = privateDirectory(path.join(directory, 'plugins', id)); + privateDirectory(path.join(root, 'backend')); privateDirectory(path.join(root, 'migrations')); + const definition = { ...require('../../../tests/fixtures/paycom-plugin.json'), id, name: id, version: pluginVersion, + frontend: null, dashboard: null, published: null, runtime: 'backend/index.js', + pages: [], actions: [], httpPrefixes: [], gatewayActions: [], services: id === 'paycom' ? ['paycom'] : [], + collectors: [], syncs: [], jobs: [], legacyProfile: null, + package: { runtime: 'backend/index.js', authentication: null, collections: 'migrations/collections.json' } }; + delete definition.settings; + atomic(path.join(root, 'dispatch-plugin.json'), definition); + fs.writeFileSync(path.join(root, 'backend/index.js'), `module.exports='${id}@${pluginVersion}';`, { mode: 0o600 }); + atomic(path.join(root, 'migrations/collections.json'), { schemaVersion: 1, collectors: [], sources: [], plans: [], syncs: [] }); + packaged.push({ pluginId: id, version: pluginVersion, digest: sealPackage(root).digest }); + privateDirectory(path.join(directory, 'code/plugins', id)); + atomic(path.join(directory, 'code/plugins', id, 'dispatch-plugin.json'), definition); + } + const manifest = { schemaVersion: 1, product, version, channel: 'development', protocol: 1, minimumProtocol: 1, + sourceDigest: 'a'.repeat(64), plugins: packaged, files: inventory(directory) }; + atomic(path.join(directory, 'release.json'), manifest); + return { directory, manifest, digest: hash(JSON.stringify(manifest)) }; + } + const core = artifact('core', '1.0.0'), before = artifact('dsp', '1.0.0', '1.0.0'), next = artifact('dsp', '1.1.0', '1.1.0'); + for (const item of [core, before, next]) await releases.stage(item.directory, item.digest); + const state = releases.state(); state.active.core = core.digest; state.defaultDsp = before.digest; releases.save(state); + privateDirectory(path.join(paths.local, 'config')); + atomic(path.join(paths.local, 'config/updates.json'), { schemaVersion: 1, devDspId: f.dsps[0].runtimeKey, apiPort: 4999 }); + const provision = dsp => withCreation(paths, 'create', assign => withLock(paths, fd => assign(dsp.runtimeKey, fd))); + for (const dsp of f.dsps) await provision(dsp); + const change = async (dsp, action) => { + const item = plugins.list(dsp.owner).items.find(item => item.id === 'paycom'); + plugins.change(dsp.owner, 'paycom', { action, expectedRevision: item.revision, idempotencyKey: `fixture:${action}:${item.revision}` }); + await plugins.runPending(); + const current = plugins.list(dsp.owner).items.find(item => item.id === 'paycom'); + if (current.failureCode || current.pending) throw new Error(JSON.stringify(current)); + return current; + }; + return { ...f, paths, roots, releases, before, next, artifact, plugins, change, provision, events, fail: digest => { failedDigest = digest; } }; +} +module.exports = { fixture }; diff --git a/core/core/updates/tests/selective-delivery.test.js b/core/core/updates/tests/selective-delivery.test.js new file mode 100644 index 0000000..8d967b4 --- /dev/null +++ b/core/core/updates/tests/selective-delivery.test.js @@ -0,0 +1,87 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), path = require('node:path'); +const { fixture } = require('./selective-delivery-fixture'); +const { installationReceipt } = require('../../../host/plugins/install'); +const { packageCatalog } = require('../../plugins/package-catalog'); +const { runtimeSource } = require('../../../host/releases/runtime'); +const pluginFile = (f, dsp, version) => path.join(f.roots.get(dsp.runtimeKey), 'plugins/paycom/versions', version, 'backend/index.js'); + +test('provisioning and delayed installs keep optional plugin bytes out of DSPs until each owner installs', async t => { + const f = await fixture(t), [dev, a, b] = f.dsps; + for (const dsp of f.dsps) { + assert.equal(fs.existsSync(path.join(path.dirname(runtimeSource(f.paths, dsp.runtimeKey)), 'plugins')), false); + assert.equal(fs.existsSync(path.join(f.roots.get(dsp.runtimeKey), 'plugins/paycom')), false); + assert.equal(fs.existsSync(path.join(f.roots.get(dsp.runtimeKey), 'plugins/sample')), false); + assert.equal(f.plugins.list(dsp.owner).items.find(item => item.id === 'paycom').state, 'uninstalled'); + } + await f.change(dev, 'install'); + await f.releases.updateDev(f.next.digest); + assert.equal(installationReceipt(f.roots.get(dev.runtimeKey), 'paycom').version, '1.1.0'); + assert.equal(fs.existsSync(pluginFile(f, a, '1.1.0')), false); + await f.provision(b); // Even a provisioning retry while Dev is newer keeps the fleet baseline. + assert.equal(packageCatalog(f.paths).latest('paycom', b.runtimeKey).version, '1.0.0'); + await f.change(a, 'install'); + assert.match(fs.readFileSync(pluginFile(f, a, '1.0.0'), 'utf8'), /paycom@1.0.0/); + assert.equal(fs.existsSync(pluginFile(f, b, '1.0.0')), false); + assert.notEqual(fs.statSync(pluginFile(f, a, '1.0.0')).ino, fs.statSync(path.join(f.paths.local, 'packages/plugins/paycom/1.0.0/backend/index.js')).ino); +}); + +test('DSP rollout upgrades enabled and disabled installed plugins one DSP at a time and skips uninstalled plugins', async t => { + const f = await fixture(t), [dev, a, b, c] = f.dsps; + for (const dsp of [dev, a, b]) await f.change(dsp, 'install'); + await f.change(b, 'disable'); + for (const dsp of f.dsps) for (const root of ['config', 'data', 'secrets']) { + fs.writeFileSync(path.join(f.roots.get(dsp.runtimeKey), root, 'sentinel'), `${dsp.runtimeKey}:${root}`, { mode: 0o600 }); + } + await f.releases.updateDev(f.next.digest); + assert.equal(installationReceipt(f.roots.get(a.runtimeKey), 'paycom').version, '1.0.0'); + await f.releases.beginRollout(f.next.digest, f.dsps.map(dsp => dsp.runtimeKey)); + const targets = f.releases.state().rollout.targets; + for (const id of targets) { + const before = f.releases.state().active.dsps; + await f.releases.step(); + const after = f.releases.state().active.dsps; + assert.equal(after[id], f.next.digest); + for (const other of f.dsps.map(dsp => dsp.runtimeKey).filter(key => key !== id)) assert.equal(after[other], before[other]); + } + for (const dsp of [dev, a, b]) assert.equal(installationReceipt(f.roots.get(dsp.runtimeKey), 'paycom').version, '1.1.0'); + assert.equal(installationReceipt(f.roots.get(b.runtimeKey), 'paycom').state, 'disabled'); + assert.equal(fs.existsSync(path.join(f.roots.get(c.runtimeKey), 'plugins/paycom')), false); + for (const dsp of f.dsps) { + assert.equal(fs.existsSync(path.join(f.roots.get(dsp.runtimeKey), 'plugins/sample')), false); + for (const root of ['config', 'data', 'secrets']) assert.equal(fs.readFileSync(path.join(f.roots.get(dsp.runtimeKey), root, 'sentinel'), 'utf8'), `${dsp.runtimeKey}:${root}`); + } + await f.change(c, 'install'); + assert.equal(installationReceipt(f.roots.get(c.runtimeKey), 'paycom').version, '1.1.0'); + assert.equal(fs.existsSync(pluginFile(f, c, '1.0.0')), false); +}); + +test('failed plugin update restores the installed version, DSP release approvals and private state', async t => { + const f = await fixture(t), [dev, other] = f.dsps; + await f.change(dev, 'install'); + const sentinel = path.join(f.roots.get(dev.runtimeKey), 'data/sentinel'); + fs.writeFileSync(sentinel, 'private before', { mode: 0o600 }); + f.fail(f.next.digest); + await assert.rejects(f.releases.updateDev(f.next.digest), /release_health_failed/); + assert.equal(installationReceipt(f.roots.get(dev.runtimeKey), 'paycom').version, '1.0.0'); + assert.equal(packageCatalog(f.paths).latest('paycom', dev.runtimeKey).version, '1.0.0'); + assert.equal(f.releases.state().active.dsps[dev.runtimeKey], f.before.digest); + assert.equal(fs.readFileSync(sentinel, 'utf8'), 'private before'); + assert.equal(fs.existsSync(pluginFile(f, dev, '1.1.0')), false); + assert.equal(fs.existsSync(pluginFile(f, other, '1.0.0')), false); + assert.equal(f.releases.state().tested, null); +}); + +test('a release cannot drop an installed plugin; an uninstalled plugin receives no later code', async t => { + const f = await fixture(t), [dev] = f.dsps; + await f.change(dev, 'install'); + const empty = f.artifact('dsp', '1.2.0'); + await f.releases.stage(empty.directory, empty.digest); + await assert.rejects(f.releases.updateDev(empty.digest), /release_installed_plugin_missing/); + await f.change(dev, 'uninstall'); + await f.releases.updateDev(empty.digest); + assert.equal(packageCatalog(f.paths).latest('paycom', dev.runtimeKey), null); + assert.equal(fs.existsSync(pluginFile(f, dev, '1.1.0')), false); + assert.equal(fs.existsSync(pluginFile(f, dev, '1.0.0')), true, 'previously installed code is retained for recovery'); +}); diff --git a/core/core/updates/transport.js b/core/core/updates/transport.js new file mode 100644 index 0000000..0d9535e --- /dev/null +++ b/core/core/updates/transport.js @@ -0,0 +1,21 @@ +'use strict'; +const path = require('node:path'); +const { PluginSdkSocket } = require('../../host/plugins/sdk-socket'); +const { createPrivateTransport } = require('../../sdk/node'); +const { result, failure, unwrap, exact } = require('../../sdk/src/protocol'); +const { privateDirectory } = require('../../host/controller/operations'); +const fileFor = paths => path.join(paths.local, 'run/updates/api.sock'); +function client(paths) { + const transport = createPrivateTransport({ socketPath: fileFor(paths), timeoutMs: 3660000 }); + return async (action, input = {}) => unwrap(await transport.request({ action, input })); +} +async function serve({ paths, execute }) { + privateDirectory(path.dirname(fileFor(paths))); + const socket = new PluginSdkSocket({ file: fileFor(paths), maximum: 2, timeoutMs: 3660000, + transport: { async request(value) { + try { exact(value, ['action', 'input']); return result(await execute(value.action, value.input)); } + catch (error) { return failure(/^release_|^directory_/.test(error.message) ? error.message : 'release_activation_failed', true); } + } } }); + await socket.start(); return socket; +} +module.exports = { client, serve }; diff --git a/core/core/updates/worker.js b/core/core/updates/worker.js new file mode 100644 index 0000000..7fb376c --- /dev/null +++ b/core/core/updates/worker.js @@ -0,0 +1,98 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const { acquireLock } = require('../../host/controller/operations'); +const { LocalReleases } = require('./local-releases'); +const { UpdateCommands } = require('./commands'); +const { PlatformGitHubReleases: GitHubReleases } = require('./platform-github'); +const { rootFor, loadConfiguration } = require('./configuration'); +const { coreHooks } = require('../../host/releases/core'); +const { client } = require('./transport'); +function authorizeOwner(paths, actorId) { + const file = path.join(paths.local, 'state/access-control/access-control.sqlite3'); + require('../../host/storage/backup-files').checked(file, false); + const db = new DatabaseSync(file, { readOnly: true }); + try { + const actor = db.prepare('SELECT platform_role,status FROM users WHERE id=?').get(actorId); + if (actor?.platform_role !== 'owner' || actor.status !== 'active') throw new Error('release_actor_forbidden'); + } finally { db.close(); } +} +class UpdateWorker { + constructor({ releases, commands, feed, invoke, authorize, clock = Date.now }) { + Object.assign(this, { releases, commands, feed, invoke, authorize, clock }); + this.lastRefresh = 0; this.pending = null; this.closed = false; + } + async initialize() { + for (const job of this.commands.list().filter(job => job.status === 'running')) { + job.status = 'failed'; job.failure = 'release_interrupted'; job.completedAt = this.clock(); this.commands.save(job); + } + const state = this.releases.state(); + if (!state.operation && state.rollout?.status === 'running') await this.releases.pause(); + this.commands.heartbeat(); + } + async execute(job) { + await this.authorize(job.actor); + if (['refresh', 'update_core', 'update_dev', 'rollout'].includes(job.action)) await this.feed.refresh(job.product); + await this.authorize(job.actor); + if (job.action === 'refresh') return; + if (job.action === 'update_core') { + if (['running', 'paused'].includes(this.releases.state().rollout?.status)) throw new Error('release_busy'); + return this.releases.updateCore(job.digest); + } + if (job.action === 'recover' && this.releases.state().operation?.product === 'core') return this.releases.recover(); + const input = { actor: job.actor }; + if (['update_dev', 'rollout'].includes(job.action)) input.digest = job.digest; + if (job.action === 'rollout') input.targets = job.targets; + await this.invoke(job.action, input); + } + tick() { + if (this.pending) return this.pending; + this.pending = this.run().finally(() => { this.pending = null; }); + return this.pending; + } + async run() { + const job = this.commands.list().find(item => item.status === 'queued'); + if (job) { + job.status = 'running'; this.commands.save(job); + try { await this.execute(job); job.status = 'completed'; job.failure = null; } + catch (error) { job.status = 'failed'; job.failure = /^release_[a-z_]+$/.test(error.message) ? error.message : 'release_operation_failed'; } + job.completedAt = this.clock(); this.commands.save(job); return; + } + const state = this.releases.state(); + if (state.operation) return; + if (state.rollout?.status === 'running') { + try { await this.authorize(state.rollout.actor); await this.invoke('step', { actor: state.rollout.actor }); } + catch { if (!this.releases.state().operation && this.releases.state().rollout?.status === 'running') await this.releases.pause(); } + return; + } + if (this.clock() - this.lastRefresh >= 300000) { + this.lastRefresh = this.clock(); + for (const product of (this.feed.isUnified ? ['core'] : ['core', 'dsp'])) { + try { await this.feed.refresh(product); } + catch { this.commands.heartbeat('feed_unavailable'); } + } + } + } + async close() { this.closed = true; await this.pending; this.commands.heartbeat('stopped'); } +} +async function startWorker(paths, dependencies = {}) { + const configuration = loadConfiguration(paths); + if (!configuration) throw new Error('release_configuration_required'); + const directory = rootFor(paths), lock = acquireLock({ local: path.join(directory, 'worker-lock') }, 'controller'); + let releases; + const commands = new UpdateCommands(directory); + releases = new LocalReleases({ directory, devDspId: configuration.devDspId, + hooks: coreHooks({ paths, configuration, releases: () => releases.state(), ...dependencies }) }); + const feed = new GitHubReleases({ directory: path.join(directory, 'downloads'), releases }); + const worker = new UpdateWorker({ releases, commands, feed, invoke: client(paths), authorize: actor => authorizeOwner(paths, actor) }); + try { await worker.initialize(); } catch (error) { fs.closeSync(lock); throw error; } + const heartbeat = setInterval(() => commands.heartbeat(), 10000); + let timer; + const loop = () => { if (!worker.closed) worker.tick().catch(() => commands.heartbeat('unavailable')).finally(() => { + if (!worker.closed) timer = setTimeout(loop, 1000); + }); }; + loop(); + let closing; + return { worker, close() { return closing ||= (async () => { clearTimeout(timer); clearInterval(heartbeat); await worker.close(); fs.closeSync(lock); })(); } }; +} +module.exports = { UpdateWorker, startWorker, authorizeOwner }; diff --git a/core/dashboard/.gitattributes b/core/dashboard/.gitattributes new file mode 100644 index 0000000..96bc401 --- /dev/null +++ b/core/dashboard/.gitattributes @@ -0,0 +1,4 @@ +# Keep generated bundles out of the default review diff. The bundled dialog +# library contains CSS template literals whose whitespace is owned upstream. +public/assets/frontend.js linguist-generated=true -whitespace +public/assets/styles.css linguist-generated=true diff --git a/core/dashboard/README.md b/core/dashboard/README.md new file mode 100644 index 0000000..041c25f --- /dev/null +++ b/core/dashboard/README.md @@ -0,0 +1,296 @@ +--- +title: Dashboard source guide +status: current +last_verified: 2026-09-07 +--- + +# Dispatch dashboard + +The dashboard presents the authenticated Dispatch UI. In split mode it serves +static assets and forwards `/api/` requests to the independently running +`dispatch-api` service. Backend composition and HTTP implementations live in +`core/api/`; account authority lives in `core/accounts/`. + +Run `bin/dispatch-dashboard --api-origin http://127.0.0.1:4311` for the UI-only +service. Without `--api-origin`, the command retains the combined compatibility +launcher for existing deployments. See [Dispatch API](../core/api/README.md) for +service ownership and migration details. Frontend plugins use `dispatch-sdk/ui` +for session-aware requests, shared controls and plugin settings. + +## Date and time + +Event timestamps are UTC instants in storage and API responses. The dashboard +formats them using the device timezone by default. Settings → Date & time offers +a validated IANA timezone override, saved per account on the current browser; +it follows the user while viewing other DSPs and synchronizes across tabs. + +Plugins can use `useTimezone()` from `frontend/src/lib/timezone.tsx` and +`dateTime(value, timeZone)` from `frontend/src/lib/date-time.ts` for event times. +The same provider covers connection checks, sync activity, audit events, +invitations, backups and release timestamps. + +Business dates are different: Paycom timecard dates, punch clock readings and +report boundaries belong to the DSP's business timezone. Use `useBusinessToday` +for the current DSP date and the calendar helpers for labels and day navigation. +Do not parse a date-only string as an event timestamp or convert a punch clock +reading into the viewer's timezone. A display preference never changes a DSP's +collector configuration or schedule. New DSP onboarding explicitly records the +business timezone, initially detected from the owner's browser. + +Existing DSP corrections must keep the organization timezone and plugin source +timezone consistent while collection is idle. Keep prior publications and run +receipts unchanged; new runs capture the corrected source configuration. + +## Start + +Read-only operational mode: + +```sh +./bin/dispatch-dashboard +``` + +Permission-gated private operator modes: + +```sh +./bin/dispatch-dashboard --operator +./bin/dispatch-dashboard --operator --installation-operator +``` + +`--operator` enables **Sync now** only for a selected member with `sync.run`. The separate `--installation-operator` capability enables fixed platform provision/retry outbox requests; it does not run the private reconciler, systemd, credential setup, or activation in the HTTP process. The service template omits `--installation-operator` by default; enabling it requires a deliberate trusted server-owner unit change and read-back. + +For local development, open `http://127.0.0.1:4310`. The installed service is instead configured for the canonical `https://dispatch.example.test` origin and is reached only through the supervised Cloudflare Tunnel. Public-origin mode requires secure cookies, exact Host and mutation Origin validation, and Cloudflare HTTPS proof; direct loopback/tailnet requests without that proxy metadata fail closed. Human login and organization authorization are always enabled. + +For already-authoritative ready managed installations, the server may receive an owner-private `DISPATCH_INSTALLATIONS_ROOT`. This is deployment configuration, not a browser/HTTP setting. Its runtime router derives and pins each managed gateway socket below that root. + +## DSP removal and restoration + +Active DSPs offer **Remove DSP**. Removal immediately signs out DSP users and prevents sign-in, pauses queued onboarding and backup work, stops and disables services, and keeps the DSP's data, credentials, service definitions, and completed backups. Existing backups are protected from expiration and are not newly exported while removed. A user can belong to only one DSP; platform support uses the separate DSP viewing context. + +The **Removed** tab contains removing, removed, restoring, and deleting DSPs, including failed operations. Once shutdown is complete, **Restore DSP** starts and verifies the retained runtime, restores its previous collection schedule, re-enables account access, and resumes normal backup scheduling and retention. Previously issued user sessions remain invalid. Earlier removals that deleted service definitions are migrated with their saved schedule; restoration reinstalls those definitions. Failed restoration keeps access blocked and remains retryable. + +**Permanently delete DSP** is available only after removal. It requires the signed-in Platform Owner's password, checked on the server with permissions, session validity, CSRF, revision checks, and password-attempt throttling. The password is not saved in lifecycle jobs or audit records. Deletion removes DSP data, files, accounts, credentials, and local and remote backups. Native DSPs remain visible until credential and Core account cleanup finishes. Full-platform backups containing the deleted DSP are also erased; other DSPs' individual backups remain. + +## Initial platform owner + +Create the initial platform-only owner from a private server terminal using the same runtime paths as the dashboard: + +```sh +./bin/dispatch-access-admin owner-create +# Recover access or replace the login email/password later: +./bin/dispatch-access-admin owner-list +./bin/dispatch-access-admin owner-recover +``` + +The commands prompt for credentials privately through `/dev/tty`; no passwords are accepted in arguments, environment values, or pipes. Recovery revokes existing sessions. The optional invitation-based `bootstrap --email OWNER_EMAIL` flow remains available and defaults to no organization. See platform administration for setup and recovery details. + +Everyone signs in on the same page. Platform owners land in the separate core-hosted console with DSPs, Updates, Backups, blank Plugins, Diagnostics, and Settings. DSP users retain their organization-scoped pages. The dashboard no longer creates a local DSP at startup. + +## Access a DSP as its owner + +Platform owners can choose **View** from a DSP's action menu or details panel to use its existing owner interface with full owner permissions for support. A persistent banner names the DSP, explains that changes are saved to it, and provides **Exit view**. Team management, roles, invitations, DSP onboarding, and available runtime actions are enabled. Team, roles, invitations, activity, and workspace details use the selected DSP's real data. Home Page and Paycom retain their current placeholder behavior. Account details continue to identify the signed-in platform owner. + +Viewing lasts up to 15 minutes, survives refresh in that tab, and leaves other tabs in their existing context. It does not create a membership or sign in as the owner. The server validates a separate, session-bound viewing reference on each request and applies the same permissions, CSRF validation, backup locks, and runtime readiness checks as DSP owner requests. The context can change only the selected DSP; exit it to use platform controls. Suspended or removed DSPs cannot be viewed. Expired or unavailable views return to the platform console with an explanation. Entering a view records `organization.view.start` under the platform account in the DSP activity history. DSP changes and sync requests are attributed to that platform account. Account security settings and sign-out operate on the signed-in platform account. + +## Diagnostics + +Platform Owners can use **Diagnostics → Deploy test DSP** to create a native DSP with synthetic employees and timecards. Each request creates a clearly named `TEST DSP`, assigns the requesting Platform Owner as its owner without sending an invitation email, and queues normal provisioning. The page shows progress; the DSP remains available until explicitly deleted through the DSPs page. The same removal, restoration, and password-confirmed permanent deletion controls apply. + +The private installation reconciler seeds only DSPs recorded in the durable diagnostics table, then uses the existing activation authority with explicitly synthetic provider evidence. The fixed private Runtime Agent command accepts only the recorded DSP identity, refuses existing non-diagnostic publications, and persists its result for retries. It is excluded from public SDK capabilities. Real provider authentication and collection are not exercised; collection schedules are manual and the sync stays stopped. + +Diagnostics requires native provisioning to be enabled. Schema 12 adds the diagnostics records while retaining existing organizations, identities, and lifecycle data. The HTTP process queues work and never executes host commands. + +Optional non-secret local-DSP configuration: + +```sh +DISPATCH_DASHBOARD_DSP_NAME='Example Delivery LLC' \ +DISPATCH_DASHBOARD_STATION='TST1' \ +./bin/dispatch-dashboard --operator +``` + +The installed service reads optional display settings from `/config/dashboard.env`. User name and role now come from the authenticated account and membership rather than display-only environment values. + +## Frontend and page scope + +The frontend uses React, TypeScript, Vite, Tailwind CSS, and locally owned shadcn/ui components. `frontend/src` owns the shared shell, authentication, DSP management, team administration, and account settings. The Updates and Backups pages mount the existing isolated controllers so their polling, rollout recovery, restore checks, and idempotent requests retain their tested behavior. + +Platform owners have **DSPs**, **Updates**, **Backups**, **Plugins**, and **Settings**. DSP members have **Home Page**, **Paycom**, **Team & Roles**, and **Settings**, subject to their existing permissions. Plugins and Home Page contain only a page title and make no workforce or bootstrap requests. Paycom provides the daily Timecard and Employees views described below. CDF and Integrations have no frontend routes; existing protected backend APIs remain available to authorized clients. + +`npm run build` type-checks the React source and produces `public/assets/frontend.js` and `styles.css`. The generated JavaScript and CSS are ignored by Git; build them before starting a fresh development checkout. The local Inter font remains tracked. Release packaging builds JavaScript and CSS from the selected commit and includes them in the Core artifact, which runs without frontend build dependencies. The server snapshots the browser assets with content-addressed URLs, preserving cache consistency across releases. A fresh per-response style nonce allows the dialog library to manage scrolling without allowing arbitrary inline scripts or styles. + +For an isolated synthetic preview: + +```sh +npm ci +npm run build +npm run preview:ui +``` + +Open `http://127.0.0.1:4339`. Fixture-only accounts are `platform@example.test` and `owner@example.test`, both with password `synthetic preview password`. The preview uses a disposable database. Backup execution is simulated; it does not send emails or change runtimes. + +Browser regression checks: + +```sh +npx playwright install chromium +npm run test:ui +``` + +The platform administration page lists DSPs with owner email, runtime health and onboarding status, plus search and Running, Onboarding and Removed filters. Create New DSP opens an email-only dialog. A single Access Control transaction creates the immutable organization and runtime identity, owner invitation and provisioning outbox request. The existing full-detail service API remains compatible with private tooling. + +After accepting the invitation, the owner supplies the DSP name, abbreviation, station and timezone. Details submitted during provisioning are retained and applied after the infrastructure worker finishes, without changing the container identity or data paths. DSP details can be completed from Team & Roles or Settings. Paycom remains blank in this frontend scope; provider enrollment is not exposed here. + +Manage updates reads the deployment-owned OCI release catalog and starts a durable fleet rollout. The private reconciler queues one verified lifecycle upgrade at a time, pauses on failure or unready DSPs, and records completion only after every current DSP is on the target and ready. New DSPs inherit the target and join an active rollout. Version identifiers remain internal to each DSP; the UI shows platform update selection, progress and history. See platform administration operations for deployment requirements and acceptance checks. + +Removal retains runtime data and backups. Permanent deletion remains a separate action. Both are available in each DSP's actions menu, using the existing session-bound control references and confirmation checks. + +## Invitation delivery + +Dashboard-created DSP-owner and member invitations are delivered through Cloudflare Email Sending as `Dispatch ` when `DISPATCH_EMAIL_ACCOUNT_ID` is present. Public-origin mode refuses invitation creation before Access Control mutation when the adapter is absent; local development without a public origin may retain the one-time manual handoff. The Dashboard reads the API token only from the fixed owner-private `/email/cloudflare-api-token` file; the token is never accepted through HTTP, argv, or an environment value. The sending request uses the authoritative invitation email, organization, role, expiry, and code-owned `https://dispatch.example.test` origin. + +A Cloudflare `delivered` or `queued` result suppresses the raw invitation link in the browser. A deterministic rejection or ambiguous transport result preserves the same one-time link as an authorized manual fallback and is never retried automatically. Platform mutation replays do not send again because Access Control does not return the raw token on replay. Cloudflare Email Preview must remain disabled because message bodies contain invitation capabilities. + +The initial platform bootstrap remains an owner-private terminal handoff so no email dependency can weaken first-owner initialization. + +## Turnstile protection for sign-in and registration + +Cloudflare Turnstile can protect `POST /api/auth/login` and `POST /api/auth/register`. +The Managed widget checks the browser while the user fills out the form. Submitting +requires a fresh token, and the server verifies success, the canonical hostname, +and the exact `login`, `register`, or `forgot_password` action before checking a password or creating an +account. Tokens never enter Access Control, audit records, or logs. Existing account +and address throttling, invitation validity, CSRF, and session checks remain in force. +Rejected login challenges count toward the existing failed-attempt limit; provider +outages do not. Signed-in invitation acceptance retains its session and CSRF checks +without another challenge. + +Activation is deployment-owned and opt-in so an unconfigured upgrade does not lock +out the existing installation: + +1. Create a **Managed** Turnstile widget in Cloudflare restricted to + `dispatch.example.test`. Leave pre-clearance disabled; Dispatch verifies each + protected submission directly. +2. Store its secret at `/turnstile/secret-key`, owned by the dashboard + service account, with directory mode `0700` and file mode `0600`. Symlinks and + hard-linked secrets are rejected. Do not put the secret in environment variables, + command arguments, frontend code, or Git. +3. Set `DISPATCH_TURNSTILE_SITE_KEY` to the widget's public site key in + `/config/dashboard.env` and restart through the normal deployment + process. The public site key alone is returned in the session response. +4. Verify a real browser login and invitation registration on the canonical hostname, + including retry after an incorrect password, then inspect Turnstile Analytics. + +When the site-key setting is absent, the existing authentication flow is retained. +When present, startup rejects incomplete or invalid configuration, noncanonical +origins, and Cloudflare dummy keys. A missing, rejected, expired, or reused token +cannot create a session or account. Siteverify requests have an eight-second timeout; +transport errors fail closed with a retryable message. The form keeps entered values +and refreshes verification after a failed submission. Core backups and host recovery +include the private Turnstile secret and its directory ownership. + +Only configured HTML responses allow the Turnstile script and iframe origin in CSP. +The widget script loads on authentication forms, not normal dashboard actions. Local +previews need no Cloudflare credentials. `tests/turnstile.test.js` exercises the real +verification adapter using simulated Siteverify responses; the isolated frontend +fixture's `DISPATCH_TURNSTILE_FIXTURE=1` option supports deterministic browser tests. +Those tests cover form behavior and enforcement, not Cloudflare's live bot detection. + +References: [widgets](https://developers.cloudflare.com/turnstile/concepts/widget/), +[server verification](https://developers.cloudflare.com/turnstile/get-started/server-side-validation/), +[CSP](https://developers.cloudflare.com/turnstile/reference/content-security-policy/). + +## Security boundary + +- Every protected route requires an authenticated opaque session. +- DSP-owner and member routes contain no organization ID; the server derives the effective DSP from the active membership. +- Platform ownership does not automatically grant workforce access to every DSP. +- Organization switching submits a membership ID, and the server resolves that membership's organization. +- Every operational call reloads current session, membership, role, organization, and installation authority before using a local or gateway connector. +- Managed connector/socket selection is server-owned, and the gateway verifies its expected runtime identity on every call. +- Session cookies are `HttpOnly` and `SameSite=Strict`; session values are hashed at rest and never stored in browser storage. +- State-changing requests require JSON, the session's CSRF token, and a fixed route. +- Platform organization controls use opaque random references in JSON bodies rather than organization or invitation IDs in URLs. References are hash-only at rest, session-bound, expiring, and re-authorized per request. +- Platform organization creation, status/invitation changes, and provision/retry requests are idempotent. Invitation-path replays return no raw handoff. +- Sync additionally requires `--operator`, a ready installation, and `sync.run`. +- Passwords use versioned scrypt hashes; plaintext passwords and invitation/session tokens are not persisted. +- Provider credentials, Auth Broker vault contents, browser endpoints, and browser sessions are never exposed. +- Workforce data remains protected business data and must not enter general logs or public issue trackers. +- One exact Cloudflare Tunnel route is the public network boundary. The origin remains loopback-only; secure host-only cookies, exact Host/Origin/HTTPS checks, application throttling, managed WAF, and managed DDoS protection protect the public login/invitation surface. Bot Fight Mode remains off during testing because its zone-wide policy challenged scripted/headless acceptance. Authenticated transactional invitation email is enabled with Cloudflare Email Preview disabled. Administrator MFA/passkeys, reviewed bot controls, and final independent review remain hardening work. Password recovery has persistent per-email, per-IP, and installation-wide throttles shared by all dashboard processes using the access database. + +See [Access Control](../core/accounts/OVERVIEW.md) for authorization and account storage. + +## Verification + +```sh +./core/accounts/scripts/verify +./runtime/gateway/scripts/verify +npm run build --prefix dashboard +npm test --prefix dashboard +``` + + +### Hotfix builds + +Published releases accept `X.Y.Z` and `X.Y.Z+hotfix.N`, where `N` is a positive integer without leading zeroes. For example, `0.0.7+hotfix.1` appears as **0.0.7 — Hotfix 1**. It is a separate GitHub tag and immutable release, with internal identity `dispatch_0.0.7_hotfix.1`. Existing tags and assets must never be replaced. + +Dispatch explicitly orders release numbers and then hotfix revisions numerically; this is an application policy because SemVer ignores build metadata when comparing precedence. A later publication date cannot make the original build supersede its hotfix. Historical prerelease catalog entries retain their publication-date fallback. Core, DSP runtime, host control, recovery, and rollout records all use the distinct internal identity. Normal verified recovery and local package retention rules still apply. + +Servers running the original 0.0.7 cannot parse hotfix catalogs. Their release discovery service and independent update supervisor must first be bootstrapped from the chosen merged hotfix source using the existing trusted release-delivery installation procedure. Queue the first hotfix rollout through that updated coordinator; subsequent hotfixes use the normal Updates flow. Verify the deployed source commit and run `core/installations/scripts/verify-live-dsps run` as root from the matching clean checkout to exercise the full live DSP lifecycle. +## Paycom workforce pages + +Connected DSPs can use two Paycom tabs. **Timecard** defaults to today's date in the DSP timezone, shows employee names and the selected day's punches/hours/status, and supports prior-date selection. Every column toggles ascending/descending sorting across the full result before pagination; names default to A–Z and empty times/hours remain at the bottom. Multiple punches remain visible in source order, with sorting based on the first punch. The page refreshes today's collected data every 30 seconds without triggering a Paycom collection. + +**Employees** searches the complete collected roster by name and opens an employee's profile and most recently collected period timecard inside that tab. Department, station, and position appear only in the employee detail. The original optional connection flow remains available to owners whose Paycom setup has not completed. + +`GET /api/paycom/daily`, `GET /api/paycom/employees`, and `GET /api/paycom/employees/:code` require `workforce.read` and resolve the runtime from the authenticated DSP context. Historical days are available only for collected periods; unavailable dates are never presented as empty employee timecards. Employee detail uses the SDK's bounded daily projection rather than raw collector records. + +Run the isolated workforce browser checks with `DISPATCH_PAYCOM_WORKFORCE_FIXTURE=1 DISPATCH_FRONTEND_PORT=4348 npm run test:ui -- tests/browser/paycom-workforce.spec.cjs`. The opt-in preview uses synthetic records through the real SDK and HTTP routes, including more than 100 employees to exercise sorting across pages. + + +## Password recovery + +The sign-in form links to `/#/forgot-password`. Recovery reuses the configured +Cloudflare email account and private API token described above; there are no new +credentials or sender domains to configure. When delivery is not configured, +recovery fails closed with the same unavailable response for every address. +Turnstile, when enabled for the installation, also verifies the `forgot_password` +action on requests. The new-password form does not load a third-party widget. + +- `POST /api/auth/forgot-password` accepts `{email, turnstileToken?}`. After + validation and address/global throttling, it returns HTTP 202 with the same + message for known, unknown, disabled, and email-throttled accounts. Account + lookup and sending happen only after the response is written. Provider delivery + failures do not change the response or disclose a reset link. +- `POST /api/auth/reset-password` accepts `{token, newPassword, confirmPassword}`. + Reset tokens are 32 random bytes, stored only as SHA-256 hashes, valid for 30 + minutes, and bound to the account's credential version and email. Within one + write transaction, a reset rechecks the token and account after scrypt hashing, + replaces the password, invalidates every reset token, revokes every session, + and records the completion audit event. It creates no login session. +- Email links use the canonical HTTPS origin and a URL fragment. The reset form + removes the fragment token from browser history on mount and keeps it only in + memory. Reloading requires reopening the email link. GET requests and email + scanner visits never consume the token; only a successful reset POST does. +- Passwords retain the existing 12–128 character policy and scrypt hashing. + A database trigger invalidates recovery links on credential-version, email, + or account-status updates, including private administrator recovery. Recovery + does not reactivate accounts, memberships, organizations, or change roles. +- Limits persist in SQLite: 5 email requests per address per hour, at least 60 + seconds apart; 20 requests per IP and 200 per installation per 15 minutes; + 30 reset submissions per IP and 100 per installation per 15 minutes. IPs and + emails are hashed in the limiter table. Its 10,000-bucket bound fails closed + instead of evicting active limits. Forwarded IPs are trusted only through the + existing canonical public-origin/loopback Cloudflare boundary. +- Email work is bounded to 16 in-flight/queued jobs and password hashing to two + recovery operations per process. The email worker keeps raw tokens only in + memory: a process restart can discard unsent mail, so users may need to request + a new link after the cooldown. Confirmation delivery is best effort and never + rolls back a completed password reset. Delivery outcomes are audited without + token, password, email body, or provider diagnostic logging. +- Schema 14 adds recovery-token and throttle tables. Sanitized Core backups remove + both tables' contents, and managed restores never restore old reset tokens. + +Security regression tests cover token replay, concurrent resets across database +connections, expiry/account changes during hashing, transaction rollback, +restart-persistent throttling, malformed/cross-site requests, delivery failure, +Turnstile action/replay checks, and browser recovery on desktop and mobile. The +browser fixture captures synthetic messages through private child-process IPC; +it does not expose an inbox endpoint or send real email. diff --git a/core/dashboard/components.json b/core/dashboard/components.json new file mode 100644 index 0000000..2ffad56 --- /dev/null +++ b/core/dashboard/components.json @@ -0,0 +1,20 @@ +{ + "$schema": "https://ui.shadcn.com/schema.json", + "style": "new-york", + "rsc": false, + "tsx": true, + "tailwind": { + "config": "", + "css": "frontend/src/styles.css", + "baseColor": "neutral", + "cssVariables": true + }, + "iconLibrary": "lucide", + "aliases": { + "components": "@/components", + "utils": "@/lib/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" + } +} diff --git a/core/dashboard/examples/frontend-preview.js b/core/dashboard/examples/frontend-preview.js new file mode 100644 index 0000000..5a41cf1 --- /dev/null +++ b/core/dashboard/examples/frontend-preview.js @@ -0,0 +1,334 @@ +#!/usr/bin/env node +"use strict"; +// Explicitly opted-in, isolated UI fixture. No runtime operations or emails. +const fs = require("node:fs"); +const os = require("node:os"); +const path = require("node:path"); +const { + AccessStore, + AccessControlService, +} = require("../../core/accounts/src"); +const { createDashboardServer } = require("../server/server"); +async function main() { + if (process.env.DISPATCH_FRONTEND_FIXTURE !== "1") + throw Error("fixture_opt_in_required"); + const root = fs.mkdtempSync(path.join(os.tmpdir(), "dispatch-frontend-")); + fs.chmodSync(root, 0o700); + const store = new AccessStore({ + databaseRoot: path.join(root, "access"), + database: path.join(root, "access/control.sqlite3"), + }); + const access = new AccessControlService(store, { + installationOperatorEnabled: true, + installationBackend: "native_service_v1", + }); + const password = "synthetic preview password"; + const bootstrap = access.createPlatformBootstrap({ + email: "platform@example.test", + }); + const platform = await access.acceptNewUser({ + token: bootstrap.token, + firstName: "Platform", + lastName: "Owner", + password, + confirmPassword: password, + }); + const organizations = []; + for (const [i, name] of [ + "Northline Logistics", + "Cedar Delivery", + "Atlas Routes", + "Harbor Logistics", + "Summit Delivery", + "Westfield Routes", + "Maple Delivery", + "River Routes", + ].entries()) { + const result = access.createOrganization(platform.session, { + idempotencyKey: `preview:frontend:dsp:${i}`, + ownerEmail: i === 0 ? "owner@example.test" : `owner${i}@example.test`, + name, + abbreviation: ["NL01", "CD02", "AR03", "HL04", "SD05", "WR06", "MD07", "RR08"][i], + stationCode: "TST1", + timezone: "America/Chicago", + }); + const id = result.organization.id; + organizations.push({ id, name, status: "ready", canBackup: true }); + store.db + .prepare( + "UPDATE installations SET status='ready' WHERE organization_id=?", + ) + .run(id); + store.updateOrganizationStatus(id, "active", Date.now()); + // Keep the optional-connection owner independent of password-change tests. + if (i >= 5) await access.acceptNewUser({ token: result.token, firstName: "Optional", lastName: "Owner", password, confirmPassword: password }); + if (i === 0) { + const owner = await access.acceptNewUser({ + token: result.token, + firstName: "Alex", + lastName: "Morgan", + password, + confirmPassword: password, + }); + store.updateOrganizationStatus(id, "active", Date.now()); + const ownerSession = access.session(owner.token); + const role = + store.roles(id).find((r) => r.key === "manager") || + store.roles(id).find((r) => r.key !== "owner"); + for (const [j, member] of [ + "Jamie Chen", + "Taylor Brooks", + "Jordan Lee", + ].entries()) { + const invite = access.createMemberInvitation(ownerSession, id, { + email: `member${j}@example.test`, + roleId: + j === 2 + ? store.roles(id).find((r) => r.key === "driver").id + : role.id, + }); + const [firstName, lastName] = member.split(" "); + await access.acceptNewUser({ + token: invite.token, + firstName, + lastName, + password, + confirmPassword: password, + }); + } + } + } + const unavailable = async () => ({ + ok: false, + status: "installation_not_ready", + data: null, + error: { code: "installation_not_ready" }, + }); + const client = { + workforce: { day: unavailable }, + sync: { status: unavailable, runNow: unavailable }, + system: { status: unavailable }, + }; + if (process.env.DISPATCH_PAYCOM_WORKFORCE_FIXTURE === "1") client.workforce = require("./paycom-workforce-fixture"); + const previewNotes = require("./grouped-changelog.json"); + const previewRelease = { version: "0.0.9", publishedAt: "2026-09-07T00:00:00.000Z", sourceCommit: "a".repeat(40), core: {}, + changelog: previewNotes.changelog.map(({kind,title,description}) => ({kind,title,description})) }; + const updates = require("../../core/accounts/src/platform-updates").createPlatformUpdates({ + store, enabled: true, releases: { "dispatch_0.0.9": {} }, platformReleases: { "dispatch_0.0.9": previewRelease }, + delivery: { view: () => null, notes: id => id === "dispatch_0.0.9" ? previewNotes : null, + history: () => ({ "dispatch_0.0.8": { version: "0.0.8", publishedAt: "2026-09-06T00:00:00.000Z", sourceCommit: "b".repeat(40), + changelog: [{kind:"improved",title:"Clearer rollout progress",description:"Follow Core and DSP update progress."}] } }) } + }); + let releasePopup = null; + if (process.env.DISPATCH_POPUP_FIXTURE === "1") { + const { authoring } = require("../../core/installations/src/release-notes"); + const { createReleasePopup } = require("../../core/accounts/src/release-popup"); + const authored = authoring(require("./popup-changelog.json")); + const release = { schemaVersion: 1, releaseId: "dispatch_0.0.9", version: "0.0.9", sourceCommit: "a".repeat(40), ...authored.popup }; + store.db.prepare("INSERT INTO platform_rollouts VALUES(?,?,?,?,'completed',?,?)") + .run("popup_fixture", release.releaseId, platform.session.user.id, "popup_fixture", Date.now(), Date.now()); + store.db.prepare("INSERT INTO platform_rollout_core VALUES(?,'succeeded',?,1,NULL,?)") + .run("popup_fixture", JSON.stringify({ ...previewRelease, changelog: authored.changelog }), Date.now()); + store.db.prepare("UPDATE installations SET release_id=?").run(release.releaseId); + releasePopup = createReleasePopup({ store, release }); + } + const now = new Date().toISOString(); + const backupData = { + enabled: true, + canBackupCore: true, + operationBlocked: null, + revision: 1, + nextBackupAt: null, + settings: { + enabled: false, + frequency: "daily", + + time: "02:00", + weekday: 0, + timezone: "America/Chicago", + retentionDays: 30, + }, + storage: { status: "connected", checkedAt: now }, + organizations, + operations: [], + backups: organizations + .filter((_, i) => i !== 4) + .map((o, i) => ({ + id: `fixture_backup_${i}`, + organizationId: o.id, + kind: "dsp", + name: o.name, + createdAt: now, + expiresAt: new Date(Date.now() + 30 * 86400000).toISOString(), + retentionDays: 30, + size: 20 * 1024 * 1024, + status: "verified", + verifiedAt: now, + trigger: "scheduled", + restoreBlocked: null, + })), + }; + backupData.schedules=['system','core',...organizations.map(o=>o.id)].map(scope=>({scope,revision:1,settings:{...backupData.settings},nextBackupAt:null})); + backupData.sets=[];backupData.deletions=[]; + backupData.backups.push({id:'fixture_core_backup',organizationId:null,kind:'core',name:'Platform Core',createdAt:now,expiresAt:null,retentionDays:null,size:1024,status:'verified',verifiedAt:now,trigger:'manual',restoreBlocked:null}); + const usageScopes=[{scope:'core',name:'Platform Core',removed:false,bytes:8*1048576,backupCount:1},...organizations.map((o,i)=>({scope:o.id,name:o.name,removed:false,bytes:i===4?0:20*1048576,backupCount:i===4?0:1})),{scope:'org_removed_fixture',name:'Pine Delivery',removed:true,bytes:5*1048576,backupCount:2}]; + backupData.storageUsage={status:'ready',checkedAt:now,bytes:usageScopes.reduce((n,s)=>n+s.bytes,0),backupCount:usageScopes.reduce((n,s)=>n+s.backupCount,0),retainedBytes:5*1048576,scopes:usageScopes,sets:[],manifestBytes:0,legacyBytes:0,other:{bytes:0,backupCount:0}}; + const backups = { + view: () => backupData, + command: (_session, input) => { + if (input.action === "settings") { + const policy=backupData.schedules.find(s=>s.scope===(input.organizationId||input.scope||'system'));policy.settings=input.settings;policy.revision++; + if(policy.scope==='system'){backupData.settings=input.settings;backupData.revision=policy.revision;} + return; + } + if(input.action==='delete'){backupData.deletions.unshift({id:'fixture_delete',backupId:input.backupId,status:'queued'});return;} + const ids = + input.scope==='system' ? [null,...organizations.map(o=>o.id)] : input.scope === "core" + ? [null] + : input.organizationIds || [input.organizationId]; + for (const id of ids) + backupData.operations.unshift({ + id: `fixture_operation_${backupData.operations.length}`, + organizationId: id, + kind: input.action === "restore" ? "restore" : id ? "backup" : "core", + status: "queued", + phase: "queued", + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + backupId: input.backupId || null, + }); + }, + }; + const runtimePlugins = new Map(); + // Core-only UI checks need catalog metadata but do not install a DSP fixture. + // Plugin browser checks use the explicitly installed DSP test package. + let paycomFixtureRoot = null; + try { paycomFixtureRoot = path.dirname(require.resolve('dispatch-dsp/plugins/paycom/dispatch-plugin.json')); } + catch (error) { if (error.code !== 'MODULE_NOT_FOUND') throw error; } + const paycomManifest = paycomFixtureRoot ? require(path.join(paycomFixtureRoot, 'dispatch-plugin.json')) + : require('../../tests/fixtures/paycom-plugin.json'); + require('../../shared/plugin-sdk/catalog').configureCatalog(() => [paycomManifest]); + require('dispatch-protocol/plugin-sdk/catalog').configureCatalog(() => [paycomManifest]); + const settingsDefinition=paycomManifest.settings; + const settingsFor=id=>require('../../core/plugins/settings-store').settingsStore(path.join(root,id),'paycom'); + const published=path.join(root,'published/paycom.sqlite3'); + if(process.env.DISPATCH_PAYCOM_WORKFORCE_FIXTURE==='1'){ + const db=require('../../shared/published/database').openDatabase(published,{write:true,journalMode:'DELETE'}); + const adapter=require('dispatch-dsp/plugins/paycom/backend/adapters/published.js');adapter.schema(db); + adapter.publishPeriod(db,require('./paycom-workforce-fixture').fixtureData,'America/Chicago');db.close(); + } + for (const [index, organization] of organizations.entries()) { + const installation = store.installation(organization.id); + settingsFor(installation.runtimeKey).initialize(settingsDefinition); + settingsFor(installation.runtimeKey).applied(0); + const installed = index < 5; + runtimePlugins.set(installation.runtimeKey, { id: 'paycom', version: paycomManifest.version, state: installed ? 'enabled' : 'uninstalled', revision: installed ? 1 : 0 }); + if (installed) store.db.prepare(`INSERT INTO dsp_plugins(organization_id,plugin_id,version,desired_state,applied_state, + revision,applied_revision,failure_code,actor_user_id,updated_at) VALUES(?,'paycom',?,'enabled','enabled',1,1,NULL,NULL,?)`) + .run(organization.id, paycomManifest.version, Date.now()); + store.db.prepare('INSERT INTO plugin_migration_checks(organization_id) VALUES(?)').run(organization.id); + } + const plugins = require('../../core/accounts/src/plugins').createPluginService({ + settingsPort:async(id,_plugin,request)=>{ + const storage=settingsFor(id); + if(request.action==='history')return storage.history(settingsDefinition,request.input); + if(request.action==='options')return new (require('dispatch-dsp/plugins/paycom/dashboard/published.js').PublishedWorkforcePort)(published).settingsOptions(); + if(request.action==='update'){const saved=storage.update(settingsDefinition,request.input,request.actor);storage.applied(saved.revision);} + return storage.read(settingsDefinition); + }, + store, access, backends: ['native_service_v1'], invoke: async (runtimeKey, action, input) => { + if (action !== 'plugins.manage') throw new Error('unexpected_fixture_action'); + const current = runtimePlugins.get(runtimeKey); + if (input.command === 'status') return { ok: true, status: 'found', data: { items: [current] }, error: null }; + if (input.revision < current.revision) throw new Error('stale_fixture_revision'); + const next = { id: input.pluginId, version: input.version, state: input.state, revision: input.revision }; + runtimePlugins.set(runtimeKey, next); + return { ok: true, status: 'applied', data: next, error: null }; + }, + }); + const pluginTimer = setInterval(() => { plugins.runPending().catch(() => {}); }, 100); + const connectionStates = new Map(); + const emailVerificationFixtures = new Set(); + const connectionInvoke = async (runtimeKey, _action, input) => { + if (!connectionStates.has(runtimeKey)) connectionStates.set(runtimeKey, new Map(['cortex', 'paycom'].map(service => [service, + { service, configured: false, state: 'not_connected', checkedAt: null, reason: null, retryAt: null }]))); + const items = connectionStates.get(runtimeKey); + if (input.command === 'list') return { ok: true, status: 'found', data: { items: [...items.values()] }, error: null }; + if (input.service === 'cortex' && input.command === 'save') { + if (input.credentials.username === 'verification-fixture') emailVerificationFixtures.add(runtimeKey); + else emailVerificationFixtures.delete(runtimeKey); + } + const previous = items.get(input.service); + if (input.command === 'verify' && previous?.verification?.id !== input.verificationId) + return { ok: false, status: 'verification_expired' }; + const view = { service: input.service, configured: input.command !== 'disconnect', + state: input.command === 'disconnect' ? 'not_connected' : 'checking', checkedAt: null, reason: null, retryAt: null }; + items.set(input.service, view); + if (view.configured) setTimeout(() => { + if (items.get(input.service) !== view) return; + if (input.service === 'cortex' && emailVerificationFixtures.has(runtimeKey) + && (input.command !== 'verify' || input.code !== '123456')) { + items.set(input.service, { ...view, state: 'verification_required', + reason: input.command === 'verify' ? 'verification_code_rejected' : 'mfa_required', + verification: previous?.verification ? { ...previous.verification, attemptsRemaining: previous.verification.attemptsRemaining - 1 } + : { id: require('node:crypto').randomBytes(16).toString('base64url'), expiresAt: new Date(Date.now() + 600000).toISOString(), attemptsRemaining: 3 } }); + } else items.set(input.service, { ...view, state: 'connected', checkedAt: new Date().toISOString() }); + }, 800).unref(); + return { ok: true, status: 'accepted', data: { ...view }, error: null }; + }; + const connections = require('../../core/accounts/src/owner-connections').createOwnerConnections({ store, access, invoke: connectionInvoke }); + const paycomSetup = require('../../core/accounts/src/owner-paycom-setup').createOwnerPaycomSetup({ store, access, invoke: unavailable }); + // Deterministic browser-test verifier. This entry point always requires the + // isolated fixture opt-in above; production main never imports it. + const usedTurnstileTokens = new Set(); + const turnstile = process.env.DISPATCH_TURNSTILE_FIXTURE === "1" ? require('../server/turnstile').createTurnstile({ + siteKey: '0x' + 'a'.repeat(24), secret: '0x' + 'b'.repeat(33), hostname: '127.0.0.1', + fetchImpl: async (_url, init) => { + const { response } = JSON.parse(init.body); + if (response === 'fixture-unavailable') throw Error('synthetic outage'); + const [prefix, action] = response.split(':'); + const success = prefix === 'fixture' && ['login', 'register', 'forgot_password'].includes(action) && !usedTurnstileTokens.has(response); + usedTurnstileTokens.add(response); + return { ok: true, json: async () => ({ success, hostname: '127.0.0.1', action }) }; + }, + }) : null; + // Synthetic inbox over the test child's private IPC channel; no real mail, + // HTTP debug endpoint, or token logging. Production never loads this fixture. + const invitationDelivery = process.env.DISPATCH_RECOVERY_FIXTURE === "1" ? { + send: async () => ({ status: 'accepted' }), + sendPasswordReset: async message => { process.send?.({ type: 'password-reset', message }); return { status: 'accepted' }; }, + sendPasswordResetConfirmation: async message => { process.send?.({ type: 'password-reset-confirmation', message }); return { status: 'accepted' }; }, + } : null; + const server = createDashboardServer({ dashboards:require('../../tooling/development-dashboard').developmentDashboards(), client, access, updates, backups, paycomSetup, connections, plugins, releasePopup, turnstile, invitationDelivery, + pluginAssets: async ({ pluginId, revision }) => { + if (pluginId !== 'paycom') throw new Error('plugin_unavailable'); + if (!paycomFixtureRoot) throw new Error('dsp_test_fixture_required'); + const directory = path.join(root, 'frontend', pluginId); + if (!fs.existsSync(path.join(directory, 'index.js'))) { + const { buildFrontend } = await import('../../tooling/build-plugin-frontend.mjs'); + await buildFrontend({ pluginRoot: paycomFixtureRoot, + output: directory, toolsRoot: path.resolve(__dirname, '..') }); + } + return { id: pluginId, version: paycomManifest.version, revision, javascript: fs.readFileSync(path.join(directory, 'index.js'), 'utf8'), stylesheet: fs.readFileSync(path.join(directory, 'styles.css'), 'utf8') }; + }, + runtimeResolver: process.env.DISPATCH_PAYCOM_WORKFORCE_FIXTURE === "1" ? installation => ({...client, + workforce:new (require('../../shared/contracts/src/workforce-client').WorkforceClient)({port:new (require('dispatch-dsp/plugins/paycom/dashboard/published.js').PublishedWorkforcePort)(published,settingsFor(installation.runtimeKey).read(settingsDefinition).values)})}) : null }); + const cleanup = () => + server.close(async () => { + clearInterval(pluginTimer); + await plugins.runPending(); + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + process.exit(0); + }); + process.once("SIGTERM", cleanup); + process.once("SIGINT", cleanup); + const port = Number(process.env.DISPATCH_FRONTEND_PORT || 4339); + if (!Number.isInteger(port) || (port !== 0 && port < 1024) || port > 65535) throw Error("invalid_preview_port"); + await new Promise((resolve) => server.listen(port, "127.0.0.1", resolve)); + process.stdout.write(`Synthetic frontend preview: http://127.0.0.1:${server.address().port}\n`); +} +main().catch((error) => { + process.stderr.write(`${error.message}\n`); + process.exitCode = 1; +}); diff --git a/core/dashboard/examples/grouped-changelog.json b/core/dashboard/examples/grouped-changelog.json new file mode 100644 index 0000000..cb03a44 --- /dev/null +++ b/core/dashboard/examples/grouped-changelog.json @@ -0,0 +1,112 @@ +{ + "groups": [ + { + "id": "backups", + "title": "Backups & recovery", + "icon": "database" + }, + { + "id": "dsps", + "title": "DSP management", + "icon": "users" + }, + { + "id": "onboarding", + "title": "Onboarding & support", + "icon": "user-plus" + }, + { + "id": "updates", + "title": "Platform updates", + "icon": "refresh-cw" + } + ], + "changelog": [ + { + "kind": "changed", + "title": "Remove and restore DSPs", + "description": "Remove active DSPs while retaining their data and backups, then restore them when needed.", + "group": "dsps", + "icon": "trash", + "details": "Remove DSP replaces the separate Suspend and Delete actions for active DSPs. Removal moves the DSP to Removed, signs its users out, blocks login, and stops its services and new backups while retaining its data and existing backups. Restore DSP brings it back into service." + }, + { + "kind": "changed", + "title": "Permanent DSP deletion with password confirmation", + "description": "Permanently delete a Removed DSP with password confirmation after shutdown completes.", + "group": "dsps", + "icon": "lock", + "details": "Permanently delete DSP is available only in Removed after shutdown finishes. Confirm with your platform-owner password. Deletion removes the DSP's users, data, services, and backups; failed cleanup remains visible and can be retried. Users can belong to only one DSP." + }, + { + "kind": "added", + "title": "Separate Core, DSP, and full-system backups", + "description": "Manage Core, DSP, and full-system backups separately with grouped recovery sets.", + "group": "backups", + "icon": "copy", + "details": "Create, restore, or delete a Core backup independently of DSP backups, or operate on one selected DSP. Full-system backups contain separate Core and DSP archives grouped into a recovery set. Removed DSPs are excluded from new backups and their existing archives remain retained. Deleting a backup does not delete live data." + }, + { + "kind": "changed", + "title": "Independent backup schedules start turned off", + "description": "Core, DSP, and full-system backup schedules are independent and start disabled.", + "group": "backups", + "icon": "calendar-clock", + "details": "Core, every DSP, and full-system backups each have their own schedule, time zone, and retention policy. Only the platform owner can adjust them. All schedules start disabled, including after upgrading from the previous shared schedule. Enable the schedules you want after updating." + }, + { + "kind": "added", + "title": "Backup storage usage", + "description": "View measured offsite storage for Core, DSP, and retained removed-DSP backups.", + "group": "backups", + "icon": "chart-column", + "details": "The Backups Storage tab shows measured offsite storage for Core, each DSP, and retained backups belonging to removed DSPs. Full-system totals count their component archives once. Measurements show when they were taken and whether they are stale or unavailable." + }, + { + "kind": "improved", + "title": "Scoped restore and recovery", + "description": "Restore Core or full-system backups with safety checks and export a recovery kit.", + "group": "backups", + "icon": "shield", + "details": "Core-only restore preserves DSP data and schedules, uses a verified safety backup, and checks Core health before completing. Full-system restore checks the complete recovery set and stops on a component failure. In-place full-system restore requires the same DSP inventory as the selected backup. Export a current recovery kit for replacement-server recovery." + }, + { + "kind": "changed", + "title": "Full-system backup completeness", + "description": "See unavailable DSPs and incomplete full-system sets instead of skipped archives.", + "group": "backups", + "icon": "check-circle", + "details": "Full-system backups report unavailable DSPs instead of silently skipping them. Deleting an individual archive makes any full-system set that references it incomplete; the remaining component archives can still be used independently." + }, + { + "kind": "improved", + "title": "Guided DSP invitation setup", + "description": "Guide DSP owners through invitations, account access, and dedicated setup.", + "group": "onboarding", + "icon": "send", + "details": "DSP owner invitations now guide new users through account creation and existing users through sign-in and acceptance, then collect the DSP name, abbreviation, station, and time zone in a dedicated setup step." + }, + { + "kind": "added", + "title": "Platform-owner DSP support access", + "description": "View a DSP with owner permissions through attributed, time-limited support access.", + "group": "onboarding", + "icon": "shield", + "details": "Platform owners can use View to work inside a DSP with owner permissions without using the DSP owner's login. A persistent banner identifies the selected DSP, and changes are attributed to the platform account. Access is limited to the current tab, expires after 15 minutes, and includes an Exit view action." + }, + { + "kind": "improved", + "title": "Clearer update rollout progress", + "description": "Track Core and DSP rollout progress, current activity, and reconnecting state.", + "group": "updates", + "icon": "refresh-cw", + "details": "Updates now shows animated Core and DSP progress, current activity, and reconnecting messages that preserve the last confirmed progress. Animations respect reduced-motion preferences and pause when progress is paused or the connection is lost." + } + ], + "afterUpdating": [ + { + "title": "Backup schedules", + "description": "Enable the backup schedules you want to run." + } + ] +} diff --git a/core/dashboard/examples/independent-updates-preview.js b/core/dashboard/examples/independent-updates-preview.js new file mode 100644 index 0000000..b526594 --- /dev/null +++ b/core/dashboard/examples/independent-updates-preview.js @@ -0,0 +1,107 @@ +#!/usr/bin/env node +'use strict'; +// Explicit synthetic fixture. The real API, command queue and release coordinator +// run here; host services and GitHub publication are simulated in private temp data. +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'); +const { AccessStore, AccessControlService } = require('../../core/accounts/src'); +const { LocalReleases } = require('../../core/updates/local-releases'); +const { UpdateCommands } = require('../../core/updates/commands'); +const { UpdateWorker } = require('../../core/updates/worker'); +const { createUpdatesService } = require('../../core/updates/service'); +const { hash, inventory } = require('../../shared/releases/package'); +const { createDashboardServer } = require('../server/server'); +async function createPreview({ port = 0, automatic = true, versionedDashboards = false } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-independent-updates-')); + const store = new AccessStore({ databaseRoot: path.join(root, 'access'), database: path.join(root, 'access/control.sqlite3') }); + const access = new AccessControlService(store, { installationOperatorEnabled: true, installationBackend: 'directory_service_v1' }); + const bootstrap = access.createPlatformBootstrap({ email: 'platform@example.test' }), password = 'synthetic preview password'; + const owner = await access.acceptNewUser({ token: bootstrap.token, firstName: 'Platform', lastName: 'Owner', password, confirmPassword: password }); + const dsps = [], owners = []; + for (const [index, name] of ['Dev DSP', 'Northline Logistics', 'Cedar Delivery'].entries()) { + const invitation = access.createOrganization(owner.session, { idempotencyKey: `preview:updates:dsp:${index}`, + ownerEmail: `owner${index}@example.test`, name, stationCode: 'TST1', timezone: 'UTC' }); + store.db.prepare("UPDATE installations SET status='ready' WHERE organization_id=?").run(invitation.organization.id); + store.updateOrganizationStatus(invitation.organization.id, 'active', Date.now()); + owners.push(await access.acceptNewUser({ token: invitation.token, firstName: 'DSP', lastName: 'Owner', password, confirmPassword: password })); + dsps.push(store.installationControl(invitation.organization.id).runtimeKey); + } + let failNext = false, events = []; + const hooks = { drain: async c => { events.push(['drain', c.product, c.dspId]); }, + snapshot: async () => ({ synthetic: true }), start: async c => { + if(versionedDashboards && c.product==='dsp')require('../../core/installations/src/release-delivery-files').atomic(require('../../host/releases/runtime').fileFor({local:path.join(root,'local')},c.dspId),{schemaVersion:1,digest:c.digest}); + }, restore: async c => { events.push(['restore', c.dspId]); }, + verify: async c => { if (failNext && c.dspId === [...dsps.slice(1)].sort()[0]) { failNext = false; return false; } return true; } }; + const releases = new LocalReleases({ directory: path.join(root, 'local/state/updates'), devDspId: dsps[0], hooks, allowDevelopment: true }); + async function publish(product, version) { + const directory = path.join(root, `${product}-${version}`); fs.mkdirSync(directory, { mode: 0o700 }); + fs.mkdirSync(path.join(directory, 'code')); fs.writeFileSync(path.join(directory, 'code/version.json'), JSON.stringify({ version })); + fs.writeFileSync(path.join(directory, 'release-notes.md'), product === 'core' + ? 'Independent platform updates\n\n• Manage Core releases separately from DSP releases.\n• See installation progress and recover interrupted updates.\n• Keep installed DSP runtimes and plugins unchanged.' + : 'A better Paycom experience\n\n• Keep each DSP’s settings and credentials independent.\n• Display employee names as First Last by default.\n• Test plugin improvements on Dev before updating your fleet.'); + if(versionedDashboards && product==='dsp'){ + fs.cpSync(path.resolve(__dirname,'../../../dsp/dashboard/public'),path.join(directory,'dashboard'),{recursive:true}); + if(version!=='0.0.1'){ + const file=path.join(directory,'dashboard/assets/frontend.js'); + fs.writeFileSync(file,fs.readFileSync(file,'utf8').replaceAll('Currently under development','Dev release preview')); + } + } + const manifest = { schemaVersion: 1, product, version, channel: 'development', protocol: 1, minimumProtocol: 1, + sourceDigest: 'a'.repeat(64), plugins: [], files: inventory(directory) }; + fs.writeFileSync(path.join(directory, 'release.json'), JSON.stringify(manifest)); + const digest = hash(JSON.stringify(manifest)); await releases.stage(directory, digest); + const state=releases.state(); + if(state.latest.core && state.latest.dsp){ + const entry={version,components:{core:{digest:state.latest.core},dsp:{digest:state.latest.dsp}}, + changes:{core:product==='core' ? 'Platform Owner dashboard and service improvements.' : 'No changes.',dsp:'DSP dashboard and runtime improvements.',plugins:'Paycom connection improvements.'},url:'https://example.test/releases/'+version}; + const history=(state.platform?.history||[]).filter(row=>row.version!==version);history.push(entry); + state.platform={latest:version,history};releases.save(state); + } + return digest; + } + const core = await publish('core', '0.0.1'), dsp = await publish('dsp', '0.0.1'); + const state = releases.state(); state.active = { core, dsps: Object.fromEntries(dsps.map(id => [id, dsp])) }; releases.save(state); + if(versionedDashboards){ + const {privateDirectory}=require('../../host/controller/operations'),{atomic}=require('../../core/installations/src/release-delivery-files'); + for(const id of dsps){const file=require('../../host/releases/runtime').fileFor({local:path.join(root,'local')},id);privateDirectory(path.dirname(file));atomic(file,{schemaVersion:1,digest:dsp});} + } + await publish('core', '0.0.2'); await publish('dsp', '0.0.2'); + const commands = new UpdateCommands(path.join(root, 'local/state/updates')); + const worker = new UpdateWorker({ releases, commands, feed: { refresh: async product => releases.state().latest[product] }, + authorize: actor => { const row = store.userById(actor); if (row?.platform_role !== 'owner' || row.status !== 'active') throw new Error('release_actor_forbidden'); }, + invoke: async (action, input) => { + if (action === 'update_dev') await releases.updateDev(input.digest); + else if (action === 'rollout') await releases.beginRollout(input.digest, input.targets, input.actor); + else if (action === 'step') await releases.step(); + else if (action === 'pause') await releases.pause(); + else if (action === 'resume') await releases.resume(); + else if (action === 'recover') await releases.recover(); + else throw new Error('release_command_invalid'); + } }); + await worker.initialize(); + const updates = createUpdatesService({ releases, commands, store, devDspId: dsps[0] }); + const unavailable = async () => ({ ok: false, status: 'installation_not_ready', data: null, error: { code: 'installation_not_ready' } }); + const server = createDashboardServer({ access, updates, ...(versionedDashboards ? {dashboards:require('../../core/updates/dashboard').dashboardProvider({paths:{local:path.join(root,'local')},store})} : {}), plugins: { catalog: () => ({ items: [] }) }, + client: { workforce: { day: unavailable }, sync: { status: unavailable, runNow: unavailable }, system: { status: unavailable } } }); + const original = server.listeners('request')[0]; server.removeAllListeners('request'); + server.on('request', async (request, response) => { + if (request.method === 'POST' && ['/__fixture/publish', '/__fixture/fail'].includes(request.url)) { + if (request.url.endsWith('/publish')) await publish('dsp', '0.0.3'); else failNext = true; + response.writeHead(200, { 'Content-Type': 'application/json' }); response.end('{}'); return; + } + original(request, response); + }); + await new Promise(resolve => server.listen(port, '127.0.0.1', resolve)); + const timer = automatic && setInterval(() => { commands.heartbeat(); worker.tick().catch(() => {}); }, 150); + return { root, store, access, owner, owners, dsps, releases, commands, worker, events, publish, server, + url: `http://127.0.0.1:${server.address().port}`, + async close() { clearInterval(timer); await worker.close(); await new Promise(resolve => server.close(resolve)); store.close(); fs.rmSync(root, { recursive: true, force: true }); } }; +} +if (require.main === module) { + if (process.env.DISPATCH_INDEPENDENT_UPDATES_FIXTURE !== '1') throw new Error('fixture_opt_in_required'); + createPreview({ port: Number(process.env.DISPATCH_UPDATES_PREVIEW_PORT || 0), versionedDashboards:process.env.DISPATCH_VERSIONED_DASHBOARD_FIXTURE==='1' }).then(app => { + const close = () => app.close().catch(() => { process.exitCode = 1; }); + process.once('SIGTERM', close); process.once('SIGINT', close); + process.stdout.write(`Synthetic updates preview: ${app.url}\n`); + }).catch(error => { process.stderr.write(error.stack + '\n'); process.exitCode = 1; }); +} +module.exports = { createPreview }; diff --git a/core/dashboard/examples/onboarding-preview.js b/core/dashboard/examples/onboarding-preview.js new file mode 100644 index 0000000..9ebbfe9 --- /dev/null +++ b/core/dashboard/examples/onboarding-preview.js @@ -0,0 +1,43 @@ +#!/usr/bin/env node +'use strict'; +// Disposable UI acceptance only. Provider transport is synthetic; it never authenticates Paycom. +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { AccessStore, AccessControlService } = require('../../core/accounts/src'); +const { createOwnerPaycomSetup } = require('../../core/accounts/src/owner-paycom-setup'); +const { success, failure } = require('../../shared/contracts/src'); +const { createDashboardServer } = require('../server/server'); +async function main() { + if (process.env.DISPATCH_ONBOARDING_UI_FIXTURE !== '1') throw new Error('explicit_fixture_opt_in_required'); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-onboarding-ui-')); fs.chmodSync(root, 0o700); + let store; + try { + store = new AccessStore({ databaseRoot: path.join(root, 'access'), database: path.join(root, 'access', 'access-control.sqlite3') }); + const access = new AccessControlService(store, { installationOperatorEnabled: true, installationBackend: 'oci_container_v1' }); + const platformInvite = access.createPlatformBootstrap({ email: 'platform@example.test' }); + const password = 'synthetic preview password'; + const platform = await access.acceptNewUser({ token: platformInvite.token, firstName: 'Platform', lastName: 'Tester', password, confirmPassword: password }); + const dsp = access.createOrganization(platform.session, { idempotencyKey: 'preview:create:paycom', name: 'Paycom Preview DSP', + abbreviation: 'PREVIEW', stationCode: 'TST1', timezone: 'America/Chicago', ownerEmail: 'dsp@example.test' }); + await access.acceptNewUser({ token: dsp.token, firstName: 'DSP', lastName: 'Tester', password, confirmPassword: password }); + store.updateInstallationControl({ organizationId: dsp.organization.id, expectedStatus: 'pending', expectedRevision: 1, + status: 'waiting_for_provider_auth', revision: 2, currentJobId: null, timestamp: Date.now() }); + const unavailable = async () => failure('installation_not_ready'); + const client = { workforce: { day: unavailable }, sync: { status: unavailable, runNow: unavailable }, system: { status: unavailable } }; + const paycomSetup = createOwnerPaycomSetup({ store, access, + invoke: async () => success('succeeded', { configured: true }), + }); + const server = createDashboardServer({ client, access, paycomSetup }); + const cleanup = () => server.close(() => { store.close(); fs.rmSync(root, { recursive: true }); process.exit(0); }); + process.once('SIGINT', cleanup); process.once('SIGTERM', cleanup); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(4327, '127.0.0.1', resolve); + }); + process.stdout.write('Synthetic onboarding UI: http://127.0.0.1:4327\n'); + } catch (error) { + store?.close(); fs.rmSync(root, { recursive: true, force: true }); throw error; + } +} +main().catch(() => { process.stderr.write('onboarding_ui_fixture_failed\n'); process.exitCode = 1; }); diff --git a/core/dashboard/examples/paycom-workforce-fixture.js b/core/dashboard/examples/paycom-workforce-fixture.js new file mode 100644 index 0000000..e5dd577 --- /dev/null +++ b/core/dashboard/examples/paycom-workforce-fixture.js @@ -0,0 +1,28 @@ +'use strict'; +// Synthetic workforce for the explicitly opted-in frontend preview only. +const { WorkforceClient } = require('dispatch-dsp/runtime/sdk/src/workforce-client.js'); +const { LocalPaycomWorkforcePort } = require('dispatch-dsp/plugins/paycom/backend/adapters/workforce.js'); +const { sourceDate } = require('../server/server'); +const timezone = 'America/Chicago'; +const today = sourceDate(new Date(), timezone); +const shift = (date, n) => { const d = new Date(date + 'T12:00:00Z'); d.setUTCDate(d.getUTCDate()+n); return d.toISOString().slice(0,10); }; +const start = shift(today, -7), end = shift(today, 6), collected = new Date().toISOString(); +const names = ['Mia Thompson','Ethan Rivera','Sofia Chen','Jordan Ellis','Noah Patel','Olivia Brooks', ...Array.from({length:99},(_,i)=>`Team member ${String(i+1).padStart(3,'0')}`), 'Zulu Avery']; +if (process.env.DISPATCH_PAYCOM_NAME_ORDER_FIXTURE === '1') { + // Canonical provider spelling for name-order browser acceptance only. + names.splice(0, 6, 'THOMPSON, MIA', 'RIVERA, ETHAN', 'CHEN, SOFIA', 'ELLIS, JORDAN', 'PATEL, NOAH', 'BROOKS, OLIVIA'); + for (let i = 6; i < names.length - 1; i++) names[i] = `TEAM ${String(i - 5).padStart(3, '0')}, MEMBER`; + names[names.length - 1] = 'AVERY, ZULU'; +} +const employees = names.map((name,i)=>({ employeeCode: `W${String(i).padStart(3,'0')}`, employeeName:name, + lifecycleStatus:'active', isActive:true, isDriverDepartment:true, departmentCode:i<100?'D1':'D2',departmentDesc:i<100?'Driver':'Dispatch', + deliveryStationCode:'DXY1',deliveryStationDesc:'Station',positionTitle:'Delivery Driver',payClass:'Hourly',payType:'Hourly',primarySupervisor:'Avery Brooks' })); +const roster = { publication:{target:end,collected_at:collected},employees }; +const resourceLinks = { publication:{period_key:`${start}_${end}`,resource_type:'paycom.timecard.summary',collected_at:collected}, rows:employees.map(e=>({employeeCode:e.employeeCode,canonicalUrl:`https://www.paycomonline.net/v4/cl/web.php/timecard/index?firstrefno=${e.employeeCode}&perioddates=${start}_${end}&formtype=SUMMARY`})) }; +const timecards = { publication:{collected_at:collected}, rows:employees.map((e,i)=>({employeeCode:e.employeeCode,employeeName:e.employeeName,observedAt:collected, record:{periodStart:start,periodEnd:end,periodTotalHours:40,days:Array.from({length:14},(_,d)=>{ + const date=shift(start,d), active=date<=today && i!==5; + const punch=(kind,time)=>({kind,displayTime:time,actualTime:time,provenanceAvailable:true}); + return {date,missingPunch:active&&i===3,totalHours:active?(i===3?null:date===today?2:10):null,unresolvedSlots:i===3?['o1']:[],punches:active?[punch('IN DAY',i===1?'08:00 AM':'09:00 AM'),...(i===3?[]:[punch('OUT LUNCH','01:00 PM')]),punch('IN LUNCH','01:30 PM'),...(date({activeWorkforce:()=>({roster,timecards,resourceLinks}),close(){}})})}); +module.exports.fixtureData={roster,timecards,resourceLinks}; diff --git a/core/dashboard/examples/popup-changelog.json b/core/dashboard/examples/popup-changelog.json new file mode 100644 index 0000000..b8ff493 --- /dev/null +++ b/core/dashboard/examples/popup-changelog.json @@ -0,0 +1,52 @@ +{ + "groups": [ + { + "id": "backups", + "title": "Backups", + "icon": "database" + }, + { + "id": "dashboard-team", + "title": "Dashboard & team", + "icon": "users" + } + ], + "changelog": [ + { + "kind": "changed", + "title": "Independent backup schedules", + "description": "Set separate schedules for Core and each DSP.", + "group": "backups", + "icon": "calendar-clock", + "details": "Platform owners can choose the frequency, time zone, and retention policy for Core, each DSP, and full-system backups independently. This keeps backup timing aligned with how your platform is used.", + "audience": "platform", + "popup": { + "title": "Choose backup schedules by scope", + "description": "Set the schedules that fit your Core and DSP workloads." + } + }, + { + "kind": "added", + "title": "Dashboard activity overview", + "description": "See recent activity across your DSP workspace at a glance.", + "group": "dashboard-team", + "icon": "chart-column", + "details": "The activity overview brings recent workspace events into one place, so DSP owners can quickly understand what changed without opening each area of the dashboard.", + "audience": "dsp", + "popup": { + "title": "See activity at a glance", + "description": "Review recent workspace events from one dashboard view." + } + }, + { + "kind": "fixed", + "title": "Team search finds every member", + "description": "Find teammates quickly by searching their name or email.", + "group": "dashboard-team", + "icon": "users", + "details": "Team search now checks the full member list and matches names and email addresses consistently, making it easier to find the person you need.", + "audience": "dsp" + } + ], + "afterUpdating": [] +} diff --git a/core/dashboard/examples/updates-preview.js b/core/dashboard/examples/updates-preview.js new file mode 100644 index 0000000..8362f9f --- /dev/null +++ b/core/dashboard/examples/updates-preview.js @@ -0,0 +1,47 @@ +#!/usr/bin/env node +'use strict'; +// Local UI fixture. Core and DSP execution are simulated; no host changes or emails. +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { AccessStore, AccessControlService } = require('../../core/accounts/src'); +const { createPlatformUpdates } = require('../../core/accounts/src/platform-updates'); +const { createDashboardServer } = require('../server/server'); +async function main() { + if (process.env.DISPATCH_UPDATES_UI_FIXTURE !== '1') throw new Error(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-updates-ui-')); fs.chmodSync(root, 0o700); + const store = new AccessStore({ databaseRoot: path.join(root, 'access'), database: path.join(root, 'access/access-control.sqlite3') }); + const access = new AccessControlService(store, { installationOperatorEnabled: true, installationBackend: 'oci_container_v1' }); + const invite = access.createPlatformBootstrap({ email: 'platform@example.test' }); + const password = 'synthetic preview password'; + const owner = await access.acceptNewUser({ token: invite.token, firstName: 'Platform', lastName: 'Owner', password, confirmPassword: password }); + for (const [index, name] of ['Northstar Delivery', 'Summit Logistics', 'Riverbend Delivery', 'Atlas Delivery', 'Horizon Logistics', 'Cedar Delivery'].entries()) { + const dsp = access.createOrganization(owner.session, { idempotencyKey: `preview:updates:${index}`, ownerEmail: `dsp${index}@example.test`, name, stationCode: 'TST1', timezone: 'UTC' }); + store.db.prepare("UPDATE installations SET status='ready' WHERE organization_id=?").run(dsp.organization.id); + store.updateOrganizationStatus(dsp.organization.id, 'active', Date.now()); + } + const releaseOptions = { releases: { dispatch_preview_2: {} }, enabled: true, + platformReleases: { dispatch_preview_2: { version: '0.0.2', publishedAt: '2026-09-05T00:00:00.000Z', core: {}, + changelog: [ + { kind: 'added', title: 'A dedicated Platform Owner workspace', description: 'Manage DSPs, updates and platform settings in one place.' }, + { kind: 'improved', title: 'Create a DSP with just an email address', description: 'Owners complete their DSP details after accepting the invitation.' }, + { kind: 'fixed', title: 'Clearer setup error messages', description: 'See what needs attention when a request cannot be completed.' }, + ] } } }; + const delivery = process.env.DISPATCH_RELEASE_DELIVERY_UI_FIXTURE === '1' + ? require('../server/release-delivery').createReleaseDelivery(root) : null; + if (delivery) { + fs.mkdirSync(path.join(root, 'config'), { mode: 0o700 }); + require('../../core/installations/src/release-delivery-files').atomic(path.join(root, 'config/release-delivery-status.json'), + { state: 'preparing', version: '0.0.2', retryable: false, changelog: releaseOptions.platformReleases.dispatch_preview_2.changelog }); + } + const updates = createPlatformUpdates({ store, ...releaseOptions, delivery, + loadCatalogs: delivery ? () => delivery.view()?.state === 'ready' ? releaseOptions : { releases: {}, platformReleases: {} } : null }); + const unavailable = async () => ({ ok: false, status: 'installation_not_ready', data: null, error: { code: 'installation_not_ready' } }); + const client = { workforce: { day: unavailable }, sync: { status: unavailable, runNow: unavailable }, system: { status: unavailable } }; + const server = createDashboardServer({ client, access, updates }); + const cleanup = () => server.close(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); process.exit(0); }); + process.once('SIGTERM', cleanup); process.once('SIGINT', cleanup); + await new Promise(resolve => server.listen(4328, '127.0.0.1', resolve)); + process.stdout.write(`Synthetic updates preview: http://127.0.0.1:4328\nFixture database: ${root}/access/access-control.sqlite3\n`); +} +main().catch(() => { process.stderr.write('updates_preview_failed\n'); process.exitCode = 1; }); diff --git a/core/dashboard/frontend/INTER-LICENSE.txt b/core/dashboard/frontend/INTER-LICENSE.txt new file mode 100644 index 0000000..40589da --- /dev/null +++ b/core/dashboard/frontend/INTER-LICENSE.txt @@ -0,0 +1,93 @@ +Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter) Inter-Italic[opsz,wght].ttf: Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/core/dashboard/frontend/src/Workspace.tsx b/core/dashboard/frontend/src/Workspace.tsx new file mode 100644 index 0000000..189f424 --- /dev/null +++ b/core/dashboard/frontend/src/Workspace.tsx @@ -0,0 +1,23 @@ +import { Building2, ArrowUpFromLine, Database, Puzzle, FlaskConical, Settings as SettingsIcon } from "lucide-react"; +import { Dsps } from "./pages/Dsps"; +import { Diagnostics } from "./pages/Diagnostics"; +import { Updates } from "./pages/Updates"; +import { ManagedPage } from "./pages/ManagedPage"; +import { Settings } from "./pages/Settings"; +import { Plugins } from "./pages/Plugins"; +import type { NavItem } from "./App"; +export function workspaceRoutes(..._args: unknown[]): NavItem[] { return [ + {id:"platform",label:"DSPs",icon:Building2}, {id:"updates",label:"Updates",icon:ArrowUpFromLine}, + {id:"backups",label:"Backups",icon:Database}, {id:"plugins",label:"Plugins",icon:Puzzle}, + {id:"diagnostics",label:"Diagnostics",icon:FlaskConical}, {id:"platform-settings",label:"Settings",icon:SettingsIcon} +]; } +export function Workspace({route,hash}: {route:string;hash:string;installed:unknown[]}) { + switch(route) { + case "platform": return ; + case "diagnostics": return ; + case "updates": return ; + case "backups": return ; + case "plugins": return ; + default: return ; + } +} diff --git a/core/dashboard/frontend/src/pages/Diagnostics.tsx b/core/dashboard/frontend/src/pages/Diagnostics.tsx new file mode 100644 index 0000000..98989c8 --- /dev/null +++ b/core/dashboard/frontend/src/pages/Diagnostics.tsx @@ -0,0 +1,128 @@ +import { useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { FlaskConical } from "lucide-react"; +import { idempotent, queryClient, request } from "@/lib/api"; +import { Button } from "@/components/ui/button"; +import { PageHeading, ErrorNotice, Loading, Notice } from "@/components/shared"; + +type DiagnosticsView = { + enabled: boolean; + dsps: { + name: string; + createdAt: string; + status: string; + installation: { state: string }; + }[]; +}; + +export function Diagnostics() { + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const data = useQuery({ + queryKey: ["platform-diagnostics"], + queryFn: () => request("/api/platform/diagnostics"), + refetchInterval: 5000, + }); + const runtime = useQuery({ + queryKey: ["platform-runtime"], + queryFn: () => request<{ enabled: boolean; storageAvailableBytes: number | null; + runtimes: { reference: string; name: string; status: string; memoryBytes: number | null; memoryLimitBytes: number | null; tasks: number | null; + storage: { limited: boolean | null; capacityBytes: number | null; availableBytes: number | null } }[] + }>("/api/platform/runtime"), + refetchInterval: 5000, + }); + async function deploy() { + if (busy) return; + setBusy(true); + setError(null); + try { + const next = await idempotent( + "diagnostics-create", + "/api/platform/diagnostics", + {}, + ); + queryClient.setQueryData(["platform-diagnostics"], next); + await queryClient.invalidateQueries({ queryKey: ["fleet"] }); + } catch (error) { + setError(error); + } finally { + setBusy(false); + } + } + return ( + <> + + + {runtime.data?.enabled && ( +
+

Runtime health

+

+ Available storage: {((runtime.data.storageAvailableBytes ?? 0) / 1024 ** 3).toFixed(1)} GiB +

+ {runtime.data.runtimes.map((item) => ( +
+ {item.name} + {item.status} · {item.memoryBytes === null ? "—" : `${Math.round(item.memoryBytes / 1024 ** 2)} MiB`} · {item.tasks ?? 0} tasks + {item.storage?.limited ? ` · ${((item.storage.availableBytes ?? 0) / 1024 ** 3).toFixed(1)} GiB storage free` + : item.storage?.limited === false ? " · Storage limit pending migration" : " · Storage unavailable"} + +
+ ))} + {!runtime.data.runtimes.length &&

No DSP runtimes yet.

} + + )} + {data.isPending ? ( + + ) : data.data ? ( +
+
+

+ Test DSP +

+

+ Deploy a DSP with synthetic employees and timecards. It stays + available until you delete it from DSPs. Provider collection stays + stopped, and no invitation email is sent. +

+ + {!data.data.enabled ? ( + + Test DSP deployment is unavailable on this installation. + + ) : null} +
+
+ {data.data.dsps.map((dsp) => ( +
+

{dsp.name}

+

+ {dsp.status === "pending" + ? "Creating DSP and preparing synthetic data…" + : dsp.status === "failed" + ? "Setup needs attention. Open DSPs to inspect or delete this test DSP." + : `Synthetic data prepared · ${dsp.installation.state === "ready" ? "Available" : dsp.installation.state}`} +

+
+ ))} +
+ + Manage test DSPs in DSPs + +
+ ) : null} + + ); +} diff --git a/core/dashboard/frontend/src/pages/Dsps.tsx b/core/dashboard/frontend/src/pages/Dsps.tsx new file mode 100644 index 0000000..c13f05c --- /dev/null +++ b/core/dashboard/frontend/src/pages/Dsps.tsx @@ -0,0 +1,567 @@ +import { DspAvatar } from "@/components/DspAvatar"; +import { useState, type FormEvent } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { Plus, Ellipsis, ArrowUpRight, Eye } from "lucide-react"; +import { request, idempotent, mutation, setDspView } from "@/lib/api"; +import { useSession } from "@/lib/session"; +import type { FleetOrganization, InvitationResult, Session } from "@/lib/types"; +import { Button } from "@/components/ui/button"; +import { FieldGroup } from "@/components/ui/field"; +import { + Table, + TableHeader, + TableBody, + TableHead, + TableRow, + TableCell, +} from "@/components/ui/table"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { + DropdownMenu, + DropdownMenuTrigger, + DropdownMenuContent, + DropdownMenuGroup, + DropdownMenuItem, +} from "@/components/ui/dropdown-menu"; +import { + PageHeading, + RefreshButton, + SearchInput, + Status, + Loading, + EmptyState, + Panel, + TextField, + ErrorNotice, + InvitationNotice, + SubmitButton, + ConfirmAction, + Notice, +} from "@/components/shared"; +const deleting = (o: FleetOrganization) => + o.installation.operation?.kind === "destroy"; +const removed = (o: FleetOrganization) => + deleting(o) || + o.installation.operation?.kind === "restore_dsp" || + ["decommissioning", "decommissioned"].includes(o.installation.state) || + o.installation.operation?.kind === "decommission"; +const running = (o: FleetOrganization) => + !removed(o) && + !deleting(o) && + o.organizationStatus === "active" && + o.installation.state === "ready"; +const onboarding = (o: FleetOrganization) => + !removed(o) && + !deleting(o) && + !running(o) && + o.organizationStatus !== "suspended"; +const labels: Record = { + ready: "Running", + pending: "Queued", + provisioning: "Creating", + waiting_for_owner: "Prepared", + waiting_for_provider_auth: "Prepared", + verifying: "Verifying", + failed: "Needs attention", + suspended: "Suspended", + decommissioning: "Removing", + decommissioned: "Removed", +}; +const actionLabels: Record = { + provision: "Start provisioning", + retry_provision: "Retry provisioning", + decommission: "Remove DSP", + destroy: "Permanently delete DSP", + restore_dsp: "Restore DSP", + suspend: "Suspend DSP", + resume: "Resume DSP", + restart: "Restart runtime", + revoke_owner_invitation: "Revoke invitation", + issue_owner_invitation: "Invite owner", +}; +const name = (o: FleetOrganization) => + o.detailsStatus === "required" ? o.ownerEmail || "New DSP" : o.name; +function onboardingLabel(o: FleetOrganization) { + return deleting(o) || removed(o) + ? "Closed" + : o.ownerStatus === "pending" + ? "Invitation pending" + : o.ownerStatus === "missing" + ? "Invite needed" + : o.detailsStatus !== "complete" + ? "DSP details needed" + : ["ready", "suspended"].includes(o.installation.state) + ? "Complete" + : "Finishing setup"; +} +function runtimeLabel(o: FleetOrganization) { + return deleting(o) + ? o.installation.operation?.status === "failed" + ? "Deletion failed" + : "Deleting" + : o.installation.operation?.kind === "restore_dsp" + ? o.installation.operation.status === "failed" + ? "Restore failed" + : "Restoring" + : labels[o.installation.state] || "Unavailable"; +} + +export function Dsps() { + const { refresh } = useSession(); + const fleet = useQuery({ + queryKey: ["fleet"], + queryFn: ({ signal }) => + request("/api/platform/organizations", { signal }), + refetchInterval: 5000, + }); + const [search, setSearch] = useState(""); + const [filter, setFilter] = useState("all"); + const [create, setCreate] = useState(false); + const [selected, setSelected] = useState(null); + const [action, setAction] = useState<{ + org: FleetOrganization; + kind: string; + } | null>(null); + const [inviteOrg, setInviteOrg] = useState(null); + const [result, setResult] = useState(null); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [notice, setNotice] = useState(""); + const organizations = fleet.data || []; + const visible = organizations.filter( + (o) => + (filter === "removed" + ? removed(o) + : filter === "running" + ? running(o) + : filter === "onboarding" + ? onboarding(o) + : !removed(o)) && + `${o.name} ${o.abbreviation || ""} ${o.ownerEmail || ""} ${o.stations.map((s) => s.code).join(" ")}` + .toLowerCase() + .includes(search.trim().toLowerCase()), + ); + const detail = fleet.error + ? undefined + : organizations.find((o) => o.continuityRef === selected); + const canView = (o: FleetOrganization) => + !removed(o) && !deleting(o) && o.organizationStatus !== "suspended"; + async function view(o: FleetOrganization) { + setBusy(true); + setError(null); + try { + const viewed = await mutation( + "/api/platform/organization/view", + "POST", + { controlRef: o.controlRef }, + ); + setDspView(viewed.dspView!.viewRef); + await refresh(viewed); + location.hash = + viewed.memberships[0].organization.status === "active" + ? "#/dashboard" + : "#/team"; + } catch (err) { + setError(err); + } finally { + setBusy(false); + } + } + async function invite(e: FormEvent) { + e.preventDefault(); + setBusy(true); + setError(null); + const ownerEmail = new FormData(e.currentTarget).get("ownerEmail"); + try { + const value = await idempotent( + inviteOrg ? `${inviteOrg.continuityRef}:invite` : "organization:create", + inviteOrg + ? "/api/platform/organization/owner-invitation" + : "/api/platform/organizations", + { + ownerEmail, + ...(inviteOrg ? { controlRef: inviteOrg.controlRef } : {}), + }, + ); + setResult(value); + setCreate(false); + setInviteOrg(null); + await fleet.refetch(); + } catch (err) { + setError(err); + } finally { + setBusy(false); + } + } + async function runAction(password?: string) { + if (!action) return; + const { org, kind } = action; + const latest = organizations.find( + (o) => o.continuityRef === org.continuityRef, + ); + if (!latest) throw Error("DSP unavailable"); + const slot = `${org.continuityRef}:${kind}`; + try { + if (kind === "revoke_owner_invitation") + await idempotent( + slot, + "/api/platform/organization/owner-invitation/revoke", + { controlRef: latest.controlRef }, + ); + else + await idempotent( + slot, + `/api/platform/installation/${({ provision: "provision", retry_provision: "retry", decommission: "remove", destroy: "delete", restore_dsp: "restore", suspend: "suspend", resume: "resume", restart: "restart" } as Record)[kind]}`, + { + controlRef: latest.controlRef, + expectedRevision: latest.installation.revision, + ...(kind === "destroy" ? { password } : {}), + }, + ); + setNotice(`${org.name}: request accepted.`); + } finally { + await fleet.refetch(); + } + } + function actions(o: FleetOrganization) { + return [...o.installation.availableActions, ...o.availableActions].filter( + (a) => + actionLabels[a] && + (!["destroy", "restore_dsp"].includes(a) || filter === "removed"), + ); + } + function pick(o: FleetOrganization, kind: string) { + setSelected(null); + setError(null); + if (kind === "issue_owner_invitation") setInviteOrg(o); + else setAction({ org: o, kind }); + } + return ( + <> + + + +
+ + {organizations.filter((o) => !removed(o)).length}{" "} + DSPs + + + {organizations.filter(running).length} running + + + {organizations.filter(onboarding).length} onboarding + +
+ + {notice && {notice}} + + + {[ + ["all", "All DSPs"], + ["running", "Running"], + ["onboarding", "Onboarding"], + ["removed", "Removed"], + ].map(([key, label]) => ( + + {label} + + ))} + + +
+ + void fleet.refetch()} + busy={fleet.isFetching} + /> +
+ + + {fleet.isPending ? ( + + ) : fleet.error ? null : ( + <> + + + + DSP + Owner + Runtime + Onboarding + + Actions + + + + + {visible.map((o) => ( + + + + + + {o.ownerEmail || "No owner assigned"} + + + + {runtimeLabel(o)} + + + + + {onboardingLabel(o)} + + + + + + + + + + void view(o)} + > + View + + {actions(o).map((a) => ( + pick(o, a)} + > + {actionLabels[a]} + + ))} + + + + + + ))} + +
+ {!visible.length && ( + + )} +

+ {visible.length} DSP{visible.length !== 1 ? "s" : ""} +

+ + )} + { + setCreate(false); + setInviteOrg(null); + }} + title={inviteOrg ? "Invite DSP owner" : "Create new DSP"} + description="Invite an owner. Their workspace will be prepared while they finish setup." + busy={busy} + > +
+ + + + +
+ + + {inviteOrg ? "Create invitation" : "Create DSP"} + +
+
+
+ setSelected(null)} + title={ + detail ? ( + + + {name(detail)} + + ) : ( + "DSP details" + ) + } + description="DSP ownership and runtime status." + > + {detail && ( +
+
+
+
Owner
+
{detail.ownerEmail || "Not assigned"}
+
+
+
Runtime
+
+ + {runtimeLabel(detail)} + +
+
+
+
Onboarding
+
{onboardingLabel(detail)}
+
+
+
Station
+
{detail.stations.map((s) => s.code).join(", ") || "—"}
+
+
+
Timezone
+
{detail.timezone || "—"}
+
+
+ {Boolean(detail.installation.failure) && ( + + This DSP needs attention. Review its setup or retry the failed + operation. + + )} +
+ + {!canView(detail) && ( +

+ Viewing is unavailable for suspended or removed DSPs. +

+ )} + + {actions(detail).map((a) => ( + + ))} +
+
+ )} +
+ {action && ( + setAction(null)} + onConfirm={runAction} + /> + )} + + ); +} diff --git a/core/dashboard/frontend/src/pages/ManagedPage.tsx b/core/dashboard/frontend/src/pages/ManagedPage.tsx new file mode 100644 index 0000000..82091b9 --- /dev/null +++ b/core/dashboard/frontend/src/pages/ManagedPage.tsx @@ -0,0 +1,100 @@ +import { useEffect, useMemo, useState } from "react"; +import { mutation, mutationKey, request, settleMutationKey } from "@/lib/api"; +import { errorMessage } from "@/lib/errors"; +import { dspIdentity } from "@/lib/identity"; +import { useTimezone } from "@/lib/timezone"; +import { PageHeading, RefreshButton, Notice } from "@/components/shared"; +const byId = (id: string) => document.getElementById(id); +const node = (tag: string, className?: string | null, text?: unknown) => { + const element = document.createElement(tag); + if (className) element.className = className; + if (text !== undefined) element.textContent = String(text); + return element; +}; +const dependencies = { + timeZone: undefined as string | undefined, + dspIdentity, + byId, + node, + mutation, + request, + mutationKey, + settleMutationKey, + errorMessage, +}; +declare global { + interface Window { + createUpdatesViews: (deps: typeof dependencies) => { + renderUpdates: () => Promise; + setUpdatesActive: (active: boolean) => void; + }; + createBackupsViews: (deps: typeof dependencies) => { + renderBackups: () => Promise; + setBackupsActive: (active: boolean) => void; + }; + showToast?: (title: string, detail?: string, type?: string) => void; + } +} +// The controllers own only these empty DOM islands. React owns the shell and +// their lifetime; the existing polling and recovery state machines stay intact. +export function ManagedPage({ + page, + hash, +}: { + page: "updates" | "backups"; + hash: string; +}) { + const { timeZone } = useTimezone(); + const controller = useMemo( + () => + page === "updates" + ? window.createUpdatesViews({ ...dependencies, timeZone }) + : window.createBackupsViews({ ...dependencies, timeZone }), + [page, timeZone], + ); + const [notice, setNotice] = useState<{ text: string; error: boolean } | null>( + null, + ); + const render = () => + "renderUpdates" in controller + ? controller.renderUpdates() + : controller.renderBackups(); + useEffect(() => { + window.showToast = (title, detail, type) => + setNotice({ + text: [title, detail].filter(Boolean).join(". "), + error: type === "error", + }); + if ("setUpdatesActive" in controller) controller.setUpdatesActive(true); + else controller.setBackupsActive(true); + void render(); + return () => { + if ("setUpdatesActive" in controller) controller.setUpdatesActive(false); + else controller.setBackupsActive(false); + delete window.showToast; + }; + }, [controller, hash]); + return ( + <> + {page === "updates" && ( + + void render()} /> + + )} + {notice && {notice.text}} +
+ + ); +} diff --git a/core/dashboard/frontend/src/pages/Updates.tsx b/core/dashboard/frontend/src/pages/Updates.tsx new file mode 100644 index 0000000..a0760ab --- /dev/null +++ b/core/dashboard/frontend/src/pages/Updates.tsx @@ -0,0 +1,132 @@ +import { useEffect, useRef, useState } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { ArrowUpCircle, FlaskConical, Pause, Play, RefreshCw, Server } from "lucide-react"; +import { idempotent, queryClient, request } from "@/lib/api"; +import { Button } from "@/components/ui/button"; +import { ErrorNotice, Loading, Notice, PageHeading } from "@/components/shared"; +import { ManagedPage } from "./ManagedPage"; + +type Product = "core" | "dsp"; +type Release = { id: string; digest: string; version: string; notes: string; publishedAt: string | null; + url: string | null; source: { commit: string } | null }; +type Track = { installedLegacy?: boolean; latest: string | null; installedVersion: string | null; installedDigest: string | null; + release: Release | null; history: { id: string; version: string }[]; tested: boolean; canUpdate: boolean }; +type UpdatesView = { platformRelease?: {version:string;changes:Record<"core"|"dsp"|"plugins",string>;url:string}; platformHistory?: {id:string;version:string}[]; latestPlatform?: string; mode: string; enabled: boolean; busy: boolean; worker: { available: boolean; status: string }; + dev: { name: string; available: boolean }; recoveryRequired: boolean; + operation: { product: Product; phase: string; dspName: string | null } | null; + tracks: Record; + rollout: { version: string; status: string; failure: string | null; updated: number; total: number; + members: { name: string; status: string }[] } | null; + jobs: { id: string; action: string; product: Product; status: string; failure: string | null }[] }; +const failureText: Record = { + release_changed: "A newer release is available. Review it and try again.", + release_dev_required: "Install the latest release on Dev before starting rollout.", + release_dsp_not_ready: "This DSP needs to be running and finish setup before it can be updated.", + release_health_failed: "Health checks failed. The previous version was restored.", + release_recovery_required: "The update needs recovery before another update can start.", + release_interrupted: "The update worker restarted. Review the state before continuing.", + release_baseline_required: "The installed version must be registered before updates can begin.", + release_fleet_changed: "The DSP list changed. Review it and start rollout again.", + release_verification_failed: "The release could not be verified. Check the worker’s GitHub connection and retry.", +}; +export function Updates({ hash }: { hash: string }) { + const [selected, setSelected] = useState(null); + const [sending, setSending] = useState(false); + const [error, setError] = useState(null); + const query = useQuery({ queryKey: ["independent-updates", selected], + queryFn: () => request(`/api/platform/updates${selected ? `?releaseId=${encodeURIComponent(selected)}` : ""}`), + refetchInterval: 3000 }); + const view = query.data; + const initialCore = useRef(undefined); + const coreDigest = view?.tracks?.core.installedDigest; + useEffect(() => { + if (view?.mode !== "independent") return; + const previous = initialCore.current; + initialCore.current = coreDigest; + if (previous !== undefined && coreDigest && previous !== coreDigest) window.location.reload(); + }, [view?.mode, coreDigest]); + if (view && view.mode !== "independent") return ; + async function command(action: string, digest: string | null = null, product: Product = "core") { + if (sending) return; + setSending(true); setError(null); + try { + await idempotent(`updates:${action}:${product}:${digest}`, "/api/platform/updates", { action, product, digest }); + await queryClient.invalidateQueries({ queryKey: ["independent-updates"] }); + } catch (cause) { setError(cause); } + finally { setSending(false); } + } + const activeJob = view?.jobs.find(job => ["queued", "running"].includes(job.status)); + const recentFailure = view?.jobs[0]?.status === "failed" ? view.jobs[0].failure : null; + const updatingCore = activeJob?.action === "update_core" || view?.operation?.product === "core"; + return <> + + + + + {updatingCore && Core is updating. This page will reconnect when it’s ready.} + {query.isPending ? : view ?
+ {!view.enabled && Updates need initial setup. Your current services will continue running.} + {view.enabled && !view.worker.available && The update worker is offline. Releases remain available to read.} + {recentFailure && {failureText[recentFailure] || "The update could not finish. Review the current state, then retry or recover."}} + {view.operation && !updatingCore && {view.operation.dspName || "Dev DSP"} is updating. Private data is being preserved.} + {view.recoveryRequired && !activeJob &&
+

Recover the interrupted update before installing another release.

+ +
} + {view.platformRelease &&
+

Release {view.platformRelease.version}

+ +
} +
+ {(["core", "dsp"] as const).map(trackName => { + const track = view.tracks[trackName], release = track.release; + const latestSelected = Boolean(release && release.digest === track.latest && (!view.platformRelease || view.platformRelease.version === view.latestPlatform)); + const rolling = view.rollout && view.rollout.status !== "completed"; + const action = trackName === "core" ? "update_core" : track.tested ? "rollout" : "update_dev"; + const label = trackName === "core" ? "Update Core" : track.tested ? "Rollout Update" : "Update Dev"; + return
+
+
+

{trackName === "core" ? "Core" : "DSP"}

+

{trackName === "core" ? "Installed" : `Installed on ${view.dev.name}`}: {track.installedVersion || "Not registered"}{track.installedLegacy ? " (legacy release)" : ""}

+ +
+

{trackName === "core" + ? "Updates the Platform Owner dashboard, shared API and Core services." + : track.tested ? "Dev has passed installation checks. Test the changes, then roll this version out to your DSPs one at a time." + : "Install this version on the permanent Dev DSP first. Your other DSPs receive it when you start rollout."}

+
+ {release ? <>
+

Version {release.version}

+ {release.url && View release on GitHub} +
{view.platformRelease ? view.platformRelease.changes[trackName] || "No changes." : release.notes}
+ {!latestSelected &&

You’re reading a previous release. Select the latest version to update.

} + :

No verified releases yet.

} +
+
+ {trackName === "dsp" && view.rollout &&
+

Rollout · {view.rollout.version}

+ {view.rollout.status === "running" && } + {view.rollout.status === "paused" && } +
+

{view.rollout.updated} of {view.rollout.total} DSPs updated · {view.rollout.status}

+ {view.rollout.status === "paused" && The rollout is paused. Resolve the affected DSP before resuming this version.} + +
    {view.rollout.members.map((member, index) =>
  • {member.name}{member.status}
  • )}
+
} +
; + })} +
+

Plugins

+

Included in the DSP update. Test on Dev before rolling out to other DSPs.

+
{view.platformRelease?.changes.plugins || "See the DSP release notes for plugin changes."}
+
+
+
: null} + ; +} diff --git a/core/dashboard/integration/systemd/dispatch-dashboard-tunnel.service.in b/core/dashboard/integration/systemd/dispatch-dashboard-tunnel.service.in new file mode 100644 index 0000000..bde25a8 --- /dev/null +++ b/core/dashboard/integration/systemd/dispatch-dashboard-tunnel.service.in @@ -0,0 +1,25 @@ +[Unit] +Description=Cloudflare Tunnel for Dispatch Dashboard +After=network-online.target dispatch-dashboard.service +Wants=network-online.target dispatch-dashboard.service +StartLimitIntervalSec=60 +StartLimitBurst=5 + +[Service] +Type=simple +Environment=PATH=@COMMAND_PATH@ +ExecStart=/usr/bin/env cloudflared --config @LOCAL_ROOT@/config/cloudflared/config.yml --no-autoupdate tunnel run +Restart=always +RestartSec=5 +KillMode=control-group +TimeoutStopSec=15 +UMask=0077 +NoNewPrivileges=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +RestrictSUIDSGID=true +LockPersonality=true +RestrictNamespaces=true +SystemCallArchitectures=native + +[Install] +WantedBy=default.target diff --git a/core/dashboard/integration/systemd/dispatch-dashboard.service.in b/core/dashboard/integration/systemd/dispatch-dashboard.service.in new file mode 100644 index 0000000..22f9853 --- /dev/null +++ b/core/dashboard/integration/systemd/dispatch-dashboard.service.in @@ -0,0 +1,30 @@ +[Unit] +Description=Dispatch Local Dashboard +After=network-online.target dispatch-auth-broker.service dispatch-collection-manager.service +Wants=network-online.target dispatch-auth-broker.service dispatch-collection-manager.service +StartLimitIntervalSec=60 +StartLimitBurst=5 + +[Service] +Type=simple +WorkingDirectory=@PROJECT_ROOT@/dashboard +Environment=DISPATCH_PROJECT_ROOT=@PROJECT_ROOT@ +Environment=DISPATCH_LOCAL_ROOT=@LOCAL_ROOT@ +Environment=NODE_NO_WARNINGS=1 +Environment=PATH=@COMMAND_PATH@ +EnvironmentFile=-@LOCAL_ROOT@/config/dashboard.env +ExecStart=@PROJECT_ROOT@/bin/dispatch-dashboard --operator --port 4310 --secure-cookies --public-origin ${DISPATCH_PUBLIC_ORIGIN} +Restart=always +RestartSec=3 +KillMode=control-group +TimeoutStopSec=15 +UMask=0077 +NoNewPrivileges=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +RestrictSUIDSGID=true +LockPersonality=true +RestrictNamespaces=true +SystemCallArchitectures=native + +[Install] +WantedBy=default.target diff --git a/core/dashboard/package-lock.json b/core/dashboard/package-lock.json new file mode 100644 index 0000000..a545278 --- /dev/null +++ b/core/dashboard/package-lock.json @@ -0,0 +1,3689 @@ +{ + "name": "dispatch-dashboard", + "version": "0.4.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "dispatch-dashboard", + "version": "0.4.0", + "dependencies": { + "@fontsource-variable/inter": "^5.3.0", + "@tanstack/react-query": "^5.102.8", + "class-variance-authority": "^0.7.1", + "clsx": "^2.1.1", + "get-nonce": "^1.0.1", + "lucide-react": "^1.41.0", + "radix-ui": "^1.6.7", + "react": "^19.2.8", + "react-dom": "^19.2.8", + "tailwind-merge": "^3.6.0", + "tw-animate-css": "^1.4.0" + }, + "devDependencies": { + "@playwright/test": "^1.63.0", + "@tailwindcss/vite": "^4.3.3", + "@types/node": "^26.4.1", + "@types/react": "^19.2.18", + "@types/react-dom": "^19.2.7", + "@vitejs/plugin-react": "^6.1.1", + "prettier": "^3.9.6", + "tailwindcss": "^4.3.3", + "typescript": "^7.0.2", + "vite": "^8.2.2" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@floating-ui/core": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", + "integrity": "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==", + "license": "MIT", + "dependencies": { + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/dom": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", + "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", + "license": "MIT", + "dependencies": { + "@floating-ui/core": "^1.8.0", + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/react-dom": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.9.tgz", + "integrity": "sha512-JDjEFGCpImxDCA7JJKviA0M9+RtmJdj0m/NVU5IMgBK+AmZouAQQ7/+2GLH0GXXY0YMw9oXPB8hKdbPYg5QLYg==", + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.8.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@floating-ui/utils": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", + "license": "MIT" + }, + "node_modules/@fontsource-variable/inter": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource-variable/inter/-/inter-5.3.0.tgz", + "integrity": "sha512-OupL48va4JNofb97w6NYeF9S7W/kHNKM0Er8Dem5nqi4jeOLrVJDoE8tZEpnMJmtkvNbB1EIPPwHcdkF6b1oUA==", + "license": "OFL-1.1" + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.148.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.148.0.tgz", + "integrity": "sha512-Nm4s/jB+4FpFsPhWGEC4h7rzksesmtnMXomo6rCMcg/b8zLQuOziRgkCS1fxDCXOlJB/6Q8oABOZ/OP6RIPj9A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@playwright/test": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@radix-ui/number": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.3.tgz", + "integrity": "sha512-Road2bidD0uu/1BGDOWNdPI06g0lIRy6IF9GZcIrDK2KGItfor8IQwQa+yM2ERgHM1MmHxaxpTzk0/Jp42lNfA==", + "license": "MIT" + }, + "node_modules/@radix-ui/primitive": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.7.tgz", + "integrity": "sha512-rqWnm76nYT8HoNNqEjpgJ7Pw/DrBj5iBTrmEPo6HTX5+VJyBNOqTdv4g89G63HuR5g0AaENoAcH7Is5fF2kZ8Q==", + "license": "MIT" + }, + "node_modules/@radix-ui/react-accessible-icon": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accessible-icon/-/react-accessible-icon-1.1.15.tgz", + "integrity": "sha512-WTQwcAvQf5sOcuUyi90lKPbhwcvQ+j55cjrSmeaN+L2vKU3DooOvlKw2MDeiJ5IkV5N905KW0/fGojKOBhD11A==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-accordion": { + "version": "1.2.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accordion/-/react-accordion-1.2.20.tgz", + "integrity": "sha512-jDhG9FvAEnlhnjrsINbNXcUa4G+L1KqSkJSunkbKEzFRcAb52jvM0PjPxPRvhe1HNc5F5yc0yzzWeeqlH4yBIg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-alert-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-alert-dialog/-/react-alert-dialog-1.1.23.tgz", + "integrity": "sha512-VAYOiQRqj3GPpYJE0I9J+X8Ip05cyVlNdKOFeiGS2Ou1HHGfpl0BxOyZm6nmVDyU+W+NF3/XLzmjHmVGydhwgA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-arrow": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.15.tgz", + "integrity": "sha512-v4zggRcjadnI+ClKDuijlQEW4tw3NoaeHc/PwpKnLoLLKNUG4InLegkstooLcRIUWCs+8L22dGURCVuFfOKfnA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-aspect-ratio": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-aspect-ratio/-/react-aspect-ratio-1.1.15.tgz", + "integrity": "sha512-fy+dyVR+90nelK8rqIznFlxzx7uPcGbhxH8Nfr2bHb4UfSe+e3hklOC0luK0hDwVwnRX7xTRySpsrQVeW+/oNQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-avatar": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-avatar/-/react-avatar-1.2.6.tgz", + "integrity": "sha512-4ULOTJ/mqy2hT9GlWa/MFHxHSvH3nJzHnZM1waNsc5Bonv7i70aNenghXmD97S6OJ81ekXONGGt4nT1r0PfEdA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-checkbox": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-checkbox/-/react-checkbox-1.3.11.tgz", + "integrity": "sha512-Gnptr9pDDQxD3hgq2dtPbtrp/c2qH1mBwIzw3X/ivrMb2e1t0jMTi606fVEqFPaQR1ggXIVQWKj3P2WW9v7zGQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collapsible": { + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collapsible/-/react-collapsible-1.1.20.tgz", + "integrity": "sha512-mcGesGplBnzN2sbvJETzpCNfSMyPnb29q1GRLU+Ib7bJrpIG2ywmRoh2V5VbA2uNvKikKUlVbAPks7JDjz4A8Q==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collection": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.15.tgz", + "integrity": "sha512-9W+B9NPF0NaaPh/1NJd3+KqsnlLqU9H7T2rvww+fp+T/evVXdNAyYcnfRQZFOjkR1ajQp3yORlqnI8soawLvNA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-compose-refs": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.5.tgz", + "integrity": "sha512-+48PbAAbq3didjJxa+OaWY2ZwgAKsNiRGyeHKszblZMQ+kcpd9pAaT11cMkGEie0vsOi3QdeTE6d5Fe3Gn61kA==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.2.2.tgz", + "integrity": "sha512-RHCUGwKHDr0hDGg4X7ma4JG4/+12qxw8rkh5QKdDldlCvtja6nUx1Ef/8HVrJze81lEsgLQlqjzjGNHantgnQA==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context-menu": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context-menu/-/react-context-menu-2.3.7.tgz", + "integrity": "sha512-CtXP35dxaB5T3zXSd+E3uHe/QpXcpYnZmxp6OaIbfthtfW4wyb77M23BG+bwIJDtsMwEP/YssdsmNyZu7jhWew==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.1.23.tgz", + "integrity": "sha512-Ksw4WeROkO4rC9k/onilX/Ao2Cr1ku1unMNH+XSCcP4jSXYu7HDsg9n4ojMjVb22XpYjAQ9qfrFlVbru1vXDUA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-direction": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.4.tgz", + "integrity": "sha512-5pzg4FGQNpExhnhT2zlrP1wZFaYCd1K0nYWoFAdcYoYK868IEigqMX3B3f8yIoRlAhAeDWciLI6ZdCKHF9P4Vg==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dismissable-layer": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.19.tgz", + "integrity": "sha512-8g4pfOL9HoKKLWGiypT+dphVqjFfmcXO5GBnhsG6zI+lxAx/8feQpr+1LSN8Re3hiZ+XkLNS4O9ztK11/LzQ6w==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-effect-event": "0.0.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dropdown-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dropdown-menu/-/react-dropdown-menu-2.1.24.tgz", + "integrity": "sha512-geq8l2rJkxvkXsT9RMgtUE3P8pITFpTsvYpbySi1IH4fZEABD/Gp85myayFgxk0ktljGMJnCbeFkyTusvSvv7g==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-guards": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.6.tgz", + "integrity": "sha512-RNOJjfZMTyBM6xYmV3IVGXkPjIhcBAuv48POevAXwrGJhkWZ9p1rFoIS1JFooPuT193AZmRsCPhpoVJxx6OPoQ==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-scope": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.16.tgz", + "integrity": "sha512-wmRZ2WWLvmt6KHy2rNPOdPUjwq5xOHY02+m+udwJTn0aNIox/rkskAvJTyTLGhPK6KgrUjlJUJpgmx/+wFiFIQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-form": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-form/-/react-form-0.1.16.tgz", + "integrity": "sha512-Q4TLEn2A7TAypxwmd6R9EwrlXDvkfYSDMrq9/887AXAGh+G1rH+kYJKSTv+Si9Y0JPKTwKYv6PviAJosysNimA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-hover-card": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-hover-card/-/react-hover-card-1.1.23.tgz", + "integrity": "sha512-H8qONfZd3ltrU3+jHCIgITbWo6e1iTKvP9DHdrvYbX48ooRM5FjEDTn16AMwdfuOGkWdZEhpl3PLL/Wk/AnHDQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-id": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.4.tgz", + "integrity": "sha512-TMQp2llA+RYn7JcjnrMnz7wN4pcVttPZnRZo52PLQsoLVKzNlVwUeHmfePgTgRluXFvlD3GD5g5MOVVTJCO0qA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-label": { + "version": "2.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-label/-/react-label-2.1.15.tgz", + "integrity": "sha512-o/rdYEwZTTo5tjknnPeyQFU45kUC4i/XyeDPP+HGyi6XqpOP6Zf5Ya5vh/Yfe9Id5JiuWnnAx2XqIeD3UYZt0g==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menu/-/react-menu-2.1.24.tgz", + "integrity": "sha512-uW7RVuU6Lp/ZtfeY4b3kL32zccgEWvPv1+cf17ubYzHa9cL8AHokmk36cG/XEiH/smbQvumnieXX9j/e9RqJWA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menubar": { + "version": "1.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menubar/-/react-menubar-1.1.24.tgz", + "integrity": "sha512-eeVs0vf7cuqXaM0qLQCPcufImiJNVBXdJDLu7ZGYl2732UH23Qat/foNGrr6vYV3/DdTsBqASoggUFgH14OcZA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-navigation-menu": { + "version": "1.2.22", + "resolved": "https://registry.npmjs.org/@radix-ui/react-navigation-menu/-/react-navigation-menu-1.2.22.tgz", + "integrity": "sha512-ou7iLEJ+yrhQndkkA4U21XIdS/CS45F4iXIkTZcb6/Ne9EMsOuDudVmCwmDnfFZZ+y1FZqXRNSIgBy+YMvZVZg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-one-time-password-field": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-one-time-password-field/-/react-one-time-password-field-0.1.16.tgz", + "integrity": "sha512-Tj9P6ntAJEw52oq/F0AGknXR4XncxEt7XU47O3xJQOiWfLzEy3d9gtgKfvjSzGxzHkfL+VzvxGu2KTFsloJqXw==", + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-password-toggle-field": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-password-toggle-field/-/react-password-toggle-field-0.1.11.tgz", + "integrity": "sha512-4gvFnmDXu3dgj21CqsufzIameRvlRd4SBqaWhcrlrNhRo0Y5i/49AmRJYe1fdAM3G2VNBbmin4b0D6cdQocwgw==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popover": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.23.tgz", + "integrity": "sha512-mw58MrBlyHWFisTOYignD0vf/3gdcgAR+9of1s9G/38CbFiUwH1nCDkc0AUM9IrXFgN5Ue8n45j9WCgyM1sbiQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popper": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.3.7.tgz", + "integrity": "sha512-UsJrrd7w4wuKKTdvd/DNERVlwSlUcyXzjhyDwBk+3aPOsCjOY6ZSbxuw8E6lZTjjfP8Cpd0J8VVkrYUWyGYXyg==", + "license": "MIT", + "dependencies": { + "@floating-ui/react-dom": "^2.0.0", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-rect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-portal": { + "version": "1.1.17", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.17.tgz", + "integrity": "sha512-vKQLcWypUnwZVvfV7UkGahH2g6ySe8M8R+zYBwPrv5byZ9QAW6cQVvNKo7GgmD+p8aYb6D9JBuvy8/WhOno2wQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-presence": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.10.tgz", + "integrity": "sha512-3wyzCQ6+ubRA+D4uv9m95JYLXxmOHp05qjrkjeA7uKHHtjpPggQzc6DAb0URl7j67oR0K2foO4ip27TiX037Bw==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-primitive": { + "version": "2.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.10.tgz", + "integrity": "sha512-MucOnzh6hR5mid6VpkbglRAMYMjKLqRnGBbjXkzjK52fuQDd1qbkx78a5P40mkcnVXJdEVxm26E9OPAiUq7nBg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-progress": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-progress/-/react-progress-1.1.16.tgz", + "integrity": "sha512-5XnomAsoZZCY+KNTxbIghpGqPruZvKFNlvcAljVAOdDRDsH4/OZQxhtwo5wdtoDM5R6MhJBb2sPnDuRFep3lzg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-radio-group": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-radio-group/-/react-radio-group-1.4.7.tgz", + "integrity": "sha512-cgYFEkntCxppHZgtSZ+7vh0wbZQ+IC7PPMw8DSnRG27B6kDd32/Zw0OJt7dGDigCoprMuWHjg2PvUn3PYvPFoQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-roving-focus": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.19.tgz", + "integrity": "sha512-V9jI6hDjT7l3jsCQD9bLNvDLM3tH/gdbOTp7Tefp3hbbgCGQoK7tUvrWiRlcoBHIZ809ElXwNQwVo0B98LuTXQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-scroll-area": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.2.18.tgz", + "integrity": "sha512-Zn5Cd171wxsO3Dfg8HaW6RifTb9CYTKQJHs/G4+LN1GfmJpaQMZQyQxMprVPHpaz7QY4l9BxK2JwQuzHsXC8nA==", + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.3.7.tgz", + "integrity": "sha512-WFGImkmbzcfxeIwq/+4HvRN0pizBwbwQUED4I13ezQsDdfl38ZntN6TmR8XaSzPBqoCToe8rF75j6NPNDSzhbg==", + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-separator": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.1.15.tgz", + "integrity": "sha512-jOLO4lssEzWpoDu7G+Ze4VjwMRUBt291pnZD0gmalREZipnTX3wadQo7Fy48GCTfe14/YRN6rw/rOJqrE85Wxw==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slider": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slider/-/react-slider-1.4.7.tgz", + "integrity": "sha512-mTSLf1GC/C0moWjTbvCM6Qn/gBjvlFt1azuWF2v7MN5C3Zq2U2J2lN3ZEYkpujuOU5Ro7A28wkviSxaKnG0BYg==", + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slot": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.3.3.tgz", + "integrity": "sha512-qx7oqnYbxnK9kYI9m317qmFmEgo6ywqWvbTogdj7cL9p3/yx4M48p7Rnw5z3H890cL/ow/EeWJsuTykeZVXP5Q==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-switch": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-switch/-/react-switch-1.3.7.tgz", + "integrity": "sha512-48tB/4dn2UVLBCYhTu9AuR63IHl73l/qLbLgxd86noTUor4/K4LFDAcYjK+isP5313qxaFpjPVogE7+Y0/V3Kw==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tabs": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.21.tgz", + "integrity": "sha512-UKxJlZid7FVtsk/WTxj4i4uSEgj2Au+KBbS7SQyTlzMhhn+86Cz3tISZdTa87bfEfcuvZezf2ZsxD4xuEKtkog==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toast": { + "version": "1.2.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toast/-/react-toast-1.2.23.tgz", + "integrity": "sha512-ofhyAsYaocRGOs/n0XWdUOSVzEAG6BfrMVM8z0c0kLEWY38w/0WuMFPTJP/HVaZPYkMvHZoKIIhNcjbTCBILPg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle": { + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle/-/react-toggle-1.1.18.tgz", + "integrity": "sha512-7lonPlKfSacd20GlOBx2ltuVKz9oqWYZz+oMQyOltw6t1y2nyftj2ZmwwUHYn49kqfDWcp8dNZm5NgV+5Z+mug==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle-group": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle-group/-/react-toggle-group-1.1.19.tgz", + "integrity": "sha512-OtnwuSVjd1Ofi+AdnvhsjQdyuhCDwYs1w9RyB5BN/OavXOVQo42SYqQjwUnbPnaiPFBpQ9aX70dWeee+v2oBLA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toolbar": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toolbar/-/react-toolbar-1.1.19.tgz", + "integrity": "sha512-Ph0IvtYw4VB12ZnZg+YtrGs8yJQsnizwo/zu0R4Y/nWugtJzA7Pg1eWeuDR9+LSqn+xjamss+UOSOJJJ4gx8jw==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-toggle-group": "1.1.19" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tooltip": { + "version": "1.2.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.2.16.tgz", + "integrity": "sha512-6EamKFRRnlpdadndbZ6LMwycfwkwPte1B42hs6QA0gYhjaOKqW4PZ4pjaW9UrlDX5eVt/OjncE7BFTPL5nmZhg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-callback-ref": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.4.tgz", + "integrity": "sha512-R6OUY2e2fA6Yn6s+VSx5KBV6Nx8LQEhu+cz7LCej18rQ1HLyg9PSC9jP/ZNx0o6FAIK9c0F1kHylzSxKsdlkrQ==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-controllable-state": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.6.tgz", + "integrity": "sha512-uEQJGT97ZA/TgP/Hydw47lHu+/vQj6z/0jA+WeTbK1o9Rx45GImjpD0tc3W5ad3D6XTSR6e1yEO0FvGq6WQfVQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-effect-event": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.5.tgz", + "integrity": "sha512-7cshFL8HGS/7HEiHH+9kL9HBwp2sa9yX18Knwek6KYWmXwM7pegMgta2AXMQKI+rq3JnfSj9x8wYqFMTdG1Jgg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-escape-keydown": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.5.tgz", + "integrity": "sha512-ge3ipobwSXTj4JyVtswQ7qZj0ZHdtbGuOno/LrgAAeSxtsJ6Vs4Gz5IkPH2bmqpjcLUFoqGhA/mueuIf63UXlA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-is-hydrated": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-is-hydrated/-/react-use-is-hydrated-0.1.3.tgz", + "integrity": "sha512-umO/aJ+82CpOnhDZUTbILCQf7kU/g0iv+oGs/Q8jw7IkhWBzaEP4sA268PhFAJTFetbwp3ICc6ktpI4TqtxcIw==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-layout-effect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.4.tgz", + "integrity": "sha512-K20DkRkUwDnxEYMBPcg3Y6voLkEy5p5QQmszZgLngKKiC7dzBR/aEuK3w1qlx2JWDUNH6FluahYdgR3BP+QbYw==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-previous": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.1.4.tgz", + "integrity": "sha512-XoSLhbRbqxFtgJoi2fNHA3C6pDlY34x508vUpUGoFZfvePfHXHbE1lC4FYFMnJWgiCRroSTw6fOsXQoVS9RwZg==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-rect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.4.tgz", + "integrity": "sha512-cSOCh6JlkmfjLyNcLiu2nB4v+nm+dkZ+Q5KHWk/soo4U7ZLiEQFKHK9/YmtBHjfCEaU43IBKQOc4/uJmCaiCTQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-size": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.4.tgz", + "integrity": "sha512-D3anSY15EJoxrihpsXI6SMrmmonnQtR2ni7arO+Lfdg3O95b9hNXxONk8jA5C8ANdF/h5HMAxejgs8PWJ6rlhw==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-visually-hidden": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.2.11.tgz", + "integrity": "sha512-NFS86RYYZb4/exihaESBGOpMJFz8MGLAfu3mOBSGByVnVPC9JPASfYubxd/8KbkQK0sYAv8lVQDEQukDX/qXvQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/rect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.3.tgz", + "integrity": "sha512-JtyZR+mqgBibTo8xea3B6ZRmzZiM/YeVBtUkas6zMuXjAlfIFIW2FgqeM9eLyvEaYX66vr6DJMK+4U6LV0KhNw==", + "license": "MIT" + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.7.tgz", + "integrity": "sha512-EypzgnYCwyVY4NDHKzGmNJT5b+XaQEBniHxsMdeIQLB/tcCzZnhqrzHpZFbX9iaxx+5RiB8caATBtfvZP7zVxQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.7.tgz", + "integrity": "sha512-l17HE9EweWaqJZhuUuNBN/FzM62xw+DECVnJyvMsxn8vJFAGLy5QfLDoYAcronkAN8VxKZHezDpulHDPx95vFw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.7.tgz", + "integrity": "sha512-8ED8ELFvHXc6OCETIn4gXObPiaR6bckM/ipXtbzlPVDRMBfEGjCKgO90F9YtfdpDatVx/ZQw7aZ1vUMf/+T3Mw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.7.tgz", + "integrity": "sha512-/WPripjtiAIZ2tWY7ddijORT0Ujg87wxWW/qcoFVCKAWVDPhtY0xr7Dj0M3GyNGz60jGwTElhro/mkF9dT7dDQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.7.tgz", + "integrity": "sha512-14DI4NcqpvbICxSnGLx3PmtDaWqRP/KGSGb6C+JLLVPeZRl6dKdHba3pGsqT3vpdTqhEYIPG0MMQ8c0xYqoJxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.7.tgz", + "integrity": "sha512-bxrWIRvHWQvbJwi+VIie/kDJmQxcNE6xxWwZdqF/ExVAigtHkv54WTLQPb+QsZdnFy18fg7JPfWGL0RH6vwIlQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.7.tgz", + "integrity": "sha512-toOY2BChBZyuxU7OYX6Tn389di4IzAqPTycVcci0O7FSfBqzRB3RZn+K5Is6ANf4tmgRd/K1yZTsNTXbkXsnLg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.7.tgz", + "integrity": "sha512-lAIXTH/aiLRLxsTgQvfhjo4K1ydWIp00+V0voOr9beb/9ZmkUFrSIb03dXNFRgMNvkE6oGsF10ioQ6UsI+vS5Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.7.tgz", + "integrity": "sha512-kdnwS28Pkenp/mZMRwjXXXwxQ7pIsm+bF919LUK93BOyhcLsrVKdP2p9fxpiPNPAbNuch8ypQt0pm2P2LYCAGg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.7.tgz", + "integrity": "sha512-516OdsyLdr5E65paF3yBF55t8mfm9+gmtCsK3xI7XKXIT7EfRlHhxL8K/NR6Hu8BWSgF5+1w74lTL0+nxcc8Qw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.7.tgz", + "integrity": "sha512-r8/z8n7GFaYRln3xmP1Cxy0HH/HLM0uBUPkEuSVEfKGDA89M0FsZRZJRSwe/tJjRx+fpH/gjorfhB8tmEbSFLA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.7.tgz", + "integrity": "sha512-pAsE8iiDxUg1xBqdhrTfg45AVDVpirjz00sblEYClGNNcMnDb+e8beQgqIAw6LvauX/APvgxUnwrgun/YYGBhw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.7.tgz", + "integrity": "sha512-lTcIYmmnQQA8Or/2DatS6oSqcdLHvendjS+zLu+FwgToynWMRSmQdpM65fTANJgIS4mjbMOo5KT2lnT9SAb96w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.7.tgz", + "integrity": "sha512-e3Gu3WxbNk/UqQhxqU7YIYO+9ZBvWNz3U+h/qRFosscMFzdRPbXYSaSWgSnklv2fz1TgzBTcti2z35c/7irsHw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.7.tgz", + "integrity": "sha512-W/jg5qoRSqjsEv0+dZi4e687mcHqmVuU0P4fK6qS/xjetW2Gmc1W8j//z5nAeNcC8Ttm0hV46IjcYeuVwYhuiw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tailwindcss/node": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.3.tgz", + "integrity": "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "enhanced-resolve": "^5.24.1", + "jiti": "^2.7.0", + "lightningcss": "1.32.0", + "magic-string": "^0.30.21", + "source-map-js": "^1.2.1", + "tailwindcss": "4.3.3" + } + }, + "node_modules/@tailwindcss/oxide": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.3.3.tgz", + "integrity": "sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20" + }, + "optionalDependencies": { + "@tailwindcss/oxide-android-arm64": "4.3.3", + "@tailwindcss/oxide-darwin-arm64": "4.3.3", + "@tailwindcss/oxide-darwin-x64": "4.3.3", + "@tailwindcss/oxide-freebsd-x64": "4.3.3", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.3.3", + "@tailwindcss/oxide-linux-arm64-gnu": "4.3.3", + "@tailwindcss/oxide-linux-arm64-musl": "4.3.3", + "@tailwindcss/oxide-linux-x64-gnu": "4.3.3", + "@tailwindcss/oxide-linux-x64-musl": "4.3.3", + "@tailwindcss/oxide-wasm32-wasi": "4.3.3", + "@tailwindcss/oxide-win32-arm64-msvc": "4.3.3", + "@tailwindcss/oxide-win32-x64-msvc": "4.3.3" + } + }, + "node_modules/@tailwindcss/oxide-android-arm64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.3.3.tgz", + "integrity": "sha512-Y85A2gmPSkl5Ve5qR86GL4HT509cFqQh1aes9p3sSkyTPwt0Pppf3GkwGe4JPACcRYjgJIEhQgM6dBClnr0NYw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-arm64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.3.3.tgz", + "integrity": "sha512-BiaWatpBcERQFDlOjRDpIVXuFK5PJez5SA4JMg6VYZdBYU+qKfV/vqjcIs+IYmtitf1xYQZTwXvU/8y4lfZUGw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-x64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.3.3.tgz", + "integrity": "sha512-fAeUqfV5ndhxRwai8cXGzdLvul9utWOmeTkv69unv4ZXixjn61Z+p9lCWdwOwA3TYboG3BwdVuN/RDjhBRl0mw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-freebsd-x64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.3.3.tgz", + "integrity": "sha512-iyf5bV6+wnAlflVeEy7R25dupxTNECZN5QMI0qNT6eT+EgaGdZcKhGkr5SdoaWiLJ3spLqIY9VCeSGrwmtg4kw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.3.3.tgz", + "integrity": "sha512-aAYUprJAJQWWbRrPvtjdroZ56Md+JM8pMiopS6xGEwDfLhqj+2ver2p4nU4Mb3CRqcMmNBjo8KkUgcxhkzVQGQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.3.3.tgz", + "integrity": "sha512-nDxldcEENOxZRzC2uu9jrutZdAAQtb+8WWDCSnWL1zvBk1+FN+x6MtDViPB5AJMfttVCUhehGWus3XBPgatM/w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-musl": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.3.3.tgz", + "integrity": "sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-gnu": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.3.3.tgz", + "integrity": "sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-musl": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.3.3.tgz", + "integrity": "sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.3.3.tgz", + "integrity": "sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==", + "bundleDependencies": [ + "@napi-rs/wasm-runtime", + "@emnapi/core", + "@emnapi/runtime", + "@tybys/wasm-util", + "@emnapi/wasi-threads", + "tslib" + ], + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.11.1", + "@emnapi/runtime": "^1.11.1", + "@emnapi/wasi-threads": "^1.2.2", + "@napi-rs/wasm-runtime": "^1.1.4", + "@tybys/wasm-util": "^0.10.2", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.3.tgz", + "integrity": "sha512-3rc292Ca2ceK6Ulcc/bAVnTs/3nDtoPhyEKlgPv+yQJQi/JS/AMJlqzxvlDacL1nekbrcf6bTqp/jV4qgnPxNQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-win32-x64-msvc": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.3.3.tgz", + "integrity": "sha512-yJ0pwIVc/nYeGoV02WtsN8KYyLQv7kyI2wDnkezyJlGGjkd4QLwDGAwl47YpPJeuI0M0ObaXGSPjvWDPeTPggw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/vite": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.3.3.tgz", + "integrity": "sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tailwindcss/node": "4.3.3", + "@tailwindcss/oxide": "4.3.3", + "tailwindcss": "4.3.3" + }, + "peerDependencies": { + "vite": "^5.2.0 || ^6 || ^7 || ^8" + } + }, + "node_modules/@tanstack/query-core": { + "version": "5.102.8", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.102.8.tgz", + "integrity": "sha512-ZNjkJ33CqvPNec/6lZBnHqLc3EVGPZ9ySLhYahU9TcuRFdmwXewuj0c4hwSWcGHqEUwcSrKeZ+oGcvPBqXcQcg==", + "license": "MIT" + }, + "node_modules/@tanstack/react-query": { + "version": "5.102.8", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.102.8.tgz", + "integrity": "sha512-TYBea4OuXWD7MhaSHq069TWbFe7rcwWN6kzT7JF0OKi1K6c1gTv2IzD6A6ExJsCMozdkqBWeuIUZmu4KQg0O5A==", + "license": "MIT", + "dependencies": { + "@tanstack/query-core": "5.102.8" + }, + "peerDependencies": { + "react": "^18 || ^19" + } + }, + "node_modules/@types/node": { + "version": "26.4.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.1.tgz", + "integrity": "sha512-k97ENvZWtvA6yqz5/FS6a7duDgOPEeOQOc2iKS/nY6mX6qJUKtLnWzQS+Xj6tXweyj6ZcTAK2Qecetnvi9nCLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/@types/react": { + "version": "19.2.18", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz", + "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.7.tgz", + "integrity": "sha512-I8bPpDLcHBv1qiIiXDCy71Rt8eQDKJP0sMSWJphDdAcdqiJ1sGpZamavoEIRZmYzjia9LuEb2HlYdDpmoENpvQ==", + "devOptional": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.2.0" + } + }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.1.1.tgz", + "integrity": "sha512-yxLaQV9gkhS8ezJqCM6+ndU7mDY6gqAg75NQ+0IjwEI8IYOmQCgkRwHKVSfWXW076DsqMo0Dk+0FK1U+M5RgFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", + "babel-plugin-react-compiler": "^1.0.0", + "oxc-transform-react": "^0.145.0", + "vite": "^8.0.0" + }, + "peerDependenciesMeta": { + "@rolldown/plugin-babel": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + }, + "oxc-transform-react": { + "optional": true + } + } + }, + "node_modules/aria-hidden": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz", + "integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/class-variance-authority": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/class-variance-authority/-/class-variance-authority-0.7.1.tgz", + "integrity": "sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg==", + "license": "Apache-2.0", + "dependencies": { + "clsx": "^2.1.1" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "devOptional": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/detect-node-es": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", + "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==", + "license": "MIT" + }, + "node_modules/enhanced-resolve": { + "version": "5.24.5", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.24.5.tgz", + "integrity": "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.3.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-nonce": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", + "integrity": "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lucide-react": { + "version": "1.41.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.41.0.tgz", + "integrity": "sha512-6lksP35l6KszDKUeRTi4LV7i6DEe0Yzl2ALJm9j4c5xEYN91GdW1xGsawGMOg2mgjF5GHBVX8pKX9kP+cWsP3Q==", + "license": "ISC", + "peerDependencies": { + "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/nanoid": { + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "dev": true, + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prettier": { + "version": "3.9.6", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.6.tgz", + "integrity": "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/radix-ui": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/radix-ui/-/radix-ui-1.6.7.tgz", + "integrity": "sha512-QBdhh1arIEUvPC0dQ5+nwWAxt7+N+oP/9jPwjJkGFoSk/sqxg32gJtSXGtFh8frAIcS6oC9cx2Q+7KYCQLOAeA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-accessible-icon": "1.1.15", + "@radix-ui/react-accordion": "1.2.20", + "@radix-ui/react-alert-dialog": "1.1.23", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-aspect-ratio": "1.1.15", + "@radix-ui/react-avatar": "1.2.6", + "@radix-ui/react-checkbox": "1.3.11", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-context-menu": "2.3.7", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-dropdown-menu": "2.1.24", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-form": "0.1.16", + "@radix-ui/react-hover-card": "1.1.23", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-menubar": "1.1.24", + "@radix-ui/react-navigation-menu": "1.2.22", + "@radix-ui/react-one-time-password-field": "0.1.16", + "@radix-ui/react-password-toggle-field": "0.1.11", + "@radix-ui/react-popover": "1.1.23", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-progress": "1.1.16", + "@radix-ui/react-radio-group": "1.4.7", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-scroll-area": "1.2.18", + "@radix-ui/react-select": "2.3.7", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-slider": "1.4.7", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-switch": "1.3.7", + "@radix-ui/react-tabs": "1.1.21", + "@radix-ui/react-toast": "1.2.23", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-toggle-group": "1.1.19", + "@radix-ui/react-toolbar": "1.1.19", + "@radix-ui/react-tooltip": "1.2.16", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-escape-keydown": "1.1.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/react": { + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", + "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", + "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.27.0" + }, + "peerDependencies": { + "react": "^19.2.8" + } + }, + "node_modules/react-remove-scroll": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz", + "integrity": "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==", + "license": "MIT", + "dependencies": { + "react-remove-scroll-bar": "^2.3.7", + "react-style-singleton": "^2.2.3", + "tslib": "^2.1.0", + "use-callback-ref": "^1.3.3", + "use-sidecar": "^1.1.3" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-remove-scroll-bar": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz", + "integrity": "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==", + "license": "MIT", + "dependencies": { + "react-style-singleton": "^2.2.2", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-style-singleton": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz", + "integrity": "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==", + "license": "MIT", + "dependencies": { + "get-nonce": "^1.0.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/rolldown": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.7.tgz", + "integrity": "sha512-g0EtLvBjTUB7jhyV0S/TCup3v/XSVl45vUIGbOGU4QPiyjTenCe4mKuFvW9fEgYmS2Fo42AUssRmNuMziXdrig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.148.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.7", + "@rolldown/binding-android-arm64": "1.2.7", + "@rolldown/binding-darwin-arm64": "1.2.7", + "@rolldown/binding-darwin-x64": "1.2.7", + "@rolldown/binding-freebsd-x64": "1.2.7", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.7", + "@rolldown/binding-linux-arm64-gnu": "1.2.7", + "@rolldown/binding-linux-arm64-musl": "1.2.7", + "@rolldown/binding-linux-ppc64-gnu": "1.2.7", + "@rolldown/binding-linux-s390x-gnu": "1.2.7", + "@rolldown/binding-linux-x64-gnu": "1.2.7", + "@rolldown/binding-linux-x64-musl": "1.2.7", + "@rolldown/binding-openharmony-arm64": "1.2.7", + "@rolldown/binding-win32-arm64-msvc": "1.2.7", + "@rolldown/binding-win32-x64-msvc": "1.2.7" + } + }, + "node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tailwind-merge": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.6.0.tgz", + "integrity": "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w==", + "license": "MIT" + }, + "node_modules/tailwindcss": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.3.tgz", + "integrity": "sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/tapable": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/tw-animate-css": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/tw-animate-css/-/tw-animate-css-1.4.0.tgz", + "integrity": "sha512-7bziOlRqH0hJx80h/3mbicLW7o8qLsH5+RaLR2t+OHM3D0JlWGODQKQ4cxbK7WlvmUxpcj6Kgu6EKqjrGFe3QQ==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/use-callback-ref": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz", + "integrity": "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sidecar": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.3.tgz", + "integrity": "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==", + "license": "MIT", + "dependencies": { + "detect-node-es": "^1.1.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/vite": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.2.tgz", + "integrity": "sha512-cFKLV/PRgAUlIRm5WjMjJ86jrftzpqcgH+Us+DS8mI3CDNiH30Whrz8uHL3+MOLPAgqbMBAqWdAHAphOAM+z/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.26", + "rolldown": "~1.2.4", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.4.0 || ^0.5.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/vite/node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + } + } + } +} diff --git a/core/dashboard/package.json b/core/dashboard/package.json new file mode 100644 index 0000000..f4e79d3 --- /dev/null +++ b/core/dashboard/package.json @@ -0,0 +1,46 @@ +{ + "name": "dispatch-dashboard", + "version": "0.4.0", + "private": true, + "description": "Authenticated tenant-scoped operations and private platform provisioning console for Dispatch", + "type": "commonjs", + "main": "server/main.js", + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "./scripts/build", + "test": "./scripts/test", + "start": "../bin/dispatch-dashboard", + "start:operator": "../bin/dispatch-dashboard --operator", + "start:private-operator": "../bin/dispatch-dashboard --operator --installation-operator", + "test:ui": "playwright test", + "preview:ui": "DISPATCH_FRONTEND_FIXTURE=1 node --no-warnings examples/frontend-preview.js", + "format": "prettier --write frontend/src" + }, + "dependencies": { + "@fontsource-variable/inter": "^5.3.0", + "@tanstack/react-query": "^5.102.8", + "class-variance-authority": "^0.7.1", + "clsx": "^2.1.1", + "get-nonce": "^1.0.1", + "lucide-react": "^1.41.0", + "radix-ui": "^1.6.7", + "react": "^19.2.8", + "react-dom": "^19.2.8", + "tailwind-merge": "^3.6.0", + "tw-animate-css": "^1.4.0" + }, + "devDependencies": { + "@playwright/test": "^1.63.0", + "@tailwindcss/vite": "^4.3.3", + "@types/node": "^26.4.1", + "@types/react": "^19.2.18", + "@types/react-dom": "^19.2.7", + "@vitejs/plugin-react": "^6.1.1", + "prettier": "^3.9.6", + "tailwindcss": "^4.3.3", + "typescript": "^7.0.2", + "vite": "^8.2.2" + } +} diff --git a/core/dashboard/playwright.config.cjs b/core/dashboard/playwright.config.cjs new file mode 100644 index 0000000..a5012f4 --- /dev/null +++ b/core/dashboard/playwright.config.cjs @@ -0,0 +1,24 @@ +const { defineConfig } = require("@playwright/test"); +const previewPort = Number(process.env.DISPATCH_FRONTEND_PORT || 4339); +module.exports = defineConfig({ + testDir: "./tests/browser", + workers: 1, + fullyParallel: false, + timeout: 30000, + outputDir: + process.env.DISPATCH_UI_ARTIFACTS || "/tmp/dispatch-ui-test-results", + use: { + baseURL: `http://127.0.0.1:${previewPort}`, + viewport: { width: 1536, height: 1024 }, + reducedMotion: "reduce", + launchOptions: process.env.DISPATCH_CHROME_EXECUTABLE + ? { executablePath: process.env.DISPATCH_CHROME_EXECUTABLE } : {}, + }, + webServer: { + command: + "DISPATCH_FRONTEND_FIXTURE=1 node --no-warnings examples/frontend-preview.js", + url: `http://127.0.0.1:${previewPort}`, + reuseExistingServer: !process.env.CI, + timeout: 30000, + }, +}); diff --git a/core/dashboard/playwright.updates.config.cjs b/core/dashboard/playwright.updates.config.cjs new file mode 100644 index 0000000..ce5258a --- /dev/null +++ b/core/dashboard/playwright.updates.config.cjs @@ -0,0 +1,8 @@ +const { defineConfig } = require('@playwright/test'); +module.exports = defineConfig({ + testDir: './tests/browser', testMatch: ['independent-updates.spec.cjs', 'updates-workspace.spec.cjs', 'dashboard-rollout.spec.cjs'], + workers: 1, timeout: 60000, + outputDir: process.env.DISPATCH_UI_ARTIFACTS || '/tmp/dispatch-updates-browser', + use: { viewport: { width: 1440, height: 1000 }, reducedMotion: 'reduce', + launchOptions: process.env.DISPATCH_CHROME_EXECUTABLE ? { executablePath: process.env.DISPATCH_CHROME_EXECUTABLE } : {} }, +}); diff --git a/core/dashboard/public/assets/backups.js b/core/dashboard/public/assets/backups.js new file mode 100644 index 0000000..c687240 --- /dev/null +++ b/core/dashboard/public/assets/backups.js @@ -0,0 +1,2333 @@ +'use strict'; + +// Shared view models keep protection, history and progress consistent across screens. +const BackupViewModel = (() => { + const active = (op) => ['queued', 'running'].includes(op.status); + const time = (value) => Date.parse(value || '') || 0; + const scope = (value) => (value.source === 'set' ? 'system' : value.organizationId || 'core'); + const sorted = (items) => [...items].sort((a, b) => time(b.createdAt) - time(a.createdAt)); + function protection(data, id) { + const backups = sorted(data.backups.filter((b) => scope(b) === id)); + const operations = sorted(data.operations.filter((op) => scope(op) === id)); + const running = operations.find(active); + const verified = backups.find((b) => b.status === 'verified'); + const latest = backups[0]; + if (running) + return { + status: 'running', + label: + running.kind === 'restore' + ? 'Restoring' + : running.status === 'queued' + ? 'Queued' + : 'Backing up', + operation: running, + verified, + latest, + }; + const failed = operations.find((op) => op.status === 'failed'); + const recoveredAt = Math.max( + time(verified?.createdAt), + ...operations + .filter((op) => op.status === 'completed') + .map((op) => time(op.updatedAt || op.createdAt)), + ); + if (failed && time(failed.updatedAt || failed.createdAt) >= recoveredAt) + return { + status: 'failed', + label: 'Needs attention', + operation: failed, + verified, + latest, + }; + if (latest?.status === 'pending') + return { status: 'pending', label: 'Uploading backup', verified, latest }; + if (verified) return { status: 'verified', label: 'Protected', verified, latest }; + return { + status: latest?.status === 'expired' ? 'expired' : 'empty', + label: latest?.status === 'expired' ? 'Backup expired' : 'No backup yet', + verified, + latest, + }; + } + function activity(data) { + const linked = new Set( + data.operations.filter((op) => op.kind !== 'restore' && op.backupId).map((op) => op.backupId), + ); + return sorted([ + ...(data.sets || []) + .filter((set) => set.status !== 'deleted') + .map((set) => { + const request = data.operations.find( + (op) => op.setId === set.id && op.kind !== 'restore', + ); + return { + ...set, + source: 'set', + event: 'Backup', + name: 'Full system', + category: request?.category || null, + }; + }), + ...data.operations.map((op) => ({ + ...op, + source: 'operation', + event: op.kind === 'restore' ? 'Restore' : 'Backup', + name: + data.organizations.find((o) => o.id === op.organizationId)?.name || + (op.organizationId ? 'Unavailable DSP' : 'Platform Core'), + })), + ...data.backups + .filter((b) => !linked.has(b.id)) + .map((b) => ({ ...b, source: 'backup', event: 'Backup' })), + ]); + } + const needsAttention = (p) => ['failed', 'empty', 'expired'].includes(p.status); + function fleet(data) { + return data.organizations.map((org) => ({ + org, + ...protection(data, org.id), + })); + } + function filterFleet(items, search, filter) { + return items.filter( + (p) => + p.org.name.toLowerCase().includes(search.trim().toLowerCase()) && + (filter === 'all' || (filter === 'attention' ? needsAttention(p) : p.status === filter)), + ); + } + function route(hash) { + try { + const parts = hash.split('?')[0].split('/').slice(2).map(decodeURIComponent); + if (!parts[0]) return { mode: 'overview' }; + if (['settings', 'history', 'storage'].includes(parts[0]) && parts.length === 1) + return { mode: parts[0] }; + if (parts[0] === 'sets' && parts.length <= 2) + return { mode: 'sets', ...(parts[1] ? { setId: parts[1] } : {}) }; + if (parts[0] === 'operations' && parts.length === 2) + return { mode: 'operation', operationId: parts[1] }; + if (parts[0] === 'core' && parts.length === 1) return { mode: 'dsp', orgId: 'core' }; + if (parts[0] === 'core' && parts[1] === 'backups' && parts.length === 3) + return { mode: 'detail', orgId: 'core', backupId: parts[2] }; + if (parts[0] === 'dsps' && parts.length === 1) return { mode: 'dsps' }; + if (parts[0] === 'dsps' && parts.length === 2) return { mode: 'dsp', orgId: parts[1] }; + if (parts[0] === 'dsps' && parts[2] === 'history' && parts.length === 3) + return { mode: 'dsp-history', orgId: parts[1] }; + if (parts[0] === 'dsps' && parts[2] === 'backups' && parts.length === 4) + return { mode: 'detail', orgId: parts[1], backupId: parts[3] }; + } catch {} + return { mode: 'missing' }; + } + function stages(op) { + const restore = op.kind === 'restore'; + const coreRestore = restore && op.organizationId === null; + const labels = coreRestore + ? [ + 'Preparing Core restore', + 'Creating safety backup', + 'Restoring Core', + 'Checking Core', + 'Complete', + ] + : restore + ? [ + 'Preparing restore', + 'Restoring DSP data and creating safety backup', + 'Checking restored DSP', + 'Complete', + ] + : ['Preparing backup', 'Creating snapshot', 'Uploading backup', 'Complete']; + const phases = coreRestore + ? { + queued: 0, + snapshotting: 1, + uploading: 2, + verifying_core: 3, + recovering_core: 3, + completed: 4, + } + : restore + ? { queued: 0, stopping: 0, restoring: 1, starting: 2, completed: 3 } + : { + queued: 0, + snapshotting: 1, + backing_up: 1, + uploading: 2, + completed: 3, + }; + const index = phases[op.phase] ?? -1; + return labels.map((label, i) => ({ + label, + status: + op.status === 'completed' + ? 'done' + : index < 0 + ? 'pending' + : i < index + ? 'done' + : i === index + ? 'active' + : 'pending', + })); + } + return { + active, + time, + scope, + sorted, + protection, + activity, + route, + stages, + fleet, + filterFleet, + needsAttention, + }; +})(); +if (typeof module !== 'undefined') module.exports = BackupViewModel; + +if (typeof window !== 'undefined') + window.createBackupsViews = function ({ + timeZone, + byId, + dspIdentity, + node, + request, + mutation, + mutationKey, + settleMutationKey, + errorMessage, + }) { + const model = BackupViewModel; + let data = null, + active = false, + poll = null, + generation = 0, + busy = false, + stale = false; + let current = { mode: 'overview' }, + drawnHash = '', + notice = '', + noticeError = false; + let search = '', + statusFilter = 'all', + scopeFilter = 'all', + eventFilter = 'all', + categoryFilter = 'all', + dateFilter = 'all', + page = 1; + let scheduleScope = 'system'; + let backupSearch = '', + historyFiltersOpen = true; + const disclosures = new Map(); + let draft = null, + draftRevision = null, + dialog = null, + refreshDialog = null; + const pageSize = 10; + const root = () => byId('platform-backups-content'); + const paths = { + users: + 'M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2M16 3a4 4 0 0 1 0 8m6 10v-2a4 4 0 0 0-3-3.87M13 7a4 4 0 1 1-8 0 4 4 0 0 1 8 0', + repeat: 'm17 2 4 4-4 4M3 11V8a2 2 0 0 1 2-2h16M7 22l-4-4 4-4m14-1v3a2 2 0 0 1-2 2H3', + server: 'M3 3h18v7H3V3m0 11h18v7H3v-7M7 6h.01M7 17h.01m4-11h6m-6 11h6', + core: 'm12 3 9 5-9 5-9-5 9-5m-9 9 9 5 9-5m-18 5 9 5 9-5', + upload: 'M7 17H6a4 4 0 0 1-.6-8A7 7 0 0 1 19 7a5 5 0 0 1 0 10h-2M12 21V11m-4 4 4-4 4 4', + settings: + 'M9 3h6l1 3 3 1 2 5-2 5-3 1-1 3H9l-1-3-3-1-2-5 2-5 3-1 1-3M12 9a3 3 0 1 0 0 6 3 3 0 0 0 0-6', + check: 'M22 11.1V12a10 10 0 1 1-5.9-9.1M22 4 12 14l-3-3', + shield: 'M12 3 3 7v5c0 5 9 9 9 9s9-4 9-9V7l-9-4m-4 9 3 3 5-6', + warning: + 'm10.3 3.9-8 14A2 2 0 0 0 4 21h16a2 2 0 0 0 1.7-3.1l-8-14a2 2 0 0 0-3.4 0M12 9v4m0 4h.01', + clock: 'M22 12a10 10 0 1 1-20 0 10 10 0 0 1 20 0M12 6v6l4 2', + calendar: 'M5 5h14a2 2 0 0 1 2 2v13H3V7a2 2 0 0 1 2-2M7 3v4m10-4v4M3 11h18m-14 4h2m3 0h2', + search: 'M21 21l-5-5M18 10a8 8 0 1 1-16 0 8 8 0 0 1 16 0', + chevron: 'm9 5 7 7-7 7', + back: 'm14 5-7 7 7 7', + close: 'm6 6 12 12M18 6 6 18', + info: 'M22 12a10 10 0 1 1-20 0 10 10 0 0 1 20 0M12 11v6m0-10h.01', + spinner: 'M22 12a10 10 0 1 1-10-10', + lock: 'M6 10h12v11H6V10m2 0V6a4 4 0 0 1 8 0v4m-4 5v2', + globe: 'M22 12a10 10 0 1 1-20 0 10 10 0 0 1 20 0M2 12h20M12 2c5 5 5 15 0 20-5-5-5-15 0-20', + }; + const messages = { + backup_settings_conflict: + 'Settings changed in another window. The saved settings have been reloaded; please apply your changes again.', + backup_dsp_unavailable: + 'This DSP is busy or unavailable. Try again when its current operation finishes.', + backup_operation_in_progress: + 'An operation is already running. Refresh and try again when it finishes.', + backup_restore_unavailable: + 'This backup cannot be restored right now. Refresh to see its current status.', + backup_confirmation_required: 'Enter the DSP name exactly to confirm.', + backup_identity_conflict: + 'The backup needs its original DSP configuration and a compatible release.', + restore_recovered_previous: + 'Restore did not pass verification. The previous DSP was recovered.', + backup_upload_failed: + 'The backup upload failed. Check backup storage, then retry the operation.', + backup_worker_interrupted: + 'The backup worker was interrupted. Retry the operation to continue.', + backup_verification_timeout: + 'The backup upload did not finish in time. Check backup storage before retrying.', + backup_download_timeout: + 'The backup could not be downloaded in time. Check backup storage before retrying.', + backup_failed: + 'The backup could not be created. Your previous uploaded backups are still available.', + backup_operation_failed: + 'The operation could not finish. Review its details before trying again.', + restore_recovery_required: + 'The restore requires attention on the server before this DSP can be used again.', + }; + const el = (tag, cls, text) => node(tag, cls, text); + function icon(name) { + const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + svg.setAttribute('viewBox', '0 0 24 24'); + svg.setAttribute('class', `backup-icon${name === 'spinner' ? ' backup-spin' : ''}`); + svg.setAttribute('aria-hidden', 'true'); + const path = document.createElementNS(svg.namespaceURI, 'path'); + path.setAttribute('d', paths[name] || paths.info); + svg.append(path); + return svg; + } + function button(text, action, variant = 'secondary', id) { + const b = el('button', `backup-button backup-button-${variant}`, text); + b.type = 'button'; + b.disabled = busy; + if (id) b.id = id; + b.addEventListener('click', action); + return b; + } + function link(text, href, name) { + const a = el('a', 'backup-link', text); + a.href = href; + if (name) a.append(icon(name)); + return a; + } + const dspPath = (id) => + id === 'core' ? '#/backups/core' : `#/backups/dsps/${encodeURIComponent(id)}`; + const detailPath = (b) => `${dspPath(model.scope(b))}/backups/${encodeURIComponent(b.id)}`; + const operationPath = (op) => `#/backups/operations/${encodeURIComponent(op.id)}`; + const scopeName = (id) => + id === 'core' + ? 'Platform Core' + : data.organizations.find((o) => o.id === id)?.name || 'Unavailable DSP'; + function navigate(hash) { + if (location.hash === hash) draw(); + else location.hash = hash; + } + const disabled = () => busy || stale || !data.enabled; + function date(value) { + if (!value || !Number.isFinite(Date.parse(value))) return 'Not available'; + return new Intl.DateTimeFormat(undefined, { + year: 'numeric', month: 'short', day: 'numeric', hour: 'numeric', + minute: '2-digit', timeZoneName: 'short', timeZone, + }).format(new Date(value)); + } + const bytes = (value) => + !Number.isFinite(value) + ? 'Not available' + : value < 1048576 + ? `${Math.ceil(value / 1024)} KB` + : value < 1073741824 + ? `${(value / 1048576).toFixed(1)} MB` + : `${(value / 1073741824).toFixed(1)} GB`; + const frequency = (value) => + ({ hourly: 'Every hour', daily: 'Every day', weekly: 'Every week' })[value] || value; + const scheduleTime = (value) => + new Intl.DateTimeFormat(undefined, { + timeStyle: 'short', + timeZone: 'UTC', + }).format(new Date(`2000-01-01T${value}:00Z`)); + const retention = (value) => (value === null ? 'Keep all backups' : `Keep for ${value} days`); + const trigger = (value) => + ({ + scheduled: 'Scheduled', + manual: 'Manual', + upgrade: 'Pre-update', + pre_update: 'Pre-update', + restore_safety: 'Safety backup', + restore: 'Before restore', + })[value] || 'Manual'; + const problem = (code) => + messages[code] || 'The operation needs attention. Its technical details are available below.'; + function status(value, label) { + const text = + label || + { + verified: 'Verified', + pending: 'Uploading backup', + running: 'In progress', + queued: 'Queued', + failed: 'Failed', + expired: 'Expired', + empty: 'No backup yet', + completed: 'Completed', + }[value] || + value; + const s = el('span', `backup-status ${value}`, text); + s.prepend( + icon( + ['verified', 'completed'].includes(value) + ? 'check' + : value === 'failed' + ? 'warning' + : ['running', 'pending'].includes(value) + ? 'spinner' + : 'clock', + ), + ); + return s; + } + function card(title, name) { + const c = el('section', 'backup-card'); + if (title) { + const h = el('h2', 'backup-card-title', title); + if (name) { + const tile = el('span', 'backup-icon-tile'); + tile.append(icon(name)); + h.prepend(tile); + } + c.append(h); + } + return c; + } + function heading(title, copy, action) { + const row = el('div', 'backup-page-title'); + const text = el('div'); + const h = el('h2', null, title); + h.tabIndex = -1; + h.id = 'backup-view-title'; + text.append(h); + if (copy) text.append(el('p', 'backup-muted', copy)); + row.append(text); + if (action) row.append(action); + root().append(row); + } + function breadcrumb(items) { + const nav = el('nav', 'backup-breadcrumb'); + nav.setAttribute('aria-label', 'Breadcrumb'); + nav.append(link('Backups', '#/backups')); + for (const [text, href] of items) + nav.append(icon('chevron'), href ? link(text, href) : el('span', null, text)); + root().append(nav); + } + function note(text, kind = 'info') { + const p = el('div', `backup-note backup-note-${kind}`); + p.append(icon(kind), el('span', null, text)); + return p; + } + function pairList(items) { + const dl = el('dl', 'backup-pairs'); + for (const [key, value] of items) { + const row = el('div'), + detail = el('dd'); + detail.append(typeof value === 'string' ? document.createTextNode(value) : value); + row.append(el('dt', null, key), detail); + dl.append(row); + } + return dl; + } + function avatar(org) { + const identity = dspIdentity(org.name); + const a = el('span', `backup-avatar ${identity.className}`, identity.initials); + a.setAttribute('aria-hidden', 'true'); + return a; + } + function searchField(id, value, change, placeholder = 'Search DSPs') { + const wrap = el('label', 'backup-search-wrap'); + wrap.append(icon('search')); + const input = el('input'); + input.type = 'search'; + input.id = id; + input.placeholder = placeholder; + input.setAttribute('aria-label', placeholder); + input.value = value; + input.addEventListener('input', () => change(input.value)); + wrap.append(input); + return wrap; + } + function select(id, options, value, change) { + const s = el('select'); + s.id = id; + for (const [key, text] of options) { + const o = el('option', null, text); + o.value = key; + s.append(o); + } + s.value = String(value); + s.addEventListener('change', () => change(s.value)); + return s; + } + function field(text, input) { + const label = el('label', 'backup-field'); + if (!input.hasAttribute('aria-label')) input.setAttribute('aria-label', text); + label.append(el('span', null, text), input); + return label; + } + function table(headers) { + const wrapper = el('div', 'backup-table-scroll'); + if (headers.at(-1) === 'Stored') wrapper.classList.add('backup-storage-table'); + wrapper.tabIndex = 0; + wrapper.setAttribute('role', 'region'); + wrapper.setAttribute('aria-label', headers[0] === 'DSP' ? 'DSP backups' : 'Backup history'); + const t = el('table', 'backup-table'), + head = el('thead'), + tr = el('tr'), + body = el('tbody'); + for (const title of headers) { + const th = el('th', null, title); + th.scope = 'col'; + tr.append(th); + } + head.append(tr); + t.append(head, body); + wrapper.append(t); + return { wrapper, body }; + } + function cell(value, cls) { + const td = el('td', cls); + td.append(typeof value === 'string' ? document.createTextNode(value) : value); + return td; + } + function paginate(items, container, render) { + const total = Math.max(1, Math.ceil(items.length / pageSize)); + page = Math.min(page, total); + for (const item of items.slice((page - 1) * pageSize, page * pageSize)) render(item); + if (items.length > pageSize) { + const footer = el('div', 'backup-pagination'); + footer.append( + el( + 'span', + null, + `${(page - 1) * pageSize + 1}–${Math.min(page * pageSize, items.length)} of ${items.length}`, + ), + ); + const controls = el('div', 'backup-actions'); + const previous = button('Previous', () => { + page--; + draw(); + }), + next = button('Next', () => { + page++; + draw(); + }); + previous.disabled = page === 1; + next.disabled = page === total; + controls.append(previous, el('span', null, `Page ${page} of ${total}`), next); + footer.append(controls); + container.append(footer); + } + } + function empty(container, title, copy, action) { + const box = el('div', 'backup-empty'); + box.append(icon('core'), el('h3', null, title), el('p', null, copy)); + if (action) box.append(action); + container.append(box); + } + function openEvent(event) { + navigate(event.source === 'backup' ? detailPath(event) : operationPath(event)); + } + const setPath = (set) => `#/backups/sets/${encodeURIComponent(set.id)}`; + function eventLink(event, text = 'View details') { + return link( + text, + event.source === 'set' + ? setPath(event) + : event.source === 'backup' + ? detailPath(event) + : operationPath(event), + 'chevron', + ); + } + function splitLayout() { + const grid = el('div', 'backup-split'); + const main = el('div', 'backup-main'); + const aside = el('aside', 'backup-rail'); + grid.append(main, aside); + root().append(grid); + return { grid, main, aside }; + } + function section(title) { + const group = el('section', 'backup-section'); + if (title) group.append(el('h2', null, title)); + return group; + } + function textAction(text, action, id) { + return button(text, action, 'text', id); + } + function editSchedule(id) { + scheduleScope = id; + draft = draftRevision = null; + navigate('#/backups/settings'); + } + function scheduleSummary(id, title = 'Schedule') { + const group = section(title); + const policy = data.schedules?.find((s) => s.scope === id); + const settings = policy?.settings || data.settings; + const summary = el('p', 'backup-schedule-state'); + summary.append( + document.createTextNode('Automatic backups '), + el('span', settings.enabled ? '' : 'backup-off', settings.enabled ? 'On' : 'Off'), + ); + group.append(summary); + if (settings.enabled) { + group.append( + el( + 'p', + 'backup-muted', + `${frequency(settings.frequency)}${settings.frequency === 'hourly' ? '' : ` at ${scheduleTime(settings.time)}`} · ${settings.timezone.replaceAll('_', ' ')}`, + ), + ); + const next = policy?.nextBackupAt || (id === 'system' ? data.nextBackupAt : null); + if (next) group.append(el('p', 'backup-muted', `Next backup: ${date(next)}`)); + } + group.append(textAction('Edit schedule', () => editSchedule(id), `backup-edit-${id}`)); + return group; + } + function disclosure(title, id) { + const details = el('details', 'backup-disclosure'); + details.id = id; + details.open = disclosures.get(id) || false; + const summary = el('summary', null, title); + summary.append(icon('chevron')); + details.append(summary); + details.addEventListener('toggle', () => { + if (details.isConnected) disclosures.set(id, details.open); + }); + return details; + } + function scopeSidebar(aside, id) { + aside.append(scheduleSummary(id, id === 'core' ? 'Core schedule' : 'DSP schedule')); + const scope = section('Backup scope'); + scope.append( + el('p', null, id === 'core' ? 'Platform accounts and settings' : `${scopeName(id)} only`), + el( + 'p', + 'backup-muted', + id === 'core' + ? 'DSP data is backed up separately.' + : 'Platform Core is backed up separately.', + ), + ); + aside.append(scope, storageSummary()); + } + function scopeOperation(container, state) { + if (!state.operation) return; + const issue = note( + state.status === 'failed' + ? problem(state.operation.failureCode) + : 'An operation is in progress for this scope.', + state.status === 'failed' ? 'warning' : 'info', + ); + issue.append( + link( + state.status === 'failed' ? 'Review issue' : 'View progress', + operationPath(state.operation), + 'chevron', + ), + ); + container.append(issue); + } + function backupHistory(container, id, withSearch = false) { + const history = section('Backup history'); + const toolbar = el('div', 'backup-toolbar'); + if (withSearch) + toolbar.append( + searchField( + 'backup-scope-search', + backupSearch, + (value) => { + backupSearch = value; + page = 1; + draw(); + }, + 'Search backups', + ), + ); + const category = select( + 'backup-scope-category', + [ + ['all', 'All categories'], + ['manual', 'Manual'], + ['scheduled', 'Scheduled'], + ['pre_update', 'Pre-update'], + ], + categoryFilter, + (value) => { + categoryFilter = value; + page = 1; + draw(); + }, + ); + category.setAttribute('aria-label', 'Category'); + toolbar.append(category); + history.append(toolbar); + const backups = model.sorted( + data.backups.filter( + (b) => + model.scope(b) === id && + (categoryFilter === 'all' || (b.category || b.trigger) === categoryFilter) && + `${date(b.createdAt)} ${trigger(b.category || b.trigger)} ${b.status}` + .toLowerCase() + .includes(backupSearch.trim().toLowerCase()), + ), + ); + backupTable(history, backups); + container.append(history); + } + + const storedBytes = (value) => + Number.isFinite(value) && value < 1024 ? `${value} B` : bytes(value); + function storageSummary() { + const group = section('Storage'); + const usage = data.storageUsage; + const connection = + data.storage.status === 'connected' + ? 'Connected' + : data.storage.status === 'attention' + ? 'Needs attention' + : 'Unavailable'; + group.append(el('p', null, `${storedBytes(usage?.bytes)} total · ${connection}`)); + if (usage?.status === 'stale') group.append(el('p', 'backup-muted', 'Last known usage')); + group.append(link('Manage storage', '#/backups/storage')); + return group; + } + function storageView() { + const { main, aside } = splitLayout(); + const connection = section('Connection'); + connection.append(storageStatus('Connected'), el('p', 'backup-muted', 'Cloudflare R2')); + const measurement = section('Measurement'); + measurement.append(el('p', null, 'Refreshes about every 5 minutes')); + aside.append( + connection, + measurement, + el( + 'p', + 'backup-muted', + 'Recovery points reuse the archives shown here. Their storage is counted once.', + ), + ); + const usage = data.storageUsage; + const summary = section('Storage usage'); + main.append(summary); + if (!usage || usage.status === 'unavailable') { + summary.append( + note( + 'Storage usage is not available yet. A measurement will appear after the backup service scans storage.', + ), + ); + return; + } + summary.append( + el('p', 'backup-storage-total', storedBytes(usage.bytes)), + el('p', 'backup-muted', `Measured ${date(usage.checkedAt)}`), + ); + if (usage.status === 'stale') + summary.append( + note( + 'Showing the last measured usage. Storage could not be refreshed; these totals may have changed.', + 'warning', + ), + ); + const scopes = section('Storage by scope'), + rows = table(['Scope', 'Stored']); + rows.wrapper.setAttribute('aria-label', 'Storage by scope'); + for (const scope of usage.scopes.filter((s) => !s.removed)) { + const row = el('tr'); + row.append(cell(scope.name), cell(storedBytes(scope.bytes))); + rows.body.append(row); + } + const extra = [usage.manifestBytes, usage.legacyBytes, usage.other?.bytes]; + const overhead = extra.every(Number.isFinite) ? extra.reduce((a, b) => a + b, 0) : null; + const other = el('tr'); + other.append(cell('Additional backup storage'), cell(storedBytes(overhead))); + rows.body.append(other); + scopes.append(rows.wrapper); + const additional = disclosure('Additional storage details', 'backup-storage-additional'); + additional.append( + pairList([ + ['Backup archives', String(usage.backupCount)], + ['Full-system manifests', storedBytes(usage.manifestBytes)], + ['Legacy and rollout safety backups', storedBytes(usage.legacyBytes)], + ['Unassigned archives', storedBytes(usage.other?.bytes)], + ]), + ); + scopes.append(additional); + main.append(scopes); + const retained = disclosure('Removed DSPs — retained backups', 'backup-storage-retained'); + const removed = table(['Scope', 'Backups', 'Stored']); + removed.wrapper.setAttribute('aria-label', 'Removed DSPs — retained backups'); + for (const scope of usage.scopes.filter((s) => s.removed)) { + const row = el('tr'); + row.append( + cell(scope.name), + cell(String(scope.backupCount)), + cell(storedBytes(scope.bytes)), + ); + removed.body.append(row); + } + retained.append( + removed.body.children.length + ? removed.wrapper + : el('p', 'backup-muted', 'No removed DSPs have retained backup storage.'), + el( + 'p', + 'backup-muted', + 'These backups remain stored while the DSP is removed. Their usage is included in the total above.', + ), + ); + const full = disclosure('Full-system backup storage', 'backup-storage-system'); + const sets = table(['Recovery point', 'Backups', 'Stored']); + sets.wrapper.setAttribute('aria-label', 'Full-system backup storage'); + for (const set of (data.sets || []).filter((s) => s.status !== 'deleted')) { + const measured = usage.sets?.find((s) => s.id === set.id), + row = el('tr'); + row.append( + cell(link(`${date(set.createdAt)} · ${set.status}`, setPath(set))), + cell(measured ? String(measured.backupCount) : 'Not measured'), + cell(storedBytes(measured?.bytes)), + ); + sets.body.append(row); + } + full.append( + sets.body.children.length + ? sets.wrapper + : el('p', 'backup-muted', 'No full-system backups yet.'), + ); + main.append(retained, full); + } + + function overview() { + const { main, aside } = splitLayout(); + const latest = model.sorted((data.sets || []).filter((s) => s.status !== 'deleted'))[0]; + const summary = section('Latest full-system backup'); + if (latest) { + const line = el('div', 'backup-latest'); + line.append(el('p', null, date(latest.createdAt)), status(latest.status)); + const dspCount = latest.members.filter((m) => m.organizationId).length; + summary.append( + line, + el('p', 'backup-muted', `Platform Core + ${dspCount} DSP${dspCount === 1 ? '' : 's'}`), + link('View details', setPath(latest), 'chevron'), + ); + if (latest.restore && latest.restore.status !== 'completed') + summary.append( + note( + latest.restore.status === 'failed' + ? 'Full-system restore is incomplete. Review the failed component.' + : 'Full-system restore in progress.', + latest.restore.status === 'failed' ? 'warning' : 'info', + ), + ); + } else summary.append(el('p', 'backup-muted', 'No full-system backups yet.')); + main.append(summary); + const fleet = model.fleet(data); + const protection = section('DSPs'); + const rows = table(['DSP', 'Last backup', 'Status']); + for (const item of fleet) { + const row = el('tr'); + row.append( + cell(link(item.org.name, dspPath(item.org.id)), 'backup-dsp-name'), + cell(item.verified ? date(item.verified.createdAt) : 'No verified backup'), + cell(status(item.status, item.label)), + ); + rows.body.append(row); + } + if (fleet.length) protection.append(rows.wrapper); + else + protection.append(el('p', 'backup-muted', 'DSPs will appear here once they are created.')); + main.append(protection); + aside.append(scheduleSummary('system'), storageSummary()); + // Failure and running states remain actionable without a permanent status dashboard. + const core = model.protection(data, 'core'); + if (core.status !== 'verified') { + const state = section('Platform Core'); + state.append(status(core.status, core.label), link('View Platform Core', '#/backups/core')); + aside.append(state); + } + } + + const isInspector = (route) => + route.mode === 'dsps' || (route.mode === 'dsp' && route.orgId !== 'core'); + function dspInspector() { + if (current.orgId && !data.organizations.some((o) => o.id === current.orgId)) + return missing('DSP not found'); + const items = + statusFilter === 'attention' + ? model.fleet(data).filter(model.needsAttention) + : model.fleet(data); + const selected = items.find((p) => p.org.id === current.orgId) || items[0]; + if (!selected) { + const { main } = splitLayout(); + empty( + main, + data.organizations.length ? 'No DSPs need attention' : 'No DSPs yet', + data.organizations.length + ? 'All DSPs have a verified backup or an operation in progress.' + : 'DSPs will appear here once they are created.', + data.organizations.length ? link('View all DSPs', '#/backups/dsps') : null, + ); + return; + } + if (current.orgId !== selected.org.id) { + const hash = + dspPath(selected.org.id) + (statusFilter === 'attention' ? '?status=attention' : ''); + history.replaceState({}, '', `${location.pathname}${location.search}${hash}`); + current = model.route(hash); + drawnHash = hash; + } + const { main, aside } = splitLayout(); + const picker = select( + 'backup-dsp-picker', + items.map((p) => [p.org.id, p.org.name]), + selected.org.id, + (id) => navigate(dspPath(id)), + ); + const chooser = field('DSP', picker); + chooser.classList.add('backup-dsp-picker'); + main.append(chooser); + if (statusFilter === 'attention') main.append(link('View all DSPs', '#/backups/dsps')); + const identity = section(); + const heading = el('div', 'backup-scope-heading'); + heading.append(el('h2', null, selected.org.name), status(selected.status, selected.label)); + identity.append( + heading, + el( + 'p', + 'backup-muted', + `Latest backup · ${selected.verified ? date(selected.verified.createdAt) : 'No verified backup'}`, + ), + ); + scopeOperation(identity, selected); + main.append(identity); + backupHistory(main, selected.org.id, true); + scopeSidebar(aside, selected.org.id); + } + + function backupTable(container, backups) { + const t = table(['Date', 'Trigger', 'Status', '']); + container.append(t.wrapper); + paginate(backups, container, (b) => { + const row = el('tr'); + row.append( + cell(date(b.createdAt)), + cell(trigger(b.category || b.trigger)), + cell(status(b.status)), + cell(link('View details', detailPath(b), 'chevron')), + ); + t.body.append(row); + }); + if (!backups.length) + empty( + container, + 'No backups available', + 'Create a backup now or wait for the next scheduled backup.', + ); + } + function dsp() { + const id = current.orgId; + if (id !== 'core' && !data.organizations.some((o) => o.id === id)) + return missing('DSP not found'); + const { main, aside } = splitLayout(); + const summary = section(scopeName(id)); + if (id === 'core') summary.append(el('p', 'backup-muted', 'Accounts and platform settings')); + const state = model.protection(data, id); + const line = el('div', 'backup-latest'); + line.append( + el('p', null, state.verified ? date(state.verified.createdAt) : 'No verified backup'), + status(state.status, state.status === 'verified' ? 'Verified' : state.label), + ); + summary.append(el('p', 'backup-latest-label backup-muted', 'Latest backup'), line); + scopeOperation(summary, state); + main.append(summary); + backupHistory(main, id); + scopeSidebar(aside, id); + } + + function settings() { + const { main, aside } = splitLayout(); + const content = section('Backup settings'); + main.append(content); + const policy = data.schedules?.find((s) => s.scope === scheduleScope) || { + settings: data.settings, + revision: data.revision, + }; + const picker = select( + 'backup-schedule-scope', + [ + ['system', 'Full system'], + ['core', 'Platform Core'], + ...data.organizations.map((o) => [o.id, o.name]), + ], + scheduleScope, + (value) => { + scheduleScope = value; + draft = null; + draftRevision = null; + draw(); + }, + ); + const scopeField = field('Schedule for', picker); + scopeField.classList.add('backup-settings-scope'); + content.append(scopeField); + const form = el('form'); + form.id = 'backup-settings-form'; + const value = draft || policy.settings; + const update = (key, v) => { + if (!draft) draftRevision = policy.revision; + draft = { ...(draft || policy.settings), [key]: v }; + }; + const title = el('div', 'backup-card-heading'); + title.append(el('h3', null, 'Automatic backups')); + const toggle = el('input', 'backup-switch'); + toggle.type = 'checkbox'; + toggle.id = 'backup-enabled'; + toggle.checked = value.enabled; + toggle.addEventListener('change', () => { + update('enabled', toggle.checked); + draw(); + }); + toggle.setAttribute('aria-label', 'Automatic backups'); + const toggleLabel = el('label', 'backup-toggle-label'); + toggleLabel.append(toggle, el('span', null, value.enabled ? 'On' : 'Off')); + title.append(toggleLabel); + form.append(title); + const fields = el('fieldset', 'backup-form-grid'); + fields.disabled = disabled() || !value.enabled; + fields.append( + field( + 'Frequency', + select( + 'backup-frequency', + [ + ['hourly', 'Every hour'], + ['daily', 'Every day'], + ['weekly', 'Every week'], + ], + value.frequency, + (v) => { + update('frequency', v); + draw(); + }, + ), + ), + ); + if (value.frequency !== 'hourly') { + const input = el('input'); + input.type = 'time'; + input.id = 'backup-time'; + input.required = true; + input.value = value.time; + input.addEventListener('input', () => update('time', input.value)); + fields.append(field('Time', input)); + } + if (value.frequency === 'weekly') + fields.append( + field( + 'Day', + select( + 'backup-weekday', + ['Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday'].map( + (d, i) => [i, d], + ), + value.weekday, + (v) => update('weekday', Number(v)), + ), + ), + ); + const zones = [ + ...new Set([ + value.timezone, + ...(Intl.supportedValuesOf + ? Intl.supportedValuesOf('timeZone') + : ['UTC', 'America/Los_Angeles', 'America/New_York']), + ]), + ]; + fields.append( + field( + 'Time zone', + select( + 'backup-timezone', + zones.map((z) => [z, z.replaceAll('_', ' ')]), + value.timezone, + (v) => update('timezone', v), + ), + ), + ); + form.append(fields); + const retained = el('div', 'backup-form-section'); + retained.append( + el('h3', null, 'Retention'), + field( + 'Keep backups', + select( + 'backup-retention', + [ + ['all', 'Keep all backups'], + ['7', 'Keep for 7 days'], + ['30', 'Keep for 30 days'], + ['90', 'Keep for 90 days'], + ['365', 'Keep for one year'], + ], + value.retentionDays ?? 'all', + (v) => update('retentionDays', v === 'all' ? null : Number(v)), + ), + ), + el( + 'p', + 'backup-muted', + 'Changes apply to future backups. Existing locked backups remain protected.', + ), + ); + form.append(retained); + form.addEventListener('submit', async (event) => { + event.preventDefault(); + if (form.reportValidity()) + await change({ + action: 'settings', + ...(scheduleScope === 'core' || scheduleScope === 'system' + ? { scope: scheduleScope } + : { organizationId: scheduleScope }), + settings: draft || policy.settings, + revision: draftRevision ?? policy.revision, + }); + }); + for (const control of form.querySelectorAll('input, select')) control.disabled = disabled(); + content.append(form); + const applies = section('Applies to'); + applies.append( + el( + 'p', + null, + scheduleScope === 'system' + ? `Platform Core + ${data.organizations.length} DSP${data.organizations.length === 1 ? '' : 's'}` + : scheduleScope === 'core' + ? 'Platform Core only' + : scopeName(scheduleScope), + ), + ); + const scope = section('Schedule scope'); + scope.append( + el( + 'p', + null, + scheduleScope === 'system' + ? 'This schedule runs a full-system backup.' + : 'This schedule backs up the selected scope.', + ), + el('p', 'backup-muted', 'Other schedules stay unchanged.'), + ); + const storage = section('Storage'); + storage.append(el('p', null, 'Cloudflare R2'), storageStatus('Connected')); + aside.append(applies, scope, storage); + } + function historyView() { + const { main, aside } = splitLayout(); + aside.id = 'backup-history-filters'; + aside.hidden = !historyFiltersOpen; + const title = section('Backup history'); + title.append(el('p', 'backup-muted', 'Backup and restore activity')); + main.append(title); + const toolbar = el('div', 'backup-toolbar'); + const filter = (key) => (value) => { + if (key === 'scope') scopeFilter = value; + else if (key === 'date') dateFilter = value; + else if (key === 'category') categoryFilter = value; + else statusFilter = value; + page = 1; + draw(); + }; + toolbar.append( + searchField( + 'backup-history-search', + search, + (value) => { + search = value; + page = 1; + draw(); + }, + 'Search activity', + ), + field( + 'Scope', + select( + 'backup-history-scope', + [ + ['all', 'All scopes'], + ['core', 'Platform Core'], + ['system', 'Full system'], + ...data.organizations.map((o) => [o.id, o.name]), + ], + scopeFilter, + filter('scope'), + ), + ), + field( + 'Date range', + select( + 'backup-history-date', + [ + ['all', 'All dates'], + ['7', 'Last 7 days'], + ['30', 'Last 30 days'], + ], + dateFilter, + filter('date'), + ), + ), + field( + 'Status', + select( + 'backup-history-status', + [ + ['all', 'All statuses'], + ['active', 'In progress'], + ['success', 'Successful'], + ['failed', 'Failed'], + ['expired', 'Expired'], + ], + statusFilter, + filter('status'), + ), + ), + ); + toolbar.append( + field( + 'Category', + select( + 'backup-history-category', + [ + ['all', 'All categories'], + ['scheduled', 'Scheduled'], + ['manual', 'Manual'], + ['pre_update', 'Pre-update'], + ], + categoryFilter, + filter('category'), + ), + ), + ); + const fields = [...toolbar.querySelectorAll('.backup-field')]; + for (const field of fields.slice(1)) aside.append(field); + const toggleFilters = textAction( + 'Filters', + () => { + historyFiltersOpen = !historyFiltersOpen; + draw(); + }, + 'backup-filter-toggle', + ); + toggleFilters.setAttribute('aria-expanded', String(historyFiltersOpen)); + toggleFilters.setAttribute('aria-controls', aside.id); + toolbar.append(toggleFilters); + aside.append( + textAction( + 'Clear filters', + () => { + search = ''; + statusFilter = scopeFilter = eventFilter = categoryFilter = dateFilter = 'all'; + page = 1; + draw(); + }, + 'backup-clear-filters', + ), + ); + main.append(toolbar); + const tabs = el('div', 'backup-filter-tabs'); + for (const [id, name] of [ + ['all', 'All'], + ['Backup', 'Backups'], + ['Restore', 'Restores'], + ]) { + const b = button(name, () => { + eventFilter = id; + page = 1; + draw(); + }); + b.setAttribute('aria-pressed', String(eventFilter === id)); + tabs.append(b); + } + main.append(tabs); + const items = model + .activity(data) + .filter( + (e) => + e.name.toLowerCase().includes(search.trim().toLowerCase()) && + (scopeFilter === 'all' || model.scope(e) === scopeFilter) && + (eventFilter === 'all' || e.event === eventFilter) && + (categoryFilter === 'all' || + (e.category || e.trigger || (e.source === 'set' ? null : 'manual')) === + categoryFilter) && + (dateFilter === 'all' || + model.time(e.createdAt) >= Date.now() - Number(dateFilter) * 86400000) && + (statusFilter === 'all' || + (statusFilter === 'active' + ? model.active(e) || e.status === 'pending' + : statusFilter === 'success' + ? ['verified', 'completed'].includes(e.status) + : e.status === statusFilter)), + ); + const c = section(), + t = table(['Scope', 'Event', 'Started', 'Status', '']); + c.append(t.wrapper); + paginate(items, c, (e) => { + const tr = el('tr'); + tr.append( + cell(e.name, 'backup-dsp-name'), + cell(activityType(e)), + cell(date(e.createdAt)), + cell(status(e.status)), + cell(eventLink(e)), + ); + t.body.append(tr); + }); + if (!items.length) empty(c, 'No activity found', 'Try different filters or start a backup.'); + c.append( + el( + 'p', + 'backup-muted backup-event-count', + `${items.length} event${items.length === 1 ? '' : 's'}`, + ), + ); + main.append(c); + } + function activityType(event) { + const label = el('span', 'backup-event-type', event.event); + if (event.event === 'Backup' && (event.source !== 'set' || event.category)) + label.append(el('small', 'backup-muted', trigger(event.category || event.trigger))); + return label; + } + function details() { + const b = data.backups.find( + (b) => b.id === current.backupId && model.scope(b) === current.orgId, + ); + if (!b) return missing('Backup not found'); + breadcrumb([[b.name, dspPath(current.orgId)], ['Backup details']]); + heading('Backup details', `Scope: ${b.name}`); + const c = card(date(b.createdAt)); + c.append( + status( + b.status, + b.status === 'verified' && !b.restoreBlocked ? 'Uploaded and ready' : null, + ), + ); + const info = pairList([ + ['Scope', b.name], + ['Backup type', trigger(b.trigger)], + ['Created', date(b.createdAt)], + ['Size', bytes(b.size)], + [ + 'Retention', + b.category === 'pre_update' + ? 'Replaced after the next pre-update backup uploads' + : retention(b.retentionDays), + ], + ['Protected until', b.expiresAt ? date(b.expiresAt) : 'No expiration'], + ]); + info.classList.add('backup-detail-pairs'); + c.append(info); + if (b.status === 'verified') + c.append( + note( + `${b.verification === 'upload' ? 'Upload confirmed' : 'Restore verification passed'}${b.verifiedAt ? ` · ${date(b.verifiedAt)}` : ''}`, + 'check', + ), + ); + root().append(c); + const recovery = card('Recovery'); + { + const actions = el('div', 'backup-card-heading'); + actions.append(el('p', null, `Restore ${b.name} to this backup.`)); + const review = button( + 'Review restore', + () => confirmRestore(b), + 'primary', + 'review-backup-restore', + ); + review.disabled = + disabled() || !!data.operationBlocked || b.status !== 'verified' || !!b.restoreBlocked; + actions.append(review); + recovery.append(actions); + if (b.restoreBlocked) recovery.append(note(b.restoreBlocked)); + } + recovery.append( + note( + 'Deleting a backup permanently erases its stored files. Live data is unchanged.', + 'lock', + ), + ); + const remove = button('Delete backup', () => confirmBackupDelete(b), 'secondary'); + remove.disabled = + disabled() || + (data.deletions || []).some((d) => d.backupId === b.id && d.status === 'queued'); + recovery.append(remove); + const deletion = (data.deletions || []).find((d) => d.backupId === b.id); + if (deletion?.status === 'failed') + recovery.append( + note( + 'Backup deletion failed. The backup remains listed until storage confirms removal. Try Delete backup again.', + 'warning', + ), + ); + else if (deletion?.status === 'queued') + recovery.append(note('Deleting backup and verifying that its stored files are gone.')); + root().append(recovery, link('Back to backups', dspPath(current.orgId), 'back')); + } + function operation() { + const op = data.operations.find((o) => o.id === current.operationId); + if (!op) return missing('Operation not found'); + const id = model.scope(op), + name = scopeName(id), + restore = op.kind === 'restore', + failed = op.status === 'failed', + complete = op.status === 'completed'; + const backup = data.backups.find((b) => b.id === op.backupId && model.scope(b) === id), + safety = data.backups.find((b) => b.id === op.safetyBackupId && model.scope(b) === id); + breadcrumb([ + [name, dspPath(id)], + [failed ? 'Operation issue' : restore ? 'Restore' : 'Backup'], + ]); + heading( + failed + ? `${restore ? 'Restore' : 'Backup'} needs attention` + : complete + ? `${restore ? 'Restore' : 'Backup'} complete` + : `${restore ? 'Restoring' : 'Backing up'} ${name}`, + `Scope: ${name}`, + ); + const grid = el('div', 'backup-overview-grid'), + main = card(), + side = card( + failed ? 'Previous uploaded backup' : complete ? 'Operation details' : 'Summary', + ); + if (failed) { + main.classList.add('backup-issue-card'); + main.append( + status('failed', `Latest ${restore ? 'restore' : 'backup'} failed`), + el('h3', null, problem(op.failureCode)), + el('p', 'backup-muted', date(op.updatedAt || op.createdAt)), + ); + const org = data.organizations.find((o) => o.id === id); + if (!restore) { + const retry = button('Retry backup', () => startBackup(id), 'primary'); + retry.disabled = disabled() || (id === 'core' ? !data.canBackupCore : !org?.canBackup); + main.append(retry); + } + const technical = el('details', 'backup-technical'); + technical.append( + el('summary', null, 'Technical details'), + pairList([ + ['Operation', op.id], + ['Failure code', op.failureCode || 'Not available'], + ]), + ); + main.append(technical); + const previous = model.protection(data, id).verified; + if (previous) + side.append( + el('h3', null, date(previous.createdAt)), + status('verified'), + el('p', 'backup-muted', bytes(previous.size)), + link('View backup', detailPath(previous), 'chevron'), + ); + else + side.append(el('p', 'backup-muted', 'No uploaded backup is available for this scope.')); + } else { + if (complete) + main.append( + status('completed', `${restore ? 'Restore' : 'Backup'} complete`), + el( + 'h3', + null, + `${name} ${restore ? 'was restored successfully.' : 'was backed up successfully.'}`, + ), + ); + else { + main.append( + status( + op.status, + { + queued: 'Queued', + snapshotting: 'Creating snapshot', + backing_up: 'Creating snapshot', + uploading: 'Uploading backup', + stopping: 'Preparing restore', + restoring: 'Restoring DSP data', + starting: 'Checking restored DSP', + recovering: 'Recovering safety backup', + restarting_previous: 'Restarting previous DSP', + }[op.phase] || 'Working', + ), + ); + const progress = el('div', 'backup-progress'); + progress.setAttribute('role', 'progressbar'); + progress.setAttribute( + 'aria-label', + restore ? 'Restore in progress' : 'Backup in progress', + ); + main.append(progress); + } + const steps = el('ol', 'backup-stages'); + for (const step of model.stages(op)) { + const li = el('li', step.status); + li.append( + icon(step.status === 'done' ? 'check' : step.status === 'active' ? 'spinner' : 'clock'), + el('span', null, step.label), + el( + 'small', + null, + step.status === 'done' + ? 'Completed' + : step.status === 'active' + ? 'In progress' + : 'Pending', + ), + ); + steps.append(li); + } + main.append(steps); + if (!complete) + main.append( + note(`You can leave this page. The ${restore ? 'restore' : 'backup'} will continue.`), + ); + side.append( + pairList([ + ['Target', name], + ['Type', restore ? 'Restore' : 'Backup'], + ['Started', date(op.createdAt)], + [complete ? 'Completed' : 'Last updated', date(op.updatedAt || op.createdAt)], + ]), + ); + if (backup) + side.append( + link( + restore + ? complete + ? 'View restored backup' + : 'View selected backup' + : 'View backup', + detailPath(backup), + 'chevron', + ), + ); + } + if (restore) { + if (safety) + side.append( + note('Safety backup preserved', 'shield'), + link('View safety backup', detailPath(safety), 'chevron'), + ); + main.append( + note('This restore is scoped to this DSP. Other DSPs and Platform Core are not changed.'), + ); + } + grid.append(main, side); + const actions = el('div', 'backup-actions'); + actions.append( + link('View backups', dspPath(id), 'back'), + link('View history', '#/backups/history'), + ); + root().append(grid, actions); + } + function missing(title = 'Page not found') { + empty( + root(), + title, + 'This item may no longer be available. Return to the backup overview.', + link('Back to backups', '#/backups'), + ); + } + function storageStatus(connectedLabel = 'Storage connected') { + const s = data.storage.status; + return status( + s === 'connected' ? 'verified' : s === 'attention' ? 'failed' : 'empty', + s === 'connected' + ? connectedLabel + : s === 'attention' + ? 'Storage needs attention' + : 'Backup storage unavailable', + ); + } + function makeDialog(title) { + if (dialog) return null; + const previous = document.activeElement, + d = el('dialog', 'backup-dialog'); + dialog = d; + const h = el('h2', null, title); + h.id = 'backup-dialog-title'; + d.setAttribute('aria-labelledby', h.id); + const top = el('div', 'backup-card-heading'), + close = button( + '', + () => { + if (!busy) d.close(); + }, + 'icon', + ); + close.append(icon('close')); + close.setAttribute('aria-label', 'Close dialog'); + top.append(h, close); + d.append(top); + d.addEventListener('cancel', (event) => { + if (busy) event.preventDefault(); + }); + d.addEventListener('close', () => { + d.remove(); + dialog = null; + refreshDialog = null; + if (previous?.isConnected) previous.focus(); + else root()?.querySelector('h1')?.focus({ preventScroll: true }); + }); + document.body.append(d); + return d; + } + function openBackup(id = null) { + if (disabled()) return; + const d = makeDialog('Back up now'); + if (!d) return; + let selection = new Set(id && id !== 'core' ? [id] : []), + selectedScope = id === 'core' ? 'core' : id === 'system' ? 'system' : 'dsps', + filter = ''; + d.append(el('p', 'backup-muted', 'Choose what to back up.')); + const tabs = el('div', 'backup-filter-tabs'), + content = el('div'), + error = el('p', 'backup-error'); + error.setAttribute('role', 'alert'); + const actions = el('div', 'backup-form-footer'), + cancel = button('Cancel', () => d.close()), + submit = button( + 'Back up now', + async () => { + if (disabled()) return; + const input = ['core', 'system'].includes(selectedScope) + ? { action: 'backup', scope: selectedScope } + : { + action: 'backup', + scope: 'dsps', + organizationIds: [...selection].sort(), + }; + const result = await change(input); + if (result) { + d.close(); + followResult(result); + } else { + error.textContent = notice; + refreshDialog?.(); + } + }, + 'primary', + 'confirm-backup-now', + ); + actions.append(cancel, submit); + function refresh() { + tabs.replaceChildren(); + for (const [key, text] of [ + ['dsps', 'DSPs'], + ['core', 'Platform Core'], + ['system', 'Full system'], + ]) { + const b = button(text, () => { + selectedScope = key; + render(); + }); + b.setAttribute('aria-pressed', String(selectedScope === key)); + tabs.append(b); + } + const unavailable = + selectedScope === 'dsps' && + [...selection].some((id) => !data.organizations.find((o) => o.id === id)?.canBackup); + submit.textContent = + selectedScope === 'system' + ? 'Back up full system' + : selectedScope === 'core' + ? 'Back up Platform Core' + : `Back up ${selection.size} DSP${selection.size === 1 ? '' : 's'}`; + submit.disabled = + disabled() || + (selectedScope === 'system' + ? !data.canBackupCore || data.organizations.some((o) => !o.canBackup) + : selectedScope === 'core' + ? !data.canBackupCore + : !selection.size || unavailable); + cancel.disabled = busy; + for (const input of content.querySelectorAll('input')) { + if (input.dataset.org) + input.disabled = + disabled() || !data.organizations.find((o) => o.id === input.dataset.org)?.canBackup; + else input.disabled = busy; + } + if (unavailable) + error.textContent = + 'A selected DSP is now busy or unavailable. Deselect it before continuing.'; + } + function render() { + content.replaceChildren(); + error.textContent = ''; + if (selectedScope === 'system') + content.append( + note( + 'Creates a separate Core backup and one backup for every active DSP, grouped into a full-system recovery point. Removed DSPs remain stopped.', + ), + ); + else if (selectedScope === 'core') + content.append( + note( + 'Creates a backup of accounts and platform settings. DSP data is not included.', + 'core', + ), + ); + else { + const searchBox = searchField('backup-selection-search', filter, (value) => { + filter = value; + renderRows(); + }); + content.append(searchBox); + const all = el('input'); + all.type = 'checkbox'; + all.id = 'backup-select-all'; + const label = field('Select all available DSPs', all); + label.classList.add('backup-check-row'); + content.append(label); + const list = el('div', 'backup-selection-list'); + content.append(list); + all.addEventListener('change', () => { + for (const org of data.organizations.filter((o) => o.canBackup)) { + if (all.checked) selection.add(org.id); + else selection.delete(org.id); + } + renderRows(); + }); + function renderRows() { + list.replaceChildren(); + const available = data.organizations.filter((o) => o.canBackup); + all.checked = available.length > 0 && available.every((o) => selection.has(o.id)); + all.indeterminate = !all.checked && available.some((o) => selection.has(o.id)); + for (const org of data.organizations.filter((o) => + o.name.toLowerCase().includes(filter.trim().toLowerCase()), + )) { + const row = el('label', 'backup-selection-row'), + checkbox = el('input'); + checkbox.type = 'checkbox'; + checkbox.dataset.org = org.id; + checkbox.checked = selection.has(org.id); + checkbox.disabled = !org.canBackup; + checkbox.addEventListener('change', () => { + if (checkbox.checked) selection.add(org.id); + else selection.delete(org.id); + renderRows(); + }); + const p = model.protection(data, org.id); + row.append( + checkbox, + avatar(org), + el('strong', null, org.name), + status( + p.status, + !org.canBackup && p.status !== 'running' ? 'Unavailable' : p.label, + ), + ); + list.append(row); + } + if (!list.children.length) + list.append(el('p', 'backup-empty', 'No DSPs match your search.')); + refresh(); + } + renderRows(); + } + refresh(); + } + refreshDialog = refresh; + d.append(tabs, content, error, actions); + render(); + d.showModal(); + } + function confirmBackupDelete(backup) { + const dialog = makeDialog('Delete backup permanently?'); + if (!dialog) return; + dialog.append( + note( + `This deletes only the selected backup for ${backup.name}. Live data and other backups stay unchanged.`, + 'warning', + ), + ); + const actions = el('div', 'backup-form-footer'), + error = el('p', 'backup-error'); + error.setAttribute('role', 'alert'); + const submit = button( + 'Delete backup', + async () => { + const result = await change({ + action: 'delete', + backupId: backup.id, + confirmation: backup.name, + ...(backup.kind === 'core' + ? { scope: 'core' } + : { organizationId: backup.organizationId }), + }); + if (result) dialog.close(); + else error.textContent = notice; + }, + 'primary', + ); + actions.append( + button('Cancel', () => dialog.close()), + submit, + ); + dialog.append(error, actions); + dialog.showModal(); + } + function systemSets() { + const section = card(current.setId ? 'Full-system backup' : 'Full-system backups'); + const sets = model.sorted( + (data.sets || []).filter( + (s) => s.status !== 'deleted' && (!current.setId || s.id === current.setId), + ), + ); + if (current.setId && !sets.length) return missing('Recovery point not found'); + for (const set of sets) { + const entry = disclosure(`${date(set.createdAt)} · ${set.status}`, `backup-set-${set.id}`); + if (current.setId) entry.open = true; + const usage = data.storageUsage?.sets?.find((s) => s.id === set.id); + if (usage) + entry.append( + el( + 'p', + 'backup-muted', + `${storedBytes(usage.bytes)} stored · included in total backup storage`, + ), + ); + if (set.status === 'deleting' && data.storage.status !== 'connected') + entry.append( + note( + 'Component backups have been deleted. Removing the full-system manifest is waiting for backup storage; it will retry automatically.', + 'warning', + ), + ); + if (set.restore) + entry.append( + note( + set.restore.status === 'failed' + ? 'Full-system restore is incomplete. Review the failed component in activity.' + : set.restore.status === 'completed' + ? 'Full-system restore completed.' + : 'Full-system restore in progress.', + ), + ); + const members = table(['Scope', 'Status', '']); + for (const member of set.members) { + const op = data.operations.find((o) => o.id === member.requestId); + const backup = data.backups.find((b) => b.id === (member.backupId || op?.backupId)); + const row = el('tr'); + const name = member.name || backup?.name || scopeName(member.organizationId || 'core'); + row.append( + cell(name), + cell(status(member.status || op?.status || backup?.status || 'pending')), + cell( + backup + ? link('View details', detailPath(backup)) + : op + ? link('View progress', operationPath(op)) + : '', + ), + ); + members.body.append(row); + } + entry.append(members.wrapper); + for (const action of ['restore', 'delete']) { + const buttonEl = button( + action === 'restore' ? 'Restore full system' : 'Delete full-system backup', + () => { + const dialog = makeDialog( + action === 'restore' ? 'Restore full system?' : 'Delete full-system backup?', + ); + if (!dialog) return; + dialog.append( + note( + action === 'restore' + ? 'Core and every DSP in this set will be restored. A failed component leaves the operation incomplete.' + : 'All component backups in this set will be permanently deleted. Live data stays unchanged.', + 'warning', + ), + ); + const fieldEl = el('input'); + fieldEl.autocomplete = 'off'; + dialog.append(field('Type Full system to confirm', fieldEl)); + const error = el('p', 'backup-error'); + error.setAttribute('role', 'alert'); + const submit = button( + action === 'restore' ? 'Restore full system' : 'Delete full-system backup', + async () => { + if (submit.disabled) return; + const result = await change({ + action, + scope: 'system', + setId: set.id, + confirmation: fieldEl.value, + }); + if (result) dialog.close(); + else error.textContent = notice; + }, + 'primary', + ); + function refresh() { + const latest = data.sets?.find((s) => s.id === set.id); + const blocked = + !latest || + latest.busy || + ['deleted', 'pending', 'deleting'].includes(latest.status) || + (action === 'restore' && (latest.status !== 'verified' || data.operationBlocked)); + submit.disabled = disabled() || !!blocked || fieldEl.value !== 'Full system'; + fieldEl.disabled = busy; + if (blocked) + error.textContent = 'This recovery point is no longer available for this action.'; + } + fieldEl.addEventListener('input', refresh); + refreshDialog = refresh; + refresh(); + const actions = el('div', 'backup-form-footer'); + actions.append( + button('Cancel', () => dialog.close()), + submit, + ); + dialog.append(error, actions); + dialog.showModal(); + fieldEl.focus(); + }, + ); + buttonEl.disabled = + disabled() || + set.busy || + (action === 'restore' && (set.status !== 'verified' || !!data.operationBlocked)) || + ['pending', 'deleting'].includes(set.status); + entry.append(buttonEl); + } + section.append(entry); + } + if (!sets.length) section.append(el('p', 'backup-muted', 'No full-system backups yet.')); + root().append(section); + } + function confirmRestore(backup) { + if (disabled() || backup.restoreBlocked || backup.status !== 'verified') return; + const org = + backup.kind === 'core' + ? { id: null, name: 'Platform Core' } + : data.organizations.find((o) => o.id === backup.organizationId); + if (!org) return; + const d = makeDialog(`Restore ${org.name}?`); + if (!d) return; + const form = el('form'); + form.append( + status( + 'verified', + backup.kind === 'core' ? 'Core only · Uploaded backup' : 'DSP only · Uploaded backup', + ), + pairList([ + ['Selected backup', date(backup.createdAt)], + ['Size', bytes(backup.size)], + ]), + note(`Current data for ${org.name} will be replaced with this backup.`, 'warning'), + el( + 'p', + null, + backup.kind === 'core' + ? 'DSP data, users, services and schedules will stay unchanged.' + : 'Other DSPs and Platform Core will not be changed.', + ), + note('A safety backup will be created before restoring.', 'shield'), + ); + const acknowledge = el('input'); + acknowledge.type = 'checkbox'; + acknowledge.id = 'backup-acknowledge'; + const label = field( + 'I understand this replaces the selected scope’s current data.', + acknowledge, + ); + label.classList.add('backup-check-row'); + form.append(label); + const name = el('input'); + name.id = 'backup-confirm-name'; + name.required = true; + name.autocomplete = 'off'; + form.append(field(`Type ${org.name} to confirm`, name)); + const error = el('p', 'backup-error'); + error.setAttribute('role', 'alert'); + const actions = el('div', 'backup-form-footer'); + const cancel = button('Cancel', () => d.close()), + submit = button( + backup.kind === 'core' ? 'Restore Core' : 'Restore DSP', + () => {}, + 'primary', + 'confirm-backup-restore', + ); + submit.type = 'submit'; + actions.append(cancel, submit); + form.append(error, actions); + function refresh() { + const latest = data.backups.find((b) => b.id === backup.id && b.organizationId === org.id); + const blocked = + !latest || latest.status !== 'verified' || latest.restoreBlocked || data.operationBlocked; + submit.disabled = + disabled() || !acknowledge.checked || name.value !== org.name || !!blocked; + cancel.disabled = busy; + name.disabled = busy; + acknowledge.disabled = busy; + if (blocked) + error.textContent = + typeof blocked === 'string' + ? blocked + : 'This backup is no longer available for restore.'; + } + name.addEventListener('input', refresh); + acknowledge.addEventListener('change', refresh); + refreshDialog = refresh; + form.addEventListener('submit', async (event) => { + event.preventDefault(); + if (submit.disabled || !form.reportValidity()) return; + const result = await change({ + action: 'restore', + ...(backup.kind === 'core' ? { scope: 'core' } : { organizationId: org.id }), + backupId: backup.id, + confirmation: name.value, + }); + if (result) { + d.close(); + followResult(result); + } else { + error.textContent = notice; + refresh(); + } + }); + d.append(form); + refresh(); + d.showModal(); + acknowledge.focus(); + } + function followResult(result) { + if (!result.follow || !active) return; + navigate( + result.operations.length === 1 ? operationPath(result.operations[0]) : '#/backups/history', + ); + } + async function startBackup(id) { + const result = await change( + id === 'core' + ? { action: 'backup', scope: 'core' } + : { action: 'backup', scope: 'dsps', organizationIds: [id] }, + ); + if (result) followResult(result); + } + async function change(input) { + if (disabled()) return false; + const startHash = location.hash; + const before = new Set(data.operations.map((op) => op.id)); + const slot = `backup:${JSON.stringify(input)}`, + body = { ...input, idempotencyKey: mutationKey(slot, 'backups') }; + busy = true; + generation++; + clearTimeout(poll); + draw(); + refreshDialog?.(); + try { + const next = await mutation('/api/platform/backups', 'POST', body); + data = next; + stale = false; + settleMutationKey(slot); + noticeError = false; + notice = + input.action === 'settings' + ? 'Backup settings saved.' + : input.action === 'delete' + ? 'Backup deletion queued.' + : input.action === 'restore' + ? 'Restore queued.' + : 'Backup queued. Selected scopes will be backed up one at a time.'; + if (input.action === 'settings') { + draft = null; + draftRevision = null; + } + // Idempotent retries may return an already-known operation after an interrupted response. + const operations = data.operations.filter((op) => !before.has(op.id)); + if (!operations.length && input.action !== 'settings') + operations.push( + ...data.operations.filter( + (op) => + model.active(op) && + (input.action === 'restore' + ? op.kind === 'restore' && op.backupId === input.backupId + : input.scope === 'core' + ? op.kind === 'core' + : input.organizationIds?.includes(op.organizationId)), + ), + ); + return { operations, follow: location.hash === startHash }; + } catch (error) { + settleMutationKey(slot, error); + noticeError = true; + notice = messages[error.code] || errorMessage(error.code); + if (error.code === 'backup_settings_conflict') { + draft = null; + draftRevision = null; + try { + data = await request('/api/platform/backups'); + } catch { + stale = true; + } + } + return false; + } finally { + busy = false; + if (active) { + draw(); + refreshDialog?.(); + poll = setTimeout(renderBackups, 5000); + } + } + } + function drawManual() { + root().classList.add('backup-streamlined'); + root().dataset.view = 'overview'; + const header = el('div', 'backup-header'); + header.append(el('h1', null, 'Backups')); + const layout = el('div', 'backup-split'), main = el('div', 'backup-main'), rail = el('aside', 'backup-rail'); + const saved = el('section', 'backup-section'), history = el('section', 'backup-section'), help = el('section', 'backup-section'); + saved.append(el('h2', null, 'Saved backups')); + if (!data.backups.length) saved.append(el('p', 'backup-muted', 'No manual backups have been created.')); + else { + const { wrapper, body } = table(['Created', 'Scope', 'Size', 'Backup reference']); + for (const backup of [...data.backups].reverse()) { + const row = el('tr'); + row.append(cell(date(backup.createdAt)), cell(backup.scope === 'platform' ? 'Full platform' : 'DSP'), + cell(`${(backup.bytes / 1024 / 1024).toFixed(1)} MB`), cell(backup.id)); + body.append(row); + } + saved.append(wrapper); + } + history.append(el('h2', null, 'Operation history')); + if (!data.operations.length) history.append(el('p', 'backup-muted', 'No manual operations yet.')); + for (const operation of [...data.operations].reverse()) { + const status = operation.status === 'complete' ? 'Complete' : operation.status === 'failed' + ? 'Failed — review the local operation report before retrying' : 'Awaiting local maintenance'; + history.append(el('p', null, `${date(operation.createdAt)} · ${operation.action === 'restore' ? 'Restore' : 'Backup'} · ${status}`)); + } + help.append(el('h2', null, 'Manual maintenance')); + help.append(el('p', null, 'Only the platform owner can create backups or restore data.')); + help.append(el('p', null, 'Suspend the affected DSPs, stop the dashboard, and use the local backup command.')); + help.append(el('p', null, 'Restores replace current data and keep DSPs suspended until you resume them.')); + help.append(el('p', 'backup-muted', 'Backups are stored privately on this server. Copy a completed backup to separate storage for protection against disk loss.')); + rail.append(help); main.append(saved, history); layout.append(main, rail); root().replaceChildren(header, layout); + } + function draw() { + if (!data || !root() || !active) return; + if (data.mode === 'manual') { drawManual(); return; } + const focused = document.activeElement, + id = focused?.id, + position = focused?.selectionStart, + within = root().contains(focused); + root().replaceChildren(); + root().classList.add('backup-streamlined'); + root().dataset.view = isInspector(current) ? 'dsps' : current.mode; + const header = el('div', 'backup-header'), + title = el('h1', null, 'Backups'); + title.tabIndex = -1; + const actions = el('div', 'backup-actions'); + const settingsLink = link('Settings', '#/backups/settings'); + if (current.mode === 'settings') settingsLink.setAttribute('aria-current', 'page'); + actions.append(settingsLink); + if (current.mode === 'settings') { + actions.append( + textAction('Cancel', () => { + draft = draftRevision = null; + navigate('#/backups'); + }), + ); + const save = button('Save settings', () => {}, 'primary', 'save-backup-settings'); + save.type = 'submit'; + save.setAttribute('form', 'backup-settings-form'); + save.disabled = disabled(); + actions.append(save); + } else { + const selected = + current.orgId || + (current.mode === 'dsps' + ? statusFilter === 'attention' + ? model.fleet(data).find(model.needsAttention)?.org.id + : data.organizations[0]?.id + : null); + const scoped = ['dsps', 'dsp', 'dsp-history', 'detail'].includes(current.mode); + const target = scoped ? selected : 'system'; + const start = button( + target === 'core' + ? 'Back up Platform Core' + : scoped + ? 'Back up DSP' + : 'Back up full system', + () => openBackup(target), + 'primary', + 'backup-now', + ); + start.disabled = + disabled() || + !!data.operationBlocked || + (target === 'core' + ? !data.canBackupCore + : target === 'system' + ? !data.canBackupCore || data.organizations.some((o) => !o.canBackup) + : !data.organizations.find((o) => o.id === target)?.canBackup); + actions.append(start); + } + const titleGroup = el('div'); + titleGroup.append(title); + header.append(titleGroup, actions); + const nav = el('nav', 'backup-tabs'); + nav.setAttribute('aria-label', 'Backup navigation'); + for (const [text, href, selected] of [ + ['Overview', '#/backups', current.mode === 'overview'], + [ + 'DSPs', + '#/backups/dsps', + ['dsps', 'dsp', 'dsp-history', 'detail', 'operation'].includes(current.mode) && + current.orgId !== 'core' && + !( + current.mode === 'operation' && + data.operations.find((o) => o.id === current.operationId)?.kind === 'core' + ), + ], + ['History', '#/backups/history', ['history', 'sets'].includes(current.mode)], + ['Storage', '#/backups/storage', current.mode === 'storage'], + [ + 'Platform Core', + '#/backups/core', + current.orgId === 'core' || + (current.mode === 'operation' && + data.operations.find((o) => o.id === current.operationId)?.kind === 'core'), + ], + ]) { + const a = link(text, href); + if (selected) a.setAttribute('aria-current', 'page'); + nav.append(a); + } + root().append(header, nav); + if (notice) { + const n = el('p', noticeError ? 'backup-error backup-notice' : 'backup-notice', notice); + n.setAttribute('role', noticeError ? 'alert' : 'status'); + root().append(n); + } + if (stale) { + const n = note( + 'Could not refresh backups. Showing the last known status. Actions are disabled until the connection recovers.', + 'warning', + ); + n.setAttribute('role', 'status'); + n.append(button('Try again', renderBackups)); + root().append(n); + } + if (data.operationBlocked) root().append(note(data.operationBlocked, 'warning')); + if (!data.enabled) + root().append( + note('The backup service is not active. Existing backups can still be viewed.'), + ); + if (current.mode === 'overview') overview(); + else if (isInspector(current)) dspInspector(); + else if (['dsp', 'dsp-history'].includes(current.mode)) dsp(); + else if (current.mode === 'settings') settings(); + else if (current.mode === 'storage') storageView(); + else if (current.mode === 'history') historyView(); + else if (current.mode === 'sets') systemSets(); + else if (current.mode === 'detail') details(); + else if (current.mode === 'operation') operation(); + else missing(); + for (const table of root().querySelectorAll('.backup-table')) { + const labels = [...table.querySelectorAll('th')].map((th) => th.textContent); + for (const row of table.querySelectorAll('tbody tr')) + [...row.children].forEach((cell, i) => { + cell.dataset.label = labels[i]; + }); + } + if (within && id && byId(id)) { + byId(id).focus({ preventScroll: true }); + try { + if (position !== null) byId(id).setSelectionRange(position, position); + } catch {} + } + } + function setBackupsActive(value) { + active = value; + if (!active) { + generation++; + clearTimeout(poll); + if (dialog) dialog.close(); + } + } + async function renderBackups() { + if (!active || !root() || !location.hash.startsWith('#/backups')) return; + clearTimeout(poll); + const hash = location.hash, + changedRoute = drawnHash !== hash; + if (changedRoute) { + notice = ''; + noticeError = false; + window.scrollTo({ top: 0, left: 0 }); + const nextRoute = model.route(hash); + const keepInspectorFilters = + isInspector(current) && + isInspector(nextRoute) && + (!nextRoute.orgId || + !data || data.mode === 'manual' || + model + .filterFleet(model.fleet(data), '', statusFilter) + .some((p) => p.org.id === nextRoute.orgId)); + if ( + nextRoute.mode === 'dsps' && + !new URLSearchParams(hash.split('?')[1] || '').has('status') + ) + statusFilter = 'all'; + current = nextRoute; + drawnHash = hash; + page = 1; + backupSearch = ''; + if (!keepInspectorFilters) { + search = ''; + statusFilter = scopeFilter = eventFilter = categoryFilter = dateFilter = 'all'; + } + if ( + isInspector(current) && + new URLSearchParams(hash.split('?')[1] || '').get('status') === 'attention' + ) + statusFilter = 'attention'; + if (dialog) dialog.close(); + if (data) draw(); + } + if (busy) return; + const requestGeneration = ++generation; + try { + const next = await request('/api/platform/backups'); + if (!active || requestGeneration !== generation) return; + const changed = stale || JSON.stringify(next) !== JSON.stringify(data); + data = next; + stale = false; + if (changed || !root().querySelector('h1')) draw(); + refreshDialog?.(); + } catch (error) { + if (!active || requestGeneration !== generation) return; + if ([401, 403].includes(error.status)) { + data = null; + if (dialog) dialog.close(); + root().replaceChildren(el('p', 'backup-error', errorMessage(error.code))); + return; + } + stale = true; + if (data) { + draw(); + refreshDialog?.(); + } else + root().replaceChildren( + el('p', 'backup-error', errorMessage(error.code)), + button('Try again', renderBackups), + ); + } finally { + if (active && requestGeneration === generation) poll = setTimeout(renderBackups, 5000); + } + } + return { renderBackups, setBackupsActive }; + }; diff --git a/core/dashboard/public/assets/inter.woff2 b/core/dashboard/public/assets/inter.woff2 new file mode 100644 index 0000000..d15208d Binary files /dev/null and b/core/dashboard/public/assets/inter.woff2 differ diff --git a/core/dashboard/public/assets/launcher.js b/core/dashboard/public/assets/launcher.js new file mode 100644 index 0000000..94ce85d --- /dev/null +++ b/core/dashboard/public/assets/launcher.js @@ -0,0 +1,33 @@ +'use strict'; +(async()=>{ + const root=document.getElementById('root'); + try { + const view=sessionStorage.getItem('dispatch-dsp-view'); + const response=await fetch('/api/dashboard',{credentials:'same-origin',headers:view?{'X-Dispatch-DSP-View':view}:{},cache:'no-store'}); + const result=await response.json(); + if(!response.ok||!result.ok) { + if(result.error?.code==='dsp_view_unavailable'&&view){sessionStorage.removeItem('dispatch-dsp-view');location.reload();return;} + throw Error('dashboard_unavailable'); + } + const {product,digest,javascript,stylesheet}=result.data; + if(!['core','dsp'].includes(product)||!/^[a-f0-9]{64}$/.test(digest)||typeof javascript!=='string'||typeof stylesheet!=='string')throw Error('dashboard_invalid'); + window.__dispatchDashboard={product,digest}; + const nonce=document.querySelector('meta[name=dispatch-style-nonce]')?.content; + const style=document.createElement('style');style.nonce=nonce;style.textContent=stylesheet;document.head.append(style); + const script=document.createElement('script');script.nonce=nonce;script.textContent=javascript;document.body.append(script); + } catch { + root.replaceChildren();const message=document.createElement('p');message.textContent='Dispatch is temporarily unavailable. Refresh to try again.'; + const button=document.createElement('button');button.textContent='Refresh';button.onclick=()=>location.reload();root.append(message,button); + } +})(); +// Also handle authority transitions in frozen pre-monorepo dashboards. +window.addEventListener('dispatch-authority-changed',()=>{ + setTimeout(async()=>{ + try{ + const view=sessionStorage.getItem('dispatch-dsp-view'); + const response=await fetch('/api/dashboard?identity=1',{credentials:'same-origin',headers:view?{'X-Dispatch-DSP-View':view}:{},cache:'no-store'}); + const result=await response.json(); + if(result.ok && window.__dispatchDashboard && result.data.product!==window.__dispatchDashboard.product)location.reload(); + }catch{ /* The dashboard owns request failure presentation. */ } + },0); +}); diff --git a/core/dashboard/public/assets/updates.js b/core/dashboard/public/assets/updates.js new file mode 100644 index 0000000..c28b580 --- /dev/null +++ b/core/dashboard/public/assets/updates.js @@ -0,0 +1,239 @@ +'use strict'; + +window.createUpdatesViews = function createUpdatesViews({ + byId, errorMessage, node, request, timeZone +}) { + const versionLabel = version => version.replace(/\+hotfix\.([1-9]\d*)$/, ' — Hotfix $1'); + let updatesPoll = null; + let updatesFingerprint = null; + let updatesGeneration = 0; + let lastUpdatesData = null; + let updatesActive = false; + let unavailableSince = null; + let searchTerm = ""; + let selectedReleaseId = null; + let nextFocus = null; + const expandedGroups = new Set(); + function setUpdatesActive(active) { + if (updatesActive === active) return; + updatesActive = active; + clearTimeout(updatesPoll); + updatesGeneration += 1; + updatesFingerprint = null; + lastUpdatesData = null; + unavailableSince = null; + selectedReleaseId = null; + nextFocus = null; + searchTerm = ""; + expandedGroups.clear(); + } + function updateButton(label, callback) { + const button = node('button', 'secondary-button', label); button.type = 'button'; + button.dataset.updateFocus = 'action'; button.addEventListener('click', callback); return button; + } + const kindLabels = { added: 'Added', changed: 'Changed', improved: 'Improved', fixed: 'Fixed', removed: 'Removed' }; + const iconPaths = { + database: 'M20 6c0 2.2-3.6 4-8 4S4 8.2 4 6s3.6-4 8-4 8 1.8 8 4ZM4 6v12c0 2.2 3.6 4 8 4s8-1.8 8-4V6M4 12c0 2.2 3.6 4 8 4s8-1.8 8-4', + users: 'M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2M22 21v-2a4 4 0 0 0-3-3.87M15 3.13a4 4 0 0 1 0 7.75M13 7a4 4 0 1 1-8 0 4 4 0 0 1 8 0Z', + 'user-plus': 'M16 21v-2a4 4 0 0 0-4-4H6a4 4 0 0 0-4 4v2M13 7a4 4 0 1 1-8 0 4 4 0 0 1 8 0ZM20 8v6M17 11h6', + copy: 'M9 9h12v12H9ZM5 15H3V3h12v2', + 'calendar-clock': 'M8 2v4M16 2v4M3 10h18M10 22H5a2 2 0 0 1-2-2V6a2 2 0 0 1 2-2h14a2 2 0 0 1 2 2v5M23 18a5 5 0 1 1-10 0 5 5 0 0 1 10 0ZM18 15v3l2 1', + 'chart-column': 'M3 13h4v9H3ZM10 8h4v14h-4ZM17 2h4v20h-4Z', + shield: 'M12 2 3 6v6c0 5 5 9 9 10 4-1 9-5 9-10V6Z', + 'check-circle': 'M22 12a10 10 0 1 1-20 0 10 10 0 0 1 20 0ZM8 12l3 3 5-6', + trash: 'M3 6h18M9 6V3h6v3M5 6l1 15h12l1-15M10 10v7M14 10v7', + lock: 'M5 10h14v12H5ZM8 10V6a4 4 0 0 1 8 0v4M12 15v3', + send: 'm22 2-7 20-4-9L2 9Zm0 0L11 13', + 'refresh-cw': 'M20 7a9 9 0 0 0-15-2L2 8M2 3v5h5M4 17a9 9 0 0 0 15 2l3-3M22 21v-5h-5', + plus: 'M12 4v16M4 12h16', pencil: 'm16 3 5 5-13 13H3v-5Zm-2 2 5 5', + 'trending-up': 'm3 17 6-6 4 4L22 6M16 6h6v6', + info: 'M22 12a10 10 0 1 1-20 0 10 10 0 0 1 20 0ZM12 11v6M12 7v.1', + warning: 'M10.3 3.9 1.8 18.6A2 2 0 0 0 3.5 21h17a2 2 0 0 0 1.7-2.4L13.7 3.9a2 2 0 0 0-3.4 0ZM12 9v4M12 17v.1', + chevron: 'm9 5 7 7-7 7', + upload: 'M12 16V3m-5 5 5-5 5 5M4 15v6h16v-6', + }; + function releaseIcon(name, className = '') { + const svg = document.createElementNS('http://www.w3.org/2000/svg', 'svg'); + for (const [key, value] of Object.entries({ viewBox: '0 0 24 24', fill: 'none', stroke: 'currentColor', + 'stroke-width': '1.7', 'stroke-linecap': 'round', 'stroke-linejoin': 'round', 'aria-hidden': 'true', class: `update-note-icon ${className}` })) svg.setAttribute(key, value); + const shape = document.createElementNS('http://www.w3.org/2000/svg', 'path'); + shape.setAttribute('d', iconPaths[name] || iconPaths.info); svg.append(shape); return svg; + } + function releaseCounts(changelog) { + return Object.keys(kindLabels).map(kind => ({ kind, count: changelog.filter(change => change.kind === kind).length })).filter(item => item.count); + } + function notesPanel(release) { + const panel = node('section', 'update-notes-panel'); panel.setAttribute('aria-label', 'Release changelog'); + const title = node('div', 'update-notes-title'); title.append(node('h2', null, `Version ${versionLabel(release.version)}`)); + panel.append(title); + const meta = node('div', 'update-release-meta'); + const date = releaseDate(release.publishedAt); + if (date) meta.append(date); + if (release.state === 'installed') meta.append(node('span', 'update-release-state', 'Installed')); + panel.append(meta); + const changes = release.notes?.changelog || release.changelog || []; + const countLabels = { added: 'addition', changed: 'change', improved: 'improvement', fixed: 'fix', removed: 'removal' }; + const counts = releaseCounts(changes); + panel.append(node('p', 'update-release-summary', changes.length + ? counts.map(({ kind, count }) => `${count} ${countLabels[kind]}${count === 1 ? '' : kind === 'fixed' ? 'es' : 's'}`).join(' · ') + : 'Release notes are not available for this version.')); + const groups = release.notes?.groups || [{ id: 'general', title: 'What’s new', icon: 'info' }]; + for (const [index, group] of groups.entries()) { + const items = changes.filter(change => !release.notes || change.group === group.id); + if (!items.length) continue; + const card = node('section', 'update-feature-group'); + const heading = node('div', 'update-feature-heading'); + const tile = node('span', 'update-group-icon'); tile.append(releaseIcon(group.icon)); + heading.append(tile, node('h3', null, group.title), node('span', 'update-group-count', `${items.length} ${items.length === 1 ? 'change' : 'changes'}`)); card.append(heading); + const list = node('ul', 'update-feature-changes'); + for (const change of items) { + const row = node('li'); const copy = node('div'); + row.append(node('span', `update-change-kind ${change.kind}`, kindLabels[change.kind])); + copy.append(node('strong', null, change.title)); + if (change.description) copy.append(node('p', null, change.description)); + row.append(copy); list.append(row); + } + card.append(list); + const detailed = items.filter(change => change.details); + if (detailed.length) { + const key = `${release.id || release.version}:${group.id}`; + const detailId = `release-details-${index}`; + const details = node('div', 'update-expanded-details'); details.id = detailId; + details.hidden = !expandedGroups.has(key); + for (const change of detailed) { const section = node('div'); section.append(node('h4', null, change.title), node('p', null, change.details)); details.append(section); } + const toggle = updateButton(details.hidden ? 'View details' : 'Hide details', () => { + if (expandedGroups.has(key)) expandedGroups.delete(key); else expandedGroups.add(key); + details.hidden = !expandedGroups.has(key); + toggle.replaceChildren(node('span', null, details.hidden ? 'View details' : 'Hide details'), releaseIcon('chevron')); + toggle.setAttribute('aria-expanded', String(!details.hidden)); + }); + toggle.className = 'update-details-toggle'; toggle.dataset.updateFocus = `details-${index}`; + toggle.setAttribute('aria-expanded', String(!details.hidden)); toggle.setAttribute('aria-controls', detailId); + toggle.append(releaseIcon('chevron')); card.append(toggle, details); + } + panel.append(card); + } + return panel; + } + function navigateRelease(id, focus) { + selectedReleaseId = id; + nextFocus = focus; + updatesFingerprint = null; + return renderUpdates(); + } + function releaseDate(value, short = false) { + if (!value || !Number.isFinite(Date.parse(value))) return null; + const date = node('time', 'update-release-date', new Date(value).toLocaleDateString('en-US', { + month: short ? 'short' : 'long', day: 'numeric', year: 'numeric', timeZone, + })); + date.dateTime = value; return date; + } + function historyNavigation(history, release) { + const navigation = node('nav', 'update-release-navigation'); navigation.setAttribute('aria-label', 'Releases'); + navigation.append(node('h2', null, 'Releases')); + const search = node('input', 'update-release-search'); + search.type = 'search'; search.placeholder = 'Find a version'; search.value = searchTerm; + search.setAttribute('aria-label', 'Find a version'); search.dataset.updateFocus = 'search'; + navigation.append(search); + const list = node('ul', 'update-release-list'); + const rows = []; + for (const item of history) { + const row = node('li'); + const button = updateButton('', () => navigateRelease(item.id, `release-${item.id}`)); + button.className = 'update-release-link'; button.dataset.updateFocus = `release-${item.id}`; + button.append(node('span', 'update-release-version', `Version ${versionLabel(item.version)}`)); + if (item.id === release?.id) button.setAttribute('aria-current', 'page'); + const date = releaseDate(item.publishedAt, true); if (date) button.append(date); + if (item.state === 'installed') button.append(node('span', 'update-release-state', 'Installed')); + row.append(button); list.append(row); rows.push({ row, version: item.version }); + } + const empty = node('p', 'update-muted', 'No matching releases.'); empty.setAttribute('role', 'status'); + const filter = () => { + let matches = 0; + for (const { row, version } of rows) { + row.hidden = !version.toLowerCase().includes(searchTerm.trim().toLowerCase()); + if (!row.hidden) matches += 1; + } + empty.hidden = matches > 0; + }; + search.addEventListener('input', () => { searchTerm = search.value; filter(); }); + filter(); navigation.append(list, empty); return navigation; + } + function afterUpdating(release) { + const notices = node('section', 'update-after-section'); notices.setAttribute('aria-label', 'After updating'); + for (const action of release.notes?.afterUpdating || []) { + const notice = node('div', 'update-after-notice'); const copy = node('div'); + copy.append(node('h3', null, 'After updating'), node('strong', release.notes.afterUpdating.length === 1 ? 'sr-only' : null, action.title), node('p', null, action.description)); + notice.append(releaseIcon('warning'), copy); notices.append(notice); + } + return notices; + } + function releaseWorkspace(data) { + const workspace = node('div', 'update-release-browser'); + const release = data.displayedRelease || data.releases?.[0]; + const history = data.releaseHistory?.length ? data.releaseHistory : data.releases || []; + if (!release && !history.length) { + const empty = node('section', 'update-empty-state'); + empty.append(node('h2', null, 'No releases yet'), node('p', 'update-muted', 'Published release notes will appear here.')); + workspace.append(empty); return workspace; + } + const layout = node('div', 'update-browser-columns'); + layout.append(historyNavigation(history, release)); + const content = node('div', 'update-browser-content'); + if (release) { + const notes = notesPanel(release); + if (release.notes?.afterUpdating?.length) notes.append(afterUpdating(release)); + content.append(notes); + } else content.append(node('p', 'update-muted', 'Choose a release to read its changelog.')); + layout.append(content); workspace.append(layout); return workspace; + } + function displayUpdates(data) { + const container = byId('platform-updates-content'); + lastUpdatesData = data; + unavailableSince = null; + container.dataset.connection = 'connected'; + container.querySelector('.update-reconnecting')?.remove(); + const fingerprint = JSON.stringify(data); + if (updatesFingerprint === fingerprint && container.children.length) return; + const focus = nextFocus || (container.contains(document.activeElement) ? document.activeElement.dataset.updateFocus : null); + const selection = focus === 'search' ? [document.activeElement?.selectionStart, document.activeElement?.selectionEnd] : null; + nextFocus = null; + updatesFingerprint = fingerprint; + container.replaceChildren(releaseWorkspace(data)); + if (focus) { + const target = Array.from(container.querySelectorAll('button,input')).find(button => button.dataset.updateFocus === focus) + || container.querySelector('.update-empty-state button'); + target?.focus({ preventScroll: true }); + if (selection && selection.every(value => Number.isInteger(value))) target?.setSelectionRange?.(...selection); + } + } + async function renderUpdates() { + clearTimeout(updatesPoll); + const generation = ++updatesGeneration; + const container = byId('platform-updates-content'); + try { + const data = await request('/api/platform/updates' + (selectedReleaseId ? `?releaseId=${encodeURIComponent(selectedReleaseId)}` : ''), { signal: AbortSignal.timeout(10000) }); + if (generation !== updatesGeneration) return; + displayUpdates(data); + } catch (error) { + if (generation !== updatesGeneration) return; + updatesFingerprint = null; + if (lastUpdatesData && (!error.status || error.status >= 500)) { + unavailableSince ??= Date.now(); + container.dataset.connection = 'reconnecting'; + if (!container.querySelector('.update-reconnecting')) { + const notice = node('p', 'update-reconnecting', 'Unable to refresh releases. Showing the last loaded changelog; retrying automatically.'); + notice.setAttribute('role', 'status'); container.prepend(notice); + } + } else { + lastUpdatesData = null; + container.replaceChildren(node('p', 'update-attention', errorMessage(error.code))); + } + } finally { + if (generation === updatesGeneration && location.hash === '#/updates') updatesPoll = setTimeout(() => { + if (location.hash === '#/updates') renderUpdates(); + }, unavailableSince === null ? 3000 : 1000); + } + } + return { renderUpdates, setUpdatesActive }; +}; diff --git a/core/dashboard/public/index.html b/core/dashboard/public/index.html new file mode 100644 index 0000000..f0d964d --- /dev/null +++ b/core/dashboard/public/index.html @@ -0,0 +1,16 @@ + + + + + + + + + Dispatch + + + + + +
+ diff --git a/core/dashboard/scripts/build b/core/dashboard/scripts/build new file mode 100755 index 0000000..87382ce --- /dev/null +++ b/core/dashboard/scripts/build @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +for file in "$ROOT"/server/*.js "$ROOT"/public/assets/*.js "$ROOT"/tests/*.js; do + # Check source scripts; the generated bundle is replaced by Vite below. + [[ "$file" == "$ROOT/public/assets/frontend.js" ]] && continue + node --no-warnings --check "$file" +done +node --no-warnings -e "require('$ROOT/server/server'); require('$ROOT/server/main')" +cd "$ROOT" +if [[ ! -x node_modules/.bin/vite ]]; then npm ci --no-audit --no-fund; fi +npx tsc --noEmit +npx vite build +printf '%s\n' '{"ok":true,"status":"built"}' diff --git a/core/dashboard/scripts/test b/core/dashboard/scripts/test new file mode 100755 index 0000000..f6e53ac --- /dev/null +++ b/core/dashboard/scripts/test @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +cd "$ROOT" +node --no-warnings --test tests/*.test.js diff --git a/core/dashboard/server/access-http.js b/core/dashboard/server/access-http.js new file mode 100644 index 0000000..5324ae1 --- /dev/null +++ b/core/dashboard/server/access-http.js @@ -0,0 +1,3 @@ +'use strict'; +// Compatibility import; HTTP implementation belongs to Dispatch API. +module.exports = require('../../core/api/access-http'); diff --git a/core/dashboard/server/core-maintenance.js b/core/dashboard/server/core-maintenance.js new file mode 100644 index 0000000..de6343d --- /dev/null +++ b/core/dashboard/server/core-maintenance.js @@ -0,0 +1,3 @@ +'use strict'; +// Compatibility import; HTTP implementation belongs to Dispatch API. +module.exports = require('../../core/api/core-maintenance'); diff --git a/core/dashboard/server/directory-platform.js b/core/dashboard/server/directory-platform.js new file mode 100644 index 0000000..d73999c --- /dev/null +++ b/core/dashboard/server/directory-platform.js @@ -0,0 +1,7 @@ +'use strict'; +const api = require('../../core/api/directory-platform'); +const { createDashboardServer } = require('./server'); +module.exports = { + startDirectoryDashboard: options => api.startDirectoryApi({ ...options, serverFactory: createDashboardServer }), + mainDirectory: options => api.mainDirectory(options, { serverFactory: createDashboardServer, compatibility: true }), +}; diff --git a/core/dashboard/server/invitation-email.js b/core/dashboard/server/invitation-email.js new file mode 100644 index 0000000..fe06d23 --- /dev/null +++ b/core/dashboard/server/invitation-email.js @@ -0,0 +1,3 @@ +'use strict'; +// Compatibility import; HTTP implementation belongs to Dispatch API. +module.exports = require('../../core/api/invitation-email'); diff --git a/core/dashboard/server/main.js b/core/dashboard/server/main.js new file mode 100644 index 0000000..0c26e64 --- /dev/null +++ b/core/dashboard/server/main.js @@ -0,0 +1,15 @@ +'use strict'; +const legacyArgs = argv => argv.includes('--port') ? argv : [...argv, '--port', '4310']; +function parseArguments(argv = process.argv.slice(2)) { return require('../../core/api/main').parseArguments(legacyArgs(argv)); } +function usage() { + return 'UI service: dispatch-dashboard --api-origin http://127.0.0.1:4311 [--port 4310] [--public-origin https://host] [--turnstile]\n\n' + + 'Without --api-origin, existing combined deployments remain supported:\n' + + require('../../core/api/main').usage().replaceAll('dispatch-api', 'dispatch-dashboard').replaceAll('4311', '4310'); +} +async function main(argv = process.argv.slice(2), dependencies = {}) { + if (argv.includes('--help')) { process.stdout.write(usage() + '\n'); return 0; } + if (argv.includes('--api-origin')) return require('./shell-main').main(argv); + return require('../../core/api/main').main(legacyArgs(argv), { ...dependencies, + serverFactory: require('./server').createDashboardServer, compatibility: true }); +} +module.exports = { parseArguments, usage, main }; diff --git a/core/dashboard/server/password-recovery-email.js b/core/dashboard/server/password-recovery-email.js new file mode 100644 index 0000000..e9b8eba --- /dev/null +++ b/core/dashboard/server/password-recovery-email.js @@ -0,0 +1,3 @@ +'use strict'; +// Compatibility import; HTTP implementation belongs to Dispatch API. +module.exports = require('../../core/api/password-recovery-email'); diff --git a/core/dashboard/server/password-recovery-http.js b/core/dashboard/server/password-recovery-http.js new file mode 100644 index 0000000..12bf8f5 --- /dev/null +++ b/core/dashboard/server/password-recovery-http.js @@ -0,0 +1,3 @@ +'use strict'; +// Compatibility import; HTTP implementation belongs to Dispatch API. +module.exports = require('../../core/api/password-recovery-http'); diff --git a/core/dashboard/server/release-delivery.js b/core/dashboard/server/release-delivery.js new file mode 100644 index 0000000..600604e --- /dev/null +++ b/core/dashboard/server/release-delivery.js @@ -0,0 +1,3 @@ +'use strict'; +// Compatibility import; HTTP implementation belongs to Dispatch API. +module.exports = require('../../core/api/release-delivery'); diff --git a/core/dashboard/server/runtime-router.js b/core/dashboard/server/runtime-router.js new file mode 100644 index 0000000..3bcf68f --- /dev/null +++ b/core/dashboard/server/runtime-router.js @@ -0,0 +1,3 @@ +'use strict'; +// Compatibility import; HTTP implementation belongs to Dispatch API. +module.exports = require('../../core/api/runtime-router'); diff --git a/core/dashboard/server/server.js b/core/dashboard/server/server.js new file mode 100644 index 0000000..9772d79 --- /dev/null +++ b/core/dashboard/server/server.js @@ -0,0 +1,14 @@ +'use strict'; +// Compatibility composition for existing deployments and frontend fixtures. +// New services use core/api/server and dashboard/server/shell independently. +const { createApiServer } = require('../../core/api/server'); +const helpers = require('../../core/api/http'); +const { DEFAULT_PUBLIC_ROOT, loadStaticFiles, sendStatic } = require('./static'); +function createDashboardServer(options = {}) { + const files = loadStaticFiles(options.publicRoot || DEFAULT_PUBLIC_ROOT); + const value=options.dashboards ? null : require('../../core/updates/dashboard').readDashboard(options.publicRoot || DEFAULT_PUBLIC_ROOT,'core'); + const dashboards=options.dashboards || ((_session,identity)=>identity?{product:value.product,digest:value.digest}:value); + return createApiServer({ ...options, dashboards, fallback: (request, response, url) => + sendStatic(response, files, url.pathname, request.method, options.turnstile) }); +} +module.exports = { ...helpers, DEFAULT_PUBLIC_ROOT, createDashboardServer }; diff --git a/core/dashboard/server/shell-main.js b/core/dashboard/server/shell-main.js new file mode 100644 index 0000000..73ea597 --- /dev/null +++ b/core/dashboard/server/shell-main.js @@ -0,0 +1,35 @@ +'use strict'; +const { createDashboardShell, checkedApiOrigin } = require('./shell'); +const { checkedPublicOrigin } = require('../../core/api/http'); +function parseArguments(argv) { + const options = { host: '127.0.0.1', port: 4310, apiOrigin: null, publicOrigin: null, turnstile: false }; + const flags = { '--host': 'host', '--port': 'port', '--api-origin': 'apiOrigin', '--public-origin': 'publicOrigin' }; + const seen = new Set(); + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]; + if (seen.has(flag)) throw new TypeError('dashboard_argument_invalid'); + seen.add(flag); + if (flag === '--turnstile') options.turnstile = true; + else if (Object.hasOwn(flags, flag) && typeof argv[i + 1] === 'string' && !argv[i + 1].startsWith('--')) options[flags[flag]] = argv[++i]; + else throw new TypeError('dashboard_argument_invalid'); + } + options.port = Number(options.port); + if (!['127.0.0.1', '::1'].includes(options.host) || !Number.isInteger(options.port) + || options.port < 1024 || options.port > 65535) throw new TypeError('dashboard_argument_invalid'); + const upstream = checkedApiOrigin(options.apiOrigin); + if (Number(upstream.port) === options.port) throw new TypeError('api_proxy_loop'); + checkedPublicOrigin(options.publicOrigin); + return options; +} +async function main(argv) { + let options; + try { options = parseArguments(argv); } + catch { process.stderr.write('Usage: dispatch-dashboard --api-origin http://127.0.0.1:4311 [--port 4310] [--public-origin https://host] [--turnstile]\n'); return 2; } + const server = createDashboardShell(options); + await new Promise((resolve, reject) => { server.once('error', reject); server.listen(options.port, options.host, resolve); }); + const close = () => server.close(); + process.once('SIGINT', close); process.once('SIGTERM', close); + process.stdout.write(JSON.stringify({ ok: true, status: 'ready', service: 'dispatch-dashboard', port: options.port }) + '\n'); + return 0; +} +module.exports = { parseArguments, main }; diff --git a/core/dashboard/server/shell.js b/core/dashboard/server/shell.js new file mode 100644 index 0000000..f985010 --- /dev/null +++ b/core/dashboard/server/shell.js @@ -0,0 +1,77 @@ +'use strict'; + +const http = require('node:http'); +const { AccessError } = require('../../core/accounts/src/validation'); +const { SERVER_OPTIONS, checkedPublicOrigin, requirePublicRequest, securityHeaders, sendJson, publicHttpFailure } = require('../../core/api/http'); +const { DEFAULT_PUBLIC_ROOT, loadStaticFiles, sendStatic } = require('./static'); + +const HOP_HEADERS = new Set(['connection', 'keep-alive', 'proxy-authenticate', 'proxy-authorization', + 'te', 'trailer', 'transfer-encoding', 'upgrade']); +function endToEndHeaders(headers) { + const excluded = new Set([...HOP_HEADERS, ...String(headers.connection || '').toLowerCase().split(',').map(x => x.trim())]); + return Object.fromEntries(Object.entries(headers).filter(([name]) => !excluded.has(name.toLowerCase()))); +} +function checkedApiOrigin(value) { + let url; + try { url = new URL(value); } catch { throw new TypeError('api_origin_invalid'); } + if (url.protocol !== 'http:' || !['127.0.0.1', '[::1]'].includes(url.hostname) + || url.origin !== value || !url.port || Number(url.port) < 1024 + || url.username || url.password || url.pathname !== '/' || url.search || url.hash) { + throw new TypeError('api_origin_invalid'); + } + return url; +} + +// The UI server owns no account store, runtime controller, or DSP credentials. +// It preserves the public request's cookies, CSRF, signed DSP view, and Host. +// Requests are streamed once, including mutations; failures are never replayed. +function createDashboardShell({ apiOrigin, publicRoot = DEFAULT_PUBLIC_ROOT, publicOrigin = null, + turnstile = false, upstreamTimeoutMs = 300_000 } = {}) { + const upstream = checkedApiOrigin(apiOrigin), origin = checkedPublicOrigin(publicOrigin); + if (!Number.isSafeInteger(upstreamTimeoutMs) || upstreamTimeoutMs < 1 || upstreamTimeoutMs > 300_000) throw new TypeError('api_timeout_invalid'); + const files = loadStaticFiles(publicRoot); + const agent = new http.Agent({ keepAlive: true, maxSockets: 128, maxFreeSockets: 16 }); + const server = http.createServer(SERVER_OPTIONS, (request, response) => { + try { + if (typeof request.url !== 'string' || !/^\/(?!\/)[^\0\r\n\\]*$/.test(request.url)) throw new AccessError('request_forbidden', 403); + const redirect = requirePublicRequest(request, origin); + if (redirect) { + response.writeHead(308, { ...securityHeaders(), 'Cache-Control': 'no-store', 'Content-Length': 0, Location: redirect }); + response.end(); return; + } + const url = new URL(request.url, 'http://127.0.0.1'); + if (url.pathname === '/api' || url.pathname.startsWith('/api/')) { + const forward = http.request({ hostname: upstream.hostname.replace(/^\[|\]$/g, ''), port: upstream.port, + path: request.url, method: request.method, headers: endToEndHeaders(request.headers), agent }, result => { + response.writeHead(result.statusCode, endToEndHeaders(result.headers)); + result.on('error', () => response.destroy()); + result.pipe(response); + }); + const timeout = setTimeout(() => forward.destroy(new Error('api_timeout')), upstreamTimeoutMs); + timeout.unref(); + const cleanup = () => { clearTimeout(timeout); if (!response.writableFinished) forward.destroy(); }; + response.once('close', cleanup); + request.once('aborted', () => forward.destroy()); + forward.once('error', () => { + clearTimeout(timeout); + if (response.destroyed) return; + if (response.headersSent) { response.destroy(); return; } + sendJson(response, 502, { ok: false, status: 'api_unavailable', data: null, error: { code: 'api_unavailable' } }); + }); + request.pipe(forward); return; + } + if (!['GET', 'HEAD'].includes(request.method)) { + sendJson(response, 405, { ok: false, status: 'method_not_allowed', data: null, error: { code: 'method_not_allowed' } }); return; + } + if (!sendStatic(response, files, url.pathname, request.method, turnstile)) { + sendJson(response, 404, { ok: false, status: 'not_found', data: null, error: { code: 'not_found' } }); + } + } catch (error) { + const { statusCode, code } = publicHttpFailure(error); + sendJson(response, statusCode, { ok: false, status: code, data: null, error: { code } }); + } + }); + server.once('close', () => agent.destroy()); + return server; +} +module.exports = { createDashboardShell, checkedApiOrigin }; diff --git a/core/dashboard/server/static.js b/core/dashboard/server/static.js new file mode 100644 index 0000000..55b8ecb --- /dev/null +++ b/core/dashboard/server/static.js @@ -0,0 +1,69 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { createHash, randomBytes } = require('node:crypto'); +const { securityHeaders } = require('../../core/api/http'); +const DEFAULT_PUBLIC_ROOT = path.resolve(__dirname, '../public'); +const STATIC_FILES = Object.freeze({ + '/': ['index.html', 'text/html; charset=utf-8', 'no-store'], + '/index.html': ['index.html', 'text/html; charset=utf-8', 'no-store'], + '/assets/launcher.js': ['assets/launcher.js', 'text/javascript; charset=utf-8', 'no-store'], + '/assets/frontend.js': ['assets/frontend.js', 'text/javascript; charset=utf-8', 'no-store'], + '/assets/inter.woff2': ['assets/inter.woff2', 'font/woff2', 'public, max-age=86400'], + '/assets/updates.js': ['assets/updates.js', 'text/javascript; charset=utf-8', 'no-store'], + '/assets/backups.js': ['assets/backups.js', 'text/javascript; charset=utf-8', 'no-store'], + '/assets/styles.css': ['assets/styles.css', 'text/css; charset=utf-8', 'no-store'], +}); + +function loadStaticFiles(publicRoot) { + // Snapshot the shell and its assets together. A changed file gets a new URL, + // so even browsers with a fresh cached copy of an older release fetch it. + const files = new Map(); + const assetUrls = new Map(); + for (const [requestPath, [relative, contentType, cacheControl]] of Object.entries(STATIC_FILES)) { + const bytes = fs.readFileSync(path.join(publicRoot, relative)); + files.set(requestPath, { bytes, contentType, cacheControl }); + if (requestPath.startsWith('/assets/')) { + const digest = createHash('sha256').update(bytes).digest('hex'); + const assetUrl = requestPath.replace(/(\.[^.]+)$/, `.${digest}$1`); + assetUrls.set(requestPath, assetUrl); + files.set(assetUrl, { bytes, contentType, cacheControl: 'public, max-age=31536000, immutable' }); + } + } + for (const requestPath of ['/', '/index.html']) { + const file = files.get(requestPath); + const html = file.bytes.toString('utf8').replace(/\b(href|src)="([^"]+)"/g, + (match, attribute, url) => assetUrls.has(url) ? `${attribute}="${assetUrls.get(url)}"` : match); + file.bytes = Buffer.from(html); + } + return files; +} + +function sendStatic(response, files, requestPath, method, turnstile = null) { + const definition = files.get(requestPath); + if (!definition) return false; + const { contentType, cacheControl } = definition; + let bytes = definition.bytes; + const headers = securityHeaders(contentType); + if (contentType.startsWith('text/html')) { + headers['Content-Security-Policy'] = headers['Content-Security-Policy'].replace("font-src 'self'", "font-src 'self' data:"); + if (turnstile) headers['Content-Security-Policy'] = headers['Content-Security-Policy'] + .replace("script-src 'self'", "script-src 'self' https://challenges.cloudflare.com") + + "; frame-src https://challenges.cloudflare.com"; + const nonce = randomBytes(18).toString('base64'); + bytes = Buffer.from(bytes.toString('utf8').replaceAll('__DISPATCH_STYLE_NONCE__', nonce)); + headers['Content-Security-Policy'] = headers['Content-Security-Policy'] + .replace("style-src 'self'", `style-src 'self' 'nonce-${nonce}'`) + .replace("script-src 'self'", `script-src 'self' 'nonce-${nonce}'`); + } + response.writeHead(200, { + ...headers, + 'Cache-Control': cacheControl, + 'Content-Length': bytes.length, + }); + response.end(method === 'HEAD' ? undefined : bytes); + return true; +} + + +module.exports = { DEFAULT_PUBLIC_ROOT, loadStaticFiles, sendStatic }; diff --git a/core/dashboard/server/turnstile.js b/core/dashboard/server/turnstile.js new file mode 100644 index 0000000..02f08d7 --- /dev/null +++ b/core/dashboard/server/turnstile.js @@ -0,0 +1,3 @@ +'use strict'; +// Compatibility import; HTTP implementation belongs to Dispatch API. +module.exports = require('../../core/api/turnstile'); diff --git a/core/dashboard/tests/backups-view.test.js b/core/dashboard/tests/backups-view.test.js new file mode 100644 index 0000000..eddea08 --- /dev/null +++ b/core/dashboard/tests/backups-view.test.js @@ -0,0 +1,179 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { + protection, + activity, + route, + stages, + fleet, + filterFleet, + needsAttention, +} = require('../public/assets/backups'); +const at = (day) => `2026-09-0${day}T12:00:00.000Z`; +const record = (id, organizationId, day, status = 'verified') => ({ + id, + organizationId, + createdAt: at(day), + status, + name: organizationId || 'Platform Core', +}); +const op = (id, organizationId, day, status = 'failed', kind = 'backup') => ({ + id, + organizationId, + createdAt: at(day), + updatedAt: at(day), + status, + kind, +}); +const data = (backups = [], operations = []) => ({ + organizations: [ + { id: 'one', name: 'DSP One' }, + { id: 'two', name: 'DSP Two' }, + ], + backups, + operations, +}); + +test('a failed request takes precedence over the last successful backup, until a newer success', () => { + const state = data([record('old', 'one', 4)], [op('failed', 'one', 5)]); + assert.equal(protection(state, 'one').status, 'failed'); + assert.equal(protection(state, 'one').verified.id, 'old'); + state.backups.push(record('new', 'one', 6)); + assert.equal(protection(state, 'one').status, 'verified'); + state.backups.reverse(); + assert.equal(protection(state, 'one').verified.id, 'new'); +}); + +test('in-progress restores and verification have distinct truthful statuses', () => { + const state = data( + [record('old', 'one', 4), record('new', 'one', 6, 'pending')], + [op('failed', 'one', 5), op('restore', 'one', 6, 'running', 'restore')], + ); + assert.equal(protection(state, 'one').label, 'Restoring'); + state.operations = []; + assert.equal(protection(state, 'one').label, 'Uploading backup'); + assert.equal(protection(state, 'one').verified.id, 'old'); +}); + +test('a failed Core operation or another DSP cannot affect the selected DSP', () => { + const state = data( + [record('one-backup', 'one', 4)], + [op('core-failed', null, 6), op('other-failed', 'two', 6)], + ); + assert.equal(protection(state, 'one').status, 'verified'); + assert.equal(protection(state, 'core').status, 'failed'); + assert.equal(protection(state, 'two').status, 'failed'); + assert.equal(protection(state, 'missing').label, 'No backup yet'); +}); + +test('expired backup does not claim protection when there is no verified recovery point', () => { + assert.equal(protection(data([record('old', 'one', 3, 'expired')]), 'one').status, 'expired'); +}); + +test('fleet summary includes unprotected DSPs without counting Core or double-counting work', () => { + const state = data( + [record('one', 'one', 4), record('core', null, 6)], + [op('active', 'one', 5, 'running')], + ); + const items = fleet(state); + assert.equal(items.length, 2); + assert.equal(items.filter(needsAttention).length, 1); + assert.equal(items.filter((p) => p.status === 'running').length, 1); + assert.equal(items.filter((p) => p.status === 'verified').length, 0); + assert.equal(filterFleet(items, '', 'attention')[0].org.id, 'two'); +}); + +test('DSP navigator combines name and status filters without crossing scopes', () => { + const items = fleet(data([record('one', 'one', 4), record('two', 'two', 3, 'expired')])); + assert.deepEqual( + filterFleet(items, ' DSP ONE ', 'verified').map((p) => p.org.id), + ['one'], + ); + assert.equal(filterFleet(items, 'one', 'attention').length, 0); + assert.equal(filterFleet(items, '', 'attention')[0].org.id, 'two'); + assert.equal(filterFleet(items, 'unmatched', 'all').length, 0); +}); + +test('history deduplicates a backup request and its artifact but retains restore events', () => { + const state = data( + [record('snapshot', 'one', 4), record('core-backup', null, 5)], + [ + { ...op('backup-op', 'one', 4, 'completed'), backupId: 'snapshot' }, + { ...op('restore-op', 'one', 6, 'completed', 'restore'), backupId: 'snapshot' }, + ], + ); + const events = activity(state); + assert.equal(events.length, 3); + assert.deepEqual( + events.map((e) => e.event), + ['Restore', 'Backup', 'Backup'], + ); + assert.equal(events[0].name, 'DSP One'); + assert.equal(events[1].name, 'Platform Core'); +}); + +test('backup routes preserve scope and reject malformed or unexpected destinations', () => { + assert.deepEqual(route('#/backups'), { mode: 'overview' }); + assert.deepEqual(route('#/backups/dsps/org_one/backups/backup_one'), { + mode: 'detail', + orgId: 'org_one', + backupId: 'backup_one', + }); + assert.deepEqual(route('#/backups/core/backups/backup_core'), { + mode: 'detail', + orgId: 'core', + backupId: 'backup_core', + }); + assert.deepEqual(route('#/backups/operations/op_one'), { + mode: 'operation', + operationId: 'op_one', + }); + assert.deepEqual(route('#/backups/dsps/org_one/history'), { + mode: 'dsp-history', + orgId: 'org_one', + }); + for (const hash of ['#/backups/dsps/%ZZ', '#/backups/unknown', '#/backups/history/extra']) + assert.equal(route(hash).mode, 'missing'); +}); + +test('progress never marks pending verification or failed recovery as complete', () => { + assert.deepEqual( + stages({ kind: 'backup', phase: 'uploading', status: 'running' }).map((s) => s.status), + ['done', 'done', 'active', 'pending'], + ); + assert.equal( + stages({ kind: 'restore', phase: 'restoring', status: 'running' })[1].status, + 'active', + ); + assert.equal( + stages({ kind: 'restore', phase: 'recovering', status: 'running' }).every( + (s) => s.status === 'pending', + ), + true, + ); + assert.equal( + stages({ kind: 'restore', phase: 'completed', status: 'completed' }).every( + (s) => s.status === 'done', + ), + true, + ); +}); + +test('full-system recovery points remain addressable and filterable without guessing their category', () => { + const state = data([], [{ ...op('request', null, 5, 'completed'), setId: 'scheduled-set', category: 'scheduled' }]); + state.sets = [ + { id: 'scheduled-set', createdAt: at(5), status: 'verified' }, + { id: 'unknown-trigger', createdAt: at(4), status: 'incomplete' }, + { id: 'removed-set', createdAt: at(3), status: 'deleted' }, + ]; + const sets = activity(state).filter(e => e.source === 'set'); + assert.equal(sets.length, 2); + assert.equal(sets[0].category, 'scheduled'); + assert.equal(sets[1].category, null); + const { scope } = require('../public/assets/backups'); + assert.equal(scope(sets[0]), 'system'); + assert.deepEqual(route('#/backups/sets/scheduled-set'), { mode: 'sets', setId: 'scheduled-set' }); + assert.deepEqual(route('#/backups/sets'), { mode: 'sets' }); + assert.equal(route('#/backups/sets/one/extra').mode, 'missing'); +}); diff --git a/core/dashboard/tests/browser/backups-streamlined.spec.cjs b/core/dashboard/tests/browser/backups-streamlined.spec.cjs new file mode 100644 index 0000000..5f34fe5 --- /dev/null +++ b/core/dashboard/tests/browser/backups-streamlined.spec.cjs @@ -0,0 +1,247 @@ +const { test, expect } = require("@playwright/test"); + +async function fixture(page) { + const errors = []; + page.on("pageerror", (e) => errors.push(e.message)); + await page.goto("/"); + await page.getByLabel("Email address").fill("platform@example.test"); + await page + .getByLabel("Password", { exact: true }) + .fill("synthetic preview password"); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await page.locator(".desktop-sidebar").waitFor(); + const data = (await (await page.request.get("/api/platform/backups")).json()) + .data; + data.operations = []; + data.settings.enabled = false; + data.schedules.forEach((s) => { + s.settings.enabled = false; + }); + const org = data.organizations[0]; + const backup = data.backups.find((b) => b.organizationId === org.id); + backup.category = "manual"; + backup.trigger = "manual"; + data.sets = [ + { + id: "streamlined-set", + createdAt: backup.createdAt, + status: "verified", + members: [ + { organizationId: org.id, backupId: backup.id, status: "verified" }, + { + organizationId: null, + backupId: "fixture_core_backup", + status: "verified", + }, + ], + }, + ]; + const inputs = []; + await page.route("**/api/platform/backups", async (route) => { + if (route.request().method() === "POST") { + const input = route.request().postDataJSON(); + inputs.push(input); + if (input.action === "settings") { + const policy = data.schedules.find( + (s) => s.scope === (input.scope || input.organizationId), + ); + policy.settings = input.settings; + policy.revision++; + if (policy.scope === "system") { + data.settings = input.settings; + data.revision = policy.revision; + } + } + } + await route.fulfill({ json: { ok: true, data } }); + }); + return { data, inputs, org, errors }; +} +const nav = (page) => + page.getByRole("navigation", { name: "Backup navigation" }); + +test("minimal overview, DSP selection, history filters and recovery links retain their scope", async ({ + page, +}) => { + const { data, org, errors } = await fixture(page); + await page.goto("/#/backups"); + await expect( + page.locator(".backup-header .backup-button-primary"), + ).toHaveCount(1); + await expect( + page.getByRole("heading", { name: "Latest full-system backup" }), + ).toBeVisible(); + await expect( + page.getByRole("heading", { name: "Recent activity" }), + ).toHaveCount(0); + await page.getByRole("link", { name: "View details", exact: true }).click(); + await expect(page).toHaveURL(/\/sets\/streamlined-set$/); + await expect( + page.getByRole("heading", { name: "Full-system backup", exact: true }), + ).toBeVisible(); + await nav(page).getByRole("link", { name: "DSPs", exact: true }).click(); + await page + .getByLabel("DSP", { exact: true }) + .selectOption(data.organizations[1].id); + await expect(page).toHaveURL(new RegExp(data.organizations[1].id + "$")); + await expect( + page.getByRole("heading", { + name: data.organizations[1].name, + exact: true, + }), + ).toBeVisible(); + await page.getByRole("button", { name: "Back up DSP", exact: true }).click(); + await expect(page.locator("#backup-select-all")).not.toBeChecked(); + await expect(page.locator("dialog input[data-org]:checked")).toHaveCount(1); + await expect(page.locator("dialog input[data-org]:checked")).toHaveAttribute( + "data-org", + data.organizations[1].id, + ); + await page.keyboard.press("Escape"); + await page.getByLabel("DSP", { exact: true }).selectOption(org.id); + await page + .getByRole("searchbox", { name: "Search backups" }) + .fill("not a backup"); + await expect(page.getByText("No backups available")).toBeVisible(); + await page.getByRole("searchbox", { name: "Search backups" }).fill(""); + await expect(page.locator(".backup-table tbody tr")).toHaveCount(1); + await nav(page).getByRole("link", { name: "History", exact: true }).click(); + await page.getByLabel("Scope", { exact: true }).selectOption("system"); + await expect(page.locator(".backup-table tbody tr")).toHaveCount(1); + await expect(page.locator(".backup-table tbody tr")).toContainText( + "Full system", + ); + await page.getByRole("button", { name: "Filters", exact: true }).click(); + await expect(page.locator("#backup-history-filters")).toBeHidden(); + await page.getByRole("button", { name: "Filters", exact: true }).click(); + await page.getByRole("button", { name: "Clear filters" }).click(); + await page.getByLabel("Scope", { exact: true }).selectOption(org.id); + await page.getByLabel("Category", { exact: true }).selectOption("manual"); + await expect(page.locator(".backup-table tbody tr")).toHaveCount(1); + await expect(page.locator(".backup-table tbody tr")).toContainText(org.name); + await page.getByRole("button", { name: "Restores", exact: true }).click(); + await expect(page.getByText("No activity found")).toBeVisible(); + expect(errors).toEqual([]); +}); + +test("schedule edits preserve drafts and scope, with a single header save action", async ({ + page, +}) => { + const { data, org, inputs, errors } = await fixture(page); + await page.goto("/#/backups/dsps/" + org.id); + await page.getByRole("button", { name: "Edit schedule" }).click(); + await expect(page.getByLabel("Schedule for")).toHaveValue(org.id); + await expect(page.getByLabel("Frequency", { exact: true })).toBeDisabled(); + await page.getByLabel("Automatic backups", { exact: true }).check(); + await page.getByLabel("Frequency", { exact: true }).selectOption("weekly"); + await page.getByLabel("Day", { exact: true }).selectOption("2"); + await page.getByLabel("Time", { exact: true }).fill("03:45"); + await page.getByLabel("Keep backups", { exact: true }).selectOption("90"); + await page.getByLabel("Automatic backups", { exact: true }).uncheck(); + await page.getByLabel("Automatic backups", { exact: true }).check(); + await expect(page.getByLabel("Time", { exact: true })).toHaveValue("03:45"); + await expect( + page.getByRole("button", { name: "Save settings", exact: true }), + ).toHaveCount(1); + await page + .getByRole("button", { name: "Save settings", exact: true }) + .click(); + await expect( + page.getByText("Backup settings saved.", { exact: true }), + ).toBeVisible(); + expect(inputs.at(-1)).toMatchObject({ + action: "settings", + organizationId: org.id, + settings: { + enabled: true, + frequency: "weekly", + weekday: 2, + time: "03:45", + retentionDays: 90, + }, + }); + expect( + data.schedules.find((s) => s.scope === "system").settings.enabled, + ).toBe(false); + await page.getByLabel("Schedule for").selectOption("core"); + await expect( + page.getByLabel("Automatic backups", { exact: true }), + ).not.toBeChecked(); + await page.getByLabel("Automatic backups", { exact: true }).check(); + await page.getByRole("button", { name: "Cancel", exact: true }).click(); + await page + .locator(".backup-header") + .getByRole("link", { name: "Settings" }) + .click(); + await expect( + page.getByLabel("Automatic backups", { exact: true }), + ).not.toBeChecked(); + expect(errors).toEqual([]); +}); + +test("system restore confirmation tracks polling, and failed or busy recovery stays guarded", async ({ + page, +}) => { + const { data, inputs, errors } = await fixture(page); + await page.goto("/#/backups/sets/streamlined-set"); + await page + .getByRole("button", { name: "Restore full system", exact: true }) + .click(); + const confirm = page + .getByRole("dialog") + .getByRole("button", { name: "Restore full system", exact: true }); + await expect(confirm).toBeDisabled(); + await page.getByLabel("Type Full system to confirm").fill("wrong"); + await expect(confirm).toBeDisabled(); + await page.getByLabel("Type Full system to confirm").fill("Full system"); + await expect(confirm).toBeEnabled(); + data.sets[0].busy = true; + await expect(confirm).toBeDisabled({ timeout: 10000 }); + await expect(page.getByRole("dialog")).toContainText("no longer available"); + expect(inputs).toHaveLength(0); + await page.keyboard.press("Escape"); + data.sets[0].busy = false; + data.sets[0].status = "incomplete"; + await page.reload(); + await expect( + page.getByRole("button", { name: "Restore full system", exact: true }), + ).toBeDisabled(); + expect(errors).toEqual([]); +}); + +test("all six surfaces fit mobile, retain visible data and disclose storage details", async ({ + page, +}) => { + const { errors } = await fixture(page); + await page.setViewportSize({ width: 390, height: 844 }); + for (const path of [ + "", + "/dsps", + "/history", + "/storage", + "/core", + "/settings", + ]) { + await page.goto("/#/backups" + path); + await expect(page.locator(".backup-split")).toBeVisible(); + expect( + await page.evaluate( + () => document.documentElement.scrollWidth <= innerWidth, + ), + ).toBe(true); + for (const cell of await page.locator(".backup-table tbody td").all()) { + const box = await cell.boundingBox(); + if (box) expect(box.x + box.width).toBeLessThanOrEqual(391); + } + } + await page.goto("/#/backups/storage"); + await page.locator("#backup-storage-retained > summary").click(); + await expect( + page.getByRole("region", { name: "Removed DSPs — retained backups" }), + ).toContainText("Pine Delivery"); + await page.locator("#backup-storage-additional > summary").click(); + await expect( + page.getByText("Backup archives", { exact: true }), + ).toBeVisible(); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/connections-persistence.spec.cjs b/core/dashboard/tests/browser/connections-persistence.spec.cjs new file mode 100644 index 0000000..9d437e8 --- /dev/null +++ b/core/dashboard/tests/browser/connections-persistence.spec.cjs @@ -0,0 +1,137 @@ +const { test, expect } = require('@playwright/test'); +const { createConnectionsStack } = require('../helpers/connections-stack.cjs'); + +async function login(page, f) { + await page.goto(`${f.base}/#/settings?tab=connections`); + await page.getByLabel('Email address').fill('owner@save.test'); + await page.getByLabel('Password', { exact: true }).fill(f.password); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await expect(page).toHaveTitle('Settings · Dispatch'); + await expect(page.getByRole('heading', { name: 'Connections', exact: true })).toBeVisible(); +} + +for (const mobile of [false, true]) test(`form credentials reach a directory DSP vault without a runtime slot (${mobile ? 'mobile' : 'desktop'})`, async ({ page }) => { + const f = await createConnectionsStack({ directoryEnrollment: true }); + let finishPaycom; + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + try { + if (mobile) await page.setViewportSize({ width: 390, height: 844 }); + await login(page, f); + await page.getByRole('button', { name: 'Connect Cortex', exact: true }).click(); + const dialog = page.getByRole('dialog'); + const credentials = { username: 'form-fixture-owner', password: 'form-fixture-password-界-"-\\' }; + await dialog.getByLabel('Amazon username').fill(credentials.username); + await dialog.getByLabel('Amazon password').fill(credentials.password); + const saved = page.waitForResponse(response => response.url().endsWith('/connections/cortex/save')); + await dialog.getByRole('button', { name: 'Save and connect' }).click(); + expect((await saved).status()).toBe(202); + await expect(dialog).toHaveCount(0); + const cortex = page.locator('[data-slot="card"]').filter({ has: page.getByText('Cortex', { exact: true }) }); + await expect(cortex.getByText('Connected', { exact: true })).toBeVisible(); + expect(f.state.broker.vault.readForAdapter('amazon-operations').credentials).toEqual(credentials); + await f.restartBroker(); + expect(f.state.broker.vault.readForAdapter('amazon-operations').credentials).toEqual(credentials); + await page.reload(); + await expect(cortex.getByText('Connected', { exact: true })).toBeVisible(); + await cortex.getByRole('button', { name: 'Update credentials' }).click(); + await expect(dialog.getByLabel('Amazon password')).toHaveValue(''); + await dialog.getByRole('button', { name: 'Cancel', exact: true }).click(); + let paycomAttempts = 0; + const pendingPaycom = new Promise(resolve => { finishPaycom = resolve; }); + f.state.authentication = async () => { paycomAttempts++; await pendingPaycom; return { status: 'authenticated' }; }; + await page.getByRole('button', { name: 'Connect Paycom', exact: true }).click(); + const paycom = { clientCode: 'form-client', username: 'form-paycom-owner', password: 'form-paycom-secret', + pin1: 'one', pin2: 'two', pin3: 'three', pin4: 'four', pin5: 'five' }; + for (const [name, label] of [['clientCode', 'Client code'], ['username', 'Username'], ['password', 'Password'], + ...[1, 2, 3, 4, 5].map(index => [`pin${index}`, `Security answer ${index}`])]) { + await dialog.getByLabel(label, { exact: true }).fill(paycom[name]); + } + const paycomSaved = page.waitForResponse(response => response.url().endsWith('/connections/paycom/save')); + await dialog.getByRole('button', { name: 'Save and connect' }).click(); + expect((await paycomSaved).status()).toBe(202); + expect(f.state.runtimeEnrollments).toBe(0); + await expect(dialog).toHaveCount(0); + const paycomCard = page.locator('[data-slot="card"]').filter({ has: page.getByText('Paycom', { exact: true }) }); + await expect(paycomCard.getByText('Checking session', { exact: true })).toBeVisible(); + await expect(paycomCard.getByText('Not verified', { exact: true })).toHaveCount(0); + expect(paycomAttempts).toBe(1); + await page.screenshot({ path: `/tmp/dispatch-paycom-checking-${mobile ? 'mobile' : 'desktop'}.png`, fullPage: true }); + finishPaycom(); + await expect(paycomCard.getByText('Connected', { exact: true })).toBeVisible(); + await f.restartBroker(); + expect(f.state.broker.vault.readForAdapter('paycom-main').credentials).toEqual(paycom); + await page.reload(); + await expect(paycomCard.getByText('Connected', { exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Update credentials', exact: true })).toHaveCount(2); + expect(await page.evaluate(() => JSON.stringify([localStorage, sessionStorage]))).not.toContain(credentials.password); + expect(await page.evaluate(() => JSON.stringify([localStorage, sessionStorage]))).not.toContain(paycom.password); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBeTruthy(); + await page.screenshot({ path: `/tmp/dispatch-save-verified-${mobile ? 'mobile' : 'desktop'}.png`, fullPage: true }); + expect(errors).toEqual([]); + } finally { finishPaycom?.(); await page.close(); await f.close(); } +}); + +for (const mobile of [false, true]) test(`an unconfirmed Paycom save stays dismissible during a stalled status check (${mobile ? 'mobile' : 'desktop'})`, async ({ page }) => { + const f = await createConnectionsStack({ directoryEnrollment: true }); + let releaseStatus, failed = false, saves = 0; + const pendingStatus = new Promise(resolve => { releaseStatus = resolve; }); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { + if (message.type() === 'error' && !message.text().includes('503 (Service Unavailable)')) errors.push(message.text()); + }); + try { + if (mobile) await page.setViewportSize({ width: 390, height: 844 }); + await login(page, f); + await page.route('**/api/organization/connections', async route => { + if (failed) await pendingStatus; + await route.continue(); + }); + await page.route('**/api/organization/connections/paycom/save', async route => { + saves++; failed = true; + await route.fulfill({ status: 503, json: { ok: false, error: { code: 'dashboard_unavailable' } } }); + }); + await page.getByRole('button', { name: 'Connect Paycom', exact: true }).click(); + const dialog = page.getByRole('dialog'); + for (const [label, value] of [['Client code', 'synthetic'], ['Username', 'synthetic'], ['Password', 'synthetic-form-secret'], + ...[1, 2, 3, 4, 5].map(index => [`Security answer ${index}`, `synthetic-${index}`])]) { + await dialog.getByLabel(label, { exact: true }).fill(value); + } + await dialog.getByRole('button', { name: 'Save and connect' }).click(); + await expect(dialog.getByText('We couldn’t confirm this save.', { exact: false })).toBeVisible(); + await expect(dialog.getByLabel('Password', { exact: true })).toHaveValue(''); + await expect(dialog.getByRole('button', { name: 'Cancel', exact: true })).toBeEnabled(); + await expect(dialog.getByRole('button', { name: 'Save and connect' })).toBeDisabled(); + await expect(page).toHaveTitle('Settings · Dispatch'); + await expect(page).toHaveURL(/#\/settings\?tab=connections$/); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBeTruthy(); + await dialog.getByRole('button', { name: 'Cancel', exact: true }).scrollIntoViewIfNeeded(); + await page.screenshot({ path: `/tmp/dispatch-save-stalled-${mobile ? 'mobile' : 'desktop'}.png` }); + await dialog.getByRole('button', { name: 'Cancel', exact: true }).click(); + await expect(dialog).toHaveCount(0); + expect(saves).toBe(1); + expect(await page.evaluate(() => JSON.stringify([localStorage, sessionStorage]))).not.toContain('synthetic-form-secret'); + expect(errors).toEqual([]); + } finally { releaseStatus(); await page.close(); await f.close(); } +}); + +test('lost acknowledgement explains uncertainty while the saved connection can be recovered', async ({ page }) => { + const f = await createConnectionsStack(); + try { + await login(page, f); + await page.getByRole('button', { name: 'Connect Cortex', exact: true }).click(); + const dialog = page.getByRole('dialog'); + await dialog.getByLabel('Amazon username').fill('reply-loss-owner'); + await dialog.getByLabel('Amazon password').fill('reply-loss-fixture-password'); + f.state.dropReply = true; + await dialog.getByRole('button', { name: 'Save and connect' }).click(); + await expect(dialog.getByText('We couldn’t confirm this save.', { exact: false })).toBeVisible(); + await expect(dialog.getByLabel('Amazon password')).toHaveValue(''); + expect(f.state.broker.vault.readForAdapter('amazon-operations').credentials.password).toBe('reply-loss-fixture-password'); + await dialog.getByRole('button', { name: 'Cancel', exact: true }).click(); + const cortex = page.locator('[data-slot="card"]').filter({ has: page.getByText('Cortex', { exact: true }) }); + await expect(cortex.getByText('Connected', { exact: true })).toBeVisible(); + } finally { await page.close(); await f.close(); } +}); diff --git a/core/dashboard/tests/browser/connections.spec.cjs b/core/dashboard/tests/browser/connections.spec.cjs new file mode 100644 index 0000000..f68bdfb --- /dev/null +++ b/core/dashboard/tests/browser/connections.spec.cjs @@ -0,0 +1,107 @@ +const { test, expect } = require('@playwright/test'); + +async function login(page, email = 'owner@example.test') { + await page.goto('/#/settings?tab=connections'); + await page.getByLabel('Email address').fill(email); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await expect(page).toHaveTitle('Settings · Dispatch'); +} + +for (const mobile of [false, true]) test(`shared Connections flow (${mobile ? 'mobile' : 'desktop'})`, async ({ page }) => { + if (mobile) await page.setViewportSize({ width: 390, height: 844 }); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + await login(page); + await expect(page.getByRole('tab', { name: 'Connections', exact: true })).toHaveAttribute('aria-selected', 'true'); + await expect(page.getByRole('heading', { name: 'Connections', exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Connect Cortex', exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Connect Paycom', exact: true })).toBeVisible(); + await page.getByRole('button', { name: 'Connect Cortex', exact: true }).click(); + const dialog = page.getByRole('dialog'); + await expect(dialog.getByRole('heading', { name: 'Cortex credentials' })).toBeVisible(); + await expect(dialog.getByLabel('Profile name')).toHaveCount(0); + await dialog.getByLabel('Amazon username').fill('synthetic-cortex-owner'); + await dialog.getByLabel('Amazon password').fill('synthetic-cortex-password'); + const submitted = page.waitForResponse(response => response.url().endsWith('/connections/cortex/save')); + await dialog.getByRole('button', { name: 'Save and connect' }).click(); + const response = await submitted; + expect(response.status()).toBe(202); + expect(await response.text()).not.toContain('synthetic-cortex-password'); + await expect(dialog).toHaveCount(0); + const cortex = page.locator('[data-slot="card"]').filter({ has: page.getByText('Cortex', { exact: true }) }); + await expect(cortex.getByText('Connected', { exact: true })).toBeVisible({ timeout: 10000 }); + await expect(cortex.getByText('Last checked:', { exact: false })).toBeVisible(); + await cortex.getByRole('button', { name: 'Update credentials' }).click(); + await expect(dialog.getByLabel('Amazon password')).toHaveValue(''); + await dialog.getByRole('button', { name: 'Cancel', exact: true }).click(); + await cortex.getByRole('button', { name: 'Test connection', exact: true }).click(); + await expect(cortex.getByText('Connected', { exact: true })).toBeVisible({ timeout: 10000 }); + await page.screenshot({ path: `/tmp/dispatch-connections-${mobile ? 'mobile' : 'desktop'}.png`, fullPage: true }); + await cortex.getByRole('button', { name: 'Disconnect', exact: true }).click(); + await expect(dialog.getByText('Previously collected data will remain available.', { exact: false })).toBeVisible(); + await dialog.getByRole('button', { name: 'Disconnect', exact: true }).click(); + await expect(cortex.getByRole('button', { name: 'Connect Cortex', exact: true })).toBeVisible(); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBeTruthy(); + expect(await page.evaluate(() => JSON.stringify([localStorage, sessionStorage]))).not.toContain('synthetic-cortex-password'); + expect(errors).toEqual([]); +}); + +test('connection verification and cooldown states explain the next step', async ({ page }) => { + await login(page); + await page.route('**/api/organization/connections', async route => { + const response = await route.fetch(); + const payload = await response.json(); + payload.data.items[0] = { service: 'cortex', configured: true, state: 'verification_required', checkedAt: new Date().toISOString(), reason: 'mfa_required', retryAt: null }; + payload.data.items[1] = { service: 'paycom', configured: true, state: 'temporarily_unavailable', checkedAt: null, reason: 'attempt_cooldown', retryAt: new Date(Date.now() + 300000).toISOString() }; + await route.fulfill({ json: payload }); + }); + await page.reload(); + await expect(page.getByText('Verification required', { exact: true })).toBeVisible(); + await expect(page.getByText('Contact your Dispatch administrator', { exact: false })).toBeVisible(); + const paycom = page.locator('[data-slot="card"]').filter({ has: page.getByText('Paycom', { exact: true }) }); + await expect(paycom.getByRole('button', { name: 'Test connection' })).toBeEnabled(); +}); + +for (const mobile of [false, true]) test(`Paycom test shows session, sign-in, CAPTCHA and final result (${mobile ? 'mobile' : 'desktop'})`, async ({ page }) => { + if (mobile) await page.setViewportSize({ width: 390, height: 844 }); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + await login(page); + let phase = 'manual'; + const connection = () => ({ service: 'paycom', configured: true, + state: phase === 'manual' ? 'verification_required' : phase === 'connected' ? 'connected' : 'checking', + checkedAt: new Date().toISOString(), reason: phase === 'manual' ? 'manual_verification_required' : null, retryAt: null, + ...(['checking_session', 'signing_in'].includes(phase) ? { check: { phase, startedAt: new Date().toISOString() } } : {}), + ...(phase === 'captcha' ? { assistance: { phase: 'solving', startedAt: new Date().toISOString() } } : {}) }); + await page.route('**/api/organization/connections', async route => { + const response = await route.fetch(), payload = await response.json(); + payload.data.items[1] = connection(); await route.fulfill({ json: payload }); + }); + await page.route('**/api/organization/connections/paycom/test', async route => { + phase = 'checking_session'; await route.fulfill({ status: 202, json: { ok: true, data: connection() } }); + }); + await page.reload(); + const paycom = page.locator('[data-slot="card"]').filter({ has: page.getByText('Paycom', { exact: true }) }); + await paycom.getByRole('button', { name: 'Test connection', exact: true }).click(); + await expect(paycom.getByText('Checking session', { exact: true })).toBeVisible(); + await expect(page.getByText('Paycom: Checking session.', { exact: true })).toBeVisible(); + await expect(paycom.getByRole('button', { name: 'Test connection', exact: true })).toBeDisabled(); + phase = 'signing_in'; await expect(paycom.getByText('Signing in', { exact: true })).toBeVisible(); + phase = 'captcha'; await expect(paycom.getByText('Completing CAPTCHA', { exact: true })).toBeVisible(); + await expect(page.getByText('Paycom: Completing CAPTCHA.', { exact: true })).toBeVisible(); + phase = 'connected'; await expect(page.getByText('Paycom: Connected.', { exact: true })).toBeVisible(); + await expect(paycom.getByText('Connected', { exact: true })).toBeVisible(); + await expect(page.getByText('Paycom connection check requested.', { exact: true })).toHaveCount(0); + await page.screenshot({ path: `/tmp/paycom-test-connection-${mobile ? 'mobile' : 'desktop'}.png`, fullPage: true }); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBeTruthy(); + expect(errors).toEqual([]); +}); + +test('staff cannot see or manage DSP connections', async ({ page }) => { + await login(page, 'member0@example.test'); + await expect(page.getByRole('tab', { name: 'Connections', exact: true })).toHaveCount(0); + expect((await page.request.get('/api/organization/connections')).status()).toBe(403); +}); diff --git a/core/dashboard/tests/browser/cortex-verification.spec.cjs b/core/dashboard/tests/browser/cortex-verification.spec.cjs new file mode 100644 index 0000000..7ec5bbe --- /dev/null +++ b/core/dashboard/tests/browser/cortex-verification.spec.cjs @@ -0,0 +1,91 @@ +'use strict'; +const { test, expect } = require('@playwright/test'); +const { SNAPSHOT, classify } = require('dispatch-dsp/runtime/auth-broker/src/adapters/amazon-logistics.js'); +const { verificationExpression } = require('dispatch-dsp/runtime/auth-broker/src/adapters/amazon-verification.js'); + +for (const platform of [false, true]) for (const mobile of [false, true]) { + test(`Cortex email code as ${platform ? 'platform owner' : 'DSP owner'} on ${mobile ? 'mobile' : 'desktop'}`, async ({ page }, testInfo) => { + if (mobile) await page.setViewportSize({ width: 390, height: 844 }); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + await page.goto('/'); + await page.getByLabel('Email address').fill(platform ? 'platform@example.test' : 'owner5@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + if (platform) { + await page.getByRole('button', { name: 'Westfield Routes WR06', exact: true }).click(); + await page.getByRole('dialog').getByRole('button', { name: 'View', exact: true }).click(); + await expect(page.getByRole('region', { name: 'DSP viewing mode' })).toBeVisible(); + } + await page.goto('/#/settings?tab=connections'); + await expect(page).toHaveTitle('Settings · Dispatch'); + const card = page.locator('[data-slot="card"]').filter({ has: page.getByText('Cortex', { exact: true }) }); + await card.getByRole('button', { name: /Connect Cortex|Update credentials/ }).click(); + await page.getByLabel('Amazon username').fill('verification-fixture'); + await page.getByLabel('Amazon password').fill('synthetic-amazon-password'); + await page.getByRole('button', { name: 'Save and connect', exact: true }).click(); + await expect(page.getByLabel('Email verification code')).toBeVisible({ timeout: 10000 }); + await page.reload(); + const code = page.getByLabel('Email verification code'); + await expect(code).toBeVisible(); + await code.fill('000000'); + await page.getByRole('button', { name: 'Verify code', exact: true }).click(); + await expect(card.getByText('Amazon didn’t accept that code. Enter the newest code from your email.')).toBeVisible({ timeout: 10000 }); + await expect(code).toHaveValue(''); + await code.scrollIntoViewIfNeeded(); + await page.screenshot({ path: testInfo.outputPath('email-code-retry.png'), fullPage: true }); + await code.fill('123456'); + const submitted = page.waitForResponse(response => response.url().endsWith('/connections/cortex/verify')); + await page.getByRole('button', { name: 'Verify code', exact: true }).click(); + const response = await submitted; + expect(response.status()).toBe(202); + if (platform) expect(Boolean(response.request().headers()['x-dispatch-dsp-view'])).toBe(true); + expect(await response.text()).not.toContain('123456'); + await expect(card.getByText('Connected', { exact: true })).toBeVisible({ timeout: 10000 }); + await expect(code).toHaveCount(0); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + expect(errors).toEqual([]); + await card.getByRole('button', { name: 'Disconnect', exact: true }).click(); + await page.getByRole('dialog').getByRole('button', { name: 'Disconnect', exact: true }).click(); + await expect(card.getByRole('button', { name: 'Connect Cortex', exact: true })).toBeVisible(); + }); +} + +test('Amazon adapter submits only the unique OTP input to the expected Amazon form', async ({ page }) => { + let submitted = null; + await page.route('**/*', async route => { + const request = route.request(); + if (request.method() === 'POST') { submitted = { url: request.url(), body: request.postData() }; await route.fulfill({ body: 'Submitted' }); return; } + await route.fulfill({ contentType: 'text/html', body: '
' }); + }); + await page.goto('https://www.amazon.com/ap/cvf/transactionapproval'); + expect((await page.evaluate(SNAPSHOT)).otpPresent).toBe(true); + expect(await page.evaluate(verificationExpression('123456'))).toEqual({ status: 'submitted' }); + await expect.poll(() => submitted).not.toBeNull(); + expect(submitted.url).toBe('https://www.amazon.com/ap/cvf/approval'); + expect(new URLSearchParams(submitted.body).get('otpCode')).toBe('123456'); + for (const scenario of ['foreign-origin', 'foreign-action', 'duplicate-field', 'wrong-route', 'get-form']) { + submitted = null; + await page.goto(scenario === 'foreign-origin' ? 'https://example.test/ap/cvf/approval' : 'https://www.amazon.com/ap/cvf/approval'); + await page.evaluate(scenario => { + if (scenario === 'foreign-action') document.forms[0].action = 'https://example.test/ap/cvf/approval'; + if (scenario === 'duplicate-field') document.forms[0].append(document.querySelector('input[name="otpCode"]').cloneNode()); + if (scenario === 'wrong-route') history.replaceState(null, '', '/ap/signin'); + if (scenario === 'get-form') document.forms[0].method = 'GET'; + }, scenario); + expect(await page.evaluate(verificationExpression('123456'))).toEqual({ status: 'manual_verification_required' }); + expect(submitted).toBeNull(); + expect(await page.locator('input[name="otpCode"]').first().inputValue()).toBe(''); + } +}); + +test('Cortex sign-in proof recognizes collapsed menus and rejects incomplete pages', async ({ page }) => { + await page.route('**/*', route => route.fulfill({ contentType: 'text/html', body: '
Choose a station
' })); + await page.goto('https://logistics.amazon.com/operations/execution/'); + expect(classify(await page.evaluate(SNAPSHOT))).toBe('authenticated'); + await page.locator('a[href="/performance"]').evaluate(element => element.remove()); + expect(classify(await page.evaluate(SNAPSHOT))).toBe('manual_verification_required'); + await page.goto('https://logistics.amazon.com/operations/execution/?unexpected=1'); + expect(classify(await page.evaluate(SNAPSHOT))).toBe('manual_verification_required'); +}); diff --git a/core/dashboard/tests/browser/dashboard-rollout.spec.cjs b/core/dashboard/tests/browser/dashboard-rollout.spec.cjs new file mode 100644 index 0000000..87f02f9 --- /dev/null +++ b/core/dashboard/tests/browser/dashboard-rollout.spec.cjs @@ -0,0 +1,32 @@ +const {test,expect}=require('@playwright/test'); +const {createPreview}=require('../../examples/independent-updates-preview'); +let app; +test.beforeAll(async()=>{app=await createPreview({versionedDashboards:true});}); +test.afterAll(async()=>{await app?.close();}); +async function login(page,email){await page.goto(app.url);await page.getByLabel('Email address').fill(email);await page.getByLabel('Password',{exact:true}).fill('synthetic preview password');await page.getByRole('button',{name:'Sign in',exact:true}).click();} +test('Dev gets its new dashboard while another DSP stays on its approved dashboard until rollout',async({browser})=>{ + const contexts=await Promise.all([browser.newContext(),browser.newContext(),browser.newContext()]); + try{ + const [owner,dev,production]=await Promise.all(contexts.map(c=>c.newPage())); + const errors=[];for(const page of [owner,dev,production])page.on('pageerror',e=>errors.push(e.message)); + await login(owner,'platform@example.test');await owner.locator('.desktop-sidebar').getByRole('link',{name:'Updates',exact:true}).click(); + await login(dev,'owner0@example.test');await login(production,'owner1@example.test'); + await expect(dev.getByRole('heading',{name:'Currently under development',exact:true})).toBeVisible(); + await expect(production.getByRole('heading',{name:'Currently under development',exact:true})).toBeVisible(); + await owner.getByRole('button',{name:'Update Dev',exact:true}).click();await expect(owner.getByRole('button',{name:'Rollout Update',exact:true})).toBeEnabled({timeout:15000}); + const stale=await dev.evaluate(async()=>{const response=await fetch('/api/auth/logout',{method:'POST',headers:{'Content-Type':'application/json','X-Dispatch-Dashboard':window.__dispatchDashboard.digest},body:'{}'});return {status:response.status,body:await response.json()};}); + expect(stale.status).toBe(409);expect(stale.body.error.code).toBe('dashboard_changed'); + await dev.reload();await production.reload(); + await expect(dev.getByRole('heading',{name:'Dev release preview',exact:true})).toBeVisible(); + await expect(production.getByRole('heading',{name:'Currently under development',exact:true})).toBeVisible(); + expect(await production.evaluate(()=>window.__dispatchDashboard.digest)).not.toEqual(await dev.evaluate(()=>window.__dispatchDashboard.digest)); + await owner.getByRole('button',{name:'Rollout Update',exact:true}).click();await expect(owner.getByText('2 of 2 DSPs updated · completed',{exact:true})).toBeVisible({timeout:15000}); + await production.reload();await expect(production.getByRole('heading',{name:'Dev release preview',exact:true})).toBeVisible(); + await expect(owner.getByRole('heading',{name:'Core',exact:true})).toBeVisible(); + await owner.locator('.desktop-sidebar').getByRole('link',{name:'DSPs',exact:true}).click(); + await owner.getByRole('button',{name:'Actions for Dev DSP',exact:true}).click();await owner.getByRole('menuitem',{name:'View',exact:true}).click(); + await expect(owner.getByRole('heading',{name:'Dev release preview',exact:true})).toBeVisible(); + await owner.getByRole('button',{name:'Exit view',exact:true}).click();await expect(owner.locator('.desktop-sidebar').getByRole('link',{name:'Updates',exact:true})).toBeVisible(); + expect(errors).toEqual([]); + }finally{await Promise.all(contexts.map(c=>c.close()));} +}); diff --git a/core/dashboard/tests/browser/dashboard.spec.cjs b/core/dashboard/tests/browser/dashboard.spec.cjs new file mode 100644 index 0000000..4c918da --- /dev/null +++ b/core/dashboard/tests/browser/dashboard.spec.cjs @@ -0,0 +1,466 @@ +const { test, expect } = require("@playwright/test"); +const password = "synthetic preview password"; +async function login(page, email = "platform@example.test") { + await page.goto("/"); + await page.getByLabel("Email address").fill(email); + await page.getByLabel("Password", { exact: true }).fill(password); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect( + page.getByRole("navigation", { name: "Primary navigation" }), + ).toBeVisible(); +} +const nav = (page) => page.locator(".desktop-sidebar").getByRole("navigation"); +test("Diagnostics queues a persistent synthetic DSP and rejects tenant access", async ({ page }) => { + await login(page); + await navigate(page, "Diagnostics"); + const response = page.waitForResponse(r => r.url().endsWith('/api/platform/diagnostics') && r.request().method() === 'POST'); + await page.getByRole('button', { name: 'Deploy test DSP', exact: true }).click(); + expect((await response).status()).toBe(202); + await expect(page.getByText('Creating DSP and preparing synthetic data…').first()).toBeVisible(); + await expect(page.getByRole('heading', { name: /^TEST DSP / }).first()).toBeVisible(); + await page.reload(); + await expect(page.getByRole('heading', { name: /^TEST DSP / }).first()).toBeVisible(); + const csrf = await page.request.post('/api/platform/diagnostics', { data: { idempotencyKey: 'browser:missing:csrf' } }); + expect(csrf.status()).toBe(403); + const current = (await (await page.request.get('/api/auth/session')).json()).data; + await page.request.post('/api/auth/logout', { headers: { 'X-Dispatch-CSRF': current.csrfToken, Origin: new URL(page.url()).origin }, data: {} }); + await login(page, 'owner@example.test'); + await expect(nav(page).getByRole('link', { name: 'Diagnostics', exact: true })).toHaveCount(0); + expect((await page.request.get('/api/platform/diagnostics')).status()).toBe(403); +}); +async function navigate(page, name) { + await nav(page).getByRole("link", { name, exact: true }).click(); + await expect( + page.getByRole("heading", { level: 1, name, exact: true }), + ).toBeVisible(); +} +test("platform page scope, searchable DSP table, creation and accessible dialogs", async ({ + page, +}) => { + const errors = []; + page.on("pageerror", (e) => errors.push(e.message)); + page.on("console", (m) => { + if (m.type() === "error" && !m.text().includes("favicon")) + errors.push(m.text()); + }); + await login(page); + await expect(nav(page).getByRole("link")).toHaveText([ + "DSPs", + "Updates", + "Backups", + "Plugins", + "Diagnostics", + "Settings", + ]); + await expect( + page.getByRole("button", { name: "Northline Logistics NL01" }), + ).toBeVisible(); + await page.screenshot({ path: "/tmp/dispatch-redesign-dsps.png" }); + await page.getByRole("searchbox").fill("northline"); + await expect(page.getByRole("row")).toHaveCount(2); + await page.getByRole("searchbox").fill("nothing-matches"); + await expect(page.getByText("No DSPs match your search")).toBeVisible(); + await page.getByRole("searchbox").fill(""); + await page.getByRole("button", { name: "Create new DSP" }).click(); + await expect(page.getByRole("dialog")).toBeVisible(); + await expect(page.getByLabel("Owner email", { exact: true })).toBeFocused(); + await page.screenshot({ path: "/tmp/dispatch-redesign-create.png" }); + await page + .getByLabel("Owner email", { exact: true }) + .fill("new-owner@example.test"); + await page.getByRole("button", { name: "Create DSP", exact: true }).click(); + await expect(page.getByRole("dialog")).toHaveCount(0); + await expect(page.getByLabel("Invitation link")).toBeVisible(); + await page.getByRole("tab", { name: "Onboarding", exact: true }).click(); + await expect( + page.getByRole("button", { + name: "new-owner@example.test Awaiting DSP details", + exact: true, + }), + ).toBeVisible(); + await navigate(page, "Plugins"); + await expect(page.locator("main")).toHaveText("Plugins"); + await page.goto("/#/cdf"); + await expect(page).toHaveURL(/#\/platform$/); + await navigate(page, "Settings"); + await expect( + page.getByText("Your platform account and security."), + ).toBeVisible(); + await expect( + page.getByText("platform@example.test", { exact: true }), + ).toBeVisible(); + await page.screenshot({ path: "/tmp/dispatch-redesign-settings.png" }); + expect(errors).toEqual([]); +}); +test("DSP pages do not fetch workforce data before connection; team roles and invitations work", async ({ + page, +}) => { + const requests = []; + const errors = []; + page.on("request", (r) => requests.push(new URL(r.url()).pathname)); + page.on("pageerror", (e) => errors.push(e.message)); + await login(page, "owner@example.test"); + await expect(nav(page).getByRole("link")).toHaveText([ + "Home Page", + "Paycom", + "Team & Roles", + "Settings", + ]); + await expect(page.getByRole("heading", { level: 1, name: "Currently under development", exact: true })).toBeVisible(); + await navigate(page, "Paycom"); + await expect(page.getByRole("heading", { name: "Paycom", exact: true })).toBeVisible(); + await expect(page.getByText("Paycom is not connected.", { exact: false })).toBeVisible(); + expect( + requests.some((p) => /^\/api\/(paycom|bootstrap|integrations)/.test(p)), + ).toBe(false); + await navigate(page, "Team & Roles"); + await expect(page.getByRole("tab", { name: "Activity", exact: true })).toHaveCount(0); + await expect(page.getByText("Jamie Chen", { exact: true })).toBeVisible(); + await page.screenshot({ path: "/tmp/dispatch-redesign-team.png" }); + await page.getByRole("tab", { name: "Roles", exact: true }).click(); + await expect(page.locator('.role-row h2')).toHaveText(['Owner', 'Manager', 'Dispatcher', 'Driver']); + await expect(page.getByRole('button', { name: 'Create role', exact: true })).toHaveCount(0); + await expect(page.getByText('Standard roles for your DSP. All roles currently have the same permissions.')).toBeVisible(); + await page.screenshot({ path: '/tmp/dispatch-fixed-roles-desktop.png' }); + await page + .getByRole("button", { name: "Invite member", exact: true }) + .click(); + await page.getByLabel("Email address").fill("invitee@example.test"); + await page + .getByLabel("Role", { exact: true }) + .selectOption({ label: "Dispatcher" }); + await expect(page.getByLabel('Role', { exact: true }).locator('option')).toHaveText(['Owner', 'Manager', 'Dispatcher', 'Driver']); + await page.screenshot({ path: "/tmp/dispatch-redesign-team-invite.png" }); + await page + .getByRole("button", { name: "Create invitation", exact: true }) + .click(); + await expect(page.getByLabel("Invitation link")).toBeVisible(); + await page.getByRole("tab", { name: /^Invitations/ }).click(); + await expect( + page.getByRole("cell", { name: "invitee@example.test", exact: true }), + ).toBeVisible(); + await page + .getByRole("button", { name: "Revoke invitation for invitee@example.test" }) + .click(); + await page + .getByRole("dialog") + .getByRole("button", { name: "Revoke invitation", exact: true }) + .click(); + await expect( + page.getByRole("cell", { name: "invitee@example.test", exact: true }), + ).toHaveCount(0); + await page.getByRole("tab", { name: "Members", exact: true }).click(); + await page.getByRole("button", { name: "Actions for Jamie Chen" }).click(); + await page.getByRole("menuitem", { name: "Change role" }).click(); + await page + .getByLabel("Role", { exact: true }) + .selectOption({ label: "Dispatcher" }); + await expect(page.getByLabel('Role', { exact: true }).locator('option')).toHaveText(['Owner', 'Manager', 'Dispatcher', 'Driver']); + await page.getByRole("button", { name: "Save role", exact: true }).click(); + await expect( + page.getByRole("row").filter({ hasText: "Jamie Chen" }), + ).toContainText("Dispatcher"); + await navigate(page, "Settings"); + await expect( + page.getByText("Northline Logistics", { exact: true }).last(), + ).toBeVisible(); + await page.getByRole("tab", { name: "Audit log", exact: true }).click(); + await expect(page).toHaveURL(/#\/settings\?tab=audit$/); + await expect(page).toHaveTitle("Settings · Dispatch"); + await expect(page.getByRole("cell", { name: "invitation revoke", exact: true }).first()).toBeVisible(); + await expect(page.getByRole("cell", { name: "organization view start", exact: true })).toHaveCount(0); + const refreshed = page.waitForResponse(r => r.url().endsWith('/api/organization/audit')); + await page.getByRole("button", { name: "Refresh", exact: true }).click(); + expect((await refreshed).status()).toBe(200); + await page.screenshot({ path: "/tmp/dispatch-settings-audit-desktop.png" }); + await page.setViewportSize({ width: 390, height: 844 }); + await expect.poll(() => page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.screenshot({ path: "/tmp/dispatch-settings-audit-mobile.png" }); + await page.setViewportSize({ width: 1536, height: 1024 }); + await page.getByRole("tab", { name: "Security", exact: true }).click(); + await page.getByLabel("Current password").fill(password); + await page + .getByLabel("New password", { exact: true }) + .fill("another synthetic password"); + await page + .getByLabel("Confirm new password") + .fill("another synthetic password"); + await page + .getByRole("button", { name: "Change password", exact: true }) + .click(); + await expect( + page.getByText("Password changed. Other sessions have been signed out."), + ).toBeVisible(); + expect(errors).toEqual([]); +}); +test("updates and backup inspection, restore guard, settings, and operation progress", async ({ + page, +}) => { + const errors = []; + page.on("pageerror", (e) => errors.push(e.message)); + await login(page); + await navigate(page, "Updates"); + await expect( + page.getByRole("heading", { name: "Version 0.0.9", exact: true }), + ).toBeVisible(); + await expect(page.locator('.update-release-summary')).toHaveText('3 additions · 4 changes · 3 improvements'); + const notesBox = await page.locator('.update-notes-panel').boundingBox(); + const navigationBox = await page.locator('.update-release-navigation').boundingBox(); + expect(navigationBox.x + navigationBox.width).toBeLessThanOrEqual(notesBox.x); + await page.screenshot({ path: "/tmp/dispatch-redesign-updates.png" }); + await expect(page.getByRole("button", { name: /Install update|Pause rollout|Resume rollout/ })).toHaveCount(0); + await navigate(page, "Backups"); + await expect( + page.getByRole("heading", { name: "DSPs", exact: true }), + ).toBeVisible(); + await page.screenshot({ path: "/tmp/dispatch-redesign-backups.png" }); + await page + .locator(".backup-tabs") + .getByRole("link", { name: "DSPs", exact: true }) + .click(); + await page.getByLabel("DSP", {exact: true}).selectOption({label: "Northline Logistics"}); + await expect( + page.getByRole("heading", { name: "Northline Logistics", exact: true }), + ).toBeVisible(); + await page.getByRole("link", { name: "View details" }).first().click(); + await page.getByRole("button", { name: "Review restore" }).click(); + await expect( + page.getByRole("button", { name: "Restore DSP", exact: true }), + ).toBeDisabled(); + await page + .getByLabel("I understand this replaces the selected scope’s current data.") + .check(); + await page + .getByLabel("Type Northline Logistics to confirm") + .fill("wrong name"); + await expect( + page.getByRole("button", { name: "Restore DSP", exact: true }), + ).toBeDisabled(); + await page + .getByLabel("Type Northline Logistics to confirm") + .fill("Northline Logistics"); + await expect( + page.getByRole("button", { name: "Restore DSP", exact: true }), + ).toBeEnabled(); + await page.getByRole("button", { name: "Cancel", exact: true }).click(); + await page + .locator(".backup-header") + .getByRole("link", { name: "Settings", exact: true }) + .click(); + await expect( + page.getByRole("button", { name: "Save settings" }), + ).toBeVisible(); + await page.getByRole("button", { name: "Save settings" }).click(); + await page.locator('.backup-tabs').getByRole('link', {name: 'Overview', exact: true}).click(); + await page.getByRole("button", { name: "Back up full system", exact: true }).click(); + await page + .getByRole("dialog") + .getByRole("button", { name: "Platform Core", exact: true }) + .click(); + await page + .getByRole("button", { name: "Back up Platform Core", exact: true }) + .click(); + await expect( + page.getByText("Preparing backup", { exact: true }), + ).toBeVisible(); + expect(errors).toEqual([]); +}); +test("mobile navigation, blank Plugins, and dialogs remain within the viewport", async ({ + page, +}) => { + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto("/"); + await page.getByLabel("Email address").fill("platform@example.test"); + await page.getByLabel("Password", { exact: true }).fill(password); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect( + page.getByRole("heading", { name: "DSPs", exact: true }), + ).toBeVisible(); + await page.screenshot({ path: "/tmp/dispatch-redesign-mobile.png" }); + await page.getByRole("button", { name: "Open navigation" }).click(); + await page + .getByRole("dialog") + .getByRole("link", { name: "Plugins", exact: true }) + .click(); + await expect(page.getByRole("dialog")).toHaveCount(0); + await expect(page.locator("main")).toHaveText("Plugins"); + await page.getByRole("button", { name: "Open navigation" }).click(); + await page + .getByRole("dialog") + .getByRole("link", { name: "DSPs", exact: true }) + .click(); + await page.getByRole("button", { name: "Create new DSP" }).click(); + await expect(page.getByLabel("Owner email", { exact: true })).toBeFocused(); + await expect.poll(async () => { + const bounds = await page.getByRole("dialog").boundingBox(); + return Boolean(bounds && bounds.x >= 0 && bounds.x + bounds.width <= 391); + }).toBe(true); + await page.getByRole("button", { name: "Cancel", exact: true }).click(); + await expect.poll( + () => page.evaluate( + () => document.documentElement.scrollWidth <= innerWidth, + ), + ).toBe(true); +}); + +test("session changes clear protected views and Driver access stays within the DSP", async ({ + page, +}) => { + await login(page); + await expect( + page.getByRole("button", { name: "Northline Logistics NL01" }), + ).toBeVisible(); + await page.route("**/api/auth/session", (route) => + route.fulfill({ + json: { + ok: true, + data: { authenticated: false, bootstrap: { initialized: true } }, + }, + }), + ); + await page.evaluate(() => window.dispatchEvent(new Event("focus"))); + await expect( + page.getByRole("heading", { name: "Sign in to Dispatch" }), + ).toBeVisible(); + await expect( + page.getByText("Northline Logistics", { exact: true }), + ).toHaveCount(0); + await page.unroute("**/api/auth/session"); + await page.getByLabel("Email address").fill("member2@example.test"); + await page.getByLabel("Password", { exact: true }).fill(password); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect(nav(page).getByRole("link")).toHaveText([ + "Home Page", + "Paycom", + "Team & Roles", + "Settings", + ]); + await page.goto("/#/platform"); + await expect(page).toHaveURL(/#\/dashboard$/); + await expect(page.getByRole("heading", { level: 1, name: "Currently under development", exact: true })).toBeVisible(); + await page.goto("/#/settings?tab=audit"); + await expect(page.getByRole("tab", { name: "Audit log", exact: true })).toBeVisible(); + await expect(page.getByRole("tab", { name: "Audit log", exact: true })).toHaveAttribute("data-state", "active"); +}); + +test("DSP removal, restoration and password deletion stay in the correct tabs", async ({ page }) => { + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + await login(page); + await expect(page).toHaveTitle('DSPs · Dispatch'); + const row = page.getByRole('row').filter({ hasText: 'Cedar Delivery' }); + const openActions = () => row.getByRole('button', { name: 'Actions for Cedar Delivery' }).click(); + await openActions(); + await expect(page.getByRole('menuitem', { name: 'Suspend DSP', exact: true })).toHaveCount(0); + await expect(page.getByRole('menuitem', { name: 'Permanently delete DSP', exact: true })).toHaveCount(0); + await page.getByRole('menuitem', { name: 'Remove DSP', exact: true }).click(); + await expect(page.getByRole('dialog')).toContainText('Existing data and backups will be retained'); + await page.getByRole('dialog').getByRole('button', { name: 'Remove DSP', exact: true }).click(); + await expect(page.getByRole('dialog')).toHaveCount(0); + await expect(row).toHaveCount(0); + await page.getByRole('tab', { name: 'Removed', exact: true }).click(); + await expect(row).toContainText('Removing'); + const fleet = await (await page.request.get('/api/platform/organizations')).json(); + const cedar = fleet.data.find(o => o.name === 'Cedar Delivery'); + expect(cedar.installation.availableActions).not.toContain('destroy'); + // The preview has no privileged host. Supply verified worker outcomes at its API boundary. + cedar.installation.state = 'decommissioned'; + cedar.installation.operation.status = 'succeeded'; + cedar.installation.availableActions = ['restore_dsp', 'destroy']; + await page.route('**/api/platform/organizations', route => route.fulfill({ json: fleet })); + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await openActions(); + await page.getByRole('menuitem', { name: 'Restore DSP', exact: true }).click(); + let restored; + await page.route('**/api/platform/installation/restore', async route => { + restored = route.request().postDataJSON(); + cedar.installation.state = 'verifying'; + cedar.installation.operation = { kind: 'restore_dsp', status: 'queued' }; + cedar.installation.availableActions = []; + await route.fulfill({ json: { ok: true, data: {} } }); + }); + await page.getByRole('dialog').getByRole('button', { name: 'Restore DSP', exact: true }).click(); + await expect(page.getByRole('dialog')).toHaveCount(0); + await expect(row).toContainText('Restoring'); + expect(restored.controlRef).toBe(cedar.controlRef); + cedar.installation.state = 'decommissioned'; + cedar.installation.operation.status = 'failed'; + cedar.installation.availableActions = ['restore_dsp', 'destroy']; + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await expect(row).toContainText('Restore failed'); + await openActions(); + await page.getByRole('menuitem', { name: 'Permanently delete DSP', exact: true }).click(); + const confirm = page.getByRole('dialog').getByRole('button', { name: 'Permanently delete DSP', exact: true }); + await expect(confirm).toBeDisabled(); + await expect(page.getByLabel('Your password')).toHaveAttribute('type', 'password'); + await expect(page.getByLabel(/Type .* to confirm/)).toHaveCount(0); + let command; + await page.route('**/api/platform/installation/delete', async route => { + command = route.request().postDataJSON(); + if (command.password !== password) return route.fulfill({ status: 403, json: { ok: false, error: { code: 'current_password_invalid' } } }); + cedar.installation.operation = { kind: 'destroy', status: 'queued' }; + cedar.installation.availableActions = []; + await route.fulfill({ json: { ok: true, data: {} } }); + }); + await page.getByLabel('Your password').fill('wrong password'); + await confirm.click(); + await expect(page.getByRole('dialog')).toContainText('The current password was not accepted'); + await expect(page.getByLabel('Your password')).toHaveValue(''); + await page.getByLabel('Your password').fill(password); + await page.screenshot({ path: '/tmp/dispatch-dsp-delete-desktop.png' }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.screenshot({ path: '/tmp/dispatch-dsp-delete-mobile.png' }); + await confirm.click(); + await expect(page.getByRole('dialog')).toHaveCount(0); + expect(command.password).toBe(password); + expect(command.confirmation).toBeUndefined(); + await expect(row).toContainText('Deleting'); + cedar.installation.state = 'failed'; + cedar.installation.operation.status = 'failed'; + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await expect(row).toContainText('Deletion failed'); + await page.getByRole('tab', { name: 'All DSPs', exact: true }).click(); + await expect(row).toHaveCount(0); + expect(errors).toEqual([]); +}); + +test('backup schedules are independent and full-system, Core and DSP actions expose their scope',async({page})=>{ + const errors=[];page.on('pageerror',e=>errors.push(e.message));await login(page);await navigate(page,'Backups'); + await page.locator('.backup-header').getByRole('link',{name:'Settings',exact:true}).click(); + const picker=page.getByLabel('Schedule for');await picker.selectOption('core');await expect(page.getByLabel('Automatic backups',{exact:true})).not.toBeChecked(); + await page.getByLabel('Automatic backups',{exact:true}).check();await page.getByRole('button',{name:'Save settings'}).click(); + await picker.selectOption('system');await expect(page.getByLabel('Automatic backups',{exact:true})).not.toBeChecked(); + await picker.selectOption({label:'Northline Logistics'});await expect(page.getByLabel('Automatic backups',{exact:true})).not.toBeChecked(); + await picker.selectOption('core');await expect(page.getByLabel('Automatic backups',{exact:true})).toBeChecked(); + await page.screenshot({path:'/tmp/dispatch-scoped-backup-schedules-desktop.png'}); + await page.locator('.backup-tabs').getByRole('link',{name:'Overview',exact:true}).click(); + await page.getByRole('button',{name:'Back up full system',exact:true}).click();await page.getByRole('dialog').getByRole('button',{name:'Full system',exact:true}).click(); + await expect(page.getByRole('dialog')).toContainText('one backup for every active DSP');await expect(page.getByRole('dialog').getByRole('button',{name:'Back up full system'})).toBeEnabled(); + await page.getByRole('dialog').getByRole('button',{name:'Cancel'}).click(); + await page.setViewportSize({width:390,height:844});await page.screenshot({path:'/tmp/dispatch-scoped-backup-schedules-mobile.png'}); + expect(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth)).toBe(true);expect(errors).toEqual([]); +}); +test('storage shows independent scopes, removed retention and non-additive system totals on desktop and mobile',async({page})=>{ + const errors=[];page.on('pageerror',error=>errors.push(error.message));await login(page);await navigate(page,'Backups'); + const payload=await (await page.request.get('/api/platform/backups')).json();const data=payload.data; + const set={id:'fixture_usage_set',createdAt:data.storageUsage.checkedAt,status:'verified',members:[]}; + data.sets=[set];data.storageUsage.sets=[{id:set.id,bytes:28*1048576,backupCount:2,manifestBytes:0}]; + await page.route('**/api/platform/backups',route=>route.fulfill({json:payload})); + await page.getByRole('navigation',{name:'Backup navigation'}).getByRole('link',{name:'Storage',exact:true}).click(); + await page.reload();await expect(page.getByRole('heading',{name:'Storage usage',exact:true})).toBeVisible(); + await expect(page.locator('.backup-storage-total')).toContainText('113.0 MB'); + await expect(page.getByRole('region',{name:'Storage by scope',exact:true}).getByRole('row').filter({hasText:'Platform Core'})).toContainText('8.0 MB'); + await expect(page.getByRole('region',{name:'Storage by scope',exact:true}).getByRole('row').filter({hasText:'Northline Logistics'})).toContainText('20.0 MB'); + await page.locator('#backup-storage-retained > summary').click(); + await page.locator('#backup-storage-system > summary').click(); + await expect(page.getByRole('region',{name:'Removed DSPs — retained backups'})).toContainText('Pine Delivery'); + await expect(page.getByRole('region',{name:'Removed DSPs — retained backups'})).toContainText('5.0 MB'); + await expect(page.getByRole('region',{name:'Full-system backup storage'})).toContainText('28.0 MB'); + await page.screenshot({path:'/tmp/dispatch-backup-storage-desktop.png'}); + await page.setViewportSize({width:390,height:844});await expect(page.getByRole('heading',{name:'Storage usage',exact:true})).toBeVisible(); + const stored=await page.getByRole('region',{name:'Storage by scope',exact:true}).getByRole('columnheader',{name:'Stored',exact:true}).boundingBox();expect(stored.x+stored.width).toBeLessThanOrEqual(390); + expect(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth)).toBe(true);await page.screenshot({path:'/tmp/dispatch-backup-storage-mobile.png',fullPage:true}); + data.storageUsage.status='stale';await page.reload();await expect(page.getByText(/Showing the last measured usage/)).toBeVisible(); + data.storageUsage={status:'unavailable',checkedAt:null};await page.reload();await expect(page.getByText(/Storage usage is not available yet/)).toBeVisible();await expect(page.locator('.backup-storage-total')).toHaveCount(0); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/dsp-view-connections.spec.cjs b/core/dashboard/tests/browser/dsp-view-connections.spec.cjs new file mode 100644 index 0000000..fdcb7f4 --- /dev/null +++ b/core/dashboard/tests/browser/dsp-view-connections.spec.cjs @@ -0,0 +1,68 @@ +'use strict'; +const { test, expect } = require('@playwright/test'); + +for (const mobile of [false, true]) test(`platform owner manages DSP connections (${mobile ? 'mobile' : 'desktop'})`, async ({ page }, testInfo) => { + if (mobile) await page.setViewportSize({ width: 390, height: 844 }); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + page.on('response', async response => { + if (response.status() >= 400) errors.push(`${response.status()} ${new URL(response.url()).pathname} ${(await response.json().catch(() => null))?.error?.code || ''}`); + }); + await page.goto('/'); + await page.getByLabel('Email address').fill('platform@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await page.getByRole('button', { name: 'Westfield Routes WR06', exact: true }).click(); + await page.getByRole('dialog').getByRole('button', { name: 'View', exact: true }).click(); + const banner = page.getByRole('region', { name: 'DSP viewing mode' }); + await expect(banner).toContainText('Viewing Westfield Routes as DSP owner'); + await page.goto('/#/plugins'); + await expect(page).toHaveTitle('Plugins · Dispatch'); + const install = page.getByRole('button', { name: 'Install Paycom', exact: true }); + if (await install.count()) await install.click(); + await expect(page.getByRole('link', { name: 'Open Paycom', exact: true })).toBeVisible(); + if (mobile) { + await page.getByRole('button', { name: 'Open navigation', exact: true }).click(); + await page.getByRole('dialog').getByRole('link', { name: 'Settings', exact: true }).click(); + } else await page.locator('.desktop-sidebar').getByRole('link', { name: 'Settings', exact: true }).click(); + await expect(page).toHaveTitle('Settings · Dispatch'); + await page.getByRole('tab', { name: 'Connections', exact: true }).click(); + await expect(page.getByRole('heading', { name: 'Connections', exact: true })).toBeVisible(); + await page.reload(); + await expect(page.getByRole('tab', { name: 'Connections', exact: true })).toHaveAttribute('aria-selected', 'true'); + await expect(banner).toBeVisible(); + for (const [service, name, credentials] of [ + ['cortex', 'Cortex', { 'Amazon username': 'synthetic-support-user', 'Amazon password': 'synthetic-support-secret' }], + ['paycom', 'Paycom', { 'Client code': 'synthetic-client', Username: 'synthetic-support-user', Password: 'synthetic-support-secret', + 'Security answer 1': 'one', 'Security answer 2': 'two', 'Security answer 3': 'three', 'Security answer 4': 'four', 'Security answer 5': 'five' }], + ]) { + await page.getByRole('button', { name: `Connect ${name}`, exact: true }).click(); + const dialog = page.getByRole('dialog'); + for (const [label, value] of Object.entries(credentials)) await dialog.getByLabel(label, { exact: true }).fill(value); + const saved = page.waitForResponse(response => response.url().endsWith(`/connections/${service}/save`)); + await dialog.getByRole('button', { name: 'Save and connect', exact: true }).click(); + const response = await saved; + expect(response.status()).toBe(202); + expect(Boolean(response.request().headers()['x-dispatch-dsp-view'])).toBe(true); + expect(await response.text()).not.toContain('synthetic-support-secret'); + await expect(dialog).toHaveCount(0); + const card = page.locator('[data-slot="card"]').filter({ has: page.getByText(name, { exact: true }) }); + await expect(card.getByText('Connected', { exact: true })).toBeVisible({ timeout: 10000 }); + await card.getByRole('button', { name: 'Test connection', exact: true }).click(); + await expect(card.getByText('Connected', { exact: true })).toBeVisible({ timeout: 10000 }); + await card.getByRole('button', { name: 'Update credentials', exact: true }).click(); + await expect(dialog.getByLabel(service === 'cortex' ? 'Amazon password' : 'Password', { exact: true })).toHaveValue(''); + await dialog.getByRole('button', { name: 'Cancel', exact: true }).click(); + await card.getByRole('button', { name: 'Disconnect', exact: true }).click(); + await dialog.getByRole('button', { name: 'Disconnect', exact: true }).click(); + await expect(card.getByRole('button', { name: `Connect ${name}`, exact: true })).toBeVisible(); + } + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.screenshot({ path: testInfo.outputPath('dsp-owner-connections.png'), fullPage: true }); + await banner.getByRole('button', { name: 'Exit view', exact: true }).click(); + await expect(page.getByRole('heading', { name: 'DSPs', exact: true })).toBeVisible(); + await page.goto('/#/settings?tab=connections'); + await expect(page.getByRole('tab', { name: 'Connections', exact: true })).toHaveCount(0); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/dsp-view.spec.cjs b/core/dashboard/tests/browser/dsp-view.spec.cjs new file mode 100644 index 0000000..ccf5270 --- /dev/null +++ b/core/dashboard/tests/browser/dsp-view.spec.cjs @@ -0,0 +1,121 @@ +const { test, expect } = require('@playwright/test'); + +async function login(page) { + await page.goto('/'); + await page.getByLabel('Email address').fill('platform@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await expect(page.getByRole('heading', { name: 'DSPs', exact: true })).toBeVisible(); +} +const nav = page => page.locator('.desktop-sidebar').getByRole('navigation'); + +test('View opens the owner interface, survives refresh, isolates tabs and exits cleanly', async ({ page, context }) => { + const errors = []; + page.on('pageerror', e => errors.push(e.message)); + page.on('console', m => { if (m.type() === 'error' && !m.text().includes('favicon')) errors.push(m.text()); }); + await login(page); + await page.getByRole('button', { name: 'Northline Logistics NL01', exact: true }).click(); + await page.getByRole('dialog').getByRole('button', { name: 'View', exact: true }).click(); + const banner = page.getByRole('region', { name: 'DSP viewing mode' }); + await expect(banner).toContainText('Viewing Northline Logistics as DSP owner'); + await expect(banner).toContainText('Full owner access. Changes are saved to this DSP.'); + await expect(page).toHaveTitle('Home Page · Dispatch'); + await expect(nav(page).getByRole('link')).toHaveText(['Home Page', 'Paycom', 'Team & Roles', 'Settings']); + await page.goto('/#/onboarding'); + await expect(page).toHaveURL(/#\/dashboard$/); + await expect(banner).toBeVisible(); + const otherTab = await context.newPage(); + await otherTab.goto('/'); + await expect(otherTab.getByRole('heading', { name: 'DSPs', exact: true })).toBeVisible(); + await expect(otherTab.getByRole('region', { name: 'DSP viewing mode' })).toHaveCount(0); + await otherTab.close(); + await nav(page).getByRole('link', { name: 'Team & Roles', exact: true }).click(); + await expect(page.getByText('Alex Morgan', { exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Invite member', exact: true })).toBeEnabled(); + await page.getByRole('button', { name: 'Actions for Jamie Chen', exact: true }).click(); + await expect(page.getByRole('menuitem', { name: 'Change role', exact: true })).toBeEnabled(); + await expect(page.getByRole('menuitem', { name: 'Remove member', exact: true })).toBeEnabled(); + await page.keyboard.press('Escape'); + await page.getByRole('tab', { name: 'Roles', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Create role', exact: true })).toHaveCount(0); + await expect(page.locator('.role-row h2')).toHaveText(['Owner', 'Manager', 'Dispatcher', 'Driver']); + await expect(page.getByText('View permissions', { exact: true })).toHaveCount(0); + await expect(page.getByText('organization · owner', { exact: true })).toHaveCount(0); + await page.getByRole('tab', { name: 'Members', exact: true }).click(); + await page.getByRole('button', { name: 'Actions for Jamie Chen', exact: true }).click(); + await page.getByRole('menuitem', { name: 'Change role', exact: true }).click(); + await page.getByLabel('Role', { exact: true }).selectOption({ label: 'Dispatcher' }); + await page.getByRole('button', { name: 'Save role', exact: true }).click(); + await expect(page.getByRole('row').filter({ hasText: 'Jamie Chen' })).toContainText('Dispatcher'); + await expect(page.getByRole('tab', { name: 'Activity', exact: true })).toHaveCount(0); + await nav(page).getByRole('link', { name: 'Settings', exact: true }).click(); + await page.getByRole('tab', { name: 'Audit log', exact: true }).click(); + await expect(page.getByRole('row').filter({ hasText: 'membership role update' }).first()).toContainText('platform@example.test'); + await expect(page.getByRole('cell', { name: 'organization view start', exact: true })).toHaveCount(0); + await page.reload(); + await expect(page.getByRole('tab', { name: 'Audit log', exact: true })).toHaveAttribute('data-state', 'active'); + await nav(page).getByRole('link', { name: 'Team & Roles', exact: true }).click(); + await expect(banner).toBeVisible(); + await expect(page.getByText('Alex Morgan', { exact: true })).toBeVisible(); + await page.screenshot({ path: '/tmp/dispatch-dsp-view-desktop.png' }); + await nav(page).getByRole('link', { name: 'Settings', exact: true }).click(); + await expect(page.locator('main').getByText('Northline Logistics', { exact: true })).toBeVisible(); + await expect(page.locator('main').getByText('platform@example.test', { exact: true })).toBeVisible(); + await page.getByRole('tab', { name: 'Security', exact: true }).click(); + await expect(page.getByLabel('Current password', { exact: true })).toBeEnabled(); + await expect(page.getByRole('button', { name: 'Change password', exact: true })).toBeEnabled(); + await banner.getByRole('button', { name: 'Exit view', exact: true }).click(); + await expect(page.getByRole('heading', { name: 'DSPs', exact: true })).toBeVisible(); + await expect(banner).toHaveCount(0); + await page.getByRole('button', { name: 'Actions for Atlas Routes', exact: true }).click(); + await page.getByRole('menuitem', { name: 'View', exact: true }).click(); + await expect(banner).toContainText('Viewing Atlas Routes as DSP owner'); + await nav(page).getByRole('link', { name: 'Team & Roles', exact: true }).click(); + await expect(page.getByText('No members found', { exact: true })).toBeVisible(); + await expect(page.getByText('Alex Morgan', { exact: true })).toHaveCount(0); + await page.getByRole('tab', { name: 'Roles', exact: true }).click(); + await expect(page.locator('.role-row h2')).toHaveText(['Owner', 'Manager', 'Dispatcher', 'Driver']); + await page.locator('.desktop-sidebar .account-button').click(); + await page.getByRole('menuitem', { name: 'Sign out', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Sign in', exact: true })).toBeVisible(); + expect(await page.evaluate(() => sessionStorage.getItem('dispatch-dsp-view'))).toBeNull(); + expect(errors).toEqual([]); +}); + +test('mobile DSP view keeps the banner and exit usable', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await login(page); + await page.getByRole('button', { name: 'Northline Logistics NL01', exact: true }).click(); + await page.getByRole('dialog').getByRole('button', { name: 'View', exact: true }).click(); + const banner = page.getByRole('region', { name: 'DSP viewing mode' }); + await expect(banner).toContainText('Northline Logistics'); + await page.getByRole('button', { name: 'Open navigation', exact: true }).click(); + await page.getByRole('dialog').getByRole('link', { name: 'Team & Roles', exact: true }).click(); + await expect(page.getByText('Alex Morgan', { exact: true })).toBeVisible(); + await expect(banner.getByRole('button', { name: 'Exit view', exact: true })).toBeInViewport(); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.getByRole('tab', { name: 'Roles', exact: true }).click(); + await expect(page.locator('.role-row h2')).toHaveText(['Owner', 'Manager', 'Dispatcher', 'Driver']); + await expect(page.getByRole('button', { name: 'Create role', exact: true })).toHaveCount(0); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.screenshot({ path: '/tmp/dispatch-fixed-roles-mobile.png', fullPage: true }); + await page.getByRole('button', { name: 'Invite member', exact: true }).click(); + await expect(page.getByLabel('Role', { exact: true }).locator('option')).toHaveText(['Owner', 'Manager', 'Dispatcher', 'Driver']); + await page.getByLabel('Role', { exact: true }).selectOption({ label: 'Owner' }); + await expect(page.getByLabel('Role', { exact: true }).locator('option:checked')).toHaveText('Owner'); + await page.getByRole('button', { name: 'Cancel', exact: true }).click(); + await banner.getByRole('button', { name: 'Exit view', exact: true }).click(); + await expect(page.getByRole('heading', { name: 'DSPs', exact: true })).toBeVisible(); +}); + + +test('an invalidated view returns to the platform with an explanation', async ({ page }) => { + await login(page); + await page.evaluate(() => sessionStorage.setItem('dispatch-dsp-view', 'x'.repeat(43))); + await page.goto('/#/team'); + await page.reload(); + await expect(page.getByRole('heading', { name: 'DSPs', exact: true })).toBeVisible(); + await expect(page.getByText('The DSP view expired or is no longer available. You’re back in the platform console.')).toBeVisible(); + await expect(page.getByRole('region', { name: 'DSP viewing mode' })).toHaveCount(0); + expect(await page.evaluate(() => sessionStorage.getItem('dispatch-dsp-view'))).toBeNull(); +}); diff --git a/core/dashboard/tests/browser/independent-updates.spec.cjs b/core/dashboard/tests/browser/independent-updates.spec.cjs new file mode 100644 index 0000000..ad108bd --- /dev/null +++ b/core/dashboard/tests/browser/independent-updates.spec.cjs @@ -0,0 +1,61 @@ +const { test, expect } = require('@playwright/test'); +const { spawn } = require('node:child_process'); +const path = require('node:path'); +let processHandle, url; +test.beforeAll(async () => { + processHandle = spawn(process.execPath, [path.resolve(__dirname, '../../examples/independent-updates-preview.js')], { + env: { ...process.env, DISPATCH_INDEPENDENT_UPDATES_FIXTURE: '1' }, stdio: ['ignore', 'pipe', 'pipe'], + }); + url = await new Promise((resolve, reject) => { + let output = '', errors = ''; + const timer = setTimeout(() => reject(Error(`Preview timed out: ${errors}`)), 15000); + processHandle.stderr.on('data', chunk => { errors += chunk; }); + processHandle.once('error', error => { clearTimeout(timer); reject(error); }); + processHandle.once('exit', code => { clearTimeout(timer); reject(Error(`Preview exited ${code}: ${errors}`)); }); + processHandle.stdout.on('data', chunk => { output += chunk; const match = /Synthetic updates preview: (http:\/\/127\.0\.0\.1:\d+)/.exec(output); + if (match) { clearTimeout(timer); resolve(match[1]); } }); + }); +}); +test.afterAll(async () => { + if (processHandle?.exitCode !== null || processHandle?.signalCode !== null) return; + await new Promise(resolve => { const timer = setTimeout(() => processHandle.kill('SIGKILL'), 5000); + processHandle.once('exit', () => { clearTimeout(timer); resolve(); }); processHandle.kill('SIGTERM'); }); +}); +test('independent Core update, new release Dev gate, failure pause and sequential resume', async ({ page }, info) => { + const errors = []; page.on('pageerror', error => errors.push(error.message)); + page.on('console', event => { if (event.type() === 'error') errors.push(`${event.text()} (${event.location().url})`); }); + await page.goto(`${url}/#/updates`); + await page.getByLabel('Email address').fill('platform@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await page.locator('.desktop-sidebar').getByRole('link', { name: 'Updates', exact: true }).click(); + await expect(page.getByRole('heading', { name: 'Core', exact: true })).toBeVisible(); + await expect(page.getByRole('heading', { name: 'DSP', exact: true })).toBeVisible(); + await expect(page.getByRole('heading', { name: 'Plugins', exact: true })).toBeVisible(); + await expect(page.getByRole('tablist', {name:'Update products'})).toHaveCount(0); + await expect(page.getByText('Installed: 0.0.1', { exact: true })).toBeVisible(); + await page.screenshot({ path: info.outputPath('core.png'), fullPage: true }); + await page.getByRole('button', { name: 'Update Core', exact: true }).click(); + await expect(page.getByText('Installed: 0.0.2', { exact: true })).toBeVisible({ timeout: 10000 }); + await expect(page.getByText('Installed on Dev DSP: 0.0.1', { exact: true })).toBeVisible(); + await page.getByRole('button', { name: 'Update Dev', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Rollout Update', exact: true })).toBeEnabled({ timeout: 10000 }); + await page.screenshot({ path: info.outputPath('dev-tested.png'), fullPage: true }); + await page.request.post(`${url}/__fixture/publish`); + await expect(page.getByRole('button', { name: 'Update Dev', exact: true })).toBeEnabled({ timeout: 10000 }); + await expect(page.getByRole('heading', { name: 'Version 0.0.3', exact: true })).toBeVisible(); + await page.getByRole('button', { name: 'Update Dev', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Rollout Update', exact: true })).toBeEnabled({ timeout: 10000 }); + await page.request.post(`${url}/__fixture/fail`); + await page.getByRole('button', { name: 'Rollout Update', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Resume rollout', exact: true })).toBeEnabled({ timeout: 10000 }); + await expect(page.getByText('0 of 2 DSPs updated · paused', { exact: true })).toBeVisible(); + await page.screenshot({ path: info.outputPath('rollout-paused.png'), fullPage: true }); + await page.getByRole('button', { name: 'Resume rollout', exact: true }).click(); + await expect(page.getByText('2 of 2 DSPs updated · completed', { exact: true })).toBeVisible({ timeout: 10000 }); + await page.setViewportSize({ width: 390, height: 844 }); + await expect.poll(() => page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.screenshot({ path: info.outputPath('mobile.png'), fullPage: true }); + await expect(page.getByText('Installed: 0.0.2', { exact: true })).toBeVisible(); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/onboarding.spec.cjs b/core/dashboard/tests/browser/onboarding.spec.cjs new file mode 100644 index 0000000..0a79a8d --- /dev/null +++ b/core/dashboard/tests/browser/onboarding.spec.cjs @@ -0,0 +1,243 @@ +const { test, expect } = require("@playwright/test"); +const { randomUUID } = require("node:crypto"); +const password = "synthetic preview password"; + +async function ownerInvitation(playwright, baseURL, email) { + const admin = await playwright.request.newContext({ baseURL }); + try { + const login = await admin.post("/api/auth/login", { + data: { email: "platform@example.test", password }, + }); + expect(login.ok()).toBeTruthy(); + const session = (await login.json()).data; + const created = await admin.post("/api/platform/organizations", { + headers: { "X-Dispatch-CSRF": session.csrfToken, Origin: baseURL }, + data: { + ownerEmail: email, + idempotencyKey: `browser:onboarding:${randomUUID()}`, + }, + }); + expect(created.status()).toBe(201); + return (await created.json()).data.invitationPath; + } finally { + await admin.dispose(); + } +} + +async function existingOwner(playwright, baseURL) { + const email = `existing-${randomUUID()}@example.test`; + const link = await ownerInvitation(playwright, baseURL, email); + const client = await playwright.request.newContext({ baseURL }); + try { + const registration = await client.post("/api/auth/register", { + data: { + token: link.split("/").at(-1), + firstName: "Existing", + lastName: "Owner", + password, + confirmPassword: password, + }, + }); + expect(registration.status()).toBe(201); + } finally { + await client.dispose(); + } + return email; +} + +async function saveDetails(page) { + await expect(page).toHaveURL(/#\/onboarding$/); + await expect(page).toHaveTitle("Set up your DSP · Dispatch"); + await expect( + page.getByRole("heading", { name: "Set up your DSP" }), + ).toBeVisible(); + await expect( + page.getByRole("button", { name: "Continue to workspace" }), + ).toHaveCount(0); + await page + .getByLabel("DSP name", { exact: true }) + .fill("Onboarding Logistics"); + await page.getByLabel("Abbreviation (optional)").fill("OL"); + await page.getByLabel("Station code").fill("TST4"); + await page.getByLabel("Business timezone").fill("America/Chicago"); + await page.getByRole("button", { name: "Save DSP details" }).click(); + await expect( + page.getByText("Your DSP details are saved.", { exact: false }), + ).toBeVisible(); + const profile = ( + await (await page.request.get("/api/organization/profile")).json() + ).data; + expect(profile.details).toEqual({ + name: "Onboarding Logistics", + abbreviation: "OL", + stationCode: "TST4", + timezone: "America/Chicago", + }); + await page.reload(); + await expect( + page.getByText("Your DSP details are saved.", { exact: false }), + ).toBeVisible(); + await page.getByRole("button", { name: "Continue to workspace" }).click(); + await expect( + page.getByRole("heading", { level: 1, name: /^(Settings|DSPs)$/ }), + ).toBeVisible(); +} + +for (const mobile of [false, true]) { + test(`new DSP owner creates account then saves DSP details (${mobile ? "mobile" : "desktop"})`, async ({ + page, + playwright, + baseURL, + }) => { + if (mobile) await page.setViewportSize({ width: 390, height: 844 }); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + page.on("console", (message) => { + if (message.type() === "error") errors.push(message.text()); + }); + const link = await ownerInvitation( + playwright, + baseURL, + `new-${randomUUID()}@example.test`, + ); + await page.goto(link); + await expect( + page.getByRole("heading", { name: "Create your DSP" }), + ).toBeVisible(); + await expect(page.getByLabel("Email address")).toHaveCount(0); + await page.getByLabel("First name").fill("New"); + await page.getByLabel("Last name").fill("Owner"); + await page.getByLabel("Password", { exact: true }).fill(password); + await page.getByLabel("Confirm password", { exact: true }).fill(password); + await page.screenshot({ + path: `/tmp/dispatch-onboarding-account-${mobile ? "mobile" : "desktop"}.png`, + fullPage: true, + }); + await page + .getByRole("button", { name: "Create account and continue" }) + .click(); + await expect(page.getByLabel("DSP name", { exact: true })).toBeVisible(); + await page.screenshot({ + path: `/tmp/dispatch-onboarding-details-${mobile ? "mobile" : "desktop"}.png`, + fullPage: true, + }); + expect( + await page.evaluate( + () => document.documentElement.scrollWidth <= window.innerWidth, + ), + ).toBeTruthy(); + await saveDetails(page); + expect(errors).toEqual([]); + await page.goto(link); + await expect( + page.getByText( + "This invitation is invalid, expired, revoked, or already used.", + ), + ).toBeVisible(); + await expect(page.getByLabel("First name")).toHaveCount(0); + }); +} + +for (const account of ["DSP owner", "platform owner"]) { + test(`existing account can join a DSP only when it has no DSP membership: ${account}`, async ({ + page, + playwright, + baseURL, + }) => { + const email = + account === "platform owner" + ? "platform@example.test" + : await existingOwner(playwright, baseURL); + const probe = await playwright.request.newContext({ baseURL }); + const initial = await probe.post('/api/auth/login', { data: { email, password } }); + const membershipCount = (await initial.json()).data.memberships.length; + await probe.dispose(); + const link = await ownerInvitation(playwright, baseURL, email); + await page.goto(link); + await expect( + page.getByText("You already have a Dispatch account.", { exact: false }), + ).toBeVisible(); + await expect(page.getByLabel("Email address")).toHaveValue(""); + await expect(page.getByLabel("First name")).toHaveCount(0); + await page.getByLabel("Email address").fill(email); + await page.getByLabel("Password", { exact: true }).fill(password); + await page.getByRole("button", { name: "Sign in and continue" }).click(); + if (membershipCount === 0) await saveDetails(page); + else { + await expect(page.getByText('This user already belongs to another DSP.')).toBeVisible(); + await expect(page).toHaveURL(new URL(link, baseURL).href); + expect((await (await page.request.get('/api/auth/session')).json()).data.memberships).toHaveLength(membershipCount); + } + }); +} + +test("wrong account cannot accept; switching accounts preserves the invitation", async ({ + page, + playwright, + baseURL, +}) => { + const email = await existingOwner(playwright, baseURL); + const link = await ownerInvitation(playwright, baseURL, email); + await page.goto(link); + await page.getByLabel("Email address").fill("platform@example.test"); + await page.getByLabel("Password", { exact: true }).fill(password); + await page.getByRole("button", { name: "Sign in and continue" }).click(); + await expect( + page.getByText( + "Sign in with the exact email address named by this invitation.", + ), + ).toBeVisible(); + await expect(page).toHaveURL(new URL(link, baseURL).href); + await page.getByRole("button", { name: "Use another account" }).click(); + await page.getByLabel("Email address").fill(email); + await page.getByLabel("Password", { exact: true }).fill(password); + await page.getByRole("button", { name: "Sign in and continue" }).click(); + await expect(page.getByText('This user already belongs to another DSP.')).toBeVisible(); + await expect(page).toHaveURL(new URL(link, baseURL).href); +}); + +test("signed-in owner cannot add a second DSP membership", async ({ + page, + playwright, + baseURL, +}) => { + const email = await existingOwner(playwright, baseURL); + const link = await ownerInvitation(playwright, baseURL, email); + await page.request.post("/api/auth/login", { data: { email, password } }); + await page.goto(link); + await expect(page.getByText(`Signed in as ${email}.`)).toBeVisible(); + await page.getByRole("button", { name: "Continue to DSP setup" }).click(); + await expect(page.getByText('This user already belongs to another DSP.')).toBeVisible(); + await expect(page).toHaveURL(new URL(link, baseURL).href); +}); + +test("password confirmation errors keep new owners in account creation", async ({ + page, + playwright, + baseURL, +}) => { + const link = await ownerInvitation( + playwright, + baseURL, + `confirmation-${randomUUID()}@example.test`, + ); + await page.goto(link); + await page.getByLabel("First name").fill("New"); + await page.getByLabel("Last name").fill("Owner"); + await page.getByLabel("Password", { exact: true }).fill(password); + await page + .getByLabel("Confirm password", { exact: true }) + .fill("a different password"); + await page + .getByRole("button", { name: "Create account and continue" }) + .click(); + await expect( + page.getByText("The password confirmation does not match."), + ).toBeVisible(); + await expect(page).toHaveURL(new URL(link, baseURL).href); + await page.getByLabel("Confirm password", { exact: true }).fill(password); + await page + .getByRole("button", { name: "Create account and continue" }) + .click(); + await expect(page.getByLabel("DSP name", { exact: true })).toBeVisible(); +}); diff --git a/core/dashboard/tests/browser/optional-paycom.spec.cjs b/core/dashboard/tests/browser/optional-paycom.spec.cjs new file mode 100644 index 0000000..0fa3f79 --- /dev/null +++ b/core/dashboard/tests/browser/optional-paycom.spec.cjs @@ -0,0 +1,238 @@ +const { test, expect } = require("@playwright/test"); +for (const mobile of [false, true]) + test(`Paycom is optional and can be connected later (${mobile ? "mobile" : "desktop"})`, async ({ + page, + }) => { + if (mobile) await page.setViewportSize({ width: 390, height: 844 }); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + page.on("console", (message) => { + // A connected account without a first collection receives the API's + // expected not_initialized 503. All other console errors still fail QA. + const expectedEmpty = message.text().includes('503') && + /\/api\/paycom\/(daily|employees)(?:\?|$)/.test(message.location().url); + if (message.type() === "error" && !expectedEmpty) errors.push(message.text()); + }); + let state = { + status: "not_started", + failureCode: null, + canSubmit: true, + canRetry: false, + }; + const submissions = []; + await page.route("**/api/organization/paycom-setup", async (route) => { + if (route.request().method() === "POST") { + submissions.push(route.request().postDataJSON()); + state = { + status: "queued", + failureCode: null, + canSubmit: false, + canRetry: false, + }; + } + await route.fulfill({ + json: { ok: true, status: "found", data: state, error: null }, + }); + }); + await page.goto("/#/plugins"); + await page.getByLabel("Email address").fill("owner5@example.test"); + await page + .getByLabel("Password", { exact: true }) + .fill("synthetic preview password"); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect(page).toHaveTitle("Plugins · Dispatch"); + const install = page.getByRole("button", { name: "Install Paycom", exact: true }); + if (await install.count()) await install.click(); + await page.getByRole("link", { name: "Open Paycom", exact: true }).click(); + await expect(page).toHaveTitle("Paycom · Dispatch"); + await expect( + page.getByText("Paycom is not connected.", { exact: false }), + ).toBeVisible(); + await expect( + page.getByText("This is optional", { exact: false }), + ).toBeVisible(); + expect(submissions).toHaveLength(0); + await page + .getByRole("button", { name: "Connect Paycom", exact: true }) + .click(); + await expect(page).toHaveURL(/settings\?tab=connections/); + await page.getByRole('button', { name: 'Connect Paycom', exact: true }).click(); + await page.route('**/api/organization/connections/paycom/save', async route => { + submissions.push(route.request().postDataJSON()); + state = { status: 'queued', failureCode: null, canSubmit: false, canRetry: false }; + await route.fulfill({ json: { ok: true, status: 'accepted', data: { service: 'paycom', configured: true, state: 'checking', checkedAt: null, retryAt: null, reason: null } } }); + }); + await page.getByLabel("Client code").fill("fixture-client"); + await page.getByRole("dialog").getByLabel("Username", { exact: true }).fill("fixture-user"); + await page.getByRole("dialog").getByLabel("Password", { exact: true }).fill("fixture-password"); + for (let n = 1; n <= 5; n++) + await page.getByLabel(`Security answer ${n}`).fill(`fixture-answer-${n}`); + const savedCredentials = page.waitForResponse(response => response.url().endsWith('/connections/paycom/save')); + await page.getByRole('button', { name: 'Save and connect' }).click(); + expect((await savedCredentials).status()).toBe(200); + expect(submissions).toHaveLength(1); + await expect(page.getByRole('dialog')).toHaveCount(0); + await page.goto('/#/paycom'); + await expect( + page.getByText("Verifying your Paycom login.", { + exact: false, + }), + ).toBeVisible(); + expect(submissions).toHaveLength(1); + expect(submissions[0].credentials.username).toBe("fixture-user"); + await expect(page.getByRole("dialog").getByLabel("Password", { exact: true })).toHaveCount(0); + state = { + status: "failed", + failureCode: "manual_verification_required", + canSubmit: true, + canRetry: false, + retryState: "manual", + }; + await expect( + page.getByText("Your DSP is still ready to use.", { exact: false }), + ).toBeVisible(); + await expect( + page.getByRole("button", { name: "Retry connection" }), + ).toHaveCount(0); + await expect(page.getByText('Paycom needs verification. Contact the Platform Owner.', { exact: true })).toBeVisible(); + await page.screenshot({ path: `/tmp/dispatch-paycom-blocked-${mobile ? "mobile" : "desktop"}.png`, fullPage: true }); + await page.getByRole('button', { name: 'Replace Paycom credentials' }).click(); + await expect(page).toHaveURL(/settings\?tab=connections/); + await page.getByRole('button', { name: 'Connect Paycom', exact: true }).click(); + await expect(page.getByText('all five distinct security answers', { exact: false })).toBeVisible(); + await expect(page.getByLabel('Security answer 5')).toBeVisible(); + await page.screenshot({ path: `/tmp/dispatch-paycom-pins-${mobile ? "mobile" : "desktop"}.png`, fullPage: true }); + await page.getByRole('button', { name: 'Cancel', exact: true }).click(); + await page.goto('/#/paycom'); + state = { status: 'failed', failureCode: 'security_answers_rejected', canSubmit: true, canRetry: false, + retryState: 'cooldown', retryAt: new Date(Date.now() + 300000).toISOString() }; + await page.reload(); + await expect(page.getByText('Another attempt is available after', { exact: false })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Retry connection' })).toHaveCount(0); + state = { ...state, canRetry: true, retryState: 'ready', retryAt: null }; + await expect(page.getByRole('button', { name: 'Retry connection' })).toBeVisible({ timeout: 8000 }); + let retries = 0; + await page.route('**/api/organization/paycom-setup/retry', async route => { + retries++; + state = { status: 'queued', failureCode: null, canSubmit: false, canRetry: false }; + await route.fulfill({ json: { ok: true, status: 'accepted', data: state, error: null } }); + }); + await page.getByRole('button', { name: 'Retry connection' }).click(); + await expect(page.getByText('Verifying your Paycom login.', { exact: false })).toBeVisible(); + expect(retries).toBe(1); + state = { status: 'succeeded', failureCode: null, canSubmit: false, canRetry: false, workforceAvailable: false }; + let workforceRequests = 0; + await page.route('**/api/paycom/**', route => { + workforceRequests++; + if (new URL(route.request().url()).pathname === '/api/paycom/sync') + return route.fulfill({ json: { ok: true, data: { + activity: 'idle', desiredState: 'running', lastSucceededAt: null, lastError: null, alerts: [], + } } }); + return route.fulfill({ status: 503, json: { ok: false, data: null, error: { code: 'not_initialized' } } }); + }); + // The queued setup poll must open the workspace without a refresh or click. + await expect(page.getByRole('tab', { name: 'Timecard', exact: true })).toBeVisible({ timeout: 8000 }); + await expect(page.getByRole('tab', { name: 'Employees', exact: true })).toBeVisible(); + await expect(page.getByText('Waiting for the first Paycom collection', { exact: true })).toBeVisible(); + await expect(page.getByText('Your Paycom account is connected', { exact: true })).toHaveCount(0); + await expect(page).toHaveURL(/#\/paycom$/); + expect(workforceRequests).toBeGreaterThan(0); + // Existing successful connections also enter the workspace on page load. + await page.reload(); + await expect(page.getByRole('tab', { name: 'Timecard', exact: true })).toBeVisible(); + await expect(page.getByText('Waiting for the first Paycom collection', { exact: true })).toBeVisible(); + await page.getByRole('tab', { name: 'Employees', exact: true }).click(); + await expect(page.getByText('Waiting for the first Paycom collection', { exact: true })).toBeVisible(); + expect(submissions).toHaveLength(1); + await page.screenshot({ + path: `/tmp/dispatch-optional-paycom-${mobile ? "mobile" : "desktop"}.png`, + fullPage: true, + }); + expect( + await page.evaluate( + () => document.documentElement.scrollWidth <= window.innerWidth, + ), + ).toBeTruthy(); + expect(errors).toEqual([]); + }); + +for (const failureCode of ['manual_verification_required', 'captcha_required']) + test(`operator recovery opens the workspace without a retry or refresh (${failureCode})`, async ({ page }) => { + await page.clock.install(); + if (failureCode === 'captcha_required') await page.setViewportSize({ width: 390, height: 844 }); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + const writes = []; + let reads = 0; + let state = { status: 'failed', failureCode, retryState: 'manual', + canSubmit: true, canRetry: false, workforceAvailable: false }; + await page.route('**/api/organization/paycom-setup', route => { + if (route.request().method() === 'GET') reads++; + else writes.push(route.request().method()); + return route.fulfill({ json: { ok: true, data: state } }); + }); + await page.route('**/api/organization/paycom-setup/retry', route => { + writes.push('retry'); + return route.fulfill({ status: 409, json: { ok: false } }); + }); + await page.route('**/api/paycom/**', route => { + if (route.request().method() !== 'GET') writes.push('workforce mutation'); + return route.fulfill({ json: { ok: true, data: + new URL(route.request().url()).pathname === '/api/paycom/sync' + ? { activity: 'queued', desiredState: 'running', lastSucceededAt: null, lastError: null, alerts: [] } + : { available: true, items: [], total: 0, offset: 0, hasMore: false }, + } }); + }); + await page.goto('/#/paycom'); + await page.getByLabel('Email address').fill('owner5@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await expect(page.getByText('Paycom needs verification. Contact the Platform Owner.', { exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Retry connection' })).toHaveCount(0); + const before = reads; + await page.clock.fastForward(6000); + await expect.poll(() => reads).toBeGreaterThan(before); + expect(writes).toEqual([]); + // The operator finishes verification and normal setup succeeds elsewhere. + state = { status: 'succeeded', failureCode: null, retryState: 'ready', + canSubmit: false, canRetry: false, workforceAvailable: false }; + await page.clock.fastForward(6000); + await expect(page.getByRole('tab', { name: 'Timecard', exact: true })).toBeVisible(); + await expect(page.getByRole('status', { name: 'Paycom sync' })).toContainText('Queued'); + await expect(page.getByText('Paycom needs verification. Contact the Platform Owner.', { exact: true })).toHaveCount(0); + await page.getByRole('tab', { name: 'Employees', exact: true }).click(); + await expect(page.getByText('No collected employees', { exact: true })).toBeVisible(); + expect(writes).toEqual([]); + expect(errors).toEqual([]); + }); + +test('an already-connected DSP opens Paycom workspace in platform viewing mode', async ({ page }) => { + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + await page.route('**/api/organization/paycom-setup', route => route.fulfill({ json: { ok: true, data: { + status: 'succeeded', workforceAvailable: false, canSubmit: false, canRetry: false, failureCode: null, + } } })); + await page.route('**/api/paycom/**', route => new URL(route.request().url()).pathname === '/api/paycom/sync' + ? route.fulfill({ json: { ok: true, data: { + activity: 'idle', desiredState: 'running', lastSucceededAt: null, lastError: null, alerts: [], + } } }) + : route.fulfill({ status: 503, json: { ok: false, data: null, error: { code: 'not_initialized' } } })); + await page.goto('/'); + await page.getByLabel('Email address').fill('platform@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await page.getByRole('button', { name: 'Northline Logistics NL01', exact: true }).click(); + await page.getByRole('dialog').getByRole('button', { name: 'View', exact: true }).click(); + const banner = page.getByRole('region', { name: 'DSP viewing mode' }); + await expect(banner).toContainText('Viewing Northline Logistics as DSP owner'); + await page.locator('.desktop-sidebar').getByRole('link', { name: 'Paycom', exact: true }).click(); + await expect(page).toHaveTitle('Paycom · Dispatch'); + await expect(page.getByRole('tab', { name: 'Timecard', exact: true })).toBeVisible(); + await expect(page.getByText('Waiting for the first Paycom collection', { exact: true })).toBeVisible(); + await page.reload(); + await expect(banner).toBeVisible(); + await expect(page.getByRole('tab', { name: 'Employees', exact: true })).toBeVisible(); + await expect(page.getByText('Your Paycom account is connected', { exact: true })).toHaveCount(0); + await page.screenshot({ path: '/tmp/dispatch-paycom-dsp-view-connected.png', fullPage: false }); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/password-recovery.spec.cjs b/core/dashboard/tests/browser/password-recovery.spec.cjs new file mode 100644 index 0000000..1755f63 --- /dev/null +++ b/core/dashboard/tests/browser/password-recovery.spec.cjs @@ -0,0 +1,132 @@ +const { test, expect } = require('@playwright/test'); +const { spawn } = require('node:child_process'); +const { once } = require('node:events'); +const path = require('node:path'); +let server, base; +const mail = []; +const original = 'synthetic preview password'; +const replacement = 'a secure replacement passphrase'; +const widgetScript = `window.turnstile = { + render(container, options) { + const id = crypto.randomUUID(); container.dataset.widget = id; + const button = document.createElement('button'); + button.type = 'button'; button.textContent = 'Complete security check'; + button.onclick = () => options.callback('fixture:' + options.action + ':' + crypto.randomUUID()); + container.replaceChildren(button); return id; + }, + remove(id) { document.querySelector('[data-widget="' + id + '"]')?.replaceChildren(); } +};`; + +test.beforeAll(async () => { + server = spawn(process.execPath, ['--no-warnings', 'examples/frontend-preview.js'], { + cwd: path.resolve(__dirname, "../.."), + env: { ...process.env, DISPATCH_FRONTEND_FIXTURE: '1', DISPATCH_RECOVERY_FIXTURE: '1', DISPATCH_TURNSTILE_FIXTURE: '1', DISPATCH_FRONTEND_PORT: '0' }, + stdio: ['ignore', 'pipe', 'pipe', 'ipc'], + }); + server.on('message', message => mail.push(message)); + base = await new Promise((resolve, reject) => { + let output = ''; + server.stdout.on('data', chunk => { + output += chunk; + const match = /http:\/\/127\.0\.0\.1:\d+/.exec(output); + if (match) resolve(match[0]); + }); + server.once('error', reject); + server.once('exit', code => reject(Error(`Recovery fixture exited: ${code}`))); + }); +}); +test.afterAll(async () => { + if (server && server.exitCode === null) { + const closed = once(server, 'exit'); server.kill('SIGTERM'); await closed; + } +}); +async function widget(page) { + await page.route('https://challenges.cloudflare.com/turnstile/v0/api.js?render=explicit', route => route.fulfill({ contentType: 'text/javascript', body: widgetScript })); +} +async function requestReset(page, email) { + await page.getByLabel('Email address').fill(email); + await expect(page.getByRole('button', { name: 'Send reset link' })).toBeDisabled(); + await page.getByRole('button', { name: 'Complete security check' }).click(); + const submitted = page.waitForRequest(request => request.url().endsWith('/api/auth/forgot-password')); + await page.getByRole('button', { name: 'Send reset link' }).click(); + expect((await submitted).postDataJSON().turnstileToken).toMatch(/^fixture:forgot_password:/); + await expect(page.getByRole('heading', { name: 'Check your email' })).toBeVisible(); + await expect(page.getByText('If an account exists for that email, we’ll send a password reset link.')).toBeVisible(); +} + +test('desktop: request email, open link, correct mismatch, reset, reject replay, and sign in with the new password', async ({ page, request }) => { + const errors = [], requests = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { + if (message.type() === 'error' && !/400 \(Bad Request\)/.test(message.text())) errors.push(message.text()); + }); + page.on('request', request => requests.push(request)); + const oldLogin = await request.post(base + '/api/auth/login', { data: { + email: 'owner@example.test', password: original, turnstileToken: 'fixture:login:recovery-old-session', + } }); + expect(oldLogin.status()).toBe(200); + await widget(page); + await page.goto(base); + await expect(page).toHaveTitle('Dispatch'); + await page.getByRole('link', { name: 'Forgot password?' }).click(); + await expect(page.getByRole('heading', { name: 'Forgot your password?' })).toBeVisible(); + await page.screenshot({ path: '/tmp/dispatch-recovery-desktop.png' }); + await requestReset(page, 'owner@example.test'); + await expect.poll(() => mail.filter(item => item.type === 'password-reset' && item.message.email === 'owner@example.test').length).toBe(1); + const token = mail.find(item => item.type === 'password-reset' && item.message.email === 'owner@example.test').message.token; + const resetUrl = base + '/#/reset-password/' + token; + await page.goto(resetUrl); + await expect(page).toHaveURL(base + '/#/reset-password'); + await expect(page.getByRole('heading', { name: 'Set a new password' })).toBeVisible(); + await expect(page.getByLabel('Security verification')).toHaveCount(0); + await page.getByLabel('New password', { exact: true }).fill(replacement); + await page.getByLabel('Confirm new password').fill('mismatched secure password'); + await page.getByRole('button', { name: 'Reset password', exact: true }).click(); + await expect(page.getByRole('alert')).toContainText('The password confirmation does not match.'); + await page.getByLabel('Confirm new password').fill(replacement); + await page.getByRole('button', { name: 'Reset password', exact: true }).click(); + await expect(page.getByRole('heading', { name: 'Password reset', exact: true })).toBeVisible(); + await page.screenshot({ path: '/tmp/dispatch-recovery-complete.png' }); + expect((await (await request.get(base + '/api/auth/session')).json()).data.authenticated).toBe(false); + await expect.poll(() => mail.filter(item => item.type === 'password-reset-confirmation').length).toBe(1); + expect(requests.every(request => !request.url().includes(token))).toBe(true); + expect(requests.filter(request => request.postData()?.includes(token)).every(request => request.url() === base + '/api/auth/reset-password')).toBe(true); + expect(await page.evaluate(secret => JSON.stringify([localStorage, sessionStorage]).includes(secret), token)).toBe(false); + await page.goto(resetUrl); + await page.getByLabel('New password', { exact: true }).fill(replacement); + await page.getByLabel('Confirm new password').fill(replacement); + await page.getByRole('button', { name: 'Reset password', exact: true }).click(); + await expect(page.getByRole('alert')).toContainText('This reset link is invalid or has expired'); + await page.getByRole('link', { name: 'Back to sign in' }).click(); + await page.getByLabel('Email address').fill('owner@example.test'); + await page.getByLabel('Password', { exact: true }).fill(replacement); + await page.getByRole('button', { name: 'Complete security check' }).click(); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await expect(page.getByRole('navigation', { name: 'Primary navigation' })).toBeVisible(); + expect(errors).toEqual([]); +}); + +test('mobile: unknown email has the same receipt; malformed and reloaded links offer recovery without overflow', async ({ page }) => { + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + await page.setViewportSize({ width: 390, height: 844 }); + await widget(page); await page.goto(base + '/#/forgot-password'); + await requestReset(page, 'unknown@example.test'); + await page.screenshot({ path: '/tmp/dispatch-recovery-mobile.png' }); + expect(mail.some(item => item.message.email === 'unknown@example.test')).toBe(false); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.goto(base + '/#/reset-password/malformed'); + await expect(page.getByText('This reset link is invalid or has expired. Request a new link to continue.')).toBeVisible(); + await expect(page.getByRole('button', { name: 'Reset password', exact: true })).toHaveCount(0); + await page.getByRole('link', { name: 'Request a new link' }).click(); + await requestReset(page, 'member0@example.test'); + await expect.poll(() => mail.some(item => item.type === 'password-reset' && item.message.email === 'member0@example.test')).toBe(true); + const token = mail.find(item => item.type === 'password-reset' && item.message.email === 'member0@example.test').message.token; + await page.goto(base + '/#/reset-password/' + token); + await expect(page.getByLabel('New password', { exact: true })).toBeVisible(); + await page.screenshot({ path: '/tmp/dispatch-recovery-mobile-password.png' }); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.reload(); + await expect(page.getByText('This reset link is invalid or has expired. Request a new link to continue.')).toBeVisible(); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/paycom-connection-state.spec.cjs b/core/dashboard/tests/browser/paycom-connection-state.spec.cjs new file mode 100644 index 0000000..adbc478 --- /dev/null +++ b/core/dashboard/tests/browser/paycom-connection-state.spec.cjs @@ -0,0 +1,60 @@ +const { test, expect } = require('@playwright/test'); + +for (const mobile of [false, true]) test(`verified Paycom opens its workspace while workforce setup continues (${mobile ? 'mobile' : 'desktop'})`, async ({ page }) => { + if (mobile) await page.setViewportSize({ width: 390, height: 844 }); + let verified = false, retries = 0; + let setup = { status: 'running', workforceAvailable: false, canSubmit: false, canRetry: false, failureCode: null }; + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { + const empty = message.text().includes('503') && /\/api\/paycom\/(daily|employees)(?:\?|$)/.test(message.location().url); + if (message.type() === 'error' && !empty) errors.push(message.text()); + }); + await page.route('**/api/organization/paycom-setup', route => route.fulfill({ json: { ok: true, data: setup } })); + await page.route('**/api/organization/connections', route => route.fulfill({ json: { ok: true, data: { + services: [{ id: 'paycom', name: 'Paycom', fields: [] }], + items: [{ service: 'paycom', configured: true, state: verified ? 'connected' : 'checking', + checkedAt: new Date().toISOString(), reason: null, retryAt: null }], + } } })); + await page.route('**/api/paycom/sync', route => route.fulfill({ json: { ok: true, data: { + activity: 'idle', desiredState: 'running', lastSucceededAt: null, lastError: null, alerts: [], + } } })); + for (const resource of ['daily', 'employees']) await page.route(`**/api/paycom/${resource}?**`, route => + route.fulfill({ status: 503, json: { ok: false, error: { code: 'not_initialized' } } })); + await page.goto('/#/paycom'); + await page.getByLabel('Email address').fill('owner@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await expect(page).toHaveTitle('Paycom · Dispatch'); + await expect(page.getByText('Verifying your Paycom login.', { exact: false })).toBeVisible(); + verified = true; + await expect(page.getByRole('tab', { name: 'Timecard', exact: true })).toBeVisible({ timeout: 10000 }); + await expect(page.getByText('Paycom is connected. Preparing workforce sync.', { exact: false })).toBeVisible(); + await expect(page.getByText('Verifying your Paycom login.', { exact: false })).toHaveCount(0); + await expect(page.getByRole('button', { name: 'Sync now', exact: true })).toHaveCount(0); + await expect(page.getByText('Waiting for the first Paycom collection', { exact: true })).toBeVisible(); + await page.getByRole('tab', { name: 'Employees', exact: true }).click(); + await expect(page.getByText('Waiting for the first Paycom collection', { exact: true })).toBeVisible(); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBeTruthy(); + await page.screenshot({ path: `/tmp/dispatch-paycom-preparing-${mobile ? 'mobile' : 'desktop'}.png`, fullPage: true }); + // A failed collection setup must not relabel a verified login as disconnected. + setup = { ...setup, status: 'failed', canRetry: true, failureCode: 'runtime_health_failed' }; + await expect(page.getByRole('button', { name: 'Retry workforce setup', exact: true })).toBeVisible(); + await expect(page.getByRole('tab', { name: 'Employees', exact: true })).toBeVisible(); + await page.route('**/api/organization/paycom-setup/retry', route => { + retries++; setup = { ...setup, status: 'queued', canRetry: false, failureCode: null }; + return route.fulfill({ json: { ok: true, data: setup } }); + }); + await page.getByRole('button', { name: 'Retry workforce setup', exact: true }).click(); + await expect(page.getByText('Paycom is connected. Preparing workforce sync.', { exact: false })).toBeVisible(); + expect(retries).toBe(1); + setup = { ...setup, status: 'succeeded' }; + await expect(page.getByRole('button', { name: 'Sync now', exact: true })).toBeVisible(); + await expect(page.getByText('Paycom is connected. Preparing workforce sync.', { exact: false })).toHaveCount(0); + await page.goto('/#/settings?tab=connections'); + await expect(page.getByText('Connected', { exact: true })).toBeVisible(); + await page.goto('/#/paycom'); + await expect(page.getByRole('tab', { name: 'Timecard', exact: true })).toBeVisible(); + await expect(page).toHaveURL(/#\/paycom$/); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/paycom-manual-sync.spec.cjs b/core/dashboard/tests/browser/paycom-manual-sync.spec.cjs new file mode 100644 index 0000000..171a328 --- /dev/null +++ b/core/dashboard/tests/browser/paycom-manual-sync.spec.cjs @@ -0,0 +1,53 @@ +const { test, expect } = require('@playwright/test'); +test.skip(process.env.DISPATCH_PAYCOM_WORKFORCE_FIXTURE !== '1', 'Requires the isolated workforce preview'); + +for (const settings of [false, true]) test(`Sync now remains clickable through authentication, paused schedules and active requests (${settings ? 'settings' : 'workforce'})`, async ({ page }) => { + await page.clock.install(); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + let activity = 'blocked', desiredState = 'running', requests = 0, release; + let lastSucceededAt = '2026-09-11T08:00:00Z'; + await page.route('**/api/organization/paycom-setup', route => route.fulfill({ json: { ok: true, data: { + status: 'succeeded', workforceAvailable: true, canSubmit: false, canRetry: false, failureCode: null, + } } })); + await page.route('**/api/paycom/sync', async route => { + if (route.request().method() === 'POST') { + requests++; + await new Promise(resolve => { release = resolve; }); + activity = 'queued'; + return route.fulfill({ status: 202, json: { ok: true, data: {} } }); + } + return route.fulfill({ json: { ok: true, data: { + activity, desiredState, lastSucceededAt, nextDueAt: null, + lastError: activity === 'blocked' ? 'manual_verification_required' : null, + alerts: activity === 'blocked' ? [{ code: 'authentication_blocked' }] : [], + } } }); + }); + await page.goto(settings ? '/#/paycom?settings' : '/#/paycom'); + await page.getByLabel('Email address').fill('owner@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + if (settings) await page.getByRole('tab', { name: /Sync/ }).click(); + await expect(page).toHaveTitle('Paycom · Dispatch'); + const button = page.getByRole('button', { name: 'Sync now', exact: true }); + await expect(button).toBeEnabled(); + for (const state of ['idle', 'queued', 'syncing', 'waiting_for_capacity', 'stopping', 'backing_off', 'blocked']) { + activity = state; desiredState = 'stopped'; + await page.clock.fastForward(5500); + await expect(button).toBeEnabled(); + } + await button.click(); + await expect.poll(() => requests).toBe(1); + await expect(button).toBeEnabled(); + await button.click(); + await expect(page.getByText('Requesting sync…', { exact: true })).toBeVisible(); + expect(requests).toBe(1); + release(); + await expect(page.getByText('Sync is in progress. Paycom will sign in automatically if needed.', { exact: true })).toBeVisible(); + await expect(button).toBeEnabled(); + activity = 'idle'; lastSucceededAt = '2026-09-11T09:00:00Z'; + await page.clock.fastForward(5500); + await expect(page.getByText('Sync completed.', { exact: true })).toBeVisible(); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/paycom-settings.spec.cjs b/core/dashboard/tests/browser/paycom-settings.spec.cjs new file mode 100644 index 0000000..40378c5 --- /dev/null +++ b/core/dashboard/tests/browser/paycom-settings.spec.cjs @@ -0,0 +1,593 @@ +const { test, expect } = require("@playwright/test"); +const path = require("node:path"); +test.skip( + process.env.DISPATCH_PAYCOM_WORKFORCE_FIXTURE !== "1", + "Requires the isolated workforce preview", +); +async function login(page, email = "owner@example.test") { + await page.route("**/api/organization/paycom-setup", (route) => + route.fulfill({ + json: { + ok: true, + data: { + status: "succeeded", + workforceAvailable: true, + canSubmit: false, + canRetry: false, + failureCode: null, + }, + }, + }), + ); + await page.route("**/api/paycom/sync", (route) => + route.fulfill({ + json: { + ok: true, + data: { + activity: "idle", + desiredState: "running", + lastSucceededAt: "2026-09-11T08:00:00Z", + nextDueAt: "2026-09-11T09:00:00Z", + lastError: null, + alerts: [], + }, + }, + }), + ); + await page.goto("/#/paycom?settings"); + await page.getByLabel("Email address").fill(email); + await page + .getByLabel("Password", { exact: true }) + .fill("synthetic preview password"); + await page.getByRole("button", { name: "Sign in", exact: true }).click(); + await expect( + page.getByRole("button", { name: "Sign in", exact: true }), + ).toHaveCount(0); +} +test("owner settings save, filter Timecards, preserve the full directory, and leave another DSP unchanged", async ({ + page, + browser, +}) => { + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + await login(page); + await expect( + page.getByRole("heading", { name: "Paycom settings", exact: true }), + ).toBeVisible(); + await expect(page).toHaveTitle("Paycom · Dispatch"); + await page + .getByRole("tab", { name: "Driver departments", exact: true }) + .click(); + await page + .getByRole("checkbox", { name: "Include all current and future options" }) + .uncheck(); + await page + .getByRole("checkbox", { name: "Driver 100", exact: true }) + .uncheck(); + await expect( + page.getByRole("status").filter({ hasText: "6 employees" }), + ).toBeVisible(); + await page.getByRole("button", { name: "Save changes", exact: true }).click(); + await expect(page.getByText("Settings saved", { exact: true })).toBeVisible(); + await page.reload(); + await page + .getByRole("tab", { name: "Driver departments", exact: true }) + .click(); + await expect( + page.getByRole("checkbox", { name: "Driver 100", exact: true }), + ).not.toBeChecked(); + await expect( + page.getByRole("checkbox", { name: "Dispatch 6", exact: true }), + ).toBeChecked(); + await page.getByRole("link", { name: "Back", exact: false }).click(); + const table = page.getByRole("table", { name: "Daily employee timecards" }); + await expect(table.locator("tbody tr")).toHaveCount(6); + await page.getByRole("tab", { name: "Employees", exact: true }).click(); + await expect( + page.getByRole("table", { name: "Employee directory" }).locator("tbody tr"), + ).toHaveCount(100); + const other = await browser.newContext({ + baseURL: new URL(page.url()).origin, + }); + const sibling = await other.newPage(); + await login(sibling, "owner5@example.test"); + await sibling.goto("/#/plugins"); + await sibling + .getByRole("button", { name: "Install Paycom", exact: true }) + .click(); + await expect( + sibling.getByRole("link", { name: "Open Paycom", exact: true }), + ).toBeVisible(); + await sibling.goto("/#/paycom?settings"); + await sibling + .getByRole("tab", { name: "Driver departments", exact: true }) + .click(); + await expect( + sibling.getByRole("checkbox", { + name: "Include all current and future options", + }), + ).toBeChecked(); + await other.close(); + await page.goto("/#/paycom?settings"); + await page + .getByRole("tab", { name: "Driver departments", exact: true }) + .click(); + await page + .getByRole("checkbox", { name: "Dispatch 6", exact: true }) + .uncheck(); + await page.getByRole("button", { name: "Save changes", exact: true }).click(); + await expect(page.getByText("Settings saved", { exact: true })).toBeVisible(); + await page.getByRole("link", { name: "Back", exact: false }).click(); + await expect( + page.getByText("No driver departments selected", { exact: true }), + ).toBeVisible(); + // Restore shared fixture defaults for other browser specifications. + await page.goto("/#/paycom?settings"); + await page + .getByRole("button", { name: "Restore defaults", exact: true }) + .click(); + await page + .getByRole("dialog") + .getByRole("button", { name: "Restore defaults", exact: true }) + .click(); + await page.getByRole("button", { name: "Save changes", exact: true }).click(); + await expect(page.getByText("Settings saved", { exact: true })).toBeVisible(); + expect(errors).toEqual([]); +}); +for (const mobile of [false, true]) + test(`settings layout and workspace preferences (${mobile ? "mobile" : "desktop"})`, async ({ + page, + }) => { + await page.setViewportSize( + mobile ? { width: 390, height: 844 } : { width: 1440, height: 1000 }, + ); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + await login(page); + await page + .getByRole("tab", { name: "Workspace view", exact: true }) + .click(); + await page.getByLabel("Rows per page", { exact: true }).selectOption("25"); + await page + .getByRole("checkbox", { name: "Lunch out", exact: true }) + .uncheck(); + await page + .getByRole("button", { name: "Save changes", exact: true }) + .click(); + await expect( + page.getByText("Settings saved", { exact: true }), + ).toBeVisible(); + expect( + await page.evaluate( + () => document.documentElement.scrollWidth <= innerWidth, + ), + ).toBe(true); + await page.evaluate(() => window.scrollTo(0, 0)); + await page.screenshot({ + path: path.join( + process.env.DISPATCH_UI_ARTIFACTS || "/tmp", + `paycom-settings-${mobile ? "mobile" : "desktop"}.png`, + ), + fullPage: true, + }); + await page.getByRole("link", { name: "Back", exact: false }).click(); + const table = page.getByRole("table", { name: "Daily employee timecards" }); + await expect(table.locator("tbody tr")).toHaveCount(25); + await expect( + table.getByRole("columnheader", { name: /Lunch out/ }), + ).toHaveCount(0); + await page.goto("/#/paycom?settings"); + await page + .getByRole("button", { name: "Restore defaults", exact: true }) + .click(); + await page + .getByRole("dialog") + .getByRole("button", { name: "Restore defaults", exact: true }) + .click(); + await page + .getByRole("button", { name: "Save changes", exact: true }) + .click(); + await expect( + page.getByText("Settings saved", { exact: true }), + ).toBeVisible(); + expect(errors).toEqual([]); + }); + +test("an open Timecard page observes department changes from another session", async ({ + page, +}) => { + await login(page); + const snapshot = await ( + await page.request.get("/api/organization/plugins/paycom/settings") + ).json(); + const session = await (await page.request.get("/api/auth/session")).json(); + await page.getByRole("link", { name: "Back", exact: false }).click(); + await expect( + page + .getByRole("table", { name: "Daily employee timecards" }) + .locator("tbody tr"), + ).toHaveCount(100); + const response = await page.request.post( + "/api/organization/plugins/paycom/settings", + { + headers: { + "x-dispatch-csrf": session.csrfToken || session.data?.csrfToken, + }, + data: { + values: { ...snapshot.data.values, driver_departments: [] }, + expectedRevision: snapshot.data.revision, + definitionVersion: snapshot.data.definitionVersion, + idempotencyKey: "browser:remote-departments", + }, + }, + ); + expect(response.status()).toBe(200); + await expect( + page.getByText("No driver departments selected", { exact: true }), + ).toBeVisible({ timeout: 20000 }); + await page.goto("/#/paycom?settings"); + await page + .getByRole("button", { name: "Restore defaults", exact: true }) + .click(); + await page + .getByRole("dialog") + .getByRole("button", { name: "Restore defaults", exact: true }) + .click(); + await page.getByRole("button", { name: "Save changes", exact: true }).click(); + await expect(page.getByText("Settings saved", { exact: true })).toBeVisible(); +}); + +for (const mobile of [false, true]) + test(`name order saves, sorts, refreshes and stays scoped to one DSP ${mobile ? "mobile" : "desktop"}`, async ({ + page, + browser, + }) => { + test.skip( + process.env.DISPATCH_PAYCOM_NAME_ORDER_FIXTURE !== "1", + "Requires canonical synthetic Paycom names", + ); + test.setTimeout(45000); + await page.setViewportSize( + mobile ? { width: 390, height: 844 } : { width: 1440, height: 1000 }, + ); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + await login(page); + await page + .getByRole("tab", { name: "Workspace view", exact: true }) + .click(); + await expect(page.getByLabel("Name order", { exact: true })).toHaveValue( + "first_last", + ); + await expect( + page.getByText("Name preview: JANE DOE", { exact: true }), + ).toBeVisible(); + await expect( + page + .getByLabel("Name order", { exact: true }) + .locator('option[value="first_last"]'), + ).toHaveText("First Last"); + await page + .getByLabel("Name order", { exact: true }) + .selectOption("last_first"); + await expect( + page.getByText("Name preview: DOE, JANE", { exact: true }), + ).toBeVisible(); + await page + .getByRole("button", { name: "Save changes", exact: true }) + .click(); + await expect( + page.getByText("Settings saved", { exact: true }), + ).toBeVisible(); + await page.reload(); + await page + .getByRole("tab", { name: "Workspace view", exact: true }) + .click(); + await expect(page.getByLabel("Name order", { exact: true })).toHaveValue( + "last_first", + ); + await expect(page.getByLabel("Rows per page", { exact: true })).toHaveValue( + "100", + ); + expect( + await page.evaluate( + () => document.documentElement.scrollWidth <= innerWidth, + ), + ).toBe(true); + await page.screenshot({ + path: path.join( + process.env.DISPATCH_UI_ARTIFACTS || "/tmp", + `name-order-${mobile ? "mobile" : "desktop"}.png`, + ), + fullPage: true, + }); + await page.getByRole("link", { name: "Back", exact: false }).click(); + const timecards = page.getByRole("table", { + name: "Daily employee timecards", + }); + await expect( + timecards.locator("tbody tr").first().locator("td").first(), + ).toHaveText("AVERY, ZULU"); + await page.getByRole("button", { name: "Next", exact: true }).click(); + await expect(timecards.locator("tbody tr")).toHaveCount(6); + await expect( + timecards.locator("tbody tr").last().locator("td").first(), + ).toHaveText("THOMPSON, MIA"); + await page.getByRole("tab", { name: "Employees", exact: true }).click(); + await page + .getByLabel("Find employee", { exact: true }) + .fill("THOMPSON, MIA"); + await page + .getByRole("button", { name: "THOMPSON, MIA", exact: true }) + .click(); + await expect( + page.getByRole("heading", { name: "THOMPSON, MIA", exact: true }), + ).toBeVisible(); + const other = await browser.newContext({ + baseURL: new URL(page.url()).origin, + }); + try { + const sibling = await other.newPage(); + await login(sibling, "owner5@example.test"); + await sibling.goto("/#/plugins"); + const install = sibling.getByRole("button", { + name: "Install Paycom", + exact: true, + }); + if (await install.count()) await install.click(); + await expect( + sibling.getByRole("link", { name: "Open Paycom", exact: true }), + ).toBeVisible(); + await sibling.goto("/#/paycom?settings"); + await sibling + .getByRole("tab", { name: "Workspace view", exact: true }) + .click(); + await expect( + sibling.getByLabel("Name order", { exact: true }), + ).toHaveValue("first_last"); + await sibling.getByRole("link", { name: "Back", exact: false }).click(); + await expect( + sibling + .getByRole("table", { name: "Daily employee timecards" }) + .locator("tbody tr") + .first() + .locator("td") + .first(), + ).toHaveText("ETHAN RIVERA"); + } finally { + await other.close(); + } + await page.getByRole("tab", { name: "Timecard", exact: true }).click(); + await expect( + timecards.locator("tbody tr").first().locator("td").first(), + ).toHaveText("AVERY, ZULU"); + const snapshot = ( + await ( + await page.request.get("/api/organization/plugins/paycom/settings") + ).json() + ).data; + const session = (await (await page.request.get("/api/auth/session")).json()) + .data; + const response = await page.request.post( + "/api/organization/plugins/paycom/settings", + { + headers: { "x-dispatch-csrf": session.csrfToken }, + data: { + values: { ...snapshot.values, name_order: "first_last" }, + expectedRevision: snapshot.revision, + definitionVersion: snapshot.definitionVersion, + idempotencyKey: `browser:remote-name-${mobile}`, + }, + }, + ); + expect(response.status()).toBe(200); + await expect( + timecards.locator("tbody tr").first().locator("td").first(), + ).toHaveText("ETHAN RIVERA", { timeout: 20000 }); + await page.getByRole("button", { name: "Next", exact: true }).click(); + await expect(timecards.locator("tbody tr")).toHaveCount(6); + await expect( + timecards.locator("tbody tr").last().locator("td").first(), + ).toHaveText("ZULU AVERY"); + await page.getByRole("tab", { name: "Employees", exact: true }).click(); + await page + .getByLabel("Find employee", { exact: true }) + .fill("MIA THOMPSON"); + await page + .getByRole("button", { name: "MIA THOMPSON", exact: true }) + .click(); + await expect( + page.getByRole("heading", { name: "MIA THOMPSON", exact: true }), + ).toBeVisible(); + expect(errors).toEqual([]); + }); + +for (const mobile of [false, true]) + test(`smarter settings preserve intent, dependencies and restore drafts (${mobile ? "mobile" : "desktop"})`, async ({ + page, + }) => { + test.skip( + process.env.DISPATCH_SMART_SETTINGS_FIXTURE !== "1", + "Requires the smarter settings scenario", + ); + test.setTimeout(60000); + await page.setViewportSize( + mobile ? { width: 390, height: 844 } : { width: 1440, height: 1000 }, + ); + const errors = []; + page.on("pageerror", (error) => errors.push(error.message)); + await login(page); + const save = async () => { + await page + .getByRole("button", { name: "Save changes", exact: true }) + .click(); + await expect( + page.getByText("Settings saved", { exact: true }), + ).toBeVisible(); + }; + await page + .getByRole("button", { name: "Restore defaults", exact: true }) + .click(); + await page + .getByRole("dialog") + .getByRole("button", { name: "Restore defaults", exact: true }) + .click(); + if ( + await page + .getByRole("button", { name: "Save changes", exact: true }) + .isEnabled() + ) + await save(); + const startingRevision = ( + await ( + await page.request.get("/api/organization/plugins/paycom/settings") + ).json() + ).data.revision; + await page + .getByRole("tab", { name: "Workspace view", exact: true }) + .click(); + await expect( + page.getByText("Name preview: JANE DOE", { exact: true }), + ).toBeVisible(); + await page + .getByRole("button", { + name: "Keep current value for Name order", + exact: true, + }) + .click(); + await save(); + await expect( + page.getByRole("button", { + name: "Use plugin default for Name order", + exact: true, + }), + ).toBeVisible(); + await page.getByRole("tab", { name: "Sync schedule", exact: true }).click(); + await page.getByLabel("Sync every", { exact: true }).selectOption("7200"); + await page + .getByRole("switch", { name: "Automatic sync", exact: true }) + .uncheck(); + await expect(page.getByLabel("Sync every", { exact: true })).toBeDisabled(); + await expect(page.getByLabel("Sync every", { exact: true })).toHaveValue( + "7200", + ); + await save(); + await expect( + page.getByText( + "The schedule is updated after saving. Running collections are allowed to finish.", + { exact: true }, + ), + ).toBeVisible(); + await page.reload(); + await expect( + page.getByRole("switch", { name: "Automatic sync", exact: true }), + ).not.toBeChecked(); + await expect(page.getByLabel("Sync every", { exact: true })).toHaveValue( + "7200", + ); + await page + .getByRole("button", { + name: "Restore Sync schedule defaults", + exact: true, + }) + .click(); + await expect( + page.getByRole("switch", { name: "Automatic sync", exact: true }), + ).toBeChecked(); + await expect(page.getByLabel("Sync every", { exact: true })).toHaveValue( + "3600", + ); + await save(); + await page + .getByRole("tab", { name: "Workspace view", exact: true }) + .click(); + await expect( + page.getByRole("button", { + name: "Use plugin default for Name order", + exact: true, + }), + ).toBeVisible(); + await page + .getByRole("button", { name: "Change history", exact: true }) + .click(); + const history = page.getByRole("region", { + name: "Settings change history", + }); + const initial = history.locator( + `:scope > details[data-revision="${startingRevision}"]`, + ); + await expect(initial).toBeVisible(); + await initial.locator(":scope > summary").click(); + await initial + .getByText("Restore an individual setting", { exact: true }) + .click(); + await initial + .getByRole("button", { name: "Restore Name order", exact: true }) + .click(); + await expect( + page.getByText( + "Restored into your draft. Review your changes before saving.", + { exact: true }, + ), + ).toBeVisible(); + await expect( + page.getByRole("button", { + name: "Keep current value for Name order", + exact: true, + }), + ).toBeVisible(); + await save(); + await page + .getByRole("button", { name: "Change history", exact: true }) + .click(); + await page + .getByLabel("Default department", { exact: true }) + .selectOption({ label: "Driver" }); + await page + .getByRole("tab", { name: "Driver departments", exact: true }) + .click(); + const all = page.getByRole("checkbox", { + name: "Include all current and future options", + }); + await expect( + page.getByRole("checkbox", { name: "Driver 100", exact: true }), + ).toBeDisabled(); + await all.uncheck(); + await page + .getByRole("checkbox", { name: "Driver 100", exact: true }) + .uncheck(); + await expect( + page.getByText( + "Your default department is excluded from Timecards. Choose an included department or update Driver departments.", + { exact: true }, + ), + ).toBeVisible(); + await expect( + page.getByRole("status").filter({ hasText: "6 employees" }), + ).toBeVisible(); + await page.getByRole("button", { name: "Discard", exact: true }).click(); + await expect( + page.getByRole("checkbox", { + name: "Include all current and future options", + }), + ).toBeChecked(); + await expect( + page.getByRole("button", { name: "Save changes", exact: true }), + ).toBeDisabled(); + expect( + await page.evaluate( + () => document.documentElement.scrollWidth <= innerWidth, + ), + ).toBe(true); + await page + .getByRole("tab", { name: "Workspace view", exact: true }) + .click(); + await page.screenshot({ + path: path.join( + process.env.DISPATCH_UI_ARTIFACTS || "/tmp", + `smarter-settings-${mobile ? "mobile" : "desktop"}.png`, + ), + fullPage: true, + }); + expect(errors).toEqual([]); + }); diff --git a/core/dashboard/tests/browser/paycom-workforce.spec.cjs b/core/dashboard/tests/browser/paycom-workforce.spec.cjs new file mode 100644 index 0000000..c860bb2 --- /dev/null +++ b/core/dashboard/tests/browser/paycom-workforce.spec.cjs @@ -0,0 +1,203 @@ +const { test, expect } = require('@playwright/test'); +test.skip(process.env.DISPATCH_PAYCOM_WORKFORCE_FIXTURE !== '1', 'Requires the isolated workforce preview fixture'); +async function login(page, connection = {status:'succeeded',workforceAvailable:false,canSubmit:false,canRetry:false,failureCode:null}) { + await page.route('**/api/organization/paycom-setup', route => route.fulfill({json:{ok:true,data:connection}})); + await page.goto('/#/paycom'); + await page.getByLabel('Email address').fill('owner@example.test'); + await page.getByLabel('Password',{exact:true}).fill('synthetic preview password'); + await page.getByRole('button',{name:'Sign in',exact:true}).click(); + await expect(page.getByRole('tab',{name:'Timecard',exact:true})).toBeVisible(); +} + +test('first collection appears automatically in the connected workspace', async ({ page }) => { + const waiting = { daily: true, employees: true }; + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { + const expectedEmpty = message.text().includes('503') && + /\/api\/paycom\/(daily|employees)(?:\?|$)/.test(message.location().url); + if (message.type() === 'error' && !expectedEmpty) errors.push(message.text()); + }); + await page.clock.install(); + await page.route('**/api/paycom/sync', route => route.fulfill({ json: { ok: true, data: { + activity: 'idle', desiredState: 'running', lastSucceededAt: null, lastError: null, alerts: [], + } } })); + for (const resource of ['daily', 'employees']) { + await page.route(`**/api/paycom/${resource}?**`, route => waiting[resource] + ? route.fulfill({ status: 503, json: { ok: false, data: null, error: { code: 'not_initialized' } } }) + : route.fallback()); + } + await login(page); + await expect(page.getByText('Waiting for the first Paycom collection', { exact: true })).toBeVisible(); + waiting.daily = false; + await page.clock.fastForward(31000); + await expect(page.getByRole('table', { name: 'Daily employee timecards' })).toBeVisible(); + await page.getByRole('tab', { name: 'Employees', exact: true }).click(); + await expect(page.getByText('Waiting for the first Paycom collection', { exact: true })).toBeVisible(); + waiting.employees = false; + await page.clock.fastForward(31000); + await expect(page.getByRole('table', { name: 'Employee directory' })).toBeVisible(); + await expect(page.getByText('Waiting for the first Paycom collection', { exact: true })).toHaveCount(0); + await expect(page.getByText('Your Paycom account is connected', { exact: true })).toHaveCount(0); + expect(errors).toEqual([]); +}); +test('daily dates, global column sorting, pagination, and employee timecards', async ({page})=>{ + const errors=[]; page.on('pageerror',e=>errors.push(e.message)); + await login(page); + await expect(page).toHaveTitle('Paycom · Dispatch'); + await expect(page.getByRole('tab')).toHaveCount(2); + await expect(page.getByRole('tab',{name:'Overview'})).toHaveCount(0); + const table=page.getByRole('table',{name:'Daily employee timecards'}); + await expect(table.locator('tbody tr').first()).toContainText('Ethan Rivera'); + await expect(table).not.toContainText('DXY1'); await expect(table).not.toContainText('Delivery Driver'); + await page.getByRole('button',{name:'Sort Employee descending',exact:true}).click(); + await expect(table.locator('tbody tr').first()).toContainText('Zulu Avery'); + await page.getByRole('button',{name:'Sort Clock in ascending',exact:true}).click(); + await expect(table.locator('tbody tr').first()).toContainText('Ethan Rivera'); + await page.getByRole('button',{name:'Next',exact:true}).click(); + await expect(table.locator('tbody tr').last()).toContainText('Olivia Brooks'); + await page.getByRole('button',{name:'Sort Hours ascending',exact:true}).click(); + await expect(table.locator('tbody tr').first()).toContainText('2.00'); + for (const label of ['Lunch out','Lunch in','Clock out','Punch status']) { + await page.getByRole('button',{name:`Sort ${label} ascending`,exact:true}).click(); + await expect(page.getByRole('columnheader').filter({has:page.getByRole('button',{name:`Sort ${label} descending`,exact:true})})).toHaveAttribute('aria-sort','ascending'); + } + const current=await page.getByLabel('Date',{exact:true}).inputValue(); + await page.getByRole('button',{name:'Previous day',exact:true}).click(); + await expect(page.getByLabel('Date',{exact:true})).not.toHaveValue(current); + await page.getByRole('button',{name:'Sort Hours ascending',exact:true}).click(); + await expect(table.locator('tbody tr').first()).toContainText('10.00'); + await page.getByLabel('Date',{exact:true}).fill('2020-01-01'); + await expect(page.getByText('No saved timecards for this date')).toBeVisible(); + await page.getByRole('button',{name:'Today',exact:true}).click(); + await expect(page.getByLabel('Date',{exact:true})).toHaveValue(current); + await page.getByRole('tab',{name:'Employees',exact:true}).click(); + await page.getByLabel('Find employee').fill('Mia'); + await page.getByRole('button',{name:'Mia Thompson',exact:true}).click(); + await expect(page.getByRole('table',{name:'Employee period timecard'}).locator('tbody tr')).toHaveCount(14); + await expect(page.getByRole('tab',{name:'Employees',exact:true})).toHaveAttribute('aria-selected','true'); + await expect(page.getByRole('link',{name:'Open in Paycom'})).toHaveAttribute('href',/firstrefno=W000/); + await page.getByRole('button',{name:'Back to employees'}).click(); + await expect(page.getByLabel('Find employee')).toHaveValue('Mia'); + await page.getByLabel('Find employee').fill('No such employee'); + await expect(page.getByText('No matching employees')).toBeVisible(); + expect(errors).toEqual([]); +}); +for(const mobile of [false,true]) test(`workforce ${mobile?'mobile':'desktop'} layout and screenshot`,async({page})=>{ + await page.setViewportSize(mobile?{width:390,height:844}:{width:1440,height:1000}); + const errors=[];page.on('pageerror',e=>errors.push(e.message)); + await login(page); + await expect(page.getByRole('table',{name:'Daily employee timecards'})).toBeVisible(); + expect(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth)).toBe(true); + await page.screenshot({path:`/tmp/paycom-workforce-${mobile?'mobile':'desktop'}.png`,fullPage:false}); + expect(errors).toEqual([]); +}); + +test('collection status distinguishes capacity, activity, authentication, and last success', async ({ page }) => { + let activity = 'waiting_for_capacity'; + let alerts = []; + await page.route('**/api/paycom/sync', route => route.fulfill({ json: { ok: true, data: { + activity, desiredState: 'running', lastSucceededAt: '2026-09-08T08:00:00Z', lastError: null, alerts, + } } })); + const errors = []; page.on('pageerror', error => errors.push(error.message)); + await login(page); + await expect(page.getByRole('status', { name: 'Paycom sync' })).toContainText('Waiting for capacity'); + await expect(page.getByRole('status', { name: 'Paycom sync' })).toContainText('Last successful sync'); + activity = 'syncing'; + await expect(page.getByRole('status', { name: 'Paycom sync' })).toContainText('Collecting', { timeout: 10000 }); + activity = 'blocked'; alerts = [{ code: 'authentication_blocked' }]; + await expect(page.getByRole('status', { name: 'Paycom sync' })).toContainText('Needs authentication', { timeout: 10000 }); + expect(errors).toEqual([]); + await page.screenshot({ path: '/tmp/dispatch-fleet-sync-status.png', fullPage: false }); +}); + +for (const mobile of [false, true]) test(`blocked sync preserves saved workforce and observes recovery (${mobile ? 'mobile' : 'desktop'})`, async ({ page }) => { + await page.setViewportSize(mobile ? { width: 390, height: 844 } : { width: 1440, height: 1000 }); + await page.clock.install(); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + let blocked = true; + let writes = 0; + let employeeReads = 0; + await page.route('**/api/paycom/employees?**', route => { employeeReads++; return route.fallback(); }); + await page.route('**/api/paycom/sync', route => { + if (route.request().method() !== 'GET') { writes++; blocked = false; } + return route.fulfill({ json: { ok: true, data: { + activity: blocked ? 'blocked' : 'idle', desiredState: 'running', + lastSucceededAt: blocked ? '2026-09-08T18:00:00Z' : '2026-09-08T18:30:00Z', + nextDueAt: '2026-09-08T19:00:00Z', + lastError: blocked ? 'manual_verification_required' : null, + alerts: blocked ? [{ code: 'authentication_blocked' }] : [], + } } }); + }); + // Historical data must stay accessible even if the current setup needs attention. + await login(page, { status: 'failed', workforceAvailable: true, canSubmit: false, canRetry: false, + failureCode: 'manual_verification_required', retryState: 'manual' }); + const status = page.getByRole('status', { name: 'Paycom sync' }); + await expect(page).toHaveTitle('Paycom · Dispatch'); + await expect(status).toContainText('Click Sync now to sign in to Paycom and sync your data.'); + await expect(status).toContainText('Previously synced data remains available below.'); + await expect(status).toContainText('Last successful sync'); + await expect(status.getByText(/Next scheduled sync/)).toHaveCount(0); + await expect(page.getByRole('button', { name: 'Sync now', exact: true })).toBeEnabled(); + await expect(page.getByRole('table', { name: 'Daily employee timecards' })).toBeVisible(); + await page.getByRole('tab', { name: 'Employees', exact: true }).click(); + await expect(page.getByRole('table', { name: 'Employee directory' })).toBeVisible(); + await page.clock.fastForward(11000); + expect(writes).toBe(0); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.screenshot({ path: `/tmp/paycom-blocked-sync-${mobile ? 'mobile' : 'desktop'}.png`, fullPage: false }); + const before = employeeReads; + // The enabled manual action requests recovery without leaving the workforce page. + await page.getByRole('button', { name: 'Sync now', exact: true }).click(); + await page.clock.fastForward(6000); + await expect(status).toContainText('Waiting for next sync'); + await expect(status.getByText(/Contact the Platform Owner/)).toHaveCount(0); + await expect(status.getByText(/Next scheduled sync/)).toBeVisible(); + await expect(page.getByRole('button', { name: 'Sync now', exact: true })).toBeEnabled(); + await expect.poll(() => employeeReads).toBeGreaterThan(before); + expect(writes).toBe(1); + expect(errors).toEqual([]); +}); + +test('manual sync queues once, preserves the scheduled time, and refreshes employee data after success', async ({ page }) => { + let activity = 'idle'; + let lastSucceededAt = '2026-09-08T18:00:00.000Z'; + let requests = 0; + let employeeReads = 0; + const nextDueAt = '2026-09-08T19:00:00.000Z'; + await page.clock.install(); + await page.route('**/api/paycom/sync', async route => { + if (route.request().method() === 'POST') { + requests++; + expect(Object.keys(route.request().postDataJSON())).toEqual(['idempotencyKey']); + expect(route.request().headers()['x-dispatch-csrf']).toBeTruthy(); + activity = 'queued'; + return route.fulfill({ status: 202, json: { ok: true, status: 'queued', data: {} } }); + } + return route.fulfill({ json: { ok: true, data: { + activity, desiredState: 'running', lastSucceededAt, nextDueAt, lastError: null, alerts: [], + } } }); + }); + await page.route('**/api/paycom/employees?**', route => { employeeReads++; return route.fallback(); }); + await login(page); + await page.getByRole('tab', { name: 'Employees', exact: true }).click(); + await expect(page.getByRole('table', { name: 'Employee directory' })).toBeVisible(); + const before = employeeReads; + const status = page.getByRole('status', { name: 'Paycom sync' }); + const scheduledText = await status.getByText(/Next scheduled sync/).textContent(); + await page.getByRole('button', { name: 'Sync now', exact: true }).click(); + await expect(status).toContainText('Queued'); + await expect(page.getByRole('button', { name: 'Sync now', exact: true })).toBeEnabled(); + expect(requests).toBe(1); + await page.getByRole('button', { name: 'Sync now', exact: true }).click(); + await expect.poll(() => requests).toBe(2); + await expect(status).toContainText('Sync is in progress.'); + expect(employeeReads).toBe(before); + activity = 'idle'; lastSucceededAt = '2026-09-08T18:10:00.000Z'; + await page.clock.fastForward(5500); + await expect(page.getByRole('button', { name: 'Sync now', exact: true })).toBeEnabled(); + await expect.poll(() => employeeReads).toBeGreaterThan(before); + await expect(status).toContainText(scheduledText); +}); diff --git a/core/dashboard/tests/browser/plugins.spec.cjs b/core/dashboard/tests/browser/plugins.spec.cjs new file mode 100644 index 0000000..64b04d7 --- /dev/null +++ b/core/dashboard/tests/browser/plugins.spec.cjs @@ -0,0 +1,65 @@ +const { test, expect } = require('@playwright/test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +async function login(page, email) { + await page.goto('/#/plugins'); + await page.getByLabel('Email address').fill(email); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await expect(page).toHaveTitle('Plugins · Dispatch'); +} +for (const mobile of [false, true]) test(`Paycom installation stays within its DSP (${mobile ? 'mobile' : 'desktop'})`, async ({ page, browser }) => { + if (mobile) await page.setViewportSize({ width: 390, height: 844 }); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + await login(page, mobile ? 'owner7@example.test' : 'owner6@example.test'); + await expect(page.getByText('Not installed', { exact: true })).toBeVisible(); + await page.goto('/#/settings?tab=connections'); + await expect(page.getByRole('button', { name: 'Connect Cortex', exact: true })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Connect Paycom', exact: true })).toHaveCount(0); + expect((await page.request.get('/api/paycom/daily')).status()).toBe(409); + await page.goto('/#/plugins'); + await page.getByRole('button', { name: 'Install Paycom', exact: true }).click(); + await expect(page.getByRole('link', { name: 'Open Paycom', exact: true })).toBeVisible(); + await page.getByRole('link', { name: 'Open Paycom', exact: true }).click(); + await expect(page).toHaveTitle('Paycom · Dispatch'); + await expect(page.getByRole('button', { name: 'Connect Paycom', exact: true })).toBeVisible(); + await page.getByRole('button', { name: 'Connect Paycom', exact: true }).click(); + await page.getByRole('button', { name: 'Connect Paycom', exact: true }).click(); + const dialog = page.getByRole('dialog'); + await dialog.getByLabel('Client code').fill('fixture-client'); + await dialog.getByLabel('Username', { exact: true }).fill('fixture-plugin-user'); + await dialog.getByLabel('Password', { exact: true }).fill('synthetic plugin connection'); + for (let index = 1; index <= 5; index++) await dialog.getByLabel(`Security answer ${index}`).fill(`fixture-answer-${index}`); + await dialog.getByRole('button', { name: 'Save and connect' }).click(); + await expect(dialog).toHaveCount(0); + await page.goto('/#/plugins'); + const output = process.env.DISPATCH_UI_ARTIFACTS || path.join(os.tmpdir(), 'dispatch-plugin-ui'); + fs.mkdirSync(output, { recursive: true }); + await page.screenshot({ path: path.join(output, `plugins-${mobile ? 'mobile' : 'desktop'}-installed.png`), fullPage: true }); + await page.getByRole('button', { name: 'Disable', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Enable', exact: true })).toBeEnabled(); + expect((await page.request.get('/api/paycom/daily')).status()).toBe(409); + const hidden = await (await page.request.get('/api/organization/connections')).json(); + expect(hidden.data.items.some(item => item.service === 'paycom')).toBe(false); + await page.getByRole('button', { name: 'Enable', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Disable', exact: true })).toBeEnabled(); + const retained = await (await page.request.get('/api/organization/connections')).json(); + expect(retained.data.items.find(item => item.service === 'paycom').configured).toBe(true); + await page.getByRole('button', { name: 'Uninstall', exact: true }).click(); + await expect(page.getByRole('dialog')).toContainText('saved credentials will stay'); + await page.getByRole('button', { name: 'Uninstall plugin', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Install Paycom', exact: true })).toBeEnabled(); + expect((await page.request.get('/api/paycom/daily')).status()).toBe(409); + const siblingContext = await browser.newContext({ baseURL: new URL(page.url()).origin }); + try { + const sibling = await siblingContext.newPage(); + await login(sibling, 'owner@example.test'); + await expect(sibling.getByRole('link', { name: 'Open Paycom', exact: true })).toBeVisible(); + } finally { await siblingContext.close(); } + await page.screenshot({ path: path.join(output, `plugins-${mobile ? 'mobile' : 'desktop'}-uninstalled.png`), fullPage: true }); + expect(await page.locator('body').evaluate(body => body.scrollWidth <= window.innerWidth)).toBe(true); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/release-popup.spec.cjs b/core/dashboard/tests/browser/release-popup.spec.cjs new file mode 100644 index 0000000..c733bed --- /dev/null +++ b/core/dashboard/tests/browser/release-popup.spec.cjs @@ -0,0 +1,122 @@ +const { test: base, expect } = require('@playwright/test'); +const { spawn } = require('node:child_process'); +const path = require('node:path'); +const test = base.extend({ + popupServer: async ({}, use) => { + const child = spawn(process.execPath, ['--no-warnings', 'examples/frontend-preview.js'], { + cwd: path.resolve(__dirname, "../.."), + env: { ...process.env, DISPATCH_FRONTEND_FIXTURE: '1', DISPATCH_POPUP_FIXTURE: '1', DISPATCH_FRONTEND_PORT: '0' }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let diagnostics = ''; + child.stderr.on('data', data => { diagnostics += data; }); + try { + const url = await new Promise((resolve, reject) => { + let output = ''; + const timeout = setTimeout(() => reject(Error('Popup fixture startup timed out: ' + diagnostics)), 15000); + child.once('exit', code => { clearTimeout(timeout); reject(Error('Fixture exit ' + code + ': ' + diagnostics)); }); + child.stdout.on('data', data => { + output += data; + const match = output.match(/Synthetic frontend preview: (http:\/\/127\.0\.0\.1:\d+)/); + if (match) { clearTimeout(timeout); resolve(match[1]); } + }); + }); + await use(url); + } finally { + if (child.exitCode === null) { + const exited = new Promise(resolve => child.once('exit', resolve)); + child.kill('SIGTERM'); + await exited; + } + } + }, + baseURL: async ({ popupServer }, use) => use(popupServer), +}); +async function login(page, email = 'platform@example.test', url = '/') { + await page.goto(url); + await page.getByLabel('Email address').fill(email); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await expect(page.getByRole('button', { name: 'Sign in', exact: true })).toHaveCount(0); +} +test('platform popup shows latest release; Got it persists across refresh and a fresh browser session', async ({ page, browser, popupServer }) => { + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + await login(page); + const dialog = page.getByRole('dialog'); + await expect(page).toHaveTitle('DSPs · Dispatch'); + await expect(dialog.getByRole('heading', { name: 'Dispatch 0.0.9' })).toBeFocused(); + await expect(dialog.getByRole('heading', { name: 'New', exact: true })).toBeVisible(); + await expect(dialog.getByRole('heading', { name: 'Improved', exact: true })).toBeVisible(); + await expect(dialog.getByRole('heading', { name: 'Fixed', exact: true })).toBeVisible(); + await expect(dialog.getByText('Choose backup schedules by scope')).toBeVisible(); + await page.mouse.click(10, 10); + await expect(dialog).toBeVisible(); + await page.screenshot({ path: '/tmp/dispatch-release-popup-desktop.png' }); + await dialog.getByRole('button', { name: 'Got it' }).click(); + await expect(dialog).toHaveCount(0); + await page.reload(); + await expect(page.getByRole('navigation', { name: 'Primary navigation' })).toBeVisible(); + expect((await (await page.request.get('/api/updates/popup')).json()).data.release).toBeNull(); + await expect(dialog).toHaveCount(0); + const context = await browser.newContext(); + try { + const other = await context.newPage(); + await login(other, 'platform@example.test', popupServer); + expect((await (await other.request.get(popupServer + '/api/updates/popup')).json()).data.release).toBeNull(); + await expect(other.getByRole('dialog')).toHaveCount(0); + } finally { await context.close(); } + expect(errors).toEqual([]); +}); +test('DSP popup is clean and scoped; X dismissal persists on mobile', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await login(page, 'owner@example.test'); + const dialog = page.getByRole('dialog'); + await expect(dialog).toBeVisible(); + await expect(dialog.getByText('See activity at a glance')).toBeVisible(); + await expect(dialog.getByText('Team search finds every member')).toBeVisible(); + const payload = (await (await page.request.get('/api/updates/popup')).json()).data; + expect(JSON.stringify(payload)).not.toMatch(/backup|audience|sourceCommit/i); + await expect(dialog.getByRole('heading', { name: 'Improved', exact: true })).toHaveCount(0); + const box = await dialog.boundingBox(); + expect(box.x).toBeGreaterThanOrEqual(0); + expect(box.x + box.width).toBeLessThanOrEqual(390); + expect(box.y + box.height).toBeLessThanOrEqual(844); + await page.screenshot({ path: '/tmp/dispatch-release-popup-mobile.png' }); + await dialog.getByRole('button', { name: 'Close update' }).click(); + await expect(dialog).toHaveCount(0); + await page.reload(); + await expect(page.getByRole('heading', { name: 'Currently under development', exact: true })).toBeVisible(); + expect((await (await page.request.get('/api/updates/popup')).json()).data.release).toBeNull(); +}); +test('Escape saves dismissal; failed save stays visible with retry and long copy scrolls', async ({ page }) => { + let failed = false; + await page.route('**/api/updates/popup', async route => { + if (route.request().method() === 'POST' && !failed) { + failed = true; + return route.fulfill({ status: 503, json: { ok: false, error: { code: 'request_failed' } } }); + } + if (route.request().method() === 'GET') { + const response = await route.fetch(); + const body = await response.json(); + if (body.data.release) body.data.release.changelog = Array.from({ length: 30 }, (_, i) => ({ + kind: 'fixed', title: `Sample fix ${i + 1}`, description: 'A detailed description of the update for this scrolling fixture.', + })); + return route.fulfill({ response, json: body }); + } + return route.continue(); + }); + await login(page); + const dialog = page.getByRole('dialog'); + await expect(dialog).toBeVisible(); + await expect(dialog.getByRole('button', { name: 'Got it' })).toBeInViewport(); + await expect(dialog.getByRole('button', { name: 'Close update' })).toBeInViewport(); + const scroll = await dialog.locator('.release-popup-body').evaluate(el => el.scrollHeight > el.clientHeight); + expect(scroll).toBe(true); + await page.keyboard.press('Escape'); + await expect(dialog.getByRole('alert')).toHaveText('We couldn’t save your dismissal. Please try again.'); + await dialog.getByRole('button', { name: 'Try again' }).click(); + await expect(dialog).toHaveCount(0); + expect((await (await page.request.get('/api/updates/popup')).json()).data.release).toBeNull(); +}); diff --git a/core/dashboard/tests/browser/rollout-backups.spec.cjs b/core/dashboard/tests/browser/rollout-backups.spec.cjs new file mode 100644 index 0000000..8aa9a67 --- /dev/null +++ b/core/dashboard/tests/browser/rollout-backups.spec.cjs @@ -0,0 +1,30 @@ +const { test, expect } = require('@playwright/test'); +test('backup categories remain available with mobile layout', async ({ page }) => { + const errors = []; page.on('pageerror', error => errors.push(error.message)); + await page.route('**/api/platform/backups', async route => { + const result = await (await route.fetch()).json(); + for (const [i, backup] of result.data.backups.entries()) { + backup.category = ['pre_update', 'scheduled', 'manual'][i % 3]; backup.trigger = backup.category; backup.verification = 'upload'; + } + return route.fulfill({ contentType: 'application/json', body: JSON.stringify(result) }); + }); + await page.goto('/'); + await page.getByLabel('Email address').fill('platform@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await page.goto('/#/backups/history'); + await expect(page.getByRole('heading', { name: 'Backup history', exact: true })).toBeVisible(); + const category = page.getByRole('combobox', { name: 'Category', exact: true }); + await category.selectOption('pre_update'); + const rows = page.locator('.backup-table tbody tr'); + await expect(rows).not.toHaveCount(0); + for (const row of await rows.all()) await expect(row).toContainText('Pre-update'); + await page.screenshot({ path: '/tmp/dispatch-backup-categories-desktop.png', fullPage: true }); + await category.selectOption('scheduled'); + for (const row of await rows.all()) await expect(row).toContainText('Scheduled'); + await category.selectOption('manual'); + for (const row of await rows.all()) await expect(row).toContainText('Manual'); + await page.setViewportSize({ width: 390, height: 844 }); + await expect.poll(() => page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/themes.spec.cjs b/core/dashboard/tests/browser/themes.spec.cjs new file mode 100644 index 0000000..69c7cc3 --- /dev/null +++ b/core/dashboard/tests/browser/themes.spec.cjs @@ -0,0 +1,130 @@ +const { test, expect } = require('@playwright/test'); + +async function login(page, email = 'platform@example.test') { + await page.goto('/'); + await page.getByLabel('Email address').fill(email); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await expect(page.locator('.desktop-sidebar')).toBeVisible(); +} +async function themeMenu(page, owner = false) { + await page.goto(`/#/${owner ? 'settings' : 'platform-settings'}?tab=theme`); + await expect(page.getByRole('radio', { name: 'Light', exact: true })).toBeVisible(); +} +async function logout(page) { + const session = (await (await page.request.get('/api/auth/session')).json()).data; + await page.request.post('/api/auth/logout', { + headers: { 'X-Dispatch-CSRF': session.csrfToken, Origin: new URL(page.url()).origin }, data: {}, + }); +} +const root = page => page.locator('html'); + +test('appearance persists locally per user and never changes workspace settings', async ({ page }) => { + await login(page); + await themeMenu(page); + const writes = []; + page.on('request', request => { if (['POST','PUT','PATCH','DELETE'].includes(request.method())) writes.push(request.url()); }); + await expect(page.getByRole('radio', { name: 'Light', exact: true })).toBeChecked(); + await page.getByRole('radio', { name: 'Dark', exact: true }).check(); + await expect(root(page)).toHaveAttribute('data-theme', 'dark'); + await page.reload(); + await expect(page.getByRole('radio', { name: 'Dark', exact: true })).toBeChecked(); + await expect(root(page)).toHaveAttribute('data-theme', 'dark'); + expect(writes).toEqual([]); + await logout(page); + await login(page, 'member0@example.test'); + await themeMenu(page, true); + await expect(root(page)).toHaveAttribute('data-theme', 'light'); + await page.getByRole('radio', { name: 'System', exact: true }).check(); + await logout(page); + await login(page); + await themeMenu(page); + await expect(page.getByRole('radio', { name: 'Dark', exact: true })).toBeChecked(); + await expect(root(page)).toHaveAttribute('data-theme', 'dark'); + const stored = await page.evaluate(() => Object.entries(localStorage).filter(([key]) => key.startsWith('dispatch:theme:'))); + expect(stored).toHaveLength(2); + expect(stored.map(([, value]) => JSON.parse(value).appearance).sort()).toEqual(['dark','system']); +}); + +test('System follows the device; explicit choices and keyboard navigation remain stable', async ({ page }) => { + await page.emulateMedia({ colorScheme: 'light' }); + await login(page); + await themeMenu(page); + await page.getByRole('radio', { name: 'System', exact: true }).check(); + await page.emulateMedia({ colorScheme: 'dark' }); + await expect(root(page)).toHaveAttribute('data-theme', 'dark'); + await page.emulateMedia({ colorScheme: 'light' }); + await expect(root(page)).toHaveAttribute('data-theme', 'light'); + await page.getByRole('radio', { name: 'System', exact: true }).focus(); + await page.keyboard.press('ArrowLeft'); + await expect(page.getByRole('radio', { name: 'Dark', exact: true })).toBeChecked(); + await expect(root(page)).toHaveAttribute('data-theme', 'dark'); + await page.emulateMedia({ colorScheme: 'dark' }); + await page.emulateMedia({ colorScheme: 'light' }); + await expect(root(page)).toHaveAttribute('data-theme', 'dark'); +}); + +test('theme updates synchronize same-account tabs and ignore other accounts', async ({ page, context }) => { + await login(page); + await themeMenu(page); + const second = await context.newPage(); + await themeMenu(second); + await page.getByRole('radio', { name: 'Dark', exact: true }).check(); + await expect(root(second)).toHaveAttribute('data-theme', 'dark'); + await second.evaluate(() => localStorage.setItem('dispatch:theme:v1:another-user', JSON.stringify({ themeId: 'precision', appearance: 'light' }))); + await expect(root(page)).toHaveAttribute('data-theme', 'dark'); + await second.getByRole('radio', { name: 'Light', exact: true }).check(); + await expect(root(page)).toHaveAttribute('data-theme', 'light'); + await second.close(); +}); + +test('invalid preferences and removed packs recover, unavailable storage does not block selection', async ({ page }) => { + await login(page); + const session = (await (await page.request.get('/api/auth/session')).json()).data; + const key = `dispatch:theme:v1:${session.user.id}`; + await page.evaluate(key => localStorage.setItem(key, '{bad json'), key); + await themeMenu(page); + await page.reload(); + await expect(root(page)).toHaveAttribute('data-theme', 'light'); + await page.evaluate(key => localStorage.setItem(key, JSON.stringify({ themeId: 'removed-pack', appearance: 'dark' })), key); + await page.reload(); + await expect(root(page)).toHaveAttribute('data-theme-pack', 'precision'); + await expect(root(page)).toHaveAttribute('data-theme', 'dark'); + await page.evaluate(() => { Storage.prototype.setItem = () => { throw new DOMException('Blocked', 'SecurityError'); }; }); + await page.getByRole('radio', { name: 'Light', exact: true }).check(); + await expect(root(page)).toHaveAttribute('data-theme', 'light'); + await expect(page.getByRole('status').filter({ hasText: 'Browser storage is unavailable' })).toBeVisible(); +}); + +test('dark pages, native backup dialogs, React sheets and mobile theme choices stay readable', async ({ page }) => { + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + await login(page); + await themeMenu(page); + await page.getByRole('radio', { name: 'Dark', exact: true }).check(); + for (const route of ['platform','updates','backups','backups/dsps','backups/history','backups/storage','backups/core','plugins','diagnostics','platform-settings']) { + await page.goto('/#/' + route); + await expect(page.locator('main h1')).toBeVisible(); + await expect(root(page)).toHaveAttribute('data-theme', 'dark'); + await expect(page.locator('body')).toHaveCSS('background-color', 'rgb(17, 21, 29)'); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + } + await page.goto('/#/platform'); + await page.getByRole('button', { name: 'Create new DSP', exact: true }).click(); + await expect(page.getByRole('dialog')).toHaveCSS('background-color', 'rgb(17, 21, 29)'); + await expect(page.getByLabel('Owner email', { exact: true })).toHaveCSS('background-color', 'rgb(17, 21, 29)'); + await page.keyboard.press('Escape'); + await page.goto('/#/backups'); + await page.getByRole('button', { name: 'Back up full system', exact: true }).click(); + await expect(page.locator('dialog')).toHaveCSS('background-color', 'rgb(17, 21, 29)'); + await page.keyboard.press('Escape'); + await themeMenu(page); + await page.setViewportSize({ width: 390, height: 844 }); + await expect(page.getByRole('radio', { name: 'Dark', exact: true })).toBeVisible(); + await expect.poll(() => page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.getByRole('button', { name: 'Open navigation' }).click(); + await expect(page.getByRole('dialog')).toHaveCSS('background-color', 'rgb(23, 28, 38)'); + await page.keyboard.press('Escape'); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/timezone.spec.cjs b/core/dashboard/tests/browser/timezone.spec.cjs new file mode 100644 index 0000000..a6c3638 --- /dev/null +++ b/core/dashboard/tests/browser/timezone.spec.cjs @@ -0,0 +1,115 @@ +const { test, expect } = require('@playwright/test'); +test.use({ timezoneId: 'America/Los_Angeles', locale: 'en-US' }); +test.skip(process.env.DISPATCH_PAYCOM_WORKFORCE_FIXTURE !== '1', 'Requires the isolated workforce preview fixture'); + +async function login(page, email = 'owner@example.test') { + await page.goto('/#/paycom'); + await page.getByLabel('Email address').fill(email); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await expect(page.getByRole('tab', { name: 'Timecard', exact: true })).toBeVisible(); +} +async function settings(page) { + await page.goto('/#/settings'); + await expect(page.getByLabel('Display timezone', { exact: true })).toBeVisible(); +} +async function logout(page) { + const session = (await (await page.request.get('/api/auth/session')).json()).data; + await page.request.post('/api/auth/logout', { + headers: { 'X-Dispatch-CSRF': session.csrfToken, Origin: new URL(page.url()).origin }, data: {}, + }); +} +test.beforeEach(async ({ page }) => { + await page.clock.install({ time: new Date('2026-09-11T02:28:00Z') }); + await page.route('**/api/auth/session', async route => { + const response = await route.fetch(); + const json = await response.json(); + for (const member of json.data?.memberships || []) member.organization.timezone = 'America/Los_Angeles'; + await route.fulfill({ response, json }); + }); + await page.route('**/api/organization/paycom-setup', route => route.fulfill({ json: { ok: true, data: { + status: 'succeeded', workforceAvailable: true, canSubmit: false, canRetry: false, failureCode: null, + } } })); + await page.route('**/api/paycom/sync', route => route.fulfill({ json: { ok: true, data: { + activity: 'idle', desiredState: 'running', lastSucceededAt: '2026-09-11T02:27:00Z', + nextDueAt: '2026-09-11T03:24:00Z', lastError: null, alerts: [], + } } })); +}); + +for (const mobile of [false, true]) test(`local evening date, sync times and preference (${mobile ? 'mobile' : 'desktop'})`, async ({ page }, testInfo) => { + await page.setViewportSize(mobile ? { width: 390, height: 844 } : { width: 1440, height: 1000 }); + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + await login(page); + await expect(page).toHaveTitle('Paycom · Dispatch'); + await expect(page).toHaveURL(/#\/paycom$/); + const date = page.getByLabel('Date', { exact: true }); + const sync = page.getByRole('status', { name: 'Paycom sync' }); + await expect(date).toHaveValue('2026-09-10'); + await expect(sync).toContainText('Sep 10, 2026, 7:27 PM PDT'); + await expect(sync).toContainText('Sep 10, 2026, 8:24 PM PDT'); + const rows = page.getByRole('table', { name: 'Daily employee timecards' }).locator('tbody'); + await expect(rows).toContainText('8:00 AM'); + const punches = await rows.textContent(); + await page.getByRole('button', { name: 'Previous day', exact: true }).click(); + await expect(date).toHaveValue('2026-09-09'); + await page.getByRole('button', { name: 'Today', exact: true }).click(); + await expect(date).toHaveValue('2026-09-10'); + await page.screenshot({ path: testInfo.outputPath('paycom-local-time.png'), fullPage: false }); + await settings(page); + await expect(page.getByLabel('Display timezone', { exact: true })).toHaveValue('automatic'); + await page.getByLabel('Display timezone', { exact: true }).selectOption('Asia/Tokyo'); + await page.reload(); + await expect(page.getByLabel('Display timezone', { exact: true })).toHaveValue('Asia/Tokyo'); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await page.screenshot({ path: testInfo.outputPath('timezone-settings.png'), fullPage: false }); + await page.goto('/#/paycom'); + await expect(sync).toContainText('Sep 11, 2026, 11:27 AM'); + await expect(date).toHaveValue('2026-09-10'); + await expect(rows).toHaveText(punches); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + expect(errors).toEqual([]); +}); + +test('Today advances at DSP midnight and a historical selection stays selected', async ({ page }) => { + await page.clock.setFixedTime(new Date('2026-09-11T06:59:59Z')); + await login(page); + const date = page.getByLabel('Date', { exact: true }); + await expect(date).toHaveValue('2026-09-10'); + await page.clock.setFixedTime(new Date('2026-09-11T07:00:01Z')); + await page.clock.fastForward(31000); + await expect(date).toHaveValue('2026-09-11'); + await page.getByRole('button', { name: 'Previous day', exact: true }).click(); + await page.clock.setFixedTime(new Date('2026-09-12T07:00:01Z')); + await page.clock.fastForward(31000); + await expect(date).toHaveValue('2026-09-10'); + await page.getByRole('button', { name: 'Today', exact: true }).click(); + await expect(date).toHaveValue('2026-09-12'); +}); + +test('preferences stay with the user; malformed preferences and blocked storage recover', async ({ page }) => { + await login(page); + await settings(page); + const input = page.getByLabel('Display timezone', { exact: true }); + await input.selectOption('UTC'); + await logout(page); + await login(page, 'member0@example.test'); + await settings(page); + await expect(input).toHaveValue('automatic'); + await logout(page); + await login(page); + await settings(page); + await expect(input).toHaveValue('UTC'); + await page.evaluate(() => { + for (const key of Object.keys(localStorage).filter(key => key.startsWith('dispatch:timezone:'))) { + localStorage.setItem(key, JSON.stringify({ timeZone: 'Mars/Base' })); + } + }); + await page.reload(); + await expect(input).toHaveValue('automatic'); + await page.evaluate(() => { Storage.prototype.setItem = () => { throw new DOMException('Blocked', 'SecurityError'); }; }); + await input.selectOption('America/Phoenix'); + await expect(input).toHaveValue('America/Phoenix'); + await expect(page.getByRole('status').filter({ hasText: 'Browser storage is unavailable' })).toBeVisible(); +}); diff --git a/core/dashboard/tests/browser/turnstile.spec.cjs b/core/dashboard/tests/browser/turnstile.spec.cjs new file mode 100644 index 0000000..7bb7c2d --- /dev/null +++ b/core/dashboard/tests/browser/turnstile.spec.cjs @@ -0,0 +1,133 @@ +const { test, expect } = require('@playwright/test'); +const { spawn } = require('node:child_process'); +const path = require('node:path'); +const { once } = require('node:events'); +let server, base; +const password = 'synthetic preview password'; + +test.beforeAll(async () => { + server = spawn(process.execPath, ['--no-warnings', 'examples/frontend-preview.js'], { + cwd: path.resolve(__dirname, "../.."), + env: { ...process.env, DISPATCH_FRONTEND_FIXTURE: '1', DISPATCH_TURNSTILE_FIXTURE: '1', DISPATCH_FRONTEND_PORT: '0' }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + base = await new Promise((resolve, reject) => { + let output = ''; + server.stdout.on('data', chunk => { + output += chunk; + const match = /http:\/\/127\.0\.0\.1:\d+/.exec(output); + if (match) resolve(match[0]); + }); + server.once('error', reject); + server.once('exit', code => reject(Error(`Turnstile fixture exited: ${code}`))); + }); +}); +test.afterAll(async () => { + if (server && server.exitCode === null) { + const closed = once(server, 'exit'); server.kill('SIGTERM'); await closed; + } +}); + +// Exercise our real form and HTTP verifier with a deterministic replacement for +// Cloudflare's third-party script/response, never a production acceptance bypass. +const widgetScript = ` +window.turnstile = { + render(container, options) { + const id = 'fixture-' + crypto.randomUUID(); + container.dataset.widget = id; + window.fixtureTurnstile = options; + const button = document.createElement('button'); + button.type = 'button'; button.textContent = 'Complete security check'; + button.addEventListener('click', () => options.callback('fixture:' + options.action + ':' + crypto.randomUUID())); + container.replaceChildren(button); + return id; + }, + remove(id) { document.querySelector('[data-widget="' + id + '"]')?.replaceChildren(); } +};`; +async function widget(page) { + await page.route('https://challenges.cloudflare.com/turnstile/v0/api.js?render=explicit', route => route.fulfill({ contentType: 'text/javascript', body: widgetScript })); +} +async function fields(page) { + await page.getByLabel('Email address').fill('platform@example.test'); + await page.getByLabel('Password', { exact: true }).fill(password); +} +const signIn = page => page.getByRole('button', { name: 'Sign in', exact: true }); +const solve = page => page.getByRole('button', { name: 'Complete security check' }).click(); + +test('login waits for verification, refreshes after wrong password, and reaches the dashboard', async ({ page }) => { + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { + if (message.type() === 'error' && !message.text().includes('401 (Unauthorized)')) errors.push(message.text()); + }); + await widget(page); await page.goto(base); await fields(page); + await expect(page).toHaveTitle('Dispatch'); + await expect(page.getByRole('heading', { name: 'Sign in to Dispatch' })).toBeVisible(); + await expect(signIn(page)).toBeDisabled(); + await page.screenshot({ path: '/tmp/dispatch-turnstile-desktop.png' }); + await page.getByLabel('Password', { exact: true }).fill('wrong password'); + await solve(page); await signIn(page).click(); + await expect(page.getByText('The email address or password was not accepted.')).toBeVisible(); + await expect(page.getByLabel('Password', { exact: true })).toHaveValue('wrong password'); + await expect(signIn(page)).toBeDisabled(); + await page.getByLabel('Password', { exact: true }).fill(password); + await solve(page); await signIn(page).click(); + await expect(page.getByRole('navigation', { name: 'Primary navigation' })).toBeVisible(); + await expect(page.getByLabel('Security verification')).toHaveCount(0); + expect(errors).toEqual([]); +}); + +test('expired token and verification outage allow retry without losing form entries', async ({ page }) => { + await widget(page); await page.setViewportSize({ width: 390, height: 844 }); + await page.goto(base); await fields(page); await solve(page); + await page.evaluate(() => window.fixtureTurnstile['expired-callback']()); + await expect(signIn(page)).toBeDisabled(); + await expect(page.getByText('Security check expired. Please verify again.')).toBeVisible(); + await page.getByRole('button', { name: 'Retry security check' }).click(); + await solve(page); + await page.evaluate(() => window.fixtureTurnstile.callback('fixture-unavailable')); + await signIn(page).click(); + await expect(page.getByText('Security verification is temporarily unavailable. Please try again shortly.')).toBeVisible(); + await expect(page.getByLabel('Email address')).toHaveValue('platform@example.test'); + await expect(page.getByLabel('Password', { exact: true })).toHaveValue(password); + await expect(signIn(page)).toBeDisabled(); + await page.screenshot({ path: '/tmp/dispatch-turnstile-mobile-retry.png' }); + expect(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await solve(page); await signIn(page).click(); + await expect(page.getByRole('heading', { name: 'DSPs', exact: true })).toBeVisible(); +}); + +test('script load failure can be retried and registration requires its own verification action', async ({ page }) => { + let blocked = true; + await page.route('https://challenges.cloudflare.com/turnstile/v0/api.js?render=explicit', route => blocked + ? route.abort('failed') : route.fulfill({ contentType: 'text/javascript', body: widgetScript })); + await page.goto(base); await fields(page); + await expect(page.getByText('Security check could not load. Check your connection and try again.')).toBeVisible(); + await expect(signIn(page)).toBeDisabled(); + blocked = false; + await page.getByRole('button', { name: 'Retry security check' }).click(); + await solve(page); await signIn(page).click(); + await expect(page.getByRole('navigation', { name: 'Primary navigation' })).toBeVisible(); + const session = (await (await page.request.get(base + '/api/auth/session')).json()).data; + const created = await page.request.post(base + '/api/platform/organizations', { + headers: { 'X-Dispatch-CSRF': session.csrfToken }, + data: { ownerEmail: `turnstile-${Date.now()}@example.test`, idempotencyKey: 'turnstile:browser:' + Date.now() }, + }); + expect(created.status()).toBe(201); + const invitation = (await created.json()).data.invitationPath; + await page.context().clearCookies(); + await page.goto(base + invitation); + await expect(page.getByLabel('First name')).toBeVisible(); + await page.getByLabel('First name').fill('Turnstile'); + await page.getByLabel('Last name').fill('Owner'); + await page.getByLabel('Password', { exact: true }).fill(password); + await page.getByLabel('Confirm password').fill(password); + const submit = page.getByRole('button', { name: 'Create account and continue' }); + await expect(submit).toBeDisabled(); + await solve(page); + const submitted = page.waitForRequest(request => request.url().endsWith('/api/auth/register')); + await submit.click(); + expect((await submitted).postDataJSON().turnstileToken).toMatch(/^fixture:register:/); + await expect(page).toHaveURL(/#\/onboarding$/); + await expect(page.getByLabel('Security verification')).toHaveCount(0); +}); diff --git a/core/dashboard/tests/browser/updates-recovery.spec.cjs b/core/dashboard/tests/browser/updates-recovery.spec.cjs new file mode 100644 index 0000000..9dfb793 --- /dev/null +++ b/core/dashboard/tests/browser/updates-recovery.spec.cjs @@ -0,0 +1,46 @@ +const { test, expect } = require('@playwright/test'); + +test('changelog remains readable across rollout phases and Core restarts without operational controls', async ({ page }) => { + const errors = []; page.on('pageerror', error => errors.push(error.message)); + const mutations = []; let disconnected = false; + const release = { id: 'dispatch_0.0.8', version: '0.0.8', publishedAt: '2026-09-08T00:00:00.000Z', state: 'rolling_out', + changelog: [{ kind: 'fixed', title: 'Saved connections recover correctly', description: 'Retry an interrupted connection.' }] }; + const rollout = { release: release.id, version: release.version, status: 'running', phase: 'backups', + core: { status: 'queued', message: null }, total: 2, updated: 0, members: [], activity: [] }; + await page.route('**/api/platform/updates*', route => { + if (route.request().method() !== 'GET') mutations.push(route.request().method()); + return route.fulfill({ status: disconnected ? 503 : 200, contentType: 'application/json', + json: disconnected ? { ok: false, error: { code: 'unavailable' } } : { ok: true, data: { + enabled: true, releases: [], releaseHistory: [release], displayedRelease: release, rollout, + } } }); + }); + await page.goto('/'); + await page.getByLabel('Email address').fill('platform@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await page.locator('.desktop-sidebar').getByRole('link', { name: 'Updates', exact: true }).click(); + const content = page.locator('#platform-updates-content'); + const refresh = () => page.getByRole('button', { name: 'Refresh', exact: true }).click(); + for (const phase of ['backups', 'core', 'verify_core', 'dsps', 'complete']) { + rollout.phase = phase; + if (phase === 'complete') { rollout.status = 'completed'; release.state = 'installed'; } + await refresh(); + await expect(content.getByText('Saved connections recover correctly')).toBeVisible(); + await expect(content.getByRole('button', { name: /^(Install update|Start rollout|Pause rollout|Resume rollout|Retry download)$/ })).toHaveCount(0); + await expect(content.locator('.update-live-rollout, .update-stage-list')).toHaveCount(0); + if (phase === 'core') { + disconnected = true; await refresh(); + await expect(content.getByRole('status')).toContainText('Showing the last loaded changelog'); + await expect(content.getByText('Saved connections recover correctly')).toBeVisible(); + disconnected = false; await refresh(); + await expect(content.locator('.update-reconnecting')).toHaveCount(0); + } + rollout.status = 'paused'; await refresh(); + await expect(content.getByRole('button', { name: 'Resume rollout' })).toHaveCount(0); + rollout.status = 'running'; + } + await expect(content.locator('.update-release-meta')).toContainText('Installed'); + await page.setViewportSize({ width: 390, height: 844 }); + await expect.poll(() => page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + expect(mutations).toEqual([]); expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/browser/updates-workspace.spec.cjs b/core/dashboard/tests/browser/updates-workspace.spec.cjs new file mode 100644 index 0000000..bf9085f --- /dev/null +++ b/core/dashboard/tests/browser/updates-workspace.spec.cjs @@ -0,0 +1,157 @@ +const { test, expect } = require('@playwright/test'); +const { spawn } = require('node:child_process'); +const path = require('node:path'); + +let preview, previewUrl; +test.beforeAll(async () => { + // Keep real release history independent of other backup tests. + preview = spawn(process.execPath, [path.resolve(__dirname, "../../examples/frontend-preview.js")], { + env: { ...process.env, DISPATCH_FRONTEND_FIXTURE: '1', DISPATCH_FRONTEND_PORT: '0' }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + previewUrl = await new Promise((resolve, reject) => { + let output = '', errors = ''; + const timeout = setTimeout(() => reject(Error(`Preview startup timed out: ${errors}`)), 20000); + const finish = (error, url) => { clearTimeout(timeout); error ? reject(error) : resolve(url); }; + preview.once('error', error => finish(error)); + preview.once('exit', code => finish(Error(`Preview exited (${code}): ${errors}`))); + preview.stderr.on('data', chunk => { errors += chunk; }); + preview.stdout.on('data', chunk => { + output += chunk; + const match = output.match(/Synthetic frontend preview: (http:\/\/127\.0\.0\.1:\d+)/); + if (match) finish(null, match[1]); + }); + }); +}); +test.afterEach(async ({ page }) => { + // Let intercepted polling/refresh responses finish before page teardown + // disposes their API responses. Keep handler errors visible to the test. + await page.unrouteAll({ behavior: 'wait' }); +}); +test.afterAll(async () => { + if (!preview || preview.exitCode !== null || preview.signalCode !== null) return; + await new Promise(resolve => { + const timeout = setTimeout(() => preview.kill('SIGKILL'), 5000); + preview.once('exit', () => { clearTimeout(timeout); resolve(); }); + preview.kill('SIGTERM'); + }); +}); + +test('grouped changelogs, details, and past releases are available through the real API', async ({ page }) => { + const errors = []; page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + await page.goto(previewUrl); + await page.getByLabel('Email address').fill('platform@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await page.locator('.desktop-sidebar').getByRole('link', { name: 'Updates', exact: true }).click(); + const mutations = []; page.on('request', request => { if (request.url().includes('/api/platform/updates') && request.method() !== 'GET') mutations.push(request.method()); }); + const content = page.locator('#platform-updates-content'); + await expect(content.getByText('3 additions · 4 changes · 3 improvements')).toBeVisible(); + await expect(content.locator('.update-feature-group')).toHaveCount(4); + await expect(content.getByRole('button', { name: /Install update|Pause rollout|Resume rollout|Retry download/ })).toHaveCount(0); + await page.screenshot({ path: '/tmp/dispatch-release-browser-qa/desktop.png', fullPage: true }); + await content.getByRole('button', { name: 'View details', exact: true }).first().click(); + await expect(content.locator('.update-expanded-details').first()).toBeVisible(); + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await expect(content.locator('.update-expanded-details').first()).toBeVisible(); + await content.getByRole('button', { name: 'Hide details', exact: true }).first().click(); + await content.getByRole('navigation', { name: 'Releases', exact: true }).getByRole('button', { name: /^Version 0\.0\.8/ }).click(); + await expect(content.getByRole('heading', { name: /^Version 0\.0\.8/ }).first()).toBeVisible(); + await expect(content.getByRole('button', { name: 'Install update' })).toHaveCount(0); + await expect(content.getByRole('heading', { name: 'What’s new' })).toBeVisible(); + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await expect(content.getByRole('button', { name: /^Version 0\.0\.8/ })).toHaveAttribute('aria-current', 'page'); + await page.screenshot({ path: '/tmp/dispatch-release-browser-qa/history.png', fullPage: true }); + const response = await page.request.get(`${previewUrl}/api/platform/updates?releaseId=dispatch_missing`); + expect(response.status()).toBe(404); + const invalid = await page.request.get(`${previewUrl}/api/platform/updates?releaseId=dispatch_0.0.8&releaseId=dispatch_0.0.9`); + expect(invalid.status()).toBe(400); + await content.getByRole('navigation', { name: 'Releases', exact: true }).getByRole('button', { name: /Version 0.0.9/ }).click(); + await expect(content.getByRole('navigation', { name: 'Releases', exact: true }).getByRole('button', { name: /Version 0.0.9/ })).toBeFocused(); + await expect(content.locator('.update-feature-group')).toHaveCount(4); + await page.setViewportSize({ width: 390, height: 844 }); + await expect.poll(() => page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + const notesBox = await content.locator('.update-notes-panel').boundingBox(); + const navigationBox = await content.getByRole('navigation', { name: 'Releases', exact: true }).boundingBox(); + expect(navigationBox.y).toBeLessThan(notesBox.y); + await page.screenshot({ path: '/tmp/dispatch-release-browser-qa/mobile.png', fullPage: true }); + await page.setViewportSize({ width: 1536, height: 1024 }); + const search = content.getByRole('searchbox', { name: 'Find a version' }); + await search.fill('0.0.8'); + await expect(content.getByRole('button', { name: /Version 0.0.9/ })).toHaveCount(0); + await expect(content.getByRole('button', { name: /^Version 0\.0\.8/ })).toBeVisible(); + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await expect(search).toHaveValue('0.0.8'); + await search.fill('missing'); + await expect(content.getByText('No matching releases.')).toBeVisible(); + await search.fill(''); + await expect(content.getByRole('button', { name: /Version 0.0.9/ })).toBeVisible(); + expect(mutations).toEqual([]); + expect(errors).toEqual([]); +}); + + +test('installed changelogs remain the default and preserve reading state during polling', async ({ page }) => { + const errors = []; + page.on('pageerror', error => errors.push(error.message)); + page.on('console', message => { if (message.type() === 'error') errors.push(message.text()); }); + let updateReads = 0; + // The real preview API supplies notes and history. Model completed installation + // in its response so this UI test never runs a host updater. + await page.route('**/api/platform/updates*', async route => { + const response = await route.fetch(); + const body = await response.json(); + if (body.data) { + updateReads += 1; + const data = body.data; + data.releases = []; + data.releaseHistory = data.releaseHistory.map(item => ({ ...item, state: item.id === 'dispatch_0.0.9' ? 'installed' : 'historical' })); + if (data.displayedRelease) data.displayedRelease.state = data.displayedRelease.id === 'dispatch_0.0.9' ? 'installed' : 'historical'; + data.rollout = { updatedAt: new Date(1788739200000 + updateReads).toISOString(), release: 'dispatch_0.0.9', version: '0.0.9', status: 'completed', phase: 'complete', + core: { status: 'succeeded', message: null }, total: 0, updated: 0, members: [], activity: [] }; + } + await route.fulfill({ response, json: body }); + }); + await page.goto(previewUrl); + await page.getByLabel('Email address').fill('platform@example.test'); + await page.getByLabel('Password', { exact: true }).fill('synthetic preview password'); + await page.getByRole('button', { name: 'Sign in', exact: true }).click(); + await page.locator('.desktop-sidebar').getByRole('link', { name: 'Updates', exact: true }).click(); + const mutations = []; page.on('request', request => { if (request.url().includes('/api/platform/updates') && request.method() !== 'GET') mutations.push(request.method()); }); + const content = page.locator('#platform-updates-content'); + await expect(page).toHaveURL(/#\/updates$/); + await expect(page).toHaveTitle(/Dispatch/); + await expect(content.getByRole('heading', { name: 'Version 0.0.9', exact: true })).toBeVisible(); + await expect(content.locator('.update-live-rollout, .update-available-banner')).toHaveCount(0); + const current = content.getByRole('navigation', { name: 'Releases', exact: true }).getByRole('button', { name: /Version 0.0.9/ }); + await expect(current).toHaveAttribute('aria-current', 'page'); + await expect(current.getByText('Installed', { exact: true })).toBeVisible(); + await expect(content.locator('.update-feature-group')).toHaveCount(4); + await expect(content.getByRole('button', { name: 'Install update' })).toHaveCount(0); + await expect(content.getByRole('region', { name: 'After updating' })).toBeVisible(); + // Polling may rebuild the DOM: retain the exact details control and disclosure. + const details = content.getByRole('button', { name: 'View details', exact: true }).first(); + await details.click(); + const readsBefore = updateReads; + await expect.poll(() => updateReads).toBeGreaterThan(readsBefore); + await expect(content.getByRole('button', { name: 'Hide details', exact: true }).first()).toBeFocused(); + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await expect(content.locator('.update-expanded-details').first()).toBeVisible(); + const older = content.getByRole('button', { name: /^Version 0\.0\.8/ }); + await older.click(); + await expect(older).toHaveAttribute('aria-current', 'page'); + await expect(content.getByRole('heading', { name: /^Version 0\.0\.8/ })).toBeVisible(); + await page.setViewportSize({ width: 390, height: 844 }); + await expect.poll(() => page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)).toBe(true); + await expect(older).toBeVisible(); + await page.screenshot({ path: '/tmp/dispatch-release-browser-qa/history-mobile.png', fullPage: true }); + await current.focus(); + await page.keyboard.press('Enter'); + await expect(content.getByRole('heading', { name: 'Version 0.0.9', exact: true })).toBeVisible(); + await expect(current).toBeFocused(); + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await expect(content.getByRole('navigation', { name: 'Releases', exact: true })).toBeVisible(); + expect(mutations).toEqual([]); + expect(errors).toEqual([]); +}); diff --git a/core/dashboard/tests/connections-persistence.test.js b/core/dashboard/tests/connections-persistence.test.js new file mode 100644 index 0000000..2c237b4 --- /dev/null +++ b/core/dashboard/tests/connections-persistence.test.js @@ -0,0 +1,285 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { createConnectionsStack } = require('./helpers/connections-stack.cjs'); +const PAYCOM = { clientCode: 'client-fixture', username: 'paycom-fixture-owner', password: 'paycom-fixture-secret', + pin1: 'answer-one', pin2: 'answer-two', pin3: 'answer-three', pin4: 'answer-four', pin5: 'answer-five' }; +async function fixture(t) { const f = await createConnectionsStack(); t.after(() => f.close()); return f; } +function assertNoPlaintext(root, secret) { + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + const file = path.join(root, entry.name); + if (entry.isDirectory()) assertNoPlaintext(file, secret); + else if (entry.isFile()) assert.equal(fs.readFileSync(file).includes(Buffer.from(secret)), false, `plaintext in ${entry.name}`); + } +} + +test('HTTP credentials persist encrypted across broker restarts for Cortex and Paycom', async t => { + const f = await fixture(t); + for (const [service, profile, credentials] of [ + ['cortex', 'amazon-operations', { username: 'cortex-fixture-owner', password: 'cortex-fixture-secret' }], + ['paycom', 'paycom-main', PAYCOM], + ]) { + const response = await f.save(service, credentials); + assert.equal(response.status, 202); + assert.equal((await response.text()).includes(credentials.password), false); + await f.state.broker.serviceConnections.close(); + await f.restartBroker(); + assert.deepEqual(f.state.broker.vault.readForAdapter(profile).credentials, credentials); + assertNoPlaintext(f.root, credentials.password); + assert.equal(fs.statSync(f.paths.database).mode & 0o777, 0o600); + assert.equal(fs.statSync(f.paths.key).mode & 0o777, 0o600); + } + assert.equal(require('../../core/accounts/src/onboarding-store').createOnboardingStore(f.store) + .latest(f.organizationId).status, 'queued'); +}); + +test('a sleeping directory DSP saves Paycom directly in its vault while runtime capacity is full', async t => { + const f = await createConnectionsStack({ directoryEnrollment: true }); t.after(() => f.close()); + const platform = f.access.session(f.platform.token); + const viewed = f.access.beginDspView(platform, { controlRef: f.access.issuePlatformControlRef(platform, f.organizationId) }); + const response = await fetch(`${f.base}/api/organization/connections/paycom/save`, { + method: 'POST', headers: { ...f.headers, Cookie: `dispatch_session=${f.platform.token}`, + 'X-Dispatch-CSRF': platform.csrfToken, 'X-Dispatch-DSP-View': viewed.dspView.viewRef }, + body: JSON.stringify({ credentials: PAYCOM }), + }); + assert.equal(response.status, 202); + assert.equal((await response.json()).data.configured, true); + assert.equal(f.state.runtimeEnrollments, 0); + assert.deepEqual(f.state.broker.vault.readForAdapter('paycom-main').credentials, PAYCOM); + const requests = require('../../core/accounts/src/onboarding-store').createOnboardingStore(f.store); + const job = requests.latest(f.organizationId); + assert.equal(job.status, 'queued'); + let capacity = false; + const worker = require('../../core/installations/src/owner-onboarding').createOwnerOnboardingWorker({ + store: f.store, backends: ['directory_service_v1'], testProvider: f.verification.poll, invoke: async (_id, _action, input) => { + assert.equal(input.step, 'sync'); + if (!capacity) return { ok: false, status: 'execution_capacity_wait' }; + return { ok: true, status: 'succeeded', data: input.step === 'sync' + ? { syncId: 'paycom-main-workforce', intervalSeconds: 3600, desiredState: 'running' } + : { profileId: 'paycom-main', provider: 'paycom', status: 'authenticated', testedAt: new Date().toISOString() } }; + }, + }); + for (let i = 0; i < 4; i++) { + const result = await worker.runPending('synthetic-worker'); + assert.equal(result.failed, 0); + assert.equal(requests.latest(f.organizationId).status, 'queued'); + assert.equal(requests.latest(f.organizationId).attempt, 0); + } + const stale = requests.claim(job.id, 'stale-worker'); requests.defer(stale); + const current = requests.claim(job.id, 'current-worker'); + assert.throws(() => requests.defer(stale), /installation_operation_in_progress/); + requests.defer(current); + capacity = true; + assert.equal((await worker.runPending('available-worker')).completed, 1); + assert.equal(requests.latest(f.organizationId).status, 'succeeded'); + await f.restartBroker(); + assert.deepEqual(f.state.broker.vault.readForAdapter('paycom-main').credentials, PAYCOM); + assertNoPlaintext(f.root, PAYCOM.password); +}); + +for (const outcome of ['authenticated', 'invalid_credentials']) test(`saving Paycom starts verification immediately and onboarding reuses ${outcome}`, async t => { + const f = await createConnectionsStack({ directoryEnrollment: true }); + let finish, attempts = 0, syncs = 0; + const pending = new Promise(resolve => { finish = resolve; }); + t.after(async () => { finish(); await f.close(); }); + f.state.authentication = async () => { + attempts++; await pending; + if (outcome !== 'authenticated') throw Object.assign(new Error(outcome), { code: outcome }); + return { status: 'authenticated' }; + }; + const response = await f.save('paycom', PAYCOM); + assert.equal(response.status, 202); + assert.equal((await response.json()).data.state, 'checking'); + assert.equal(attempts, 1); + assert.equal((await f.list()).find(item => item.service === 'paycom').state, 'checking'); + assert.equal(f.state.runtimeEnrollments, 0); + const worker = require('../../core/installations/src/owner-onboarding').createOwnerOnboardingWorker({ + store: f.store, backends: ['directory_service_v1'], testProvider: f.verification.poll, + delay: async () => { finish(); await f.state.broker.serviceConnections.close(); }, + invoke: async (_id, _action, input) => { + assert.equal(input.step, 'sync'); syncs++; + return { ok: true, status: 'succeeded', data: { syncId: 'paycom-main-workforce', intervalSeconds: 3600, desiredState: 'running' } }; + }, + }); + const result = await worker.runPending('immediate-check'); + assert.equal(attempts, 1); + assert.equal(syncs, outcome === 'authenticated' ? 1 : 0); + assert.equal(result.completed, syncs); + await f.restartBroker(); + assert.equal((await f.list()).find(item => item.service === 'paycom').state, + outcome === 'authenticated' ? 'connected' : 'credentials_rejected'); + assertNoPlaintext(f.root, PAYCOM.password); + if (outcome === 'invalid_credentials') { + f.state.authentication = async () => { attempts++; return { status: 'authenticated' }; }; + await f.paycomSetup.retry(f.owner.session, {}); + await f.state.broker.serviceConnections.close(); + assert.equal(attempts, 2); + assert.equal((await worker.runPending('owner-retry')).completed, 1); + assert.equal(attempts, 2); + } +}); + +for (const recovered of [true, false]) test(`an unsent initial check exposes pending work and its eventual ${recovered ? 'success' : 'failure'}`, async t => { + const f = await createConnectionsStack({ directoryEnrollment: true }); t.after(() => f.close()); + let attempts = 0; + f.state.authentication = async () => { attempts++; return { status: 'authenticated' }; }; + f.verification.start = async () => { throw new Error('initial verification delivery unavailable'); }; + const response = await f.save('paycom', PAYCOM); + assert.equal(response.status, 202); + assert.equal((await response.json()).data.state, 'checking'); + assert.equal(attempts, 0); + assert.equal(f.state.broker.serviceConnections.view('paycom').state, 'not_verified'); + const worker = require('../../core/installations/src/owner-onboarding').createOwnerOnboardingWorker({ + store: f.store, backends: ['directory_service_v1'], + testProvider: recovered ? f.verification.poll : async () => ({ ok: false, status: 'provider_setup_failed' }), + delay: async () => { await f.state.broker.serviceConnections.close(); }, + invoke: async (_id, _action, input) => { + assert.equal(input.step, 'sync'); + return { ok: true, status: 'succeeded', data: { syncId: 'paycom-main-workforce', intervalSeconds: 3600, desiredState: 'running' } }; + }, + }); + assert.equal((await worker.runPending('recover-check')).completed, recovered ? 1 : 0); + assert.equal(attempts, recovered ? 1 : 0); + const paycom = (await f.list()).find(item => item.service === 'paycom'); + assert.equal(paycom.state, recovered ? 'connected' : 'temporarily_unavailable'); + assert.equal(paycom.reason, recovered ? null : 'auth_unavailable'); +}); + +test('a directory enrollment with a lost response stays recoverable without a runtime slot', async t => { + const f = await createConnectionsStack({ directoryEnrollment: true }); t.after(() => f.close()); + f.state.dropReply = true; + assert.equal((await f.save('paycom', PAYCOM)).status, 503); + await f.restartBroker(); + assert.deepEqual(f.state.broker.vault.readForAdapter('paycom-main').credentials, PAYCOM); + assert.equal((await f.save('paycom', { ...PAYCOM, password: 'synthetic-replacement' })).status, 202); + assert.equal(f.state.runtimeEnrollments, 0); + assert.equal(f.state.broker.vault.readForAdapter('paycom-main').credentials.password, 'synthetic-replacement'); +}); + +test('platform owner DSP view saves Cortex and Paycom through HTTP into the selected encrypted vault', async t => { + const f = await fixture(t), platform = f.access.session(f.platform.token); + const viewed = f.access.beginDspView(platform, { controlRef: f.access.issuePlatformControlRef(platform, f.organizationId) }); + const headers = { 'Content-Type': 'application/json', Cookie: `dispatch_session=${f.platform.token}`, + 'X-Dispatch-CSRF': platform.csrfToken, 'X-Dispatch-DSP-View': viewed.dspView.viewRef }; + for (const [service, profile, credentials] of [ + ['cortex', 'amazon-operations', { username: 'synthetic-support-owner', password: 'synthetic-cortex-support-secret' }], + ['paycom', 'paycom-main', PAYCOM], + ]) { + const response = await fetch(`${f.base}/api/organization/connections/${service}/save`, { + method: 'POST', headers, body: JSON.stringify({ credentials }), + }); + assert.equal(response.status, 202); + assert.equal((await response.text()).includes(credentials.password), false); + await f.state.broker.serviceConnections.close(); + await f.restartBroker(); + assert.deepEqual(f.state.broker.vault.readForAdapter(profile).credentials, credentials); + assertNoPlaintext(f.root, credentials.password); + } + const audited = f.store.db.prepare("SELECT actor_user_id,organization_id FROM audit_events WHERE action='connection.save'").all(); + assert.equal(audited.length, 2); + assert.ok(audited.every(row => row.actor_user_id === platform.user.id && row.organization_id === f.organizationId)); +}); + +test('maximum valid multibyte and JSON-escaped credentials survive every request boundary', async t => { + const f = await fixture(t); + for (const character of ['界', '\u0001']) { + const credentials = { username: '界'.repeat(320), password: character.repeat(4096) }; + assert.equal((await f.save('cortex', credentials)).status, 202); + await f.state.broker.serviceConnections.close(); + assert.deepEqual(f.state.broker.vault.readForAdapter('amazon-operations').credentials, credentials); + } + assert.equal((await f.save('cortex', { username: 'owner', password: 'a'.repeat(4097) })).status, 400); + assert.equal((await f.save('cortex', { username: 'owner', password: 'a'.repeat(33000) })).status, 413); + assert.equal(f.state.broker.vault.readForAdapter('amazon-operations').credentials.password, '\u0001'.repeat(4096)); +}); + +test('failed SQLite replacement preserves the previous account and can be retried', async t => { + const f = await fixture(t); + const original = { username: 'old-owner', password: 'old-fixture-secret' }; + const replacement = { username: 'new-owner', password: 'new-fixture-secret' }; + assert.equal((await f.save('cortex', original)).status, 202); + await f.state.broker.serviceConnections.close(); + f.state.broker.vault.db.exec("CREATE TEMP TRIGGER fail_save BEFORE UPDATE ON credential_profiles BEGIN SELECT RAISE(ABORT, 'injected write failure'); END"); + assert.notEqual((await f.save('cortex', replacement)).status, 202); + assert.deepEqual(f.state.broker.vault.readForAdapter('amazon-operations').credentials, original); + assert.notEqual((await f.list())[0].state, 'connected'); + f.state.broker.vault.db.exec('DROP TRIGGER fail_save'); + assert.equal((await f.save('cortex', replacement)).status, 202); + await f.state.broker.serviceConnections.close(); + await f.restartBroker(); + assert.deepEqual(f.state.broker.vault.readForAdapter('amazon-operations').credentials, replacement); +}); + +test('an existing Paycom vault account without an onboarding receipt can be updated from Connections', async t => { + const f = await fixture(t); + f.state.broker.vault.put('paycom-main', 'paycom', { ...PAYCOM, password: 'previous-fixture-secret' }); + const response = await f.save('paycom', PAYCOM); + assert.equal(response.status, 202); + assert.deepEqual(f.state.broker.vault.readForAdapter('paycom-main').credentials, PAYCOM); +}); + +test('a lost save acknowledgement does not lose credentials and Paycom enrollment can recover', async t => { + const f = await fixture(t); + f.state.dropReply = true; + assert.notEqual((await f.save('paycom', PAYCOM)).status, 202); + await f.restartBroker(); + assert.deepEqual(f.state.broker.vault.readForAdapter('paycom-main').credentials, PAYCOM); + assert.equal((await f.save('paycom', PAYCOM)).status, 202); + assertNoPlaintext(f.root, PAYCOM.password); +}); + +test('a Core audit write failure after Paycom persistence reports an unconfirmed save, not invalid credentials', async t => { + const f = await fixture(t); + f.access.audit = () => { throw new Error('injected Core audit write failure'); }; + const response = await f.save('paycom', PAYCOM); + assert.equal(response.status, 503); + assert.deepEqual(f.state.broker.vault.readForAdapter('paycom-main').credentials, PAYCOM); +}); + +test('failure to invalidate old verification cannot replace the saved account', async t => { + const f = await fixture(t); + const original = { username: 'old-owner', password: 'old-diagnostic-fixture' }; + assert.equal((await f.save('cortex', original)).status, 202); + await f.state.broker.serviceConnections.close(); + const diagnostics = f.state.broker.sessions.lastAuthentication; + const remove = diagnostics.delete; + diagnostics.delete = () => { throw new Error('injected diagnostic storage failure'); }; + try { + assert.notEqual((await f.save('cortex', { username: 'replacement', password: 'new-diagnostic-fixture' })).status, 202); + assert.deepEqual(f.state.broker.vault.readForAdapter('amazon-operations').credentials, original); + } finally { diagnostics.delete = remove; } + await f.restartBroker(); + assert.notEqual((await f.list())[0].state, 'connected'); +}); + +test('competing saves and disconnects cannot overwrite a connection being verified', async t => { + const f = await fixture(t); + let finish; + const checking = new Promise(resolve => { finish = () => resolve({ status: 'authenticated' }); }); + f.state.authentication = () => checking; + const original = { username: 'concurrent-owner', password: 'concurrent-fixture-secret' }; + try { + assert.equal((await f.save('cortex', original)).status, 202); + const [save, disconnect] = await Promise.all([ + f.save('cortex', { username: 'other-owner', password: 'other-fixture-secret' }), + fetch(`${f.base}/api/organization/connections/cortex/disconnect`, { method: 'POST', headers: f.headers, body: '{}' }), + ]); + assert.equal(save.status, 409); + assert.equal(disconnect.status, 409); + assert.deepEqual(f.state.broker.vault.readForAdapter('amazon-operations').credentials, original); + } finally { finish(); } +}); + +test('restart during login verification preserves the saved account without reporting connected', async t => { + const f = await fixture(t); + f.state.authentication = (_browser, _credentials, { signal }) => new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(Object.assign(new Error('cancelled'), { code: 'acquisition_cancelled' })), { once: true }); + }); + const credentials = { username: 'restart-owner', password: 'restart-fixture-secret' }; + assert.equal((await f.save('cortex', credentials)).status, 202); + await f.restartBroker(); + assert.deepEqual(f.state.broker.vault.readForAdapter('amazon-operations').credentials, credentials); + assert.notEqual((await f.list())[0].state, 'connected'); +}); diff --git a/core/dashboard/tests/cortex-verification.test.js b/core/dashboard/tests/cortex-verification.test.js new file mode 100644 index 0000000..fbefdb4 --- /dev/null +++ b/core/dashboard/tests/cortex-verification.test.js @@ -0,0 +1,129 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const { createConnectionsStack } = require('./helpers/connections-stack.cjs'); +const credentials = { username: 'verification-owner', password: 'verification-private-password' }; +const code = '827194'; +const reject = reason => { throw Object.assign(new Error(reason), { code: reason }); }; +async function fixture(t) { + const f = await createConnectionsStack(); t.after(() => f.close()); + f.state.authentication = async (_browser, _credentials, { onSubmit }) => { onSubmit(); reject('mfa_required'); }; + f.post = (action, body = {}, headers = f.headers) => fetch(`${f.base}/api/organization/connections/cortex/${action}`, { + method: 'POST', headers, body: JSON.stringify(body), + }); + f.finish = () => f.state.broker.serviceConnections.close(); + f.view = async () => (await f.list()).find(item => item.service === 'cortex'); + assert.equal((await f.save('cortex', credentials)).status, 202); await f.finish(); + return f; +} +function noCode(root) { + for (const item of fs.readdirSync(root, { withFileTypes: true })) { + const file = path.join(root, item.name); + if (item.isDirectory()) noCode(file); + else if (item.isFile()) assert.equal(fs.readFileSync(file).includes(Buffer.from(code)), false, item.name); + } +} +test('owner submits an emailed code to the same browser, retries a wrong code and persists only success metadata', async t => { + const f = await fixture(t), initial = await f.view(); + assert.equal(initial.state, 'verification_required'); assert.equal(initial.reason, 'mfa_required'); + assert.equal(initial.verification.attemptsRemaining, 3); + assert.equal(f.state.browsers.length, 1); assert.equal(f.state.browsers[0].closed, false); + let accepted = false; + f.state.verification = async (browser, input) => { + assert.equal(browser, f.state.browsers[0]); assert.equal(input.code, code); + if (!accepted) reject('verification_code_rejected'); + return { status: 'authenticated' }; + }; + const body = { verificationId: initial.verification.id, code }; + assert.equal((await f.post('verify', body)).status, 202); await f.finish(); + const wrong = await f.view(); + assert.equal(wrong.reason, 'verification_code_rejected'); assert.equal(wrong.verification.attemptsRemaining, 2); + assert.equal(f.state.browsers[0].closed, false); + accepted = true; + const response = await f.post('verify', body); assert.equal(response.status, 202); + assert.equal((await response.text()).includes(code), false); await f.finish(); + assert.equal((await f.view()).state, 'connected'); assert.equal((await f.view()).verification, undefined); + assert.equal(f.state.browsers[0].closed, true); assert.equal(f.state.browsers.length, 1); + assert.equal(f.state.broker.sessions.attemptGuard.status('amazon-operations'), null); + noCode(f.root); await f.restartBroker(); assert.equal((await f.view()).state, 'connected'); +}); +test('verification HTTP requires CSRF and DSP ownership, including an authenticated platform-owner view', async t => { + const f = await fixture(t), pending = await f.view(); + const body = { verificationId: pending.verification.id, code }; + let calls = 0; f.state.verification = async () => { calls++; return { status: 'authenticated' }; }; + const missingCsrf = { ...f.headers }; delete missingCsrf['X-Dispatch-CSRF']; + assert.equal((await f.post('verify', body, missingCsrf)).status, 403); + assert.equal((await f.post('verify', { ...body, code: 'bad' })).status, 400); + assert.equal((await f.post('verify', { ...body, runtimeKey: 'other' })).status, 400); + assert.equal((await f.post('verify', { ...body, verificationId: 'a'.repeat(22) })).status, 409); + const platform = f.access.session(f.platform.token); + const headers = { ...f.headers, Cookie: `dispatch_session=${f.platform.token}`, 'X-Dispatch-CSRF': platform.csrfToken }; + assert.equal((await f.post('verify', body, headers)).status, 409); + const view = f.access.beginDspView(platform, { controlRef: f.access.issuePlatformControlRef(platform, f.organizationId) }); + headers['X-Dispatch-DSP-View'] = view.dspView.viewRef; + assert.equal((await f.post('verify', body, headers)).status, 202); await f.finish(); + assert.equal(calls, 1); assert.equal((await f.view()).state, 'connected'); + const audit = f.store.db.prepare("SELECT actor_user_id,organization_id FROM audit_events WHERE action='connection.verify'").get(); + assert.equal(audit.actor_user_id, platform.user.id); assert.equal(audit.organization_id, f.organizationId); + noCode(f.root); +}); +test('expiry and broker restart discard the challenge; stale codes cannot reach a newer browser', async t => { + const f = await fixture(t), original = await f.view(); + const pending = f.state.broker.sessions.verifications.entries.get('amazon-operations'); + pending.deadline = 0; + assert.equal((await f.view()).verification, undefined); + await f.state.broker.sessions.verifications.cancel('amazon-operations'); + assert.equal(f.state.browsers[0].closed, true); + assert.equal((await f.post('verify', { verificationId: original.verification.id, code })).status, 409); + assert.equal((await f.post('test')).status, 202); await f.finish(); + const next = await f.view(); assert.ok(next.verification); assert.notEqual(next.verification.id, original.verification.id); + assert.equal((await f.post('verify', { verificationId: original.verification.id, code })).status, 409); + await f.restartBroker(); assert.equal((await f.view()).verification, undefined); + assert.equal((await f.view()).reason, 'verification_expired'); + assert.ok(f.state.browsers.every(browser => browser.closed)); +}); +test('replacing or disconnecting credentials cancels the waiting browser and its code', async t => { + const f = await fixture(t), pending = await f.view(); + assert.equal((await f.save('cortex', { ...credentials, username: 'replacement-owner' })).status, 202); await f.finish(); + assert.equal(f.state.browsers[0].closed, true); + assert.equal((await f.post('verify', { verificationId: pending.verification.id, code })).status, 409); + const replacement = await f.view(); + assert.equal((await f.post('disconnect')).status, 202); + assert.equal((await f.post('verify', { verificationId: replacement.verification.id, code })).status, 409); + assert.equal((await f.view()).state, 'not_connected'); + assert.ok(f.state.browsers.every(browser => browser.closed)); +}); +test('a pending code submission is exclusive and cannot outlive revoked credentials', async t => { + const f = await fixture(t), pending = await f.view(); let finish; + f.state.verification = async () => new Promise(resolve => { finish = resolve; }); + const body = { verificationId: pending.verification.id, code }; + assert.equal((await f.post('verify', body)).status, 202); + assert.equal((await f.post('verify', body)).status, 409); + assert.equal((await f.post('disconnect')).status, 409); + f.state.broker.vault.put('amazon-operations', 'amazon-logistics', { ...credentials, username: 'changed-directly' }, { operation: 'replace' }); + finish({ status: 'authenticated' }); await f.finish(); + assert.notEqual((await f.view()).state, 'connected'); +}); +test('three rejected codes close the challenge without repeating the login', async t => { + const f = await fixture(t), pending = await f.view(); + f.state.verification = async () => reject('verification_code_rejected'); + for (let attempt = 0; attempt < 3; attempt++) { + assert.equal((await f.post('verify', { verificationId: pending.verification.id, code })).status, 202); await f.finish(); + } + assert.equal((await f.view()).reason, 'verification_expired'); + assert.equal(f.state.browsers.length, 1); assert.equal(f.state.browsers[0].closed, true); +}); +test('an owner check observes an existing completed sign-in without retrying latched credentials', async t => { + const f = await fixture(t); + const sessions = f.state.broker.sessions; + await sessions.verifications.cancel('amazon-operations', 'manual_verification_required'); + sessions.attemptGuard.lock('amazon-operations'); + f.state.broker.vault.readForAdapter = () => { throw new Error('must not read credentials during recovery'); }; + let observed = 0; + sessions.adapters['amazon-logistics'].recover = async () => { observed++; return { status: 'authenticated' }; }; + assert.equal((await f.post('test')).status, 202); await f.finish(); + assert.equal(observed, 1); assert.equal((await f.view()).state, 'connected'); + assert.equal(sessions.attemptGuard.status('amazon-operations'), null); +}); diff --git a/core/dashboard/tests/dashboard.test.js b/core/dashboard/tests/dashboard.test.js new file mode 100644 index 0000000..e7bed8c --- /dev/null +++ b/core/dashboard/tests/dashboard.test.js @@ -0,0 +1,1294 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); + +const { AccessStore, AccessControlService } = require('../../core/accounts/src'); +const { createDashboardServer, sourceDate, publicSyncView } = require('../server/server'); +const { parseArguments } = require('../server/main'); + +const COLLECTED = '2026-08-30T18:00:00.000Z'; + +test('private installation control is an explicit capability separate from sync operation', () => { + assert.deepEqual(parseArguments(['--operator', '--installation-operator']), { + host: '127.0.0.1', port: 4310, operator: true, installationOperator: true, + installationBackend: 'native_service_v1', secureCookies: false, publicOrigin: null, + }); + assert.equal(parseArguments([ + '--secure-cookies', '--public-origin', 'https://dispatch.example.test', + ]).publicOrigin, 'https://dispatch.example.test'); + assert.throws(() => parseArguments(['--public-origin', 'https://dispatch.example.test'])); + assert.equal(parseArguments(['--secure-cookies', '--public-origin', 'https://other.example']).publicOrigin, 'https://other.example'); + assert.throws(() => parseArguments(['--secure-cookies', '--public-origin', 'https://other.example/path'])); + assert.equal(parseArguments([]).installationOperator, false); + assert.equal(parseArguments(['--installation-backend', 'oci_container_v1']).installationBackend, + 'oci_container_v1'); + assert.throws(() => parseArguments(['--installation-backend', 'unknown'])); + assert.throws(() => parseArguments(['--installation-backend', 'systemd_user'])); + const serviceTemplate = fs.readFileSync( + path.join(__dirname, "../integration/systemd/dispatch-dashboard.service.in"), 'utf8', + ); + assert.doesNotMatch(serviceTemplate, /--installation-operator/); + assert.match(serviceTemplate, /--secure-cookies --public-origin \$\{DISPATCH_PUBLIC_ORIGIN\}/); +}); + +test('public origin enforces canonical host, mutation origin, and secure cookies', async t => { + const origin = 'https://dispatch.example.test'; + const running = await runningServer({ secureCookies: true, publicOrigin: origin }); + t.after(running.close); + const cloudflareHttps = { 'CF-Visitor': '{"scheme":"https"}' }; + const request = ({ pathname, method = 'GET', headers = {}, body = null }) => new Promise((resolve, reject) => { + const local = http.request(new URL(pathname, running.base), { method, headers }, response => { + const chunks = []; + response.on('data', chunk => chunks.push(chunk)); + response.once('end', () => resolve({ + status: response.statusCode, + headers: response.headers, + body: Buffer.concat(chunks).toString('utf8'), + })); + }); + local.once('error', reject); + if (body !== null) local.end(body); else local.end(); + }); + + const wrongHost = await request({ + pathname: '/api/auth/session', headers: { Host: 'untrusted.example', ...cloudflareHttps }, + }); + assert.equal(wrongHost.status, 403); + + const page = await request({ + pathname: '/', headers: { Host: 'dispatch.example.test', ...cloudflareHttps }, + }); + assert.equal(page.status, 200); + assert.equal(page.headers['cache-control'], 'no-store'); + assert.equal(page.headers['strict-transport-security'], 'max-age=31536000'); + + const insecurePage = await request({ + pathname: '/api/auth/session?probe=1', + headers: { Host: 'dispatch.example.test', 'CF-Visitor': '{"scheme":"http"}' }, + }); + assert.equal(insecurePage.status, 308); + assert.equal(insecurePage.headers.location, `${origin}/api/auth/session?probe=1`); + + const credentials = JSON.stringify({ + email: 'owner@example.test', password: 'correct horse battery staple', + }); + const missingOrigin = await request({ + pathname: '/api/auth/login', + method: 'POST', + headers: { Host: 'dispatch.example.test', 'Content-Type': 'application/json', ...cloudflareHttps }, + body: credentials, + }); + assert.equal(missingOrigin.status, 403); + + const login = await request({ + pathname: '/api/auth/login', + method: 'POST', + headers: { + Host: 'dispatch.example.test', Origin: origin, 'Content-Type': 'application/json', ...cloudflareHttps, + }, + body: credentials, + }); + assert.equal(login.status, 200); + assert.match(login.headers['set-cookie'][0], /^__Host-dispatch_session=/); + assert.match(login.headers['set-cookie'][0], /; Path=\/; HttpOnly; SameSite=Strict;/); + assert.match(login.headers['set-cookie'][0], /; Secure/); + assert.doesNotMatch(login.headers['set-cookie'][0], /; Domain=/i); + + const authenticated = JSON.parse(login.body).data; + const publicCookie = login.headers['set-cookie'][0].split(';')[0]; + const publicHeaders = { + Host: 'dispatch.example.test', + Origin: origin, + 'Content-Type': 'application/json', + 'X-Dispatch-CSRF': authenticated.csrfToken, + Cookie: publicCookie, + ...cloudflareHttps, + }; + const before = JSON.parse((await request({ + pathname: '/api/platform/organizations', + headers: { Host: 'dispatch.example.test', Cookie: publicCookie, ...cloudflareHttps }, + })).body).data.length; + const unavailable = await request({ + pathname: '/api/platform/organizations', + method: 'POST', + headers: publicHeaders, + body: JSON.stringify({ + idempotencyKey: 'dashboard:organization:email-required', + name: 'Must Not Exist', abbreviation: null, stationCode: 'DWA9', + timezone: 'America/Los_Angeles', ownerEmail: 'blocked@example.test', + }), + }); + assert.equal(unavailable.status, 503); + assert.equal(JSON.parse(unavailable.body).status, 'invitation_email_unavailable'); + const after = JSON.parse((await request({ + pathname: '/api/platform/organizations', + headers: { Host: 'dispatch.example.test', Cookie: publicCookie, ...cloudflareHttps }, + })).body).data.length; + assert.equal(after, before); +}); + +test('email-only creation reports disabled provisioning without creating records or sending mail', async t => { + const deliveries = []; + const running = await runningServer({ platformOnly: true, invitationDelivery: { + send: async value => { deliveries.push(value); return { status: 'accepted' }; }, + } }); + t.after(running.close); + const before = running.store.db.prepare('SELECT count(*) AS count FROM invitations').get().count; + const create = () => fetch(`${running.base}/api/platform/organizations`, { + method: 'POST', + headers: { Cookie: running.cookie, 'Content-Type': 'application/json', 'X-Dispatch-CSRF': running.csrfToken }, + body: JSON.stringify({ idempotencyKey: 'dashboard:organization:disabled', ownerEmail: 'new-owner@example.test' }), + }); + const response = await create(); + assert.equal(response.status, 503); + assert.equal((await response.json()).error.code, 'installation_operator_disabled'); + assert.equal(running.store.db.prepare('SELECT count(*) AS count FROM organizations').get().count, 0); + assert.equal(running.store.db.prepare('SELECT count(*) AS count FROM invitations').get().count, before); + assert.equal(deliveries.length, 0); + + // Only the explicit AccessError allowlist is public; unexpected 5xx details stay private. + for (const error of [ + Object.assign(new Error('private database path'), { statusCode: 503, code: 'installation_operator_disabled' }), + new (require('../../core/accounts/src').AccessError)('private_internal_failure', 503), + ]) { + running.access.createOrganization = () => { throw error; }; + const failure = await create(); + assert.equal(failure.status, 503); + const payload = await failure.json(); + assert.equal(payload.error.code, 'dashboard_unavailable'); + assert.doesNotMatch(JSON.stringify(payload), /private|installation_operator_disabled/); + } +}); + +test('dashboard failure views discard upstream details and raw exception messages', async t => { + assert.deepEqual(publicSyncView({ + ok: false, + status: 'sync_unavailable', + error: { code: 'sync_unavailable', recoverable: true, detail: '/private/runtime' }, + data: null, + }), { + ok: false, status: 'sync_unavailable', + error: { code: 'sync_unavailable', recoverable: true }, data: null, + }); + assert.deepEqual(publicSyncView({ + ok: false, status: '/private/runtime', error: { code: '/private/runtime' }, data: null, + }), { + ok: false, status: 'sync_unavailable', + error: { code: 'sync_unavailable', recoverable: false }, data: null, + }); + + const client = fixtureClient(); + client.workforce.day = async () => { throw Object.assign(new Error('/private/runtime'), { statusCode: 400 }); }; + const running = await runningServer({ client }); + t.after(running.close); + const response = await fetch(`${running.base}/api/paycom/daily?date=2026-08-30`, { + headers: { Cookie: running.cookie }, + }); + const body = await response.json(); + assert.equal(response.status, 400); + assert.equal(body.status, 'invalid_input'); + assert.equal(JSON.stringify(body).includes('/private/runtime'), false); +}); + +function fixtureClient() { + const calls = []; + const syncData = { + id: 'paycom-main-workforce', desiredState: 'running', activity: 'idle', + lastSucceededAt: COLLECTED, lastError: null, + businessContext: { date: '2026-08-30', timezone: 'America/Los_Angeles' }, + alerts: [], activeRun: null, queuedRunCount: 0, + }; + return { + calls, + workforce: { + day: async query => { + calls.push(['day', query]); + return { + contractVersion: 1, ok: true, status: 'found', + data: { + kind: 'workforce_day', target: '2026-09-05', businessDate: query.date, + businessTimezone: 'America/Los_Angeles', periodStart: '2026-08-23', periodEnd: '2026-09-05', + available: true, collectedAt: COLLECTED, + summary: { + employees: 1, activeEmployees: 1, inDayPunches: 1, completeTimecards: 0, + needsReview: 0, noActivity: 0, missingOutDay: 1, incompleteLunch: 0, unclassifiedPunches: 0, + }, + items: [{ employeeCode: 'A001', employeeName: 'Fixture Employee' }], + total: 1, limit: query.limit, offset: query.offset, hasMore: false, + }, + }; + }, + }, + sync: { + status: async id => { + calls.push(['sync.status', id]); + return { contractVersion: 1, ok: true, status: 'found', data: syncData }; + }, + runNow: async (id, options) => { + calls.push(['sync.runNow', id, options]); + return { contractVersion: 1, ok: true, status: 'queued', data: { sync: syncData, run: null } }; + }, + }, + system: { + status: async () => ({ + contractVersion: 1, ok: true, status: 'degraded', + data: { + components: { + auth: { healthy: false, ready: false, status: 'stopped', data: null, error: null }, + collections: { healthy: true, ready: false, status: 'stopped', data: { counts: { queued: 0, running: 0 } }, error: null }, + paycom: { healthy: true, ready: true, status: 'ready', data: {}, error: null }, + }, + summary: { ready: 1, degraded: 2, failed: 0 }, + }, + }), + }, + }; +} + +async function runningServer(options = {}) { + const client = options.client || fixtureClient(); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-dashboard-access-')); + fs.chmodSync(root, 0o700); + const store = new AccessStore({ + databaseRoot: path.join(root, 'access-control'), + database: path.join(root, 'access-control', 'access-control.sqlite3'), + }); + const now = () => new Date('2026-09-01T12:00:00.000Z'); + const access = new AccessControlService(store, { + clock: now, + installationOperatorEnabled: options.installationOperator ?? false, + ...(options.installationBackend ? { installationBackend: options.installationBackend } : {}), + }); + if (!options.platformOnly) access.ensureLocalOrganization({ + organization: { id: 'local-dsp', name: 'Fixture DSP' }, + site: { id: 'local-site', code: 'TST1' }, + timezone: 'America/Los_Angeles', + }); + if (!options.platformOnly) require('../../core/accounts/tests/plugin-fixture').enableFixturePlugin(store, 'local-dsp'); + const invitation = access.createPlatformBootstrap({ email: 'owner@example.test', organizationId: options.platformOnly ? null : 'local-dsp' }); + const authenticated = await access.acceptNewUser({ + token: invitation.token, firstName: 'Fixture', lastName: 'Owner', + password: 'correct horse battery staple', confirmPassword: 'correct horse battery staple', + }); + // These legacy tenant API tests explicitly select their existing DSP membership. + if (!options.platformOnly) access.selectMembership(authenticated.session, authenticated.session.memberships[0].id); + const server = createDashboardServer({ + client, + access, + publicRoot: options.publicRoot, + coreIdentity: options.coreIdentity, + coreMaintenance: options.coreMaintenance, + operator: options.operator ?? false, + secureCookies: options.secureCookies ?? false, + publicOrigin: options.publicOrigin ?? null, + invitationDelivery: options.invitationDelivery ?? null, + turnstile: options.turnstile ?? null, + paycomSetup: options.paycomInvoke ? require('../../core/accounts/src/owner-paycom-setup').createOwnerPaycomSetup({ + store, access, invoke: options.paycomInvoke, clock: () => now().getTime(), + }) : null, + connections: options.connectionsInvoke ? require('../../core/accounts/src/owner-connections').createOwnerConnections({ + store, access, invoke: options.connectionsInvoke, clock: () => now().getTime(), + }) : null, + backups: options.backups ? require('../../core/accounts/src/platform-backups').createPlatformBackups({store,enabled:true}) : null, + updates: options.updates ? require('../../core/accounts/src/platform-updates').createPlatformUpdates({ + store, releases: { dispatch_update_2: {} }, platformReleases: { dispatch_update_2: { version: '0.0.2', publishedAt: '2026-09-05T00:00:00.000Z', changelog: [], core: {} } }, enabled: true, + }) : null, + now, + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', resolve); + }); + const address = server.address(); + return { + access, + store, + client, + server, + cookie: `dispatch_session=${authenticated.token}`, + csrfToken: authenticated.session.csrfToken, + base: `http://127.0.0.1:${address.port}`, + close: () => new Promise(resolve => server.close(() => { store.close(); fs.rmSync(root, { recursive: true, force: true }); resolve(); })), + }; +} + +test('dashboard serves the secure application shell and a source-local bootstrap', async t => { + const running = await runningServer(); + t.after(running.close); + const page = await fetch(`${running.base}/`); + assert.equal(page.status, 200); + assert.match(page.headers.get('content-security-policy'), /default-src 'self'/); + assert.match(await page.text(), /
<\/div>/); + + const anonymous = await (await fetch(`${running.base}/api/auth/session`)).json(); + assert.equal(anonymous.data.authenticated, false); + const blocked = await fetch(`${running.base}/api/bootstrap`); + assert.equal(blocked.status, 401); + const authenticated = await (await fetch(`${running.base}/api/auth/session`, { headers: { Cookie: running.cookie } })).json(); + assert.equal(authenticated.data.authenticated, true); + assert.equal(authenticated.data.memberships[0].organization.id, 'local-dsp'); + assert.equal(JSON.stringify(authenticated).includes('runtimeKey'), false); + + const bootstrap = await (await fetch(`${running.base}/api/bootstrap`, { headers: { Cookie: running.cookie } })).json(); + assert.equal(bootstrap.ok, true); + assert.equal(bootstrap.data.today, '2026-09-01'); + assert.equal(bootstrap.data.timezone, 'America/Los_Angeles'); + assert.equal(bootstrap.data.operatorActions, false); + assert.equal(typeof bootstrap.data.csrfToken, 'string'); + assert.equal(sourceDate(new Date('2026-09-01T02:00:00.000Z'), 'America/Los_Angeles'), '2026-08-31'); +}); + +test('shell references content-addressed assets with matching GET and HEAD responses', async t => { + const running = await runningServer(); + t.after(running.close); + const page = await fetch(`${running.base}/`); + const html = await page.text(); + assert.equal(page.headers.get('cache-control'), 'no-store'); + const normalizeNonce = value => value.replace(/( match[1]); + assert.deepEqual(urls.map(url => url.split('/').pop().split('.')[0]), [ + 'updates', 'backups', 'launcher', + ]); + for (const url of urls) { + assert.match(url, /^\/assets\/(frontend|updates|backups|styles|launcher)\.[a-f0-9]{64}\.(js|css)$/); + const response = await fetch(`${running.base}${url}`); + assert.equal(response.status, 200); + assert.equal(response.headers.get('cache-control'), 'public, max-age=31536000, immutable'); + const bytes = Buffer.from(await response.arrayBuffer()); + const digest = require('node:crypto').createHash('sha256').update(bytes).digest('hex'); + assert.equal(url.includes(digest), true); + const head = await fetch(`${running.base}${url}`, { method: 'HEAD' }); + assert.equal(await head.text(), ''); + for (const header of ['content-type', 'content-length', 'cache-control']) { + assert.equal(head.headers.get(header), response.headers.get(header)); + } + const legacyUrl = url.replace(/\.[a-f0-9]{64}\./, '.'); + const legacy = await fetch(`${running.base}${legacyUrl}`); + assert.equal(legacy.headers.get('cache-control'), 'no-store'); + assert.deepEqual(Buffer.from(await legacy.arrayBuffer()), bytes); + } + assert.equal((await fetch(`${running.base}/assets/styles.${'0'.repeat(64)}.css`)).status, 404); +}); + +test('deployments bypass previously cached assets and keep each running shell consistent', async t => { + const publicRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-dashboard-assets-')); + t.after(() => fs.rmSync(publicRoot, { recursive: true, force: true })); + fs.cpSync(path.join(__dirname, "../public"), publicRoot, { recursive: true }); + const before = await runningServer({ publicRoot }); + t.after(before.close); + const htmlBefore = await (await fetch(`${before.base}/`)).text(); + const styleUrl = html => html.match(/src="([^\"]*launcher[^\"]+\.js)"/)[1]; + const urlBefore = styleUrl(htmlBefore); + const cssBefore = await (await fetch(`${before.base}${urlBefore}`)).text(); + // Model the still-fresh browser cache retained across a deployment. + const cache = new Map([['/assets/launcher.js', cssBefore], [urlBefore, cssBefore]]); + fs.appendFileSync(path.join(publicRoot, 'assets/launcher.js'), '\n// next launcher build\n'); + const normalizeNonce = value => value.replace(/( { + const running = await runningServer({ platformOnly: true, updates: true }); + t.after(running.close); + const session = await (await fetch(`${running.base}/api/auth/session`, { headers: { Cookie: running.cookie } })).json(); + assert.equal(session.data.activeOrganizationId, null); + assert.deepEqual(session.data.memberships, []); + assert.equal(running.store.organizations().length, 0); + const headers = { Cookie: running.cookie }; + assert.equal((await fetch(`${running.base}/api/platform/organizations`, { headers })).status, 200); + assert.equal((await fetch(`${running.base}/api/platform/updates`, { headers })).status, 200); + assert.equal((await fetch(`${running.base}/api/bootstrap`, { headers })).status, 409); + assert.equal(running.client.calls.length, 0, 'platform login must not call a DSP runtime'); + const { administerOwner } = require('../../core/accounts/src/owner-admin'); + await administerOwner(running.store, 'owner-recover', { email: 'owner@example.test', newEmail: 'recovered@example.test', + password: 'new platform recovery password', confirmPassword: 'new platform recovery password' }); + assert.equal((await fetch(`${running.base}/api/platform/organizations`, { headers })).status, 401); + const login = await fetch(`${running.base}/api/auth/login`, { method: 'POST', + headers: { 'Content-Type': 'application/json', Origin: running.base }, + body: JSON.stringify({ email: 'recovered@example.test', password: 'new platform recovery password' }) }); + assert.equal(login.status, 200); + const recovered = await login.json(); + assert.equal(recovered.data.activeOrganizationId, null); + assert.deepEqual(recovered.data.memberships, []); +}); + +test('platform organization and owner controls use opaque session-bound references', async t => { + const running = await runningServer(); + t.after(running.close); + const login = await fetch(`${running.base}/api/auth/login`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ email: 'owner@example.test', password: 'correct horse battery staple' }), + }); + assert.equal(login.status, 200); + assert.match(login.headers.get('set-cookie'), /HttpOnly/); + assert.match(login.headers.get('set-cookie'), /SameSite=Strict/); + + const createBody = { + idempotencyKey: 'dashboard:organization:create:second', + name: 'Second DSP', abbreviation: 'SDSP', stationCode: 'DWA1', + timezone: 'America/Los_Angeles', ownerEmail: 'second-owner@example.test', + }; + const provisioned = await fetch(`${running.base}/api/platform/organizations`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Cookie: running.cookie, 'X-Dispatch-CSRF': running.csrfToken }, + body: JSON.stringify(createBody), + }); + const created = await provisioned.json(); + assert.equal(provisioned.status, 201); + assert.equal(Object.hasOwn(created.data.organization, 'id'), false); + assert.equal(JSON.stringify(created).includes('runtimeKey'), false); + const invitationToken = /^#\/invitation\/([A-Za-z0-9_-]{43})$/.exec(new URL(created.data.invitationPath, running.base).hash)?.[1]; + assert.equal(typeof invitationToken, 'string'); + const replay = await (await fetch(`${running.base}/api/platform/organizations`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Cookie: running.cookie, 'X-Dispatch-CSRF': running.csrfToken }, + body: JSON.stringify(createBody), + })).json(); + assert.equal(replay.status, 'replayed'); + assert.equal(replay.data.invitationPath, null); + + const registration = await fetch(`${running.base}/api/auth/register`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + token: invitationToken, firstName: 'Second', lastName: 'Owner', + password: 'second owner secure password', confirmPassword: 'second owner secure password', + }), + }); + assert.equal(registration.status, 201); + let ownerCookie = registration.headers.get('set-cookie').split(';')[0]; + const organizations = await (await fetch(`${running.base}/api/platform/organizations`, { + headers: { Cookie: running.cookie }, + })).json(); + const second = organizations.data.find(item => item.name === 'Second DSP'); + assert.match(second.controlRef, /^[A-Za-z0-9_-]{43}$/); + assert.match(second.continuityRef, /^[A-Za-z0-9_-]{43}$/); + assert.equal(Object.hasOwn(second, 'id'), false); + assert.equal(JSON.stringify(second).includes('runtimeKey'), false); + assert.equal(second.availableActions.includes('suspend'), false); + assert.deepEqual(second.installation.availableActions, []); + const disabledProvision = await fetch(`${running.base}/api/platform/installation/provision`, { + method: 'POST', + headers: { Cookie: running.cookie, 'Content-Type': 'application/json', 'X-Dispatch-CSRF': running.csrfToken }, + body: JSON.stringify({ + controlRef: second.controlRef, + idempotencyKey: 'dashboard:installation:disabled:second', + expectedRevision: second.installation.revision, + }), + }); + assert.equal(disabledProvision.status, 503); + + const suspended = await fetch(`${running.base}/api/platform/organization/status`, { + method: 'POST', + headers: { Cookie: running.cookie, 'Content-Type': 'application/json', 'X-Dispatch-CSRF': running.csrfToken }, + body: JSON.stringify({ + controlRef: second.controlRef, idempotencyKey: 'dashboard:organization:suspend:second', suspended: true, + }), + }); + assert.equal(suspended.status, 200); + const suspendedProjection = (await (await fetch(`${running.base}/api/platform/organizations`, { + headers: { Cookie: running.cookie }, + })).json()).data.find(item => item.name === 'Second DSP'); + assert.equal(suspendedProjection.availableActions.includes('resume'), false); + assert.equal((await fetch(`${running.base}/api/organization/administration`, { headers: { Cookie: ownerCookie } })).status, 401); + const resumed = await fetch(`${running.base}/api/platform/organization/status`, { + method: 'POST', + headers: { Cookie: running.cookie, 'Content-Type': 'application/json', 'X-Dispatch-CSRF': running.csrfToken }, + body: JSON.stringify({ + controlRef: second.controlRef, idempotencyKey: 'dashboard:organization:resume:second', suspended: false, + }), + }); + assert.equal(resumed.status, 200); + assert.equal((await fetch(`${running.base}/api/organization/administration`, { headers: { Cookie: ownerCookie } })).status, 401); + const relogin = await fetch(`${running.base}/api/auth/login`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ email: createBody.ownerEmail, password: 'second owner secure password' }) }); + assert.equal(relogin.status, 200); + ownerCookie = relogin.headers.get('set-cookie').split(';')[0]; + const ownerAdministration = await fetch(`${running.base}/api/organization/administration`, { headers: { Cookie: ownerCookie } }); + assert.equal(ownerAdministration.status, 200); + const ownerAdministrationBody = await ownerAdministration.json(); + assert.notEqual(ownerAdministrationBody.data.organization.id, 'local-dsp'); + assert.equal(ownerAdministrationBody.data.organization.name, 'Second DSP'); + const tenantOverride = await fetch(`${running.base}/api/organization/administration?organizationId=local-dsp`, { headers: { Cookie: ownerCookie } }); + assert.equal(tenantOverride.status, 400); + const legacyPlatformPath = await fetch(`${running.base}/api/platform/organizations/local-dsp/status`, { + method: 'POST', headers: { Cookie: running.cookie, 'Content-Type': 'application/json', 'X-Dispatch-CSRF': running.csrfToken }, body: '{}', + }); + assert.equal(legacyPlatformPath.status, 405); + const pendingRuntime = await fetch(`${running.base}/api/bootstrap`, { headers: { Cookie: ownerCookie } }); + assert.equal(pendingRuntime.status, 409); +}); + +test('invitation email uses authoritative fields, hides accepted links, and does not resend replays', async t => { + const deliveries = []; + const running = await runningServer({ + invitationDelivery: { + send: async value => { + deliveries.push(value); + return { status: value.email.startsWith('unknown-') ? 'unknown' : 'accepted' }; + }, + }, + }); + t.after(running.close); + const headers = { + 'Content-Type': 'application/json', Cookie: running.cookie, 'X-Dispatch-CSRF': running.csrfToken, + }; + const acceptedRequest = { + idempotencyKey: 'dashboard:organization:create:emailed', + name: 'Emailed DSP', abbreviation: 'MAIL', stationCode: 'DWA2', + timezone: 'America/New_York', ownerEmail: 'emailed-owner@example.test', + }; + const acceptedResponse = await fetch(`${running.base}/api/platform/organizations`, { + method: 'POST', headers, body: JSON.stringify(acceptedRequest), + }); + const accepted = await acceptedResponse.json(); + assert.equal(acceptedResponse.status, 201); + assert.deepEqual(accepted.data.delivery, { status: 'accepted' }); + assert.equal(accepted.data.invitationPath, null); + assert.equal(deliveries.length, 1); + assert.equal(deliveries[0].email, acceptedRequest.ownerEmail); + assert.equal(deliveries[0].organizationName, acceptedRequest.name); + assert.equal(deliveries[0].roleName, 'Owner'); + assert.match(deliveries[0].token, /^[A-Za-z0-9_-]{43}$/); + assert.equal(JSON.stringify(accepted).includes(deliveries[0].token), false); + + const replay = await (await fetch(`${running.base}/api/platform/organizations`, { + method: 'POST', headers, body: JSON.stringify(acceptedRequest), + })).json(); + assert.equal(replay.status, 'replayed'); + assert.deepEqual(replay.data.delivery, { status: 'already_processed' }); + assert.equal(replay.data.invitationPath, null); + assert.equal(deliveries.length, 1); + + const unknownRequest = { + idempotencyKey: 'dashboard:organization:create:email-unknown', + name: 'Unknown Delivery DSP', abbreviation: null, stationCode: 'DWA3', + timezone: 'America/Chicago', ownerEmail: 'unknown-owner@example.test', + }; + const unknownResponse = await fetch(`${running.base}/api/platform/organizations`, { + method: 'POST', headers, body: JSON.stringify(unknownRequest), + }); + const unknown = await unknownResponse.json(); + assert.equal(unknownResponse.status, 201); + assert.deepEqual(unknown.data.delivery, { status: 'unknown' }); + assert.match(unknown.data.invitationPath, /^\/#\/invitation\/[A-Za-z0-9_-]{43}$/); + assert.equal(deliveries.length, 2); + assert.equal(unknown.data.invitationPath.endsWith(deliveries[1].token), true); +}); + +test('platform provisioning request and owner setup status remain browser-safe and runtime-free', async t => { + const running = await runningServer({ installationOperator: true }); + t.after(running.close); + const createdResponse = await fetch(`${running.base}/api/platform/organizations`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Cookie: running.cookie, 'X-Dispatch-CSRF': running.csrfToken }, + body: JSON.stringify({ + idempotencyKey: 'dashboard:organization:create:setup', + name: 'Setup DSP', abbreviation: 'SETUP', stationCode: 'DWA6', + timezone: 'America/Chicago', ownerEmail: 'setup-owner@example.test', + }), + }); + const created = await createdResponse.json(); + assert.equal(createdResponse.status, 201); + const invitationToken = /^#\/invitation\/([A-Za-z0-9_-]{43})$/.exec( + new URL(created.data.invitationPath, running.base).hash, + )?.[1]; + assert.equal(typeof invitationToken, 'string'); + const inspectedInvitation = await (await fetch(`${running.base}/api/auth/invitation/inspect`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ token: invitationToken }), + })).json(); + assert.equal(Object.hasOwn(inspectedInvitation.data, 'id'), false); + assert.equal(Object.hasOwn(inspectedInvitation.data.organization, 'id'), false); + assert.equal(Object.hasOwn(inspectedInvitation.data.role, 'id'), false); + const ownerRegistration = await fetch(`${running.base}/api/auth/register`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + token: invitationToken, firstName: 'Setup', lastName: 'Owner', + password: 'setup owner secure password', confirmPassword: 'setup owner secure password', + }), + }); + const ownerCookie = ownerRegistration.headers.get('set-cookie').split(';')[0]; + const setup = await (await fetch(`${running.base}/api/organization/setup`, { + headers: { Cookie: ownerCookie }, + })).json(); + assert.equal(setup.data.installationState, 'pending'); + assert.equal(setup.data.setupState, 'waiting_for_platform'); + assert.equal(setup.data.operationalAccess, 'unavailable'); + assert.equal(JSON.stringify(setup).includes('runtimeKey'), false); + assert.equal((await fetch(`${running.base}/api/organization/setup?organizationId=local-dsp`, { + headers: { Cookie: ownerCookie }, + })).status, 400); + assert.equal((await fetch(`${running.base}/api/organization/setup`)).status, 401); + + const organizations = await (await fetch(`${running.base}/api/platform/organizations`, { + headers: { Cookie: running.cookie }, + })).json(); + const row = organizations.data.find(item => item.name === 'Setup DSP'); + const request = { + controlRef: row.controlRef, + idempotencyKey: 'dashboard:installation:provision:setup', + expectedRevision: row.installation.revision, + }; + const missingCsrf = await fetch(`${running.base}/api/platform/installation/provision`, { + method: 'POST', headers: { 'Content-Type': 'application/json', Cookie: running.cookie }, + body: JSON.stringify(request), + }); + assert.equal(missingCsrf.status, 403); + const callerSelectedTarget = await fetch(`${running.base}/api/platform/installation/provision`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Cookie: running.cookie, 'X-Dispatch-CSRF': running.csrfToken }, + body: JSON.stringify({ ...request, organizationId: 'local-dsp' }), + }); + assert.equal(callerSelectedTarget.status, 400); + const callerSubmittedContinuity = await fetch(`${running.base}/api/platform/installation/provision`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Cookie: running.cookie, 'X-Dispatch-CSRF': running.csrfToken }, + body: JSON.stringify({ ...request, continuityRef: row.continuityRef }), + }); + assert.equal(callerSubmittedContinuity.status, 400); + const accepted = await fetch(`${running.base}/api/platform/installation/provision`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Cookie: running.cookie, 'X-Dispatch-CSRF': running.csrfToken }, + body: JSON.stringify(request), + }); + const receipt = await accepted.json(); + assert.equal(accepted.status, 202); + assert.deepEqual(receipt.data, { + action: 'provision', status: 'accepted', installationState: 'provisioning', + installationRevision: 2, replayed: false, + }); + const replay = await (await fetch(`${running.base}/api/platform/installation/provision`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Cookie: running.cookie, 'X-Dispatch-CSRF': running.csrfToken }, + body: JSON.stringify(request), + })).json(); + assert.equal(replay.data.status, 'replayed'); + const latest = await (await fetch(`${running.base}/api/platform/organizations`, { + headers: { Cookie: running.cookie }, + })).json(); + const pending = latest.data.find(item => item.name === 'Setup DSP'); + assert.equal(pending.installation.state, 'provisioning'); + assert.deepEqual(pending.installation.operation, { kind: 'provision', status: 'pending' }); + assert.equal(JSON.stringify(pending).includes('jobId'), false); + const ownerAfterRequest = await (await fetch(`${running.base}/api/organization/administration`, { + headers: { Cookie: ownerCookie }, + })).json(); + assert.equal(JSON.stringify(ownerAfterRequest.data.audit).includes('targetId'), false); + assert.equal(JSON.stringify(ownerAfterRequest.data.audit).includes('organizationId'), false); +}); + +test('daily API forwards only the closed SDK query and returns sync freshness', async t => { + const running = await runningServer(); + t.after(running.close); + const response = await fetch(`${running.base}/api/paycom/daily?date=2026-08-30&search=Fixture&attention=incomplete&limit=10&offset=0`, { headers: { Cookie: running.cookie } }); + const body = await response.json(); + assert.equal(response.status, 200); + assert.equal(body.data.day.kind, 'workforce_day'); + assert.equal(body.data.day.summary.missingOutDay, 1); + assert.equal(body.data.sync.data.desiredState, 'running'); + assert.deepEqual(running.client.calls.find(call => call[0] === 'day'), ['day', { + date: '2026-08-30', search: 'Fixture', attention: 'incomplete', limit: 10, offset: 0, + }]); + + await fetch(`${running.base}/api/paycom/daily?date=2026-08-30`, { headers: { Cookie: running.cookie } }); + assert.deepEqual(running.client.calls.filter(call => call[0] === 'day').at(-1), ['day', { + date: '2026-08-30', limit: 100, offset: 0, + }]); + + const invalid = await fetch(`${running.base}/api/paycom/daily?date=bad`, { headers: { Cookie: running.cookie } }); + assert.equal(invalid.status, 400); +}); + +test('employee routes require workforce authority, validate closed inputs, and sanitize failures', async t => { + const client = fixtureClient(); + client.workforce.employees = async query => { client.calls.push(['employees',query]); return { ok: true, status: 'found', data: { items: [], total: 0, hasMore: false } }; }; + client.workforce.employee = async code => { client.calls.push(['employee',code]); return { ok: false, status: 'employee_not_found', error: { code: 'employee_not_found', message: '/private/source' }, data: null }; }; + const running = await runningServer({ client }); t.after(running.close); + const headers = { Cookie: running.cookie }; + assert.equal((await fetch(`${running.base}/api/paycom/employees`)).status, 401); + assert.equal((await fetch(`${running.base}/api/paycom/employees`, { headers })).status, 200); + assert.deepEqual(client.calls.find(c=>c[0]==='employees')[1], { limit: 100, offset: 0, lifecycleStatus: null }); + const response = await fetch(`${running.base}/api/paycom/employees/a001`, { headers }); + assert.equal(response.status, 404); + assert.doesNotMatch(await response.text(), /private/); + assert.deepEqual(client.calls.find(c=>c[0]==='employee'), ['employee','A001']); + for (const suffix of ['employees?runtime=other','employees?limit=101','employees?offset=1&offset=2','employees/bad','employees/A001?tenant=other','daily?date=2026-02-30','daily?date=2026-08-30&sort=private','daily?date=2026-08-30&direction=random']) { + assert.equal((await fetch(`${running.base}/api/paycom/${suffix}`, { headers })).status, 400, suffix); + } + const before = client.calls.length; + const ownerRole = running.store.roles('local-dsp').find(role => role.key === 'owner'); + running.store.db.prepare('DELETE FROM role_permissions WHERE role_id=? AND permission=?').run(ownerRole.id, 'workforce.read'); + for (const suffix of ['employees', 'employees/A001', 'daily?date=2026-08-30']) { + assert.equal((await fetch(`${running.base}/api/paycom/${suffix}`, { headers })).status, 403); + } + assert.equal(client.calls.length, before); +}); + +test('Sync now works without operator mode and requires authentication, tenant permission, and session CSRF', async t => { + const running = await runningServer({ operator: false }); + t.after(running.close); + const unauthenticated = await fetch(`${running.base}/api/paycom/sync`, { method: 'POST', body: '{}' }); + assert.equal(unauthenticated.status, 401); + const forbidden = await fetch(`${running.base}/api/paycom/sync`, { + method: 'POST', headers: { 'Content-Type': 'application/json', Cookie: running.cookie }, body: '{}', + }); + assert.equal(forbidden.status, 403); + + const bootstrap = await (await fetch(`${running.base}/api/bootstrap`, { headers: { Cookie: running.cookie } })).json(); + const queried = await fetch(`${running.base}/api/paycom/sync?runtime=other`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Dispatch-CSRF': bootstrap.data.csrfToken, Cookie: running.cookie }, + body: '{}', + }); + assert.equal(queried.status, 400); + assert.equal((await queried.json()).status, 'invalid_request'); + assert.equal(running.client.calls.some(item => item[0] === 'sync.runNow'), false); + const queued = await fetch(`${running.base}/api/paycom/sync`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Dispatch-CSRF': bootstrap.data.csrfToken, Cookie: running.cookie }, + body: JSON.stringify({ idempotencyKey: 'dashboard:fixture-sync-request-1' }), + }); + const receipt = await queued.json(); + assert.equal(queued.status, 202); + assert.equal(receipt.status, 'queued'); + const call = running.client.calls.find(item => item[0] === 'sync.runNow'); + assert.equal(call[1], 'paycom-main-workforce'); + assert.equal(call[2].idempotencyKey, 'dashboard:fixture-sync-request-1'); + const count = running.client.calls.length; + const role = running.store.roles('local-dsp').find(item => item.key === 'owner'); + running.store.db.prepare('DELETE FROM role_permissions WHERE role_id=? AND permission=?').run(role.id, 'sync.run'); + const denied = await fetch(`${running.base}/api/paycom/sync`, { + method: 'POST', headers: { 'Content-Type': 'application/json', 'X-Dispatch-CSRF': bootstrap.data.csrfToken, Cookie: running.cookie }, + body: JSON.stringify({ idempotencyKey: 'dashboard:fixture-sync-request-2' }), + }); + assert.equal(denied.status, 403); + assert.equal(running.client.calls.length, count); + +}); + +test('DSP removal requires platform authority and CSRF, queues once and blocks retained invitations', async t => { + const running = await runningServer({ installationOperator: true }); + t.after(running.close); + const headers = { 'Content-Type': 'application/json', Cookie: running.cookie, 'X-Dispatch-CSRF': running.csrfToken }; + const create = await (await fetch(`${running.base}/api/platform/organizations`, { + method: 'POST', headers, body: JSON.stringify({ idempotencyKey: 'http:create:removal:alpha', + name: 'Removal Test DSP', abbreviation: 'REMOVAL', stationCode: 'DWA6', + timezone: 'America/Chicago', ownerEmail: 'removal@example.test' }), + })).json(); + const rows = await (await fetch(`${running.base}/api/platform/organizations`, { headers })).json(); + const row = rows.data.find(item => item.name === 'Removal Test DSP'); + const input = { controlRef: row.controlRef, idempotencyKey: 'http:removal:alpha', + expectedRevision: row.installation.revision }; + const remove = (body, requestHeaders = headers) => fetch(`${running.base}/api/platform/installation/remove`, { + method: 'POST', headers: requestHeaders, body: JSON.stringify(body), + }); + assert.equal((await remove(input, { 'Content-Type': 'application/json' })).status, 401); + assert.equal((await remove(input, { 'Content-Type': 'application/json', Cookie: running.cookie })).status, 403); + assert.equal((await remove({ ...input, confirmation: 'wrong' })).status, 400); + assert.equal((await remove({ ...input, organizationId: 'local-dsp' })).status, 400); + const response = await remove(input); + assert.equal(response.status, 202); + const result = await response.json(); + assert.equal(result.data.installationState, 'decommissioning'); + assert.equal((await (await remove(input)).json()).data.replayed, true); + const invitationToken = new URL(create.data.invitationPath, running.base).hash.split('/').at(-1); + assert.equal((await fetch(`${running.base}/api/auth/invitation/inspect`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ token: invitationToken }), + })).status, 404); + const latest = await (await fetch(`${running.base}/api/platform/organizations`, { headers })).json(); + const removed = latest.data.find(item => item.name === row.name); + assert.deepEqual(removed.installation.operation, { kind: 'decommission', status: 'queued' }); + assert.deepEqual(removed.availableActions, []); + assert.equal(latest.data.find(item => item.name === 'Fixture DSP').installation.state, 'ready'); +}); + + +test('owner Paycom HTTP setup requires owner membership, CSRF and a closed body before credential delivery', async t => { + const calls = []; + const running = await runningServer({ installationOperator: true, installationBackend: 'oci_container_v1', + paycomInvoke: async (...args) => { calls.push(args); return { ok: true, status: 'succeeded', data: { configured: true }, error: null }; } }); + t.after(running.close); + const platform = running.access.session(running.cookie.slice('dispatch_session='.length)); + const dsp = running.access.createOrganization(platform, { idempotencyKey: 'http:paycom:create', name: 'HTTP Paycom DSP', + abbreviation: 'HTTP', stationCode: 'TST1', timezone: 'UTC', ownerEmail: 'http-dsp@example.test' }); + const owner = await running.access.acceptNewUser({ token: dsp.token, firstName: 'HTTP', lastName: 'Owner', + password: 'fixture owner password', confirmPassword: 'fixture owner password' }); + running.store.updateInstallationControl({ organizationId: dsp.organization.id, expectedStatus: 'pending', expectedRevision: 1, + status: 'waiting_for_provider_auth', revision: 2, currentJobId: null, timestamp: Date.now() }); + require('../../core/accounts/tests/plugin-fixture').enableFixturePlugin(running.store, dsp.organization.id); + const input = { idempotencyKey: 'http:paycom:enroll', intent: 'create', credentials: { + clientCode: 'fixture', username: 'fixture', password: 'fixture', pin1: '1', pin2: '2', pin3: '3', pin4: '4', pin5: '5', + } }; + const endpoint = `${running.base}/api/organization/paycom-setup`; + const headers = { 'Content-Type': 'application/json', Cookie: `dispatch_session=${owner.token}`, 'X-Dispatch-CSRF': owner.session.csrfToken }; + assert.equal((await fetch(endpoint, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(input) })).status, 401); + assert.equal((await fetch(endpoint, { method: 'POST', headers: { ...headers, 'X-Dispatch-CSRF': 'wrong' }, body: JSON.stringify(input) })).status, 403); + assert.equal((await fetch(endpoint, { method: 'POST', headers, body: JSON.stringify({ ...input, runtimeKey: 'local' }) })).status, 400); + const invalidCredentials = await fetch(endpoint, { method: 'POST', headers, body: JSON.stringify({ ...input, credentials: { ...input.credentials, pin5: '1' } }) }); + assert.equal(invalidCredentials.status, 400); + assert.equal((await invalidCredentials.json()).error.code, 'paycom_credentials_invalid'); + assert.equal(calls.length, 0); + const result = await fetch(endpoint, { method: 'POST', headers, body: JSON.stringify(input) }); + assert.equal(result.status, 202); + assert.equal((await result.json()).data.status, 'queued'); + assert.equal(calls.length, 1); + assert.equal(calls[0][0], running.store.installationControl(dsp.organization.id).runtimeKey); + const replay = await fetch(endpoint, { method: 'POST', headers, body: JSON.stringify(input) }); + assert.equal((await replay.json()).data.replayed, true); + assert.equal(calls.length, 1); +}); + +test('email-first creation and rollout HTTP routes enforce platform permissions, CSRF and fixed targets', async t => { + const running = await runningServer({ installationOperator: true, installationBackend: 'oci_container_v1', updates: true }); + t.after(running.close); + const headers = { 'Content-Type': 'application/json', Cookie: running.cookie, 'X-Dispatch-CSRF': running.csrfToken }; + const createdResponse = await fetch(`${running.base}/api/platform/organizations`, { method: 'POST', headers, + body: JSON.stringify({ ownerEmail: 'email-first@example.test', idempotencyKey: 'dashboard:email-first:create' }) }); + assert.equal(createdResponse.status, 201); + const created = (await createdResponse.json()).data; + const row = running.store.organizations().find(o => o.name === 'New DSP'); + assert.equal(running.store.installationControl(row.id).status, 'provisioning'); + assert.equal(running.store.db.prepare('SELECT count(*) n FROM installation_provisioning_requests').get().n, 1); + const token = new URL(created.invitationPath, running.base).hash.split('/').at(-1); + const registered = await fetch(`${running.base}/api/auth/register`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ token, firstName: 'New', lastName: 'Owner', password: 'new owner password', confirmPassword: 'new owner password' }) }); + assert.equal(registered.status, 201); + const ownerHeaders = { Cookie: registered.headers.get('set-cookie').split(';')[0] }; + assert.equal((await fetch(`${running.base}/api/platform/updates`, { headers: ownerHeaders })).status, 403); + assert.equal((await fetch(`${running.base}/api/platform/updates`)).status, 401); + assert.equal((await fetch(`${running.base}/api/organization/profile`, { headers: ownerHeaders })).status, 200); + assert.equal((await fetch(`${running.base}/api/organization/profile?organizationId=local-dsp`, { headers: ownerHeaders })).status, 400); + const start = { action: 'start', releaseId: 'dispatch_update_2', idempotencyKey: 'dashboard:rollout:start' }; + assert.equal((await fetch(`${running.base}/api/platform/updates`, { method: 'POST', headers: { 'Content-Type': 'application/json', Cookie: running.cookie }, body: JSON.stringify(start) })).status, 403); + assert.equal((await fetch(`${running.base}/api/platform/updates`, { method: 'POST', headers, body: JSON.stringify({ ...start, releaseId: 'https://attacker.test/image' }) })).status, 409); + // A fleet still provisioning cannot enter the shared backup phase. + assert.equal((await fetch(`${running.base}/api/platform/updates`, { method: 'POST', headers, body: JSON.stringify(start) })).status, 409); + running.store.db.prepare("UPDATE installations SET status='ready' WHERE organization_id=?").run(row.id); + running.store.updateOrganizationStatus(row.id, 'active', Date.now()); + const response = await fetch(`${running.base}/api/platform/updates`, { method: 'POST', headers, body: JSON.stringify(start) }); + assert.equal(response.status, 200); + const data = (await response.json()).data; + assert.equal(data.rollout.total, 2); + assert.equal(data.rollout.status, 'running'); + for (const forbidden of ['runtimeKey', 'organizationId', 'imageDigest', 'socket', 'databaseRoot']) assert.equal(JSON.stringify(data).includes(forbidden), false); +}); + + +test('Core readiness reports the running bundle only when identity and central storage are ready', async t => { + const legacy = await runningServer(); t.after(legacy.close); + assert.equal((await fetch(`${legacy.base}/api/platform/core-health`)).status, 503); + const identity = { releaseId: 'dispatch_update_2', version: '0.0.2', sourceCommit: 'a'.repeat(40) }; + const running = await runningServer({ coreIdentity: identity }); t.after(running.close); + const ready = await fetch(`${running.base}/api/platform/core-health`); + assert.equal(ready.status, 200); + assert.deepEqual((await ready.json()).data, identity); + running.store.db.exec('ALTER TABLE users RENAME TO unavailable_users'); + assert.equal((await fetch(`${running.base}/api/platform/core-health`)).status, 500); +}); + + +test('Core verification blocks authenticated and public traffic without modifying business or session data', async t => { + let maintenance = { rolloutId: 'rollout_' + 'a'.repeat(32), nonce: 'b'.repeat(64) }; + const running = await runningServer({ coreIdentity: { releaseId: 'dispatch_update_2', version: '0.0.2', sourceCommit: 'c'.repeat(40) }, + coreMaintenance: () => maintenance }); + t.after(running.close); + const before = running.store.db.prepare('SELECT * FROM sessions').all(); + for (const endpoint of ['/', '/login', '/api/auth/session', '/api/workforce/day']) { + const response = await fetch(running.base + endpoint, { headers: { cookie: running.cookie } }); + assert.equal(response.status, 503); + assert.equal(response.headers.get('retry-after'), '10'); + } + const post = await fetch(running.base + '/api/auth/login', { method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ email: 'owner@example.test', password: 'correct horse battery staple' }) }); + assert.equal(post.status, 503); + assert.deepEqual(running.store.db.prepare('SELECT * FROM sessions').all(), before); + const normal = await (await fetch(running.base + '/api/platform/core-health')).json(); + assert.equal(normal.data.recoveryProbe, undefined); + const probe = await (await fetch(running.base + '/api/platform/core-health', { headers: { 'X-Dispatch-Recovery-Probe': maintenance.nonce } })).json(); + assert.equal(probe.data.recoveryProbe, 'passed'); + assert.equal(running.store.db.prepare("SELECT count(*) AS n FROM sqlite_master WHERE name='dispatch_recovery_probe'").get().n, 0); + maintenance = null; + assert.equal((await fetch(running.base + '/', { headers: { cookie: running.cookie } })).status, 200); +}); + +test('invalid maintenance records keep traffic closed and cannot authorize a probe', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-maintenance-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + fs.mkdirSync(path.join(root, 'config')); + const { createCoreMaintenance, probeAllowed } = require('../server/core-maintenance'); + const read = createCoreMaintenance(root); + assert.equal(read(), null); + fs.writeFileSync(path.join(root, 'config/core-maintenance.json'), '{invalid', { mode: 0o600 }); + assert.deepEqual(read(), { nonce: null }); + assert.equal(probeAllowed(read(), 'b'.repeat(64)), false); + assert.equal(probeAllowed({ nonce: 'short' }, 'b'.repeat(64)), false); +}); + + +test('backup HTTP routes protect schedules and restore commands with owner authorization, CSRF and strict inputs', async t => { + const running = await runningServer({installationOperator:true,installationBackend:'oci_container_v1',backups:true}); + t.after(running.close); + const endpoint = `${running.base}/api/platform/backups`; + const headers = {'Content-Type':'application/json',Cookie:running.cookie,'X-Dispatch-CSRF':running.csrfToken}; + assert.equal((await fetch(endpoint)).status,401); + const view=await (await fetch(endpoint,{headers})).json(); + assert.equal(view.data.settings.enabled,false); + const input={action:'settings',idempotencyKey:'http:backups:settings',revision:view.data.revision,settings:{...view.data.settings,enabled:true,retentionDays:30}}; + assert.equal((await fetch(endpoint,{method:'POST',headers:{...headers,'X-Dispatch-CSRF':'wrong'},body:JSON.stringify(input)})).status,403); + assert.equal((await fetch(endpoint+'?organizationId=other',{headers})).status,400); + assert.equal((await fetch(endpoint,{method:'POST',headers,body:JSON.stringify({...input,path:'/etc/passwd'})})).status,400); + const saved=await fetch(endpoint,{method:'POST',headers,body:JSON.stringify(input)});assert.equal(saved.status,200); + assert.equal((await saved.json()).data.settings.retentionDays,30); + const platform=running.access.session(running.cookie.slice('dispatch_session='.length)); + const invitation=running.access.createOrganization(platform,{ownerEmail:'backup-dsp@example.test',idempotencyKey:'http:backup:dsp:create'}); + const owner=await running.access.acceptNewUser({token:invitation.token,firstName:'DSP',lastName:'Owner',password:'synthetic owner password',confirmPassword:'synthetic owner password'}); + assert.equal((await fetch(endpoint,{headers:{Cookie:`dispatch_session=${owner.token}`}})).status,403); + assert.equal((await fetch(endpoint,{method:'POST',headers:{...headers,Cookie:`dispatch_session=${owner.token}`,'X-Dispatch-CSRF':owner.session.csrfToken},body:JSON.stringify(input)})).status,403); +}); + + +test('DSP audit filters platform access before limiting and enforces independent audit permission', async t => { + const running = await runningServer({ platformOnly: true, installationOperator: true, installationBackend: 'oci_container_v1' }); + t.after(running.close); + const { access, store, base, cookie } = running; + const platform = access.session(cookie.split('=')[1]); + const invitation = access.createOrganization(platform, { + ownerEmail: 'audit-owner@example.test', idempotencyKey: 'http:audit:dsp:create', + }); + const owner = await access.acceptNewUser({ + token: invitation.token, firstName: 'Audit', lastName: 'Owner', + password: 'synthetic audit password', confirmPassword: 'synthetic audit password', + }); + const organizationId = invitation.organization.id; + const headers = { Cookie: `dispatch_session=${owner.token}` }; + const endpoint = `${base}/api/organization/audit`; + const timestamp = access.now() + 1000; + access.audit({ organizationId, action: 'role.create', targetType: 'role', actorUserId: owner.session.user.id, timestamp }); + access.audit({ organizationId, action: 'installation.upgrade.complete', targetType: 'installation', timestamp }); + // Actual support changes stay attributable; entering a DSP is internal-only. + access.audit({ organizationId, action: 'role.update', targetType: 'role', actorUserId: platform.user.id, timestamp }); + for (let i = 0; i < 105; i++) { + access.audit({ organizationId, action: 'organization.view.start', targetType: 'organization', actorUserId: platform.user.id, timestamp: timestamp + i + 1 }); + } + access.ensureLocalOrganization({ organization: { id: 'other-dsp', name: 'Other DSP' }, site: { code: 'TST1' }, timezone: 'UTC' }); + access.audit({ organizationId: 'other-dsp', action: 'other.tenant.event', targetType: 'organization', timestamp }); + assert.equal((await fetch(endpoint)).status, 401); + assert.equal((await fetch(`${endpoint}?organizationId=other-dsp`, { headers })).status, 400); + const response = await fetch(endpoint, { headers }); + assert.equal(response.status, 200); + const { data } = await response.json(); + assert.equal(data.audit.some(event => event.action.startsWith('organization.view.')), false); + assert.equal(data.audit.some(event => event.action === 'other.tenant.event'), false); + assert.equal(data.audit.find(event => event.action === 'role.create').actor, owner.session.user.email); + assert.equal(data.audit.find(event => event.action === 'installation.upgrade.complete').actor, 'System'); + assert.equal(data.audit.find(event => event.action === 'role.update').actor, platform.user.email); + assert.ok(data.audit.length <= 100); + const legacy = await (await fetch(`${base}/api/organization/administration`, { headers })).json(); + assert.deepEqual(legacy.data.audit, data.audit); + assert.equal(store.audits(organizationId, 200).filter(event => event.action === 'organization.view.start').length, 105); + + const auditRole = store.roleByKey(organizationId, 'driver'); + const memberInvite = access.createMemberInvitation(owner.session, organizationId, { + email: 'auditor@example.test', roleId: auditRole.id, + }); + const auditor = await access.acceptNewUser({ + token: memberInvite.token, firstName: 'DSP', lastName: 'Auditor', + password: 'synthetic auditor password', confirmPassword: 'synthetic auditor password', + }); + const auditorHeaders = { Cookie: `dispatch_session=${auditor.token}` }; + assert.equal((await fetch(endpoint, { headers: auditorHeaders })).status, 200); + assert.equal((await fetch(`${base}/api/organization/administration`, { headers: auditorHeaders })).status, 200); + // Permission checks still consult storage on every request for future role changes. + store.db.prepare("DELETE FROM role_permissions WHERE role_id=? AND permission='audit.read'").run(auditRole.id); + assert.equal((await fetch(endpoint, { headers: auditorHeaders })).status, 403); + store.updateOrganizationStatus(organizationId, 'suspended', access.now()); + assert.equal((await fetch(endpoint, { headers })).status, 401); +}); + +test('DSP viewing gives a platform owner scoped owner writes with actor attribution and normal CSRF checks', async t => { + const running = await runningServer({ platformOnly: true, operator: true }); + t.after(running.close); + const { access, store, base, cookie, csrfToken } = running; + const org = access.ensureLocalOrganization({ organization: { id: 'local-dsp', name: 'Viewed DSP' }, site: { code: 'TST1' }, timezone: 'UTC' }); + require('../../core/accounts/tests/plugin-fixture').enableFixturePlugin(store, 'local-dsp'); + const platform = access.session(cookie.split('=')[1]); + const controlRef = access.issuePlatformControlRef(platform, org.id); + const headers = { Cookie: cookie, 'Content-Type': 'application/json', 'X-Dispatch-CSRF': csrfToken }; + assert.equal((await fetch(`${base}/api/bootstrap`, { headers })).status, 409); + assert.equal((await fetch(`${base}/api/platform/organization/view`, { + method: 'POST', headers: { Cookie: cookie, 'Content-Type': 'application/json' }, body: JSON.stringify({ controlRef }), + })).status, 403); + const started = await fetch(`${base}/api/platform/organization/view`, { method: 'POST', headers, body: JSON.stringify({ controlRef }) }); + assert.equal(started.status, 200); + const viewed = (await started.json()).data; + assert.equal(viewed.user.id, platform.user.id); + assert.equal(viewed.dspView.access, 'owner'); + assert.equal(viewed.activeOrganizationId, org.id); + assert.equal(viewed.memberships[0].roleKey, 'owner'); + assert.equal(store.membership(platform.user.id, org.id), null); + assert.equal(store.audits(org.id).find(a => a.action === 'organization.view.start').actor, platform.user.email); + const scoped = { ...headers, 'X-Dispatch-DSP-View': viewed.dspView.viewRef }; + const session = (await (await fetch(`${base}/api/auth/session`, { headers: scoped })).json()).data; + assert.equal(session.activeOrganizationId, org.id); + for (const pathname of ['/api/bootstrap', '/api/paycom/daily?date=2026-08-30', '/api/integrations', '/api/organization/administration', '/api/organization/profile']) { + const result = await fetch(`${base}${pathname}`, { headers: scoped }); + assert.equal(result.status, 200, pathname); + if (pathname === '/api/organization/administration') assert.equal((await result.json()).data.organization.id, org.id); + } + const command = async (pathname, method, body, expected = 200) => { + const response = await fetch(`${base}${pathname}`, { method, headers: scoped, body: JSON.stringify(body) }); + const result = await response.json(); + assert.equal(response.status, expected, `${pathname}: ${JSON.stringify(result)}`); + return result.data; + }; + const roleBody = { name: 'Support role', description: 'Created by platform support', permissions: ['dashboard.view', 'workforce.read'] }; + const missingCsrf = await fetch(`${base}/api/organization/roles`, { method: 'POST', headers: { Cookie: cookie, 'Content-Type': 'application/json', 'X-Dispatch-DSP-View': viewed.dspView.viewRef }, body: JSON.stringify(roleBody) }); + assert.equal(missingCsrf.status, 403); + assert.equal((await missingCsrf.json()).error.code, 'csrf_invalid'); + await command('/api/organization/roles', 'POST', roleBody, 409); + const role = store.roleByKey(org.id, 'dispatcher'); + await command(`/api/organization/roles/${role.id}`, 'PUT', { ...roleBody, name: 'Updated support role' }, 409); + const invite = await command('/api/organization/invitations', 'POST', { email: 'supported-member@example.test', roleId: role.id }, 201); + const invitationToken = invite.invitationPath.split('/').at(-1); + const member = await access.acceptNewUser({ token: invitationToken, firstName: 'Supported', lastName: 'Member', password: 'synthetic member password', confirmPassword: 'synthetic member password' }); + const membership = store.membership(member.session.user.id, org.id); + const viewer = store.roles(org.id).find(r => r.key === 'driver'); + await command(`/api/organization/members/${membership.id}/role`, 'PUT', { roleId: viewer.id }); + await command(`/api/organization/members/${membership.id}`, 'DELETE', {}); + await command(`/api/organization/roles/${role.id}`, 'DELETE', {}, 409); + store.createLifecycleJob({ id: 'life_support_backup', organizationId: org.id, operation: 'backup', + startingState: 'ready', installationState: 'ready', installationRevision: 1, manifestRevision: 1, + runtimeKey: 'local', releaseId: 'dispatch_current_1', targetReleaseId: null, backupId: null, + safetyBackupId: null, authorityScope: 'platform', idempotencyKey: 'test:support:backup', stages: [], timestamp: access.now() }); + const locked = await fetch(`${base}/api/paycom/sync`, { method: 'POST', headers: scoped, body: JSON.stringify({ idempotencyKey: 'test:support:sync-request' }) }); + assert.equal(locked.status, 409); + assert.equal((await locked.json()).error.code, 'backup_operation_in_progress'); + store.db.prepare("UPDATE installation_lifecycle_jobs SET status='succeeded',finished_at=1,result_json='{}' WHERE id='life_support_backup'").run(); + await command('/api/paycom/sync', 'POST', { idempotencyKey: 'test:support:sync-request' }, 202); + assert.equal(running.client.calls.filter(call => call[0] === 'sync.runNow').length, 1); + for (const action of ['membership.role.update', 'membership.remove', 'sync.run.request']) { + assert.equal(store.audits(org.id).find(a => a.action === action).actor, platform.user.email, action); + } + for (const pathname of ['/api/platform/organization/status', '/api/platform/organizations', '/api/platform/updates', '/api/platform/backups', '/api/platform/diagnostics', '/api/auth/select-organization', '/api/auth/register', '/api/auth/login']) { + const result = await fetch(`${base}${pathname}`, { method: 'POST', headers: scoped, body: '{}' }); + assert.equal(result.status, 403, pathname); + assert.equal((await result.json()).error.code, 'dsp_view_scope', pathname); + } + assert.equal((await fetch(`${base}/api/platform/organizations`, { headers: scoped })).status, 403); + const internal = access.dspViewSession(platform, viewed.dspView.viewRef); + assert.throws(() => access.requirePermission(internal, 'other-dsp', 'members.manage'), /organization_forbidden/); + // Complete actual DSP onboarding through the same owner endpoint. + store.db.prepare('INSERT INTO organization_profiles(organization_id,owner_email) VALUES(?,?)').run(org.id, 'dsp-owner@example.test'); + store.db.prepare("UPDATE installations SET status='waiting_for_provider_auth' WHERE organization_id=?").run(org.id); + await command('/api/organization/profile', 'POST', { name: 'Supported DSP', abbreviation: 'SUP', stationCode: 'TST1', timezone: 'UTC' }); + assert.equal(store.organization(org.id).name, 'Supported DSP'); + assert.equal(store.audits(org.id).find(a => a.action === 'organization.details.submit').actor, platform.user.email); + // A separate tab and exiting view retain the original platform session. + const original = (await (await fetch(`${base}/api/auth/session`, { headers })).json()).data; + assert.equal(original.dspView, undefined); + assert.equal(original.activeOrganizationId, null); + assert.deepEqual(original.memberships, []); + assert.equal((await fetch(`${base}/api/platform/organizations`, { headers })).status, 200); +}); + +test('DSP viewing rejects forged, cross-session, tenant, expired, suspended and removed scopes', async t => { + const running = await runningServer({ platformOnly: true }); + t.after(running.close); + const { access, store, base, cookie, csrfToken } = running; + access.ensureLocalOrganization({ organization: { id: 'local-dsp', name: 'Viewed DSP' }, site: { code: 'TST1' }, timezone: 'UTC' }); + const platform = access.session(cookie.split('=')[1]); + const ref = access.issuePlatformControlRef(platform, 'local-dsp'); + const viewed = access.beginDspView(platform, { controlRef: ref }); + const request = (viewRef, sessionCookie = cookie) => fetch(`${base}/api/organization/administration`, { headers: { Cookie: sessionCookie, 'X-Dispatch-DSP-View': viewRef } }); + assert.equal((await request(ref)).status, 403); // A platform control is not a viewing capability. + assert.equal((await request('x'.repeat(43))).status, 403); + const second = access.createSession(platform.user.id); + assert.equal((await request(viewed.dspView.viewRef, `dispatch_session=${second.token}`)).status, 403); + const tenantInvite = access.createOrganization(platform, { idempotencyKey: 'test:dsp-view:tenant', name: 'Tenant DSP', stationCode: 'DOT6', timezone: 'UTC', ownerEmail: 'tenant@example.test' }); + assert.throws(() => access.requirePermission(viewed, tenantInvite.organization.id, 'members.manage'), /organization_forbidden/); + const foreignRole = store.roles(tenantInvite.organization.id).find(r => r.key === 'driver'); + assert.throws(() => access.updateRole(viewed, 'local-dsp', foreignRole.id, { name: 'Wrong DSP', description: '', permissions: [] }), /role_not_found/); + const oldView = access.issuePlatformControlRef(platform, 'local-dsp', 'dispatch_dsp_view_v1'); + assert.equal((await request(oldView)).status, 403); + const tenant = await access.acceptNewUser({ token: tenantInvite.token, firstName: 'Tenant', lastName: 'Owner', password: 'synthetic tenant password', confirmPassword: 'synthetic tenant password' }); + assert.equal((await request(viewed.dspView.viewRef, `dispatch_session=${tenant.token}`)).status, 403); + const tenantStart = await fetch(`${base}/api/platform/organization/view`, { method: 'POST', headers: { Cookie: `dispatch_session=${tenant.token}`, 'Content-Type': 'application/json', 'X-Dispatch-CSRF': tenant.session.csrfToken }, body: JSON.stringify({ controlRef: ref }) }); + assert.equal(tenantStart.status, 403); + store.updateOrganizationStatus('local-dsp', 'suspended', access.now()); + assert.equal((await request(viewed.dspView.viewRef)).status, 403); + assert.throws(() => access.beginDspView(platform, { controlRef: ref }), /dsp_view_unavailable/); + store.updateOrganizationStatus('local-dsp', 'active', access.now()); + store.db.prepare("UPDATE installations SET status='decommissioned' WHERE organization_id='local-dsp'").run(); + assert.equal((await request(viewed.dspView.viewRef)).status, 403); + store.db.prepare("UPDATE installations SET status='ready' WHERE organization_id='local-dsp'").run(); + const timestamp = access.now(); + access.clock = () => new Date(timestamp + 16 * 60 * 1000); + assert.equal((await request(viewed.dspView.viewRef)).status, 403); + access.clock = () => new Date(timestamp); + access.signOut(platform); + assert.equal((await request(viewed.dspView.viewRef)).status, 401); +}); + + +test('Turnstile gates password checks and invitation registration, enforces single-use tokens, and preserves sessions', async t => { + const { createTurnstile } = require('../server/turnstile'); + const used = new Set(); + const turnstile = createTurnstile({ siteKey: '0x' + 'a'.repeat(24), secret: '0x' + 'b'.repeat(33), + hostname: 'dispatch.example.test', fetchImpl: async (_url, init) => { + const { response } = JSON.parse(init.body); + if (response === 'unavailable') throw Error('provider failure'); + const success = !used.has(response) && response !== 'expired'; used.add(response); + return { ok: true, json: async () => ({ success, hostname: 'dispatch.example.test', action: response.split(':')[0] }) }; + } }); + const running = await runningServer({ platformOnly: true, installationOperator: true, installationBackend: 'native_service_v1', turnstile }); + t.after(running.close); + let passwordChecks = 0, registrations = 0; + const signIn = running.access.signIn.bind(running.access), accept = running.access.acceptNewUser.bind(running.access); + running.access.signIn = async body => { passwordChecks++; assert.equal(body.turnstileToken, undefined); return signIn(body); }; + running.access.acceptNewUser = async body => { registrations++; assert.equal(body.turnstileToken, undefined); return accept(body); }; + const post = async (route, body) => { + const response = await fetch(running.base + route, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + return { response, body: await response.json() }; + }; + const login = { email: 'owner@example.test', password: 'correct horse battery staple' }; + for (const [token, code] of [[undefined, 'turnstile_required'], ['expired', 'turnstile_invalid'], ['register:wrong-action', 'turnstile_invalid'], ['unavailable', 'turnstile_unavailable']]) { + const result = await post('/api/auth/login', { ...login, ...(token ? { turnstileToken: token } : {}) }); + assert.equal(result.body.error.code, code); + assert.equal(result.response.headers.get('set-cookie'), null); + } + assert.equal(passwordChecks, 0); + const deniedPassword = await post('/api/auth/login', { ...login, password: 'incorrect password', turnstileToken: 'login:1' }); + assert.equal(deniedPassword.body.error.code, 'invalid_credentials'); + assert.equal((await post('/api/auth/login', { ...login, turnstileToken: 'login:1' })).body.error.code, 'turnstile_invalid'); + assert.equal(passwordChecks, 1); + const success = await post('/api/auth/login', { ...login, turnstileToken: 'login:2' }); + assert.equal(success.response.status, 200); + assert.match(success.response.headers.get('set-cookie'), /dispatch_session=/); + const invitation = running.access.createOrganization(running.access.session(running.cookie.split('=')[1]), { + idempotencyKey: 'turnstile:registration:fixture', ownerEmail: 'new-turnstile@example.test', + }); + const registration = { token: invitation.token, firstName: 'New', lastName: 'Owner', password: login.password, confirmPassword: login.password }; + for (const token of [undefined, 'login:wrong-action', 'expired', 'unavailable']) { + const denied = await post('/api/auth/register', { ...registration, ...(token ? { turnstileToken: token } : {}) }); + assert.ok(denied.response.status >= 400); + assert.equal(denied.response.headers.get('set-cookie'), null); + } + assert.equal(registrations, 0); + assert.equal((await post('/api/auth/register', { ...registration, turnstileToken: 'register:1' })).response.status, 201); + assert.equal(registrations, 1); + const session = await fetch(running.base + '/api/auth/session'); + assert.deepEqual((await session.json()).data.turnstile, turnstile.publicConfig); + assert.equal(session.headers.get('cache-control'), 'no-store'); + const page = await fetch(running.base + '/'); + assert.match(page.headers.get('content-security-policy'), /script-src 'self' 'nonce-[A-Za-z0-9+/]+={0,2}' https:\/\/challenges.cloudflare.com/); + assert.doesNotMatch(page.headers.get('content-security-policy'), /'unsafe-inline'|'unsafe-eval'/); + assert.match(page.headers.get('content-security-policy'), /frame-src https:\/\/challenges.cloudflare.com/); + const cookie = success.response.headers.get('set-cookie').split(';')[0]; + assert.equal((await (await fetch(running.base + '/api/auth/session', { headers: { Cookie: cookie } })).json()).data.authenticated, true); +}); + +test('Turnstile rejections use the login attempt limit while provider outages do not lock accounts', async t => { + const { createTurnstile } = require('../server/turnstile'); + let checks = 0; + const turnstile = createTurnstile({ siteKey: '0x' + 'a'.repeat(24), secret: '0x' + 'b'.repeat(33), + hostname: 'dispatch.example.test', fetchImpl: async (_url, init) => { + checks++; + if (JSON.parse(init.body).response === 'outage') throw Error('synthetic outage'); + return { ok: true, json: async () => ({ success: false }) }; + } }); + const running = await runningServer({ platformOnly: true, turnstile }); t.after(running.close); + const attempt = async (email, token) => fetch(running.base + '/api/auth/login', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ email, password: 'irrelevant password', turnstileToken: token }), + }); + for (let i = 0; i < 9; i++) assert.equal((await attempt('outage@example.test', 'outage')).status, 503); + for (let i = 0; i < 8; i++) assert.equal((await attempt('owner@example.test', 'invalid')).status, 403); + const throttled = await attempt('owner@example.test', 'invalid'); + assert.equal(throttled.status, 429); + assert.equal((await throttled.json()).error.code, 'login_rate_limited'); + assert.equal(checks, 17); +}); + +test('collection status uses the authenticated DSP and rejects scope overrides', async t => { + const running = await runningServer(); + t.after(running.close); + const endpoint = `${running.base}/api/paycom/sync`; + assert.equal((await fetch(endpoint)).status, 401); + const headers = { Cookie: running.cookie }; + const response = await fetch(endpoint, { headers }); + assert.equal(response.status, 200); + const value = await response.json(); + assert.equal(value.data.activity, 'idle'); + assert.equal(value.data.lastSucceededAt, COLLECTED); + assert.equal((await fetch(`${endpoint}?runtime=another-dsp`, { headers })).status, 400); +}); + + +test('Connections HTTP protects owner mutations, CSRF, DSP selection, and response secrecy', async t => { + const calls = []; + const view = service => ({ service, configured: true, state: 'checking', checkedAt: null, reason: null, retryAt: null }); + const running = await runningServer({ installationOperator: true, installationBackend: 'native_service_v1', + connectionsInvoke: async (...args) => { calls.push(args); return { ok: true, status: args[2].command === 'list' ? 'found' : 'accepted', + data: args[2].command === 'list' ? { items: ['cortex', 'paycom'].map(view) } : view(args[2].service), error: null }; } }); + t.after(running.close); + const platform = running.access.session(running.cookie.slice('dispatch_session='.length)); + const dsp = running.access.createOrganization(platform, { idempotencyKey: 'http:connections:create', name: 'Connections DSP', + abbreviation: 'HTTP', stationCode: 'TST1', timezone: 'UTC', ownerEmail: 'connections-dsp@example.test' }); + const owner = await running.access.acceptNewUser({ token: dsp.token, firstName: 'HTTP', lastName: 'Owner', + password: 'fixture owner password', confirmPassword: 'fixture owner password' }); + running.store.updateInstallationControl({ organizationId: dsp.organization.id, expectedStatus: 'pending', expectedRevision: 1, + status: 'ready', revision: 2, currentJobId: null, timestamp: Date.now() }); + require('../../core/accounts/tests/plugin-fixture').enableFixturePlugin(running.store, dsp.organization.id); + running.store.updateOrganizationStatus(dsp.organization.id, 'active', Date.now()); + const endpoint = `${running.base}/api/organization/connections/cortex/save`; + const input = { credentials: { username: 'fixture', password: 'private-http-connection' } }; + const headers = { 'Content-Type': 'application/json', Cookie: `dispatch_session=${owner.token}`, 'X-Dispatch-CSRF': owner.session.csrfToken }; + assert.equal((await fetch(endpoint, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(input) })).status, 401); + assert.equal((await fetch(endpoint, { method: 'POST', headers: { ...headers, 'X-Dispatch-CSRF': 'wrong' }, body: JSON.stringify(input) })).status, 403); + assert.equal((await fetch(endpoint, { method: 'POST', headers, body: JSON.stringify({ ...input, runtimeKey: 'another' }) })).status, 400); + assert.equal((await fetch(endpoint, { method: 'POST', headers: { ...headers, Cookie: running.cookie }, body: JSON.stringify(input) })).status, 403); + assert.equal(calls.length, 0); + const result = await fetch(endpoint, { method: 'POST', headers, body: JSON.stringify(input) }); + assert.equal(result.status, 202); + assert.equal((await result.text()).includes(input.credentials.password), false); + assert.equal(calls[0][0], running.store.installationControl(dsp.organization.id).runtimeKey); + const response = await fetch(`${running.base}/api/organization/connections`, { headers }); + assert.equal(response.status, 200); + const data = (await response.json()).data; + assert.deepEqual(data.services.map(item => item.id), ['cortex', 'paycom']); + assert.equal(JSON.stringify(data).includes('profile'), false); + const viewed = running.access.beginDspView(platform, { + controlRef: running.access.issuePlatformControlRef(platform, dsp.organization.id), + }); + const scoped = { ...headers, Cookie: running.cookie, 'X-Dispatch-CSRF': platform.csrfToken, + 'X-Dispatch-DSP-View': viewed.dspView.viewRef }; + const platformList = await fetch(`${running.base}/api/organization/connections`, { headers: scoped }); + assert.equal(platformList.status, 200); + assert.equal((await fetch(endpoint, { method: 'POST', headers: { ...scoped, 'X-Dispatch-CSRF': 'wrong' }, body: JSON.stringify(input) })).status, 403); + const supported = await fetch(endpoint, { method: 'POST', headers: scoped, body: JSON.stringify(input) }); + assert.equal(supported.status, 202); + assert.equal((await supported.text()).includes(input.credentials.password), false); + for (const command of ['test', 'disconnect']) { + assert.equal((await fetch(`${running.base}/api/organization/connections/cortex/${command}`, { + method: 'POST', headers: scoped, body: '{}', + })).status, 202); + } + assert.ok(calls.every(call => call[0] === running.store.installationControl(dsp.organization.id).runtimeKey)); +}); diff --git a/core/dashboard/tests/date-time.test.js b/core/dashboard/tests/date-time.test.js new file mode 100644 index 0000000..2caba59 --- /dev/null +++ b/core/dashboard/tests/date-time.test.js @@ -0,0 +1,46 @@ +'use strict'; +const { test, before } = require('node:test'); +const assert = require('node:assert/strict'); +const { stripTypeScriptTypes } = require('node:module'); +const fs = require('node:fs'); +const path = require('node:path'); +let calendarDateAt, moveCalendarDate, calendarDateLabel, dateTime, validTimeZone; +before(async () => { + const source = fs.readFileSync(path.join(__dirname, '../frontend/src/lib/date-time.ts'), 'utf8'); + ({ calendarDateAt, moveCalendarDate, calendarDateLabel, dateTime, validTimeZone } = + await import(`data:text/javascript;base64,${Buffer.from(stripTypeScriptTypes(source)).toString('base64')}`)); +}); +const { dateInTimezone } = require('dispatch-dsp/plugins/paycom/backend/src/collector.js'); +const zone = 'America/Los_Angeles'; + +test('dashboard and Paycom collector agree before and after Pacific midnight, including DST', () => { + for (const [instant, expected] of [ + ['2026-09-11T02:28:00Z', '2026-09-10'], + ['2026-09-11T06:59:59Z', '2026-09-10'], ['2026-09-11T07:00:00Z', '2026-09-11'], + ['2026-01-11T07:59:59Z', '2026-01-10'], ['2026-01-11T08:00:00Z', '2026-01-11'], + ['2026-03-08T09:59:59Z', '2026-03-08'], ['2026-03-08T10:00:00Z', '2026-03-08'], + ['2026-11-01T08:30:00Z', '2026-11-01'], ['2026-11-01T09:30:00Z', '2026-11-01'], + ]) { + assert.equal(calendarDateAt(zone, new Date(instant)), expected); + assert.equal(dateInTimezone(zone, new Date(instant)), expected); + } +}); +test('events shift for the viewer while calendar labels and calendar navigation preserve dates', () => { + assert.match(dateTime('2026-09-11T02:28:00Z', zone), /Sep 10, 2026.*7:28 PM PDT/); + assert.match(dateTime('2026-09-11T02:28:00Z', 'Asia/Tokyo'), /Sep 11, 2026.*11:28 AM/); + assert.match(dateTime('2026-01-11T02:28:00Z', zone), /Jan 10, 2026.*6:28 PM PST/); + assert.match(calendarDateLabel('2026-09-10'), /Sep 10, 2026/); + for (const [date, next] of [['2026-03-08','2026-03-09'], ['2026-11-01','2026-11-02'], ['2028-02-28','2028-02-29'], ['2026-12-31','2027-01-01']]) { + assert.equal(moveCalendarDate(date, 1), next); + assert.equal(moveCalendarDate(next, -1), date); + } +}); +test('invalid dates and preferences are rejected without treating epoch zero as absent', () => { + assert.equal(validTimeZone('Mars/Base'), false); + assert.equal(validTimeZone(null), false); + assert.equal(validTimeZone(zone), true); + assert.equal(dateTime('invalid', zone), '—'); + assert.match(dateTime(0, 'UTC'), /Jan 1, 1970/); + assert.throws(() => moveCalendarDate('2026-02-30', 1)); + assert.throws(() => calendarDateLabel('2026-13-01')); +}); diff --git a/core/dashboard/tests/directory-platform.test.js b/core/dashboard/tests/directory-platform.test.js new file mode 100644 index 0000000..bec798c --- /dev/null +++ b/core/dashboard/tests/directory-platform.test.js @@ -0,0 +1,598 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { platformPaths } = require('../../shared/paths/platform-paths'); +const { startDirectoryDashboard } = require('../server/directory-platform'); +const { DirectoryManager } = require('../../host/controller/manager'); +const { createAccessInstallationLifecycleAuthority } = require('../../core/accounts/src/installation-lifecycle'); +const { createInstallationProvisioningReconciler } = require('../../core/accounts/src/installation-provisioning'); +const { AccessStore } = require('../../core/accounts/src/store'); + +async function fixture(t, overrides = {}) { + const { prepare = () => {}, ...settings } = overrides; + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'directory-dashboard-')); + for (const name of ['live', 'local', 'dev', 'dsps', 'worktrees']) fs.mkdirSync(path.join(root, name), { mode: 0o700 }); + const paths = platformPaths(root), active = new Set(), starts = [], errors = []; + await prepare(paths); + const host = { prepare: async () => {}, start: async id => { + if (host.failStart) throw Object.assign(new Error('injected'), { code: 'directory_host_operation_failed' }); + if (!active.has(id)) { active.add(id); starts.push(id); } + }, stop: async id => active.delete(id) }; + const runtimeFactory = async options => { + const bridges = new Set(); + const hub = { connected: id => bridges.has(id) && active.has(id) && options.authorityCatalog.resolve(id) !== null, + invoke: async id => { assert.ok(hub.connected(id)); return { ok: true }; } }; + const manager = new DirectoryManager({ ...options, hub, host }); + manager.bridge = async record => bridges.add(record.id); + await manager.recover(options.select); + return { manager, hub, journal: options.journal, close: async () => { bridges.clear(); await manager.close(); } }; + }; + const options = { paths, installation: {}, port: 0, installationOperator: true, runtimeFactory, environment: {}, onError: error => errors.push(error), ...settings }; + let app = await startDirectoryDashboard(options); + await app.worker.close(); // Drive the durable queue explicitly in these tests. + const invitation = app.access.createPlatformBootstrap({ email: 'platform@example.test' }); + const login = await app.access.acceptNewUser({ token: invitation.token, firstName: 'Platform', lastName: 'Fixture', + password: 'synthetic owner password', confirmPassword: 'synthetic owner password' }); + const headers = { Cookie: `dispatch_session=${login.token}`, 'Content-Type': 'application/json', 'X-Dispatch-CSRF': login.session.csrfToken }; + t.after(async () => { await app.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const value = { paths, active, starts, host, errors, headers, login, get app() { return app; }, + async restart() { await app.close(); app = await startDirectoryDashboard(options); await app.worker.close(); }, + async post(body, authenticated = true) { + const response = await fetch(`http://127.0.0.1:${app.server.address().port}/api/platform/organizations`, { + method: 'POST', headers: authenticated ? headers : { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + return { status: response.status, body: await response.json() }; + }, + create: suffix => value.post({ ownerEmail: `${suffix}@example.test`, idempotencyKey: `directory:create:${suffix}` }), + }; + return value; +} + +async function deletionFixture(t) { + const c = await fixture(t); + c.app.access.platformDiagnostics(c.login.session, { idempotencyKey: 'erase:target:fixture' }); + c.app.access.platformDiagnostics(c.login.session, { idempotencyKey: 'erase:neighbor:fixture' }); + await c.app.worker.runPending(); + const rows = c.app.store.db.prepare('SELECT organization_id,runtime_key FROM installations ORDER BY rowid').all(); + c.target = rows[0]; c.neighbor = rows[1]; + c.remove = async row => { + const control = c.app.store.installationControl(row.organization_id); + const org = c.app.access.platformOrganizations(c.login.session).find(item => item.name === c.app.store.organization(row.organization_id).name); + await c.app.access.requestPlatformRemoval(c.login.session, { controlRef: org.controlRef, + expectedRevision: control.revision, idempotencyKey: 'remove:' + row.runtime_key }, 'decommission'); + await c.app.lifecycle.runPending(); + }; + c.command = (row = c.target) => { + const org = c.app.access.platformOrganizations(c.login.session).find(item => item.name === c.app.store.organization(row.organization_id).name); + return { controlRef: org.controlRef, expectedRevision: org.installation.revision, + idempotencyKey: 'erase:' + row.runtime_key, password: 'synthetic owner password' }; + }; + c.app.deletions.eraseFiles = async (paths, job) => { + assert.equal(c.active.has(job.runtimeKey), false); + fs.rmSync(path.join(paths.dsps, job.runtimeKey), { recursive: true, force: true }); + }; + return c; +} + +test('permanent deletion requires removal and password, erases tenant access and files, and preserves a sibling', async t => { + const c = await deletionFixture(t), db = c.app.store.db; + await assert.rejects(c.app.access.requestPlatformRemoval(c.login.session, c.command(), 'destroy'), /installation_operation_not_allowed/); + const owner = db.prepare('SELECT user_id FROM memberships WHERE organization_id=?').get(c.target.organization_id).user_id; + db.exec('CREATE TABLE tenant_extension(id TEXT PRIMARY KEY,organization_id TEXT REFERENCES organizations(id),value TEXT)'); + db.prepare('INSERT INTO tenant_extension VALUES(?,?,?)').run('target-extra', c.target.organization_id, 'synthetic target content'); + db.prepare('INSERT INTO tenant_extension VALUES(?,?,?)').run('neighbor-extra', c.neighbor.organization_id, 'synthetic neighbor content'); + const data = path.join(c.paths.dsps, c.target.runtime_key, 'data/erasure-fixture'); fs.writeFileSync(data, 'synthetic private data', { mode: 0o600 }); + await c.remove(c.target); + assert.equal(fs.existsSync(data), true); + const input = c.command(); + c.app.deletions.enabled = false; + assert.ok(!c.app.access.installationConsoleStatus(c.target.organization_id).availableActions.includes('destroy')); + await assert.rejects(c.app.access.requestPlatformRemoval(c.login.session, input, 'destroy'), /installation_operation_not_allowed/); + c.app.deletions.enabled = true; + await assert.rejects(c.app.access.requestPlatformRemoval(c.login.session, { ...input, password: 'incorrect' }, 'destroy'), /current_password_invalid/); + await assert.rejects(c.app.access.requestPlatformRemoval(c.login.session, { ...input, expectedRevision: input.expectedRevision - 1 }, 'destroy'), /installation_operation_not_allowed/); + await c.app.access.requestPlatformRemoval(c.login.session, input, 'destroy'); + assert.equal(c.app.access.installationConsoleStatus(c.target.organization_id).operation.kind, 'destroy'); + await assert.rejects(c.app.access.requestPlatformRemoval(c.login.session, { controlRef: input.controlRef, + expectedRevision: input.expectedRevision, idempotencyKey: 'restore:during:erase' }, 'resume'), /installation_operation_not_allowed/); + await c.app.deletions.runPending(); + assert.deepEqual(c.errors, []); + assert.equal(fs.existsSync(path.dirname(data)), false); + assert.equal(c.app.store.organization(c.target.organization_id), null); + assert.equal(c.app.store.userById(owner), null); + assert.equal(db.prepare('SELECT count(*) n FROM tenant_extension').get().n, 1); + assert.equal(db.prepare('SELECT value FROM tenant_extension').get().value, 'synthetic neighbor content'); + assert.ok(c.app.store.organization(c.neighbor.organization_id)); + assert.equal(c.active.has(c.neighbor.runtime_key), true); + assert.deepEqual(db.prepare('PRAGMA foreign_key_check').all(), []); + assert.equal(c.app.runtime.manager.journal.record(c.target.runtime_key), null); + assert.throws(() => require('../../host/storage/storage').ensureDsp(c.paths, c.target.runtime_key, 'create_' + 'a'.repeat(32)), /directory_dsp_deleted/); + await assert.rejects(c.app.runtime.manager.apply('create', 'revive:erased:dsp', c.target.runtime_key)); + await c.app.deletions.runPending(); + assert.equal(c.app.deletions.get(c.target.organization_id).status, 'complete'); +}); + +test('deletion scrubs shared platform backups, removes DSP backups, and blocks old backup resurrection', async t => { + const c = await deletionFixture(t), backups = c.app.backups; + await c.remove(c.target); await c.remove(c.neighbor); + fs.mkdirSync(path.join(c.paths.local, 'config'), { mode: 0o700 }); + fs.mkdirSync(path.join(c.paths.local, 'secrets'), { mode: 0o700 }); + fs.writeFileSync(path.join(c.paths.local, 'config/platform.json'), JSON.stringify({ version: 1, platformRoot: c.paths.platformRoot }), { mode: 0o600 }); + fs.writeFileSync(path.join(c.paths.dsps, c.target.runtime_key, 'data/target.txt'), 'synthetic target contents', { mode: 0o600 }); + fs.writeFileSync(path.join(c.paths.dsps, c.neighbor.runtime_key, 'data/neighbor.txt'), 'synthetic neighbor contents', { mode: 0o600 }); + const { DatabaseSync } = require('node:sqlite'); + for (const row of [c.target, c.neighbor]) { + const file = path.join(c.paths.dsps, row.runtime_key, 'data/snapshot.sqlite3'); + const db = new DatabaseSync(file); + db.exec("PRAGMA journal_mode=WAL; CREATE TABLE entries(value TEXT); INSERT INTO entries VALUES('synthetic retained data')"); + db.close(); fs.chmodSync(file, 0o600); + } + backups.volumes = { ensure: async () => ({ limited: false }) }; + const records = c.app.runtime.manager.journal.all(); + const platformId = 'mbk_' + '1'.repeat(32), dspId = 'mbk_' + '2'.repeat(32); + await backups.create({ scope: 'platform', organizationId: null, records, backupId: platformId }, null); + await backups.create({ scope: 'dsp', organizationId: c.target.organization_id, records: [records.find(r => r.id === c.target.runtime_key)], backupId: dspId }, null); + const original = fs.readFileSync(path.join(backups.root, platformId, 'manifest.json')); + for (const row of [c.target, c.neighbor]) { + const file = path.join(backups.root, platformId, 'payload', row.runtime_key + '_data/snapshot.sqlite3'); + const reader = new DatabaseSync(file, { readOnly: true }); + reader.prepare('SELECT value FROM entries').get(); reader.close(); + for (const suffix of ['-wal', '-shm']) { + fs.chmodSync(file + suffix, 0o600); + const later = (JSON.parse(original).createdAt + 1000) / 1000; + fs.utimesSync(file + suffix, later, later); + } + } + await c.app.access.requestPlatformRemoval(c.login.session, c.command(), 'destroy'); + assert.throws(() => backups.inspect(platformId), /directory_deletion_in_progress/); + await c.app.deletions.runPending(); + assert.deepEqual(c.errors, []); + assert.equal(fs.existsSync(path.join(backups.root, dspId)), false); + const retained = backups.inspect(platformId); + assert.deepEqual(retained.dsps.map(dsp => dsp.id), [c.neighbor.runtime_key]); + assert.equal(fs.readFileSync(path.join(backups.root, platformId, 'payload', c.neighbor.runtime_key + '_data/neighbor.txt'), 'utf8'), 'synthetic neighbor contents'); + assert.equal(fs.existsSync(path.join(backups.root, platformId, 'payload', c.neighbor.runtime_key + '_data/snapshot.sqlite3-shm')), false); + const saved = new (require('node:sqlite').DatabaseSync)(path.join(backups.root, platformId, 'payload/core/access-control.sqlite3'), { readOnly: true }); + assert.equal(saved.prepare('SELECT 1 FROM organizations WHERE id=?').get(c.target.organization_id), undefined); + assert.ok(saved.prepare('SELECT 1 FROM organizations WHERE id=?').get(c.neighbor.organization_id)); saved.close(); + fs.writeFileSync(path.join(backups.root, platformId, 'manifest.json'), original); + assert.throws(() => backups.inspect(platformId), /directory_dsp_deleted/); +}); + +test('backup integrity failures retain their code and deletion retries after the original data is restored', async t => { + const c = await deletionFixture(t), backups = c.app.backups; + await c.remove(c.target); + const data = path.join(c.paths.dsps, c.target.runtime_key, 'data/target.txt'); + fs.writeFileSync(data, 'synthetic original data', { mode: 0o600 }); + backups.volumes = { ensure: async () => ({ limited: false }) }; + const backupId = 'mbk_' + '3'.repeat(32); + await backups.create({ scope: 'dsp', organizationId: c.target.organization_id, + records: [c.app.runtime.manager.journal.record(c.target.runtime_key)], backupId }, null); + const saved = path.join(backups.root, backupId, 'payload', c.target.runtime_key + '_data/target.txt'); + fs.writeFileSync(saved, 'changed data'); + const input = c.command(); + await c.app.access.requestPlatformRemoval(c.login.session, input, 'destroy'); + await c.app.deletions.runPending(); + assert.equal(c.app.deletions.get(c.target.organization_id).phase, 'backups'); + assert.equal(c.app.deletions.get(c.target.organization_id).failureCode, 'directory_backup_changed'); + assert.equal(c.errors.at(-1).code, 'directory_backup_changed'); + assert.equal(fs.existsSync(data), true); + assert.ok(c.app.store.organization(c.target.organization_id)); + fs.writeFileSync(saved, 'synthetic original data'); + await c.app.access.requestPlatformRemoval(c.login.session, input, 'destroy'); + await c.app.deletions.runPending(); + assert.equal(c.app.deletions.get(c.target.organization_id).status, 'complete'); + assert.equal(fs.existsSync(path.join(backups.root, backupId)), false); + assert.ok(c.app.store.organization(c.neighbor.organization_id)); +}); + +test('an interrupted deletion remains blocked from restore and resumes its recorded phase', async t => { + const c = await deletionFixture(t); await c.remove(c.target); + const erase = c.app.deletions.eraseFiles; let calls = 0; + c.app.deletions.eraseFiles = async (...args) => { calls++; if (calls === 1) throw Error('synthetic interruption'); return erase(...args); }; + const input = c.command(); await c.app.access.requestPlatformRemoval(c.login.session, input, 'destroy'); await c.app.deletions.runPending(); + assert.equal(c.app.deletions.get(c.target.organization_id).phase, 'storage'); + assert.equal(c.app.deletions.get(c.target.organization_id).status, 'failed'); + await c.app.access.requestPlatformRemoval(c.login.session, input, 'destroy'); + await c.app.deletions.runPending(); + assert.equal(c.app.deletions.get(c.target.organization_id).status, 'complete'); + assert.equal(calls, 2); +}); + +test('deletion retains an account referenced by a different DSP', async t => { + const c = await deletionFixture(t), db = c.app.store.db; + const user = db.prepare('SELECT user_id FROM memberships WHERE organization_id=?').get(c.target.organization_id).user_id; + // Historical creator identity can be shared even though current membership + // policy permits a user to belong to only one DSP. + db.prepare('UPDATE organizations SET created_by=? WHERE id=?').run(user, c.neighbor.organization_id); + const before = c.app.store.userById(user); + await c.remove(c.target); await c.app.access.requestPlatformRemoval(c.login.session, c.command(), 'destroy'); + await c.app.deletions.runPending(); + assert.deepEqual(c.errors, []); assert.deepEqual(c.app.store.userById(user), before); + assert.equal(db.prepare('SELECT count(*) n FROM memberships WHERE user_id=?').get(user).n, 0); + assert.equal(db.prepare('SELECT created_by FROM organizations WHERE id=?').get(c.neighbor.organization_id).created_by, user); +}); + +test('authenticated dashboard creation maps one opaque DSP, replays safely and hides host identity', async t => { + const c = await fixture(t); + assert.equal((await c.post({ ownerEmail: 'blocked@example.test', idempotencyKey: 'directory:blocked' }, false)).status, 401); + assert.equal((await c.post({ ownerEmail: 'blocked@example.test', idempotencyKey: 'directory:blocked', dspId: 'dsp_' + 'a'.repeat(32) })).status, 400); + assert.equal(c.app.store.organizations().length, 0); + const created = await c.create('first'); assert.equal(created.status, 201); + const [row] = c.app.store.db.prepare('SELECT * FROM installations').all(); + assert.equal(row.backend, 'directory_service_v1'); assert.match(row.runtime_key, /^dsp_[a-f0-9]{32}$/); + await c.app.worker.runPending(); + assert.equal(c.app.store.installationControl(row.organization_id).status, 'waiting_for_owner'); + assert.equal(c.app.runtime.hub.connected(row.runtime_key), true); + const metadata = JSON.parse(fs.readFileSync(path.join(c.paths.dsps, row.runtime_key, 'config/installation.json'))); + assert.equal(metadata.organizationId, row.organization_id); assert.equal(metadata.runtimeKey, row.runtime_key); + assert.deepEqual(Object.keys(metadata).sort(), ['organizationId', 'runtimeKey', 'version']); + const authority = c.app.store.activeRuntimeAgentAuthority(row.runtime_key); + assert.equal(authority.tokenHash, c.app.runtime.journal.record(row.runtime_key).tokenHash); + const replay = await c.create('first'); assert.equal(replay.status, 200); + await c.app.worker.runPending(); assert.equal(c.starts.length, 1); + assert.equal(c.app.store.organizations().length, 1); assert.equal(fs.readdirSync(c.paths.dsps).length, 1); + const listing = c.app.access.platformOrganizations(c.app.access.session(c.login.token)); + assert.deepEqual(listing[0].installation.availableActions, ['suspend', 'decommission']); + for (const response of [created.body, replay.body, listing]) { + const raw = JSON.stringify(response); + assert.equal(raw.includes(row.runtime_key), false); assert.equal(raw.includes(c.paths.platformRoot), false); + assert.equal(raw.includes(authority.tokenHash), false); + } + assert.deepEqual(c.errors, []); +}); + +test('a failed directory start can be retried through the existing provisioning request with the same identity and token', async t => { + const c = await fixture(t); c.host.failStart = true; + await c.create('retry'); await c.app.worker.runPending(); + const [row] = c.app.store.db.prepare('SELECT * FROM installations').all(); + assert.equal(row.status, 'failed'); + const tokenFile = path.join(c.paths.dsps, row.runtime_key, 'secrets/runtime-agent/registration-token'); + const token = fs.readFileSync(tokenFile); + assert.equal(c.app.access.installationConsoleStatus(row.organization_id).failure.recoverable, true); + c.host.failStart = false; + c.app.access.requestInstallationRetry(c.app.access.session(c.login.token), row.organization_id, + { idempotencyKey: 'directory:retry:request', expectedRevision: row.revision }); + await c.app.worker.runPending(); + assert.equal(c.app.store.installationControl(row.organization_id).status, 'waiting_for_owner'); + assert.deepEqual(fs.readFileSync(tokenFile), token); assert.equal(fs.readdirSync(c.paths.dsps).length, 1); + assert.equal(c.app.runtime.hub.connected(row.runtime_key), true); assert.deepEqual(c.errors, []); +}); + +test('restart reconciles a healthy runtime when the Access completion did not commit', async t => { + const c = await fixture(t); await c.create('crash'); + c.app.store.finishProvisioningRequest = () => { throw new Error('injected completion interruption'); }; + await c.app.worker.runPending(); + const [row] = c.app.store.db.prepare('SELECT * FROM installations').all(); + assert.equal(c.app.store.latestProvisioningRequest(row.organization_id).status, 'dispatched'); + assert.equal(c.starts.length, 1); + await c.restart(); await c.app.worker.runPending(); + assert.equal(c.app.store.latestProvisioningRequest(row.organization_id).status, 'completed'); + assert.equal(c.app.runtime.hub.connected(row.runtime_key), true); + assert.equal(c.starts.length, 1); assert.equal(fs.readdirSync(c.paths.dsps).length, 1); +}); + +test('legacy workers and lifecycle commands cannot claim directory DSPs', async t => { + const c = await fixture(t); await c.create('boundary'); + const [row] = c.app.store.db.prepare('SELECT * FROM installations').all(); + let calls = 0; + const legacy = createInstallationProvisioningReconciler({ store: c.app.store, provisionerFactory: () => { calls++; throw new Error('legacy worker called'); } }); + assert.equal(legacy.runPending('worker_fixture').processed, 0); assert.equal(calls, 0); + const pending = c.app.store.latestProvisioningRequest(row.organization_id); + assert.throws(() => legacy.dispatch(pending.id), { code: 'installation_operation_not_allowed' }); + assert.equal(calls, 0); + await c.app.worker.runPending(); + const authority = createAccessInstallationLifecycleAuthority({ store: c.app.store, organizationId: row.organization_id, + authorityScope: 'platform_installation' }); + assert.throws(() => authority.request({ operation: 'decommission', expectedRevision: c.app.store.installationControl(row.organization_id).revision, + idempotencyKey: 'directory:remove:blocked' }), { code: 'installation_operation_not_allowed' }); + assert.equal(c.app.store.db.prepare('SELECT count(*) n FROM installation_lifecycle_jobs').get().n, 0); + assert.equal(c.active.has(row.runtime_key), true); +}); + +test('completion replay cannot override a newer operator stop or report a stopped runtime as prepared', async t => { + const c = await fixture(t); await c.create('stopped'); + const finish = c.app.store.finishProvisioningRequest; + c.app.store.finishProvisioningRequest = () => { throw new Error('injected completion interruption'); }; + await c.app.worker.runPending(); + const [row] = c.app.store.db.prepare('SELECT * FROM installations').all(); + await c.app.runtime.manager.apply('stop', 'operator_stopped_runtime', row.runtime_key); + c.app.store.finishProvisioningRequest = finish; + await c.app.worker.runPending(); + assert.equal(c.app.store.installationControl(row.organization_id).status, 'failed'); + assert.equal(c.active.has(row.runtime_key), false); + assert.equal(c.app.runtime.journal.record(row.runtime_key).desiredState, 'stopped'); + assert.equal(c.starts.length, 1); +}); + +test('a changed installation revision fences completion after an awaited host operation', async t => { + const c = await fixture(t); await c.create('fenced'); + const [row] = c.app.store.db.prepare('SELECT * FROM installations').all(); + const start = c.host.start; + c.host.start = async id => { + await start(id); + c.app.store.db.prepare('UPDATE installations SET revision=revision+1 WHERE organization_id=?').run(row.organization_id); + }; + await c.app.worker.runPending(); + assert.equal(c.app.store.latestProvisioningRequest(row.organization_id).status, 'dispatched'); + assert.equal(c.app.store.installationControl(row.organization_id).status, 'provisioning'); + assert.equal(c.errors.length, 1); assert.equal(c.errors[0].code, 'directory_access_changed'); +}); + +test('schema upgrade preserves the existing backend, foreign keys, identity and immutability trigger', async t => { + const c = await fixture(t); await c.create('schema'); await c.app.worker.runPending(); + const { db, paths } = c.app.store; + const row = db.prepare('SELECT * FROM installations').get(); + // Construct the prior reviewed backend CHECK with an existing native row. + const sql = db.prepare("SELECT sql FROM sqlite_schema WHERE name='installations'").get().sql; + const dependents = db.prepare("SELECT sql FROM sqlite_schema WHERE tbl_name='installations' AND type IN ('index','trigger') AND sql IS NOT NULL").all(); + db.exec('PRAGMA foreign_keys=OFF; BEGIN IMMEDIATE'); + db.exec(sql.replace(/CREATE TABLE "?installations"?/, 'CREATE TABLE installations_prior').replace(",'directory_service_v1'", '')); + const columns = Object.keys(row).join(','); + row.backend = 'native_service_v1'; + db.prepare(`INSERT INTO installations_prior(${columns}) VALUES(${Object.keys(row).map(() => '?').join(',')})`).run(...Object.values(row)); + db.exec('DROP TABLE installations; ALTER TABLE installations_prior RENAME TO installations;'); + for (const item of dependents) db.exec(item.sql); + db.exec('PRAGMA user_version=14; COMMIT; PRAGMA foreign_keys=ON;'); + await c.app.close(); + const upgraded = new AccessStore(paths); + try { + assert.deepEqual({ ...upgraded.db.prepare('SELECT * FROM installations').get() }, { ...row }); + assert.equal(upgraded.db.prepare('PRAGMA user_version').get().user_version, require('../../core/accounts/src/schema').SCHEMA_VERSION); + assert.deepEqual(upgraded.db.prepare('PRAGMA foreign_key_check').all(), []); + assert.throws(() => upgraded.db.prepare("UPDATE installations SET backend='directory_service_v1'").run(), /installation_backend_immutable/); + assert.ok(upgraded.db.prepare("SELECT sql FROM sqlite_schema WHERE name='installations'").get().sql.includes("'directory_service_v1'")); + } finally { upgraded.close(); } +}); + +async function lifecycleRequest(c, action, key, options = {}) { + const listing = c.app.access.platformOrganizations(c.app.access.session(c.login.token)); + const org = listing.find(item => item.controlRef === options.controlRef) || listing[0]; + const route = { decommission: 'remove', restore_dsp: 'restore' }[action] || action; + const body = { controlRef: org.controlRef, expectedRevision: org.installation.revision, + idempotencyKey: key, ...options }; + const response = await fetch(`http://127.0.0.1:${c.app.server.address().port}/api/platform/installation/${route}`, + { method: 'POST', headers: c.headers, body: JSON.stringify(body) }); + return { status: response.status, body: await response.json(), request: body }; +} + +test('dashboard suspension revokes authority before stopping and resume preserves onboarding state and credentials', async t => { + const c = await fixture(t); await c.create('lifecycle'); await c.app.worker.runPending(); + const row = c.app.store.db.prepare('SELECT * FROM installations').get(); + const tokenFile = path.join(c.paths.dsps, row.runtime_key, 'secrets/runtime-agent/registration-token'); + const token = fs.readFileSync(tokenFile); + const suspended = await lifecycleRequest(c, 'suspend', 'directory:suspend:fixture'); + assert.equal(suspended.status, 202, JSON.stringify(suspended.body)); + assert.equal(c.app.store.organization(row.organization_id).status, 'suspended'); + assert.equal(c.app.runtime.hub.connected(row.runtime_key), false); + assert.equal(c.active.has(row.runtime_key), true); // Host work happens asynchronously. + await c.app.worker.runPending(); + assert.equal(c.active.has(row.runtime_key), false); + assert.equal(c.app.lifecycle.authority.latest(row.organization_id).status, 'succeeded'); + const replay = await lifecycleRequest(c, 'suspend', 'directory:suspend:fixture', suspended.request); + assert.equal(replay.status, 202); assert.equal(replay.body.data.replayed, true); + const resumed = await lifecycleRequest(c, 'resume', 'directory:resume:fixture'); + assert.equal(resumed.status, 202, JSON.stringify(resumed.body)); + await c.app.worker.runPending(); + assert.equal(c.app.store.installationControl(row.organization_id).status, 'waiting_for_owner'); + assert.equal(c.app.store.installationControl(row.organization_id).currentJobId, null); + assert.equal(c.app.store.organization(row.organization_id).status, 'pending_owner'); + assert.equal(c.app.runtime.hub.connected(row.runtime_key), true); + assert.deepEqual(fs.readFileSync(tokenFile), token); + assert.deepEqual(c.errors, []); +}); + +test('dashboard removal retains data without automatic backup and restoration restores the previous suspension', async t => { + const c = await fixture(t); await c.create('retained'); await c.app.worker.runPending(); + const row = c.app.store.db.prepare('SELECT * FROM installations').get(); + const retained = path.join(c.paths.dsps, row.runtime_key, 'data/retained'); fs.writeFileSync(retained, 'synthetic retained data'); + await lifecycleRequest(c, 'suspend', 'directory:remove:suspend'); await c.app.worker.runPending(); + const removed = await lifecycleRequest(c, 'decommission', 'directory:remove:request'); + assert.equal(removed.status, 202, JSON.stringify(removed.body)); + assert.equal(c.app.access.removalStarted(row.organization_id), true); + await c.app.worker.runPending(); + assert.equal(c.app.store.installationControl(row.organization_id).status, 'decommissioned'); + assert.equal(c.active.has(row.runtime_key), false); + assert.equal(fs.readFileSync(retained, 'utf8'), 'synthetic retained data'); + assert.equal(c.app.store.db.prepare('SELECT count(*) n FROM installation_backups').get().n, 0); + assert.equal(c.app.store.db.prepare('SELECT count(*) n FROM platform_backup_requests').get().n, 0); + const restored = await lifecycleRequest(c, 'restore_dsp', 'directory:restore:request'); + assert.equal(restored.status, 202, JSON.stringify(restored.body)); + await c.app.worker.runPending(); + assert.equal(c.app.store.installationControl(row.organization_id).status, 'suspended'); + assert.equal(c.app.store.organization(row.organization_id).status, 'suspended'); + assert.equal(c.active.has(row.runtime_key), false); assert.equal(c.app.access.removalStarted(row.organization_id), false); + assert.equal(c.app.access.installationConsoleStatus(row.organization_id).availableActions.includes('resume'), true); + await lifecycleRequest(c, 'resume', 'directory:restored:resume'); await c.app.worker.runPending(); + assert.equal(c.active.has(row.runtime_key), true); assert.deepEqual(c.errors, []); +}); + +test('lifecycle completion resumes after a Core commit interruption and rejects stale revisions or injected fields', async t => { + const c = await fixture(t); await c.create('interruption'); await c.app.worker.runPending(); + const row = c.app.store.db.prepare('SELECT * FROM installations').get(); + assert.equal((await lifecycleRequest(c, 'suspend', 'directory:bad:revision', { expectedRevision: 1 })).status, 409); + assert.equal((await lifecycleRequest(c, 'suspend', 'directory:bad:fields', { runtimeKey: row.runtime_key })).status, 400); + await lifecycleRequest(c, 'suspend', 'directory:interrupted:suspend'); + const update = c.app.store.updateInstallationControl; + c.app.store.updateInstallationControl = () => { throw new Error('injected commit interruption'); }; + await c.app.worker.runPending(); assert.equal(c.active.has(row.runtime_key), false); + assert.equal(c.app.lifecycle.authority.latest(row.organization_id).status, 'running'); + c.app.store.updateInstallationControl = update; + await c.restart(); await c.app.worker.runPending(); + assert.equal(c.app.lifecycle.authority.latest(row.organization_id).status, 'succeeded'); + assert.equal(c.active.has(row.runtime_key), false); assert.equal(c.starts.length, 1); +}); + +test('a failed host stop remains visible and an explicit retry completes the same DSP', async t => { + const c = await fixture(t); await c.create('stop-failure'); await c.app.worker.runPending(); + const row = c.app.store.db.prepare('SELECT * FROM installations').get(); + const stop = c.host.stop; c.host.stop = async () => { throw new Error('injected stop failure'); }; + await lifecycleRequest(c, 'suspend', 'directory:failed:suspend'); await c.app.worker.runPending(); + assert.equal(c.app.store.installationControl(row.organization_id).status, 'failed'); + assert.equal(c.app.runtime.hub.connected(row.runtime_key), false); + assert.equal(c.app.access.installationConsoleStatus(row.organization_id).failure.recoverable, true); + c.host.stop = stop; + const retry = await lifecycleRequest(c, 'suspend', 'directory:retry:suspend'); + assert.equal(retry.status, 202, JSON.stringify(retry.body)); + await c.app.worker.runPending(); + assert.equal(c.app.store.installationControl(row.organization_id).status, 'suspended'); + assert.equal(c.active.has(row.runtime_key), false); assert.equal(fs.readdirSync(c.paths.dsps).length, 1); + assert.deepEqual(c.errors, []); +}); + +test('directory diagnostics create separate synthetic app owners and keep provider egress disabled', async t => { + const c = await fixture(t), seeded = []; + c.app.runtime.hub.invoke = async (key, action, input) => { + if (action === 'diagnostics.seed') { + seeded.push({ key, input }); + return { ok: true, status: 'succeeded', data: { roster: { publicationId: 'fixture_roster' }, timecards: { publicationId: 'fixture_timecards' } } }; + } + return { ok: true }; + }; + for (const name of ['first', 'second']) { + const result = c.app.access.platformDiagnostics(c.app.access.session(c.login.token), { idempotencyKey: `directory:diagnostic:${name}` }); + assert.equal(result.enabled, true); + await c.app.worker.runPending(); + } + const rows = c.app.store.db.prepare('SELECT i.runtime_key,d.status FROM diagnostic_dsps d JOIN installations i ON i.organization_id=d.organization_id').all(); + assert.equal(rows.length, 2); assert.equal(seeded.length, 2); + assert.equal(c.app.store.membershipsForUser(c.login.session.user.id).length, 0); + for (const row of rows) { + assert.equal(row.status, 'ready'); + assert.equal(c.app.runtime.manager.networkPermitted(row.runtime_key), false); + assert.equal(c.active.has(row.runtime_key), true); + } + assert.equal(c.app.store.db.prepare('SELECT count(*) n FROM installation_activation_jobs').get().n, 0); + assert.deepEqual(c.errors, []); +}); + +test('runtime monitoring requires platform ownership and hides host paths and runtime identity', async t => { + const c = await fixture(t); await c.create('monitor'); await c.app.worker.runPending(); + const endpoint = `http://127.0.0.1:${c.app.server.address().port}/api/platform/runtime`; + assert.equal((await fetch(endpoint)).status, 401); + const result = await fetch(endpoint, { headers: c.headers }); assert.equal(result.status, 200); + const view = await result.json(); assert.equal(view.data.enabled, true); assert.equal(view.data.runtimes.length, 1); + const raw = JSON.stringify(view); + const row = c.app.store.db.prepare('SELECT runtime_key FROM installations').get(); + assert.equal(raw.includes(c.paths.platformRoot), false); assert.equal(raw.includes(row.runtime_key), false); + assert.equal(raw.includes('.service'), false); +}); + +test('directory dashboard behind Cloudflare enforces its origin and secure authenticated sessions', async t => { + const origin = 'https://dispatch.example.test'; + const c = await fixture(t, { publicOrigin: origin, secureCookies: true }); + const base = `http://127.0.0.1:${c.app.server.address().port}`; + const http = require('node:http'); + const request = (route, options = {}) => new Promise((resolve, reject) => { + const outgoing = http.request(base + route, options, response => { + let body = ''; response.setEncoding('utf8'); response.on('data', chunk => { body += chunk; }); + response.on('end', () => resolve({ status: response.statusCode, headers: response.headers, body })); + }); + outgoing.on('error', reject); outgoing.end(options.body); + }); + const headers = { Host: 'dispatch.example.test', 'CF-Visitor': '{"scheme":"https"}', Origin: origin, 'Content-Type': 'application/json' }; + assert.equal((await request('/api/auth/session')).status, 403); + const redirected = await request('/', { headers: { ...headers, 'CF-Visitor': '{"scheme":"http"}' } }); + assert.equal(redirected.status, 308); assert.equal(redirected.headers.location, origin + '/'); + const login = { email: 'platform@example.test', password: 'synthetic owner password' }; + const post = extra => request('/api/auth/login', { method: 'POST', headers: { ...headers, ...extra }, body: JSON.stringify(login) }); + assert.equal((await post({ Origin: 'https://other.example.test' })).status, 403); + assert.equal((await post({ Host: 'other.example.test' })).status, 403); + const signedIn = await post({}); assert.equal(signedIn.status, 200); + const current = JSON.parse(signedIn.body).data; + const cookie = signedIn.headers['set-cookie'][0]; + assert.match(cookie, /; Secure/); assert.match(cookie, /; HttpOnly/); assert.match(cookie, /SameSite=Strict/); + const session = await request('/api/auth/session', { headers: { ...headers, Cookie: cookie.split(';')[0] } }); + assert.equal(session.status, 200); assert.equal(JSON.parse(session.body).data.user.email, login.email); + const created = await request('/api/platform/organizations', { method: 'POST', + headers: { ...headers, Cookie: cookie.split(';')[0], 'X-Dispatch-CSRF': current.csrfToken }, + body: JSON.stringify({ ownerEmail: 'manual@example.test', idempotencyKey: 'directory:create:manual' }) }); + assert.equal(created.status, 503); + assert.equal(JSON.parse(created.body).error.code, 'invitation_email_unavailable'); + assert.equal(c.app.store.organizations().length, 0); + assert.equal(c.app.store.db.prepare("SELECT count(*) n FROM invitations WHERE kind='organization_owner'").get().n, 0); +}); + +test('directory email configuration sends the owner invitation once and preserves a failed handoff', async t => { + const origin = 'https://dispatch.example.test', requests = []; + const c = await fixture(t, { + publicOrigin: origin, secureCookies: true, + environment: { DISPATCH_EMAIL_ACCOUNT_ID: 'a'.repeat(32), DISPATCH_EMAIL_FROM_ADDRESS: 'invites@example.test' }, + prepare: paths => { + const secrets = path.join(paths.local, 'secrets/email'); + fs.mkdirSync(secrets, { recursive: true, mode: 0o700 }); + fs.writeFileSync(path.join(secrets, 'cloudflare-api-token'), 'b'.repeat(48), { mode: 0o600 }); + }, + invitationFetchImpl: async (url, options) => { + const payload = JSON.parse(options.body); + requests.push({ url, payload }); + return payload.to === 'rejected@example.test' + ? { ok: false, status: 403, json: async () => ({ success: false }) } + : { ok: true, status: 200, json: async () => ({ success: true, result: { queued: [payload.to] } }) }; + }, + }); + const create = suffix => new Promise((resolve, reject) => { + const request = require('node:http').request({ hostname: '127.0.0.1', port: c.app.server.address().port, + path: '/api/platform/organizations', method: 'POST', headers: { ...c.headers, + Cookie: `__Host-dispatch_session=${c.login.token}`, Host: 'dispatch.example.test', + Origin: origin, 'CF-Visitor': '{"scheme":"https"}' } }, response => { + let body = ''; response.setEncoding('utf8'); response.on('data', chunk => { body += chunk; }); + response.on('end', () => resolve({ status: response.statusCode, body: JSON.parse(body) })); + }); + request.on('error', reject); + request.end(JSON.stringify({ ownerEmail: `${suffix}@example.test`, idempotencyKey: `directory:create:${suffix}` })); + }); + const created = await create('emailed'); + assert.equal(created.status, 201); + assert.equal(created.body.data.delivery.status, 'accepted'); + assert.equal(created.body.data.invitationPath, null); + assert.equal(requests.length, 1); + assert.equal(requests[0].url, `https://api.cloudflare.com/client/v4/accounts/${'a'.repeat(32)}/email/sending/send`); + assert.equal(requests[0].payload.to, 'emailed@example.test'); + assert.deepEqual(requests[0].payload.from, { address: 'invites@example.test', name: 'Dispatch' }); + assert.match(requests[0].payload.text, /DSP OWNER INVITATION/); + const token = requests[0].payload.text.match(/\/#\/invitation\/([A-Za-z0-9_-]{43})/)[1]; + assert.equal(c.app.access.inspectInvitation(token).email, 'e*****d@example.test'); + const replay = await create('emailed'); + assert.equal(replay.status, 200); + assert.equal(replay.body.data.delivery.status, 'already_processed'); + assert.equal(requests.length, 1); + const failed = await create('rejected'); + assert.equal(failed.status, 201); + assert.equal(failed.body.data.delivery.status, 'failed'); + assert.match(failed.body.data.invitationPath, /^\/#\/invitation\//); + assert.equal(requests.length, 2); +}); + +test('local release history is readable without enabling remote downloads or legacy rollout backups', async t => { + const c = await fixture(t), endpoint = `http://127.0.0.1:${c.app.server.address().port}/api/platform/updates`; + assert.equal((await fetch(endpoint)).status, 401); + const response = await fetch(endpoint, { headers: c.headers }); + assert.equal(response.status, 200); + const view = (await response.json()).data; + assert.equal(view.enabled, false); assert.equal(view.mode, 'independent'); + assert.deepEqual(view.tracks.core.history, []); assert.deepEqual(view.tracks.dsp.history, []); + const update = await fetch(endpoint, { method: 'POST', headers: c.headers, + body: JSON.stringify({ action: 'start', releaseId: 'synthetic_release', idempotencyKey: 'directory:unconfigured:release' }) }); + assert.equal(update.status, 503); + assert.equal((await update.json()).error.code, 'release_worker_unavailable'); + assert.equal(c.app.store.db.prepare('SELECT count(*) n FROM platform_backup_requests').get().n, 0); +}); + +test('manual backup history is owner-only and the dashboard cannot start offline backup effects', async t => { + const c = await fixture(t), endpoint = `http://127.0.0.1:${c.app.server.address().port}/api/platform/backups`; + assert.equal((await fetch(endpoint)).status, 401); + const response = await fetch(endpoint, { headers: c.headers }); + assert.equal(response.status, 200); + const view = (await response.json()).data; + assert.deepEqual(view, { mode: 'manual', ownerOnly: true, offline: true, backups: [], operations: [] }); + const command = await fetch(endpoint, { method: 'POST', headers: c.headers, body: JSON.stringify({ action: 'backup', scope: 'platform' }) }); + assert.equal(command.status, 409); + assert.equal((await command.json()).error.code, 'manual_backup_requires_offline_command'); + c.app.store.db.prepare('UPDATE users SET platform_role=NULL WHERE id=?').run(c.login.session.user.id); + assert.equal((await fetch(endpoint, { headers: c.headers })).status, 403); + assert.equal(c.app.store.db.prepare('SELECT count(*) n FROM platform_backup_requests').get().n, 0); +}); diff --git a/core/dashboard/tests/helpers/connections-stack.cjs b/core/dashboard/tests/helpers/connections-stack.cjs new file mode 100644 index 0000000..d2c3280 --- /dev/null +++ b/core/dashboard/tests/helpers/connections-stack.cjs @@ -0,0 +1,114 @@ +'use strict'; + +// Isolated integration stack. Only the external website/browser adapter is fake. +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { AccessStore, AccessControlService } = require('../../../core/accounts/src'); +const { createOwnerConnections } = require('../../../core/accounts/src/owner-connections'); +const { createOwnerPaycomSetup } = require('../../../core/accounts/src/owner-paycom-setup'); +const { createDashboardServer } = require('../../server/server'); +const { defaultPaths } = require('dispatch-dsp/runtime/auth-broker/src/paths.js'); +const { AuthBrokerServer } = require('dispatch-dsp/runtime/auth-broker/src/server.js'); +const { RuntimeGatewayServer } = require('dispatch-dsp/runtime/gateway/src/server.js'); +const { createRuntimeGatewayDispatchClient } = require('../../../shared/gateway/client'); +const { createRuntimeConnections } = require('dispatch-runtime-kit/supervisor/src/connections'); +const { createContainerPaycomSetup } = require('dispatch-dsp/plugins/paycom/backend/runtime/setup.js'); + +async function createConnectionsStack({ directoryEnrollment = false } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-save-')); + fs.chmodSync(root, 0o700); + const oldEnvironment = ['DISPATCH_MANAGED_RUNTIME', 'DISPATCH_PROJECT_ROOT'].map(key => [key, process.env[key]]); + process.env.DISPATCH_MANAGED_RUNTIME = '1'; + process.env.DISPATCH_PROJECT_ROOT = '/opt/dispatch'; + const paths = defaultPaths({ databaseRoot: path.join(root, 'vault'), secretRoot: path.join(root, 'secret'), + stateRoot: path.join(root, 'state'), runtimeRoot: path.join(root, 'run') }); + const store = new AccessStore({ databaseRoot: path.join(root, 'core'), database: path.join(root, 'core', 'access.sqlite3') }); + const access = new AccessControlService(store, { installationOperatorEnabled: true, + installationBackend: directoryEnrollment ? 'directory_service_v1' : 'native_service_v1' }); + const password = 'isolated dashboard owner password'; + const platformInvite = access.createPlatformBootstrap({ email: 'platform@save.test' }); + const platform = await access.acceptNewUser({ token: platformInvite.token, firstName: 'Platform', lastName: 'Owner', + password, confirmPassword: password }); + const dsp = access.createOrganization(platform.session, { idempotencyKey: 'save:fixture:create', name: 'Credential Test DSP', + abbreviation: 'SAVE', stationCode: 'TST1', timezone: 'UTC', ownerEmail: 'owner@save.test' }); + const owner = await access.acceptNewUser({ token: dsp.token, firstName: 'Credential', lastName: 'Owner', + password, confirmPassword: password }); + store.updateInstallationControl({ organizationId: dsp.organization.id, expectedStatus: 'pending', expectedRevision: 1, + status: 'ready', revision: 2, currentJobId: null, timestamp: Date.now() }); + store.updateOrganizationStatus(dsp.organization.id, 'active', Date.now()); + require('../../../core/accounts/tests/plugin-fixture').enableFixturePlugin(store, dsp.organization.id); + const runtimeKey = store.installationControl(dsp.organization.id).runtimeKey; + const state = { dropReply: false, authentication: null, verification: null, browsers: [], broker: null, runtimeEnrollments: 0 }; + const options = { + browserRuntime: { launch: async () => { + const browser = { endpoint: 'http://127.0.0.1:43210', closed: false, + async close() { this.closed = true; }, isAlive() { return !this.closed; } }; + state.browsers.push(browser); return browser; + } }, + adapters: Object.fromEntries(['amazon-logistics', 'paycom'].map(provider => [provider, { provider, + authenticate: async (...args) => state.authentication ? state.authentication(...args) : { status: 'authenticated' }, + ...(provider === 'amazon-logistics' ? { completeVerification: async (...args) => state.verification ? state.verification(...args) : { status: 'authenticated' } } : {}), + }])), + }; + state.broker = new AuthBrokerServer(paths, options); + await state.broker.start(); + const unused = async () => { throw new Error('unexpected feature call'); }; + const config = { runtimeKey, paths: { projectRoot: path.resolve(__dirname, '../../..'), + dataRoot: path.join(root, 'data'), stateRoot: paths.stateRoot, stagingRoot: path.join(root, 'staging'), + auth: { socket: paths.socket } }, layout: { directories: { stateRoot: paths.stateRoot } } }; + const socketPath = path.join(paths.runtimeRoot, 'runtime-gateway.sock'); + const gateway = new RuntimeGatewayServer({ socketPath, runtimeKey, client: { + workforce: { day: unused }, sync: { status: unused, runNow: unused, start: unused, stop: unused }, + collections: { health: unused }, system: { status: unused }, + connectionsManage: createRuntimeConnections(config), paycomSetup: createContainerPaycomSetup(config, {}), + } }); + await gateway.start(); + const transport = createRuntimeGatewayDispatchClient({ socketPath, runtimeKey }); + const invoke = async (key, action, input) => { + if (key !== runtimeKey) throw new Error('wrong DSP'); + if (input.command === 'enroll') { + state.runtimeEnrollments++; + if (directoryEnrollment) return { ok: false, status: 'execution_capacity_wait' }; + } + const result = await (action === 'connections.manage' ? transport.connectionsManage(input) : transport.paycomSetup(input)); + if (state.dropReply && (input.command === 'save' || input.command === 'enroll')) { + state.dropReply = false; + throw new Error('simulated lost acknowledgement after persistence'); + } + return result; + }; + const backend = { async request(id, operation, input, options) { + if (id !== runtimeKey || operation !== 'auth.request') throw new Error('wrong DSP'); + const result = await require('dispatch-runtime-kit/auth-broker/src/client').request(paths.socket, input, options); + if (state.dropReply && input.action === 'enroll-paycom') { state.dropReply = false; throw new Error('lost enrollment response'); } + return result; + } }; + const verification = require('../../../host/controller/paycom-verification').createPaycomVerification({ backend }); + const enroll = directoryEnrollment ? require('../../../host/controller/paycom-enrollment').createPaycomEnrollment({ backend, verification }) : undefined; + const paycomSetup = createOwnerPaycomSetup({ store, access, invoke, + ...(enroll ? { enroll, beginVerification: verification.start, readReadiness: verification.readiness } : {}) }); + const connections = createOwnerConnections({ store, access, invoke, paycomSetup }); + const server = createDashboardServer({ access, connections, paycomSetup, client: { + workforce: { day: unused }, sync: { status: unused, runNow: unused }, system: { status: unused }, + } }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const base = `http://127.0.0.1:${server.address().port}`; + const headers = { Cookie: `dispatch_session=${owner.token}`, 'X-Dispatch-CSRF': owner.session.csrfToken, 'Content-Type': 'application/json' }; + return { + root, paths, store, access, platform, owner, state, base, headers, password, verification, paycomSetup, + organizationId: dsp.organization.id, + save: (id, credentials) => fetch(`${base}/api/organization/connections/${id}/save`, { + method: 'POST', headers, body: JSON.stringify({ credentials }), + }), + list: async () => (await (await fetch(`${base}/api/organization/connections`, { headers })).json()).data.items, + async restartBroker() { await state.broker.close(); state.broker = new AuthBrokerServer(paths, options); await state.broker.start(); }, + async close() { + await new Promise(resolve => server.close(resolve)); + await gateway.close(); await state.broker.close(); store.close(); + fs.rmSync(root, { recursive: true, force: true }); + for (const [key, value] of oldEnvironment) { if (value === undefined) delete process.env[key]; else process.env[key] = value; } + }, + }; +} +module.exports = { createConnectionsStack }; diff --git a/core/dashboard/tests/invitation-email.test.js b/core/dashboard/tests/invitation-email.test.js new file mode 100644 index 0000000..3341e1f --- /dev/null +++ b/core/dashboard/tests/invitation-email.test.js @@ -0,0 +1,251 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); + +const { + readPrivateApiToken, + invitationMessage, + CloudflareInvitationDelivery, + invitationDeliveryFromEnvironment, +} = require('../server/invitation-email'); + +const ACCOUNT_ID = 'a'.repeat(32); +const API_TOKEN = `cfat_${'b'.repeat(48)}`; +const INVITATION_TOKEN = 'c'.repeat(43); +const PUBLIC_ORIGIN = 'https://dispatch.example.test'; + +function invitation(overrides = {}) { + return { + email: 'invitee@example.test', + organizationName: 'Example DSP', + roleName: 'Manager', + expiresAt: '2026-09-06T12:00:00.000Z', + token: INVITATION_TOKEN, + ...overrides, + }; +} + +test('Cloudflare invitation delivery sends one canonical HTML and text message', async () => { + const requests = []; + const delivery = new CloudflareInvitationDelivery({ + accountId: ACCOUNT_ID, + apiToken: API_TOKEN, + publicOrigin: PUBLIC_ORIGIN, + fetchImpl: async (...args) => { + requests.push(args); + return { + ok: true, + status: 200, + json: async () => ({ + success: true, + result: { + delivered: ['invitee@example.test'], queued: [], permanent_bounces: [], suppressed_recipients: [], + }, + }), + }; + }, + }); + + assert.deepEqual(await delivery.send(invitation({ organizationName: 'Example ', roleName: 'Manager & Admin' })), { + status: 'accepted', + }); + assert.equal(requests.length, 1); + const [url, options] = requests[0]; + assert.equal(url, `https://api.cloudflare.com/client/v4/accounts/${ACCOUNT_ID}/email/sending/send`); + assert.equal(options.method, 'POST'); + assert.equal(options.redirect, 'error'); + assert.equal(options.headers.Authorization, `Bearer ${API_TOKEN}`); + const body = JSON.parse(options.body); + assert.equal(body.to, 'invitee@example.test'); + assert.deepEqual(body.from, { address: `invites@${new URL(PUBLIC_ORIGIN).hostname}`, name: 'Dispatch' }); + assert.equal(body.subject, "You're invited to Dispatch"); + assert.match(body.text, new RegExp(`${PUBLIC_ORIGIN}/#/invitation/${INVITATION_TOKEN}`)); + assert.match(body.html, /Example <DSP>/); + assert.match(body.html, /Manager & Admin/); + assert.doesNotMatch(body.html, /Example /); + assert.equal(Object.hasOwn(body, 'reply_to'), false); + + await delivery.send(invitation({ kind: 'organization_member', organizationName: 'Example ', roleName: 'Manager & Admin' })); + const teamBody = JSON.parse(requests[1][1].body); + assert.match(teamBody.html, /TEAM INVITATION/); + assert.match(teamBody.text, /TEAM INVITATION/); + assert.match(teamBody.html, /Example <DSP>/); + assert.match(teamBody.html, /Manager & Admin/); + assert.doesNotMatch(teamBody.html, /Example /); + assert.match(teamBody.text, /DSP: Example \nYour role: Manager & Admin/); +}); + +test('invitation delivery never retries an ambiguous request and returns closed statuses', async () => { + let ambiguousCalls = 0; + const ambiguous = new CloudflareInvitationDelivery({ + accountId: ACCOUNT_ID, + apiToken: API_TOKEN, + publicOrigin: PUBLIC_ORIGIN, + fetchImpl: async () => { ambiguousCalls += 1; throw new Error('network detail must not escape'); }, + }); + assert.deepEqual(await ambiguous.send(invitation()), { status: 'unknown' }); + assert.equal(ambiguousCalls, 1); + + const rejected = new CloudflareInvitationDelivery({ + accountId: ACCOUNT_ID, + apiToken: API_TOKEN, + publicOrigin: PUBLIC_ORIGIN, + fetchImpl: async () => ({ ok: false, status: 403, json: async () => ({ success: false }) }), + }); + assert.deepEqual(await rejected.send(invitation()), { status: 'failed' }); + + const suppressed = new CloudflareInvitationDelivery({ + accountId: ACCOUNT_ID, + apiToken: API_TOKEN, + publicOrigin: PUBLIC_ORIGIN, + fetchImpl: async () => ({ + ok: true, + status: 200, + json: async () => ({ + success: true, + result: { + delivered: [], queued: [], permanent_bounces: [], suppressed_recipients: ['invitee@example.test'], + }, + }), + }), + }); + assert.deepEqual(await suppressed.send(invitation()), { status: 'failed' }); +}); + +test('email configuration reads only an exact-mode owner-private token file', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-email-')); + const emailRoot = path.join(root, 'email'); + fs.chmodSync(root, 0o700); + fs.mkdirSync(emailRoot, { mode: 0o700 }); + const tokenFile = path.join(emailRoot, 'cloudflare-api-token'); + fs.writeFileSync(tokenFile, `${API_TOKEN}\n`, { mode: 0o600 }); + try { + assert.equal(readPrivateApiToken(tokenFile), API_TOKEN); + assert.equal(invitationDeliveryFromEnvironment({ + environment: {}, paths: { secretsRoot: root }, publicOrigin: null, + }), null); + const configured = invitationDeliveryFromEnvironment({ + environment: { DISPATCH_EMAIL_ACCOUNT_ID: ACCOUNT_ID }, + paths: { secretsRoot: root }, + publicOrigin: PUBLIC_ORIGIN, + fetchImpl: async () => { throw new Error('unused'); }, + }); + assert.equal(configured.accountId, ACCOUNT_ID); + assert.equal(configured.publicOrigin, PUBLIC_ORIGIN); + + fs.chmodSync(tokenFile, 0o644); + assert.throws(() => readPrivateApiToken(tokenFile), /invitation_email_config_invalid/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('invitation messages reject non-canonical origins and malformed authoritative input', () => { + const input = { ...invitation(), recipient: 'invitee@example.test' }; + const message = invitationMessage({ ...input, publicOrigin: PUBLIC_ORIGIN }); + assert.equal(message.invitationUrl, `${PUBLIC_ORIGIN}/#/invitation/${INVITATION_TOKEN}`); + assert.throws(() => invitationMessage({ ...input, publicOrigin: 'http://dispatch.example.test' })); + assert.throws(() => invitationMessage({ ...input, publicOrigin: 'https://other.example/untrusted-path' })); + assert.throws(() => invitationMessage({ ...input, token: 'short', publicOrigin: PUBLIC_ORIGIN })); +}); + +test('owner invitations explain DSP setup without exposing provisional organization details', () => { + const input = { ...invitation(), kind: 'organization_owner', recipient: 'invitee@example.test', publicOrigin: PUBLIC_ORIGIN }; + const message = invitationMessage({ ...input, organizationName: 'Internal provisional DSP', roleName: 'Internal owner role' }); + for (const body of [message.text, message.html]) { + assert.match(body, /DSP OWNER INVITATION/); + assert.match(body, /You’ll enter your DSP details during setup\./); + assert.match(body, /create an account or sign in/); + assert.match(body, /September 6, 2026 at 12:00 PM UTC/); + assert.doesNotMatch(body, /Internal provisional DSP|Internal owner role/); + assert.ok(body.includes(message.invitationUrl)); + } + assert.deepEqual(invitationMessage({ ...input, organizationName: null, roleName: null }), message); + assert.deepEqual(invitationMessage({ ...input, organizationName: undefined, roleName: undefined }), message); + assert.throws(() => invitationMessage({ ...input, publicOrigin: 'https://other.example/untrusted-path' })); + assert.throws(() => invitationMessage({ ...input, token: 'short' })); + assert.throws(() => invitationMessage({ ...input, recipient: 'invalid' })); + assert.throws(() => invitationMessage({ ...input, expiresAt: 'invalid' })); +}); + +test('owner template selection uses invitation kind rather than a role display name', () => { + for (const kind of ['organization_member', 'platform_owner', undefined]) { + const message = invitationMessage({ ...invitation(), kind, roleName: 'Owner', recipient: 'invitee@example.test', publicOrigin: PUBLIC_ORIGIN }); + for (const body of [message.text, message.html]) { + assert.match(body, /Example DSP/); + assert.doesNotMatch(body, /DSP OWNER INVITATION|enter your DSP details/); + } + assert.throws(() => invitationMessage({ ...invitation(), kind, organizationName: null, recipient: 'invitee@example.test', publicOrigin: PUBLIC_ORIGIN })); + } +}); + +test('delivery forwards owner invitation kind to both HTML and plain text templates', async () => { + const requests = []; + const delivery = new CloudflareInvitationDelivery({ + accountId: ACCOUNT_ID, apiToken: API_TOKEN, publicOrigin: PUBLIC_ORIGIN, + fetchImpl: async (...args) => { + requests.push(args); + return { ok: true, status: 200, json: async () => ({ success: true, result: { queued: ['invitee@example.test'] } }) }; + }, + }); + assert.deepEqual(await delivery.send(invitation({ kind: 'organization_owner', organizationName: null, roleName: null })), { status: 'accepted' }); + assert.equal(requests.length, 1); + const payload = JSON.parse(requests[0][1].body); + assert.equal(payload.subject, "You're invited to Dispatch"); + assert.match(payload.html, /DSP OWNER INVITATION/); + assert.match(payload.text, /DSP OWNER INVITATION/); + assert.deepEqual(payload.from, { address: `invites@${new URL(PUBLIC_ORIGIN).hostname}`, name: 'Dispatch' }); +}); + +test('team invitations use the assigned DSP and role without owner setup instructions', () => { + for (const roleName of ['Manager', 'Dispatcher', 'Driver']) { + const message = invitationMessage({ ...invitation(), kind: 'organization_member', roleName, + recipient: 'invitee@example.test', publicOrigin: PUBLIC_ORIGIN }); + for (const body of [message.html, message.text]) { + assert.match(body, /TEAM INVITATION/); + assert.ok(body.includes(roleName)); + assert.equal(body.split('Example DSP').length - 1, 1); + assert.ok(body.includes(message.invitationUrl)); + assert.match(body, /Create an account or sign in to join\./); + assert.match(body, /Expires Sep 6, 2026 · 12:00 PM UTC/); + assert.doesNotMatch(body, /DSP OWNER INVITATION|enter your DSP details/); + } + assert.equal((message.html.match(/ { + const input = { ...invitation(), kind: 'organization_member', recipient: 'invitee@example.test', publicOrigin: PUBLIC_ORIGIN }; + const message = invitationMessage({ ...input, organizationName: '', roleName: 'Driver' }); + assert.doesNotMatch(message.html, /Driver/); + assert.match(message.html, /<img src=x onerror="bad\(\)">/); + assert.match(message.html, /<b>Driver<\/b>/); + for (const overrides of [{ organizationName: null }, { roleName: '' }, { roleName: 'x'.repeat(65) }, + { organizationName: 'x'.repeat(121) }, { token: 'short' }, { publicOrigin: 'https://other.example/untrusted-path' }]) { + assert.throws(() => invitationMessage({ ...input, ...overrides })); + } +}); + + +test('deployment origin and sender are configurable without embedding a personal domain', async () => { + const requests = []; + const delivery = new CloudflareInvitationDelivery({ accountId: ACCOUNT_ID, apiToken: API_TOKEN, + publicOrigin: 'https://portal.example.invalid', senderAddress: 'notify@example.invalid', + fetchImpl: async (_url, options) => { requests.push(JSON.parse(options.body)); + return { ok: true, json: async () => ({ success: true, result: { queued: ['invitee@example.test'] } }) }; }, + }); + await delivery.send(invitation()); + assert.deepEqual(requests[0].from, { address: 'notify@example.invalid', name: 'Dispatch' }); + assert.ok(requests[0].text.includes('https://portal.example.invalid/#/invitation/')); + assert.throws(() => new CloudflareInvitationDelivery({ accountId: ACCOUNT_ID, apiToken: API_TOKEN, + publicOrigin: 'https://portal.example.invalid', senderAddress: 'invalid\\r\\nheader' })); + for (const publicOrigin of ['http://portal.example.invalid', 'https://portal.example.invalid/path', + 'https://portal.example.invalid/', 'https://user:password@portal.example.invalid', + 'https://portal.example.invalid?redirect=1', 'https://portal.example.invalid#fragment']) { + assert.throws(() => new CloudflareInvitationDelivery({ accountId: ACCOUNT_ID, apiToken: API_TOKEN, publicOrigin })); + } +}); diff --git a/core/dashboard/tests/on-demand-fixture.js b/core/dashboard/tests/on-demand-fixture.js new file mode 100644 index 0000000..d4d8fd2 --- /dev/null +++ b/core/dashboard/tests/on-demand-fixture.js @@ -0,0 +1,57 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { fixture: accounts, enableFixturePlugin } = require('../../core/accounts/tests/plugin-fixture'); +const { DirectoryExecution } = require('../../host/controller/execution'); +const { createDashboardServer } = require('../server/server'); +const { createInstallationRuntimeResolver } = require('../server/runtime-router'); +const { createRuntimeAgentDispatchClient } = require('../../core/agents/src/client'); +const { createOwnerConnections } = require('../../core/accounts/src/owner-connections'); +const { createOwnerPaycomSetup } = require('../../core/accounts/src/owner-paycom-setup'); +const { openDatabase } = require('../../shared/published/database'); +const { saveStatus } = require('../../shared/published/status'); +const { success } = require('../../shared/contracts/src/result'); +const { publishPeriod, schema } = require('dispatch-dsp/plugins/paycom/backend/adapters/published.js'); +const { workforceFixture } = require('dispatch-dsp/plugins/paycom/backend/tests/published-fixture.js'); + +async function fixture(t) { + const f = await accounts(t), roots = new Map(); + const local = path.join(f.root, 'local'); + for (const directory of [local, path.join(local, 'state'), path.join(local, 'config')]) fs.mkdirSync(directory, { mode: 0o700 }); + let runtimeCalls = 0; + const hub = { connected: () => false, invoke: async () => { runtimeCalls++; throw new Error('sleeping'); } }; + const execution = new DirectoryExecution({ paths: { local }, accessStore: f.store, hub, + publishedReader: ({directory}) => require('dispatch-dsp/plugins/paycom/dashboard/published.js').createPublishedClient({directory}), + configuration: { version: 1, enabled: true }, manager: { journal: { record: id => roots.get(id) }, checkedDsp: record => record } }); + execution.wake = () => {}; + for (const [index, dsp] of f.dsps.entries()) { + enableFixturePlugin(f.store, dsp.id); + const requests = require('../../core/accounts/src/onboarding-store').createOnboardingStore(f.store); + const setup = requests.begin(dsp.id, dsp.owner.user.id, 'fixture:connected', 'create', 1); + requests.enrolled(setup.id); requests.finish(requests.claim(setup.id, 'fixture')); + const root = path.join(f.root, dsp.runtimeKey); + fs.mkdirSync(root, { mode: 0o700 }); fs.mkdirSync(path.join(root, 'data'), { mode: 0o700 }); + roots.set(dsp.runtimeKey, { id: dsp.runtimeKey, root }); + const directory = path.join(root, 'data/published'), db = openDatabase(path.join(directory, 'paycom.sqlite3'), { write: true }); + schema(db); publishPeriod(db, workforceFixture({ name: index ? 'Cedar' : 'Northline' }), 'America/Chicago'); db.close(); + saveStatus(directory, { + 'sync:paycom-main-workforce': success('found', { id: 'paycom-main-workforce', desiredState: 'running', activity: 'idle', queuedRunCount: 0, + lastSucceededAt: '2026-09-11T12:00:00.000Z', nextDueAt: '2026-09-11T13:00:00.000Z', lastError: null, alerts: [], activeRun: null, businessContext: null }), + system: success('ready', { components: {}, summary: { ready: 2, degraded: 0, failed: 0 } }), + connections: success('found', { items: ['paycom', 'cortex'].map(service => ({ service, configured: false, + state: 'not_connected', checkedAt: null, retryAt: null, reason: null })) }), + }); + await execution.enroll(dsp.runtimeKey); + execution.store.update(dsp.runtimeKey, { state: 'sleeping', snapshot_ready: 1, check_at: null }, Date.now()); + } + const proxy = { invoke: (id, action, input) => execution.invoke(id, action, input) }; + const client = createRuntimeAgentDispatchClient({ hub: proxy, runtimeKey: 'unassigned' }); + const connections = createOwnerConnections({ store: f.store, access: f.access, invoke: proxy.invoke }); + const paycomSetup = createOwnerPaycomSetup({ store: f.store, access: f.access, invoke: proxy.invoke }); + const server = createDashboardServer({ access: f.access, client, plugins: f.plugins, connections, paycomSetup, + runtimeResolver: createInstallationRuntimeResolver({ localClient: client, runtimeAgentHub: proxy }) }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(async () => { await new Promise(resolve => server.close(resolve)); await execution.close(); }); + return { ...f, execution, server, url: `http://127.0.0.1:${server.address().port}`, runtimeCalls: () => runtimeCalls }; +} +module.exports = { fixture }; diff --git a/core/dashboard/tests/on-demand.test.js b/core/dashboard/tests/on-demand.test.js new file mode 100644 index 0000000..2b0d604 --- /dev/null +++ b/core/dashboard/tests/on-demand.test.js @@ -0,0 +1,32 @@ +'use strict'; +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { fixture } = require('./on-demand-fixture'); + +test('authenticated saved-data pages remain available with every DSP runtime stopped', async t => { + const f = await fixture(t), [a, b] = f.dsps; + const headers = { cookie: `dispatch_session=${a.token}` }; + for (const endpoint of ['/api/bootstrap', '/api/paycom/employees', '/api/paycom/employees/0000', + '/api/paycom/daily?date=2026-09-06', '/api/paycom/sync', '/api/integrations', '/api/organization/connections', '/api/organization/paycom-setup']) { + const response = await fetch(f.url + endpoint, { headers }); + assert.equal(response.status, 200, endpoint + ' ' + await response.clone().text()); + const raw = await response.text(); + assert.equal(raw.includes('Cedar'), false, endpoint); + assert.equal(raw.includes(a.runtimeKey), false, endpoint); + } + assert.equal((await fetch(f.url + '/api/paycom/employees')).status, 401); + assert.equal((await fetch(f.url + '/api/paycom/employees?runtimeKey=' + b.runtimeKey, { headers })).status, 400); + const platform = f.access.session(f.platform.token); + const view = f.access.beginDspView(platform, { controlRef: f.access.issuePlatformControlRef(platform, b.id) }); + const response = await fetch(f.url + '/api/paycom/employees', { headers: { cookie: `dispatch_session=${f.platform.token}`, 'x-dispatch-dsp-view': view.dspView.viewRef } }); + const raw = await response.text(); assert.equal(response.status, 200); assert.equal(raw.includes('Cedar'), true); assert.equal(raw.includes('Northline'), false); + assert.equal(f.runtimeCalls(), 0); + const request = await fetch(f.url + '/api/paycom/sync', { method: 'POST', headers: { ...headers, 'content-type': 'application/json', 'x-dispatch-csrf': a.owner.csrfToken }, + body: JSON.stringify({ idempotencyKey: 'offline-read-test-sync' }) }); + assert.equal(request.status, 202); + assert.equal(f.execution.store.pending(a.runtimeKey), 1); + assert.equal((await (await fetch(f.url + '/api/paycom/sync', { headers })).json()).data.activity, 'queued'); + assert.equal(f.runtimeCalls(), 0, 'queue admission does not synchronously launch a runtime'); + f.store.db.prepare("UPDATE dsp_plugins SET desired_state='disabled' WHERE organization_id=?").run(a.id); + assert.equal((await fetch(f.url + '/api/paycom/employees', { headers })).status, 409); +}); diff --git a/core/dashboard/tests/password-recovery.test.js b/core/dashboard/tests/password-recovery.test.js new file mode 100644 index 0000000..29e90c6 --- /dev/null +++ b/core/dashboard/tests/password-recovery.test.js @@ -0,0 +1,202 @@ +'use strict'; +const CANONICAL_PUBLIC_ORIGIN = 'https://dispatch.example.test'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const http = require('node:http'); +const { AccessStore, AccessControlService } = require('../../core/accounts/src'); +const { createDashboardServer } = require('../server/server'); +const { GENERIC_MESSAGE, createPasswordRecoveryHttp } = require('../server/password-recovery-http'); +const { CloudflareInvitationDelivery } = require('../server/invitation-email'); +const { passwordRecoveryMessage } = require('../server/password-recovery-email'); +const { createTurnstile } = require('../server/turnstile'); +const PASSWORD = 'original account passphrase'; +const NEXT = 'replacement account passphrase'; +const tick = () => new Promise(resolve => setImmediate(resolve)); + +async function fixture(t, options = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-recovery-http-')); + fs.chmodSync(root, 0o700); + const store = new AccessStore({ databaseRoot: path.join(root, 'access'), database: path.join(root, 'access/db.sqlite') }); + const access = new AccessControlService(store); + const owner = await access.acceptNewUser({ token: access.createPlatformBootstrap({ email: 'owner@example.test' }).token, + firstName: 'Test', lastName: 'Owner', password: PASSWORD, confirmPassword: PASSWORD }); + const messages = [], notifications = []; + const invitationDelivery = { send: async () => ({ status: 'accepted' }), + sendPasswordReset: async message => { messages.push(message); return { status: 'accepted' }; }, + sendPasswordResetConfirmation: async message => { notifications.push(message); return { status: 'accepted' }; } }; + const unavailable = async () => { throw Error('unused'); }; + const server = createDashboardServer({ access, invitationDelivery, + client: { workforce: { day: unavailable }, sync: { status: unavailable, runNow: unavailable }, system: { status: unavailable } }, + ...options }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(async () => { await tick(); await new Promise(resolve => server.close(resolve)); store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + const base = `http://127.0.0.1:${server.address().port}`; + const post = (route, body, headers = {}) => new Promise((resolve, reject) => { + const request = http.request(base + '/api/auth/' + route, { + method: 'POST', headers: { 'Content-Type': 'application/json', ...headers }, + }, response => { + const chunks = []; + response.on('data', chunk => chunks.push(chunk)); + response.once('end', () => resolve(new Response(Buffer.concat(chunks), { status: response.statusCode, headers: response.headers }))); + }); + request.once('error', reject); request.end(JSON.stringify(body)); + }); + return { access, store, owner, messages, notifications, base, post }; +} + +test('forgot/reset HTTP flow is anonymous, reveals no secret, revokes sessions and sends notification', async t => { + const c = await fixture(t); + for (const email of ['owner@example.test', 'missing@example.test', 'OWNER@example.test']) { + const response = await c.post('forgot-password', { email }); + assert.equal(response.status, 202); + assert.equal(response.headers.get('cache-control'), 'no-store'); + assert.equal(response.headers.get('referrer-policy'), 'no-referrer'); + assert.equal(response.headers.get('set-cookie'), null); + assert.deepEqual(await response.json(), { ok: true, status: 'accepted', data: { message: GENERIC_MESSAGE }, error: null }); + } + await tick(); + assert.equal(c.messages.length, 1); + const { token } = c.messages[0]; + assert.ok(c.access.session(c.owner.token)); + // Loading the page (including a mail scanner visit) never consumes a token. + assert.equal((await fetch(c.base + '/#/reset-password/' + token)).status, 200); + assert.equal(c.store.db.prepare('SELECT count(*) AS n FROM password_reset_tokens').get().n, 1); + const reset = await c.post('reset-password', { token, newPassword: NEXT, confirmPassword: NEXT }); + assert.equal(reset.status, 200); + assert.equal(reset.headers.get('set-cookie'), null); + assert.equal(JSON.stringify(await reset.json()).includes(token), false); + assert.equal(c.access.session(c.owner.token), null); + await tick(); + assert.equal(c.notifications.length, 1); + assert.deepEqual(Object.keys(c.notifications[0]).sort(), ['email', 'userId']); + assert.equal((await c.post('reset-password', { token, newPassword: NEXT, confirmPassword: NEXT })).status, 400); + assert.equal((await c.post('login', { email: 'owner@example.test', password: PASSWORD })).status, 401); + assert.equal((await c.post('login', { email: 'owner@example.test', password: NEXT })).status, 200); +}); + +test('unconfigured delivery fails closed equally for every email', async t => { + const c = await fixture(t, { invitationDelivery: null }); + for (const email of ['owner@example.test', 'missing@example.test']) { + const response = await c.post('forgot-password', { email }); + assert.equal(response.status, 503); + assert.equal((await response.json()).error.code, 'password_recovery_unavailable'); + } + assert.equal(c.store.db.prepare('SELECT count(*) AS n FROM password_reset_tokens').get().n, 0); +}); + +test('generic response precedes any account lookup and does not await email delivery', async t => { + const c = await fixture(t); + let responded = false, lookedUp = false, release; + const held = new Promise(resolve => { release = resolve; }); + const delivery = { sendPasswordReset: () => held, sendPasswordResetConfirmation: async () => ({ status: 'accepted' }) }; + const recovery = createPasswordRecoveryHttp({ + access: { store: c.store, audit: () => {}, requestPasswordReset: () => { + assert.equal(responded, true); lookedUp = true; return { email: 'owner@example.test', token: 'a'.repeat(43), userId: c.owner.session.user.id }; + } }, delivery, turnstile: null, requestAddress: () => '127.0.0.1', clock: () => new Date(), + }); + await recovery.route({ method: 'POST', headers: { 'content-type': 'application/json' } }, {}, + new URL('https://dispatch.example.test/api/auth/forgot-password'), { + readJson: async () => ({ email: 'owner@example.test' }), sendJson: (_response, status, body) => { + assert.equal(status, 202); assert.equal(body.data.message, GENERIC_MESSAGE); responded = true; + }, + }); + assert.equal(lookedUp, false); + await tick(); assert.equal(lookedUp, true); + release({ status: 'failed' }); await tick(); +}); + +test('provider exceptions leave generic responses unchanged and never expose provider diagnostics', async t => { + const c = await fixture(t, { invitationDelivery: { send: async () => {}, + sendPasswordReset: async () => { throw Error('private provider details'); }, + sendPasswordResetConfirmation: async () => { throw Error('private provider details'); }, + } }); + const response = await c.post('forgot-password', { email: 'owner@example.test' }); + assert.equal((await response.json()).data.message, GENERIC_MESSAGE); + await tick(); + const audit = JSON.stringify(c.store.db.prepare('SELECT * FROM audit_events').all()); + assert.match(audit, /account.password.reset.email.failed/); + assert.doesNotMatch(audit, /private provider details/); +}); + +test('slow email delivery cannot exceed the worker bound and recovery resumes after capacity frees', async t => { + let release; + const held = new Promise(resolve => { release = resolve; }); + const c = await fixture(t, { invitationDelivery: { send: async () => {}, + sendPasswordReset: () => held, sendPasswordResetConfirmation: () => held } }); + // Synthetic recipients isolate capacity from per-account throttling. + c.access.requestPasswordReset = ({ email }) => ({ email, userId: c.owner.session.user.id, token: 'a'.repeat(43) }); + try { + for (let i = 0; i < 16; i++) assert.equal((await c.post('forgot-password', { email: `owner${i}@example.test` })).status, 202); + const busy = await c.post('forgot-password', { email: 'missing@example.test' }); + assert.equal(busy.status, 503); + assert.equal((await busy.json()).error.code, 'password_recovery_busy'); + } finally { release({ status: 'unknown' }); await tick(); } + assert.equal((await c.post('forgot-password', { email: 'owner@example.test' })).status, 202); +}); + +test('both endpoints reject cross-site requests, token queries, wrong content types and extra identity fields', async t => { + const c = await fixture(t); + for (const route of ['forgot-password', 'reset-password']) { + assert.equal((await c.post(route, {}, { 'Sec-Fetch-Site': 'cross-site' })).status, 403); + assert.equal((await c.post(route + '?token=secret', {})).status, 400); + assert.equal((await c.post(route, {}, { 'Content-Type': 'text/plain' })).status, 415); + assert.equal((await fetch(c.base + '/api/auth/' + route)).status, 405); + } + assert.equal((await c.post('forgot-password', { email: 'owner@example.test', redirect: 'https://evil.test' })).status, 400); + assert.equal((await c.post('reset-password', { token: 'a'.repeat(43), newPassword: NEXT, confirmPassword: NEXT, userId: 'victim' })).status, 400); +}); + +test('public recovery enforces the canonical host and same-origin JSON mutations', async t => { + const c = await fixture(t, { secureCookies: true, publicOrigin: CANONICAL_PUBLIC_ORIGIN }); + const headers = { Host: 'dispatch.example.test', 'CF-Visitor': '{"scheme":"https"}' }; + for (const route of ['forgot-password', 'reset-password']) { + assert.equal((await c.post(route, {}, headers)).status, 403); + assert.equal((await c.post(route, {}, { ...headers, Origin: 'https://evil.test' })).status, 403); + assert.equal((await c.post(route, {}, { ...headers, Host: 'evil.test', Origin: CANONICAL_PUBLIC_ORIGIN })).status, 403); + } + assert.equal((await c.post('forgot-password', { email: 'owner@example.test' }, { ...headers, Origin: CANONICAL_PUBLIC_ORIGIN })).status, 202); +}); + +test('IP limits cannot be bypassed by changing the email or spoofing Cloudflare headers locally', async t => { + const c = await fixture(t); + for (let i = 0; i < 20; i++) assert.equal((await c.post('forgot-password', { email: `missing${i}@example.test` }, { 'CF-Connecting-IP': `192.0.2.${i}` })).status, 202); + const denied = await c.post('forgot-password', { email: 'owner@example.test' }); + assert.equal(denied.status, 429); + assert.equal((await denied.json()).error.code, 'password_recovery_rate_limited'); + for (let i = 0; i < 30; i++) assert.equal((await c.post('reset-password', { token: 'x'.repeat(43), newPassword: NEXT, confirmPassword: NEXT })).status, 400); + assert.equal((await c.post('reset-password', { token: 'y'.repeat(43), newPassword: NEXT, confirmPassword: NEXT })).status, 429); +}); + +test('recovery requires a fresh Turnstile token with the forgot_password action', async t => { + const consumed = new Set(), payloads = []; + const turnstile = createTurnstile({ siteKey: 'a'.repeat(24), secret: 'b'.repeat(24), hostname: 'dispatch.example.test', + fetchImpl: async (_url, init) => { + const body = JSON.parse(init.body); payloads.push(body); + const success = !consumed.has(body.response); consumed.add(body.response); + return { ok: true, json: async () => ({ success, hostname: 'dispatch.example.test', action: body.response.split(':')[0] }) }; + } }); + const c = await fixture(t, { turnstile }); + assert.equal((await c.post('forgot-password', { email: 'owner@example.test' })).status, 400); + assert.equal((await c.post('forgot-password', { email: 'owner@example.test', turnstileToken: 'login:one' })).status, 403); + assert.equal((await c.post('forgot-password', { email: 'owner@example.test', turnstileToken: 'forgot_password:one' })).status, 202); + assert.equal((await c.post('forgot-password', { email: 'owner@example.test', turnstileToken: 'forgot_password:one' })).status, 403); + assert.doesNotMatch(JSON.stringify(payloads), /owner@example|newPassword|email/); +}); + +test('reset emails use canonical HTTPS fragment links and confirmations contain no bearer token or password', async () => { + const token = 'a'.repeat(43), sent = []; + const delivery = new CloudflareInvitationDelivery({ accountId: 'a'.repeat(32), apiToken: 'b'.repeat(40), publicOrigin: CANONICAL_PUBLIC_ORIGIN, + fetchImpl: async (_url, init) => { sent.push(JSON.parse(init.body)); return { ok: true, json: async () => ({ success: true, result: { queued: ['owner@example.test'] } }) }; } }); + assert.deepEqual(await delivery.sendPasswordReset({ email: 'owner@example.test', token }), { status: 'accepted' }); + assert.deepEqual(await delivery.sendPasswordResetConfirmation({ email: 'owner@example.test' }), { status: 'accepted' }); + assert.ok(sent[0].html.includes(`${CANONICAL_PUBLIC_ORIGIN}/#/reset-password/${token}`)); + assert.match(sent[0].text, /30 minutes/); + assert.match(sent[1].text, /All existing sessions/); + assert.equal(JSON.stringify(sent[1]).includes(token), false); + assert.equal(JSON.stringify(sent).includes(NEXT), false); + assert.throws(() => passwordRecoveryMessage({ token, publicOrigin: 'https://invalid.example/path' }), /invalid/); + assert.throws(() => passwordRecoveryMessage({ token: '`; + try { + browser = await new ChromeBrowserRuntime({ stateRoot: path.join(root, 'sessions'), executable: chrome, transport: 'tcp' }).launch({ nativeInput: true }); + const target = await createTarget(browser.endpoint, 'about:blank'); + connection = await CdpConnection.connect(target.webSocketDebuggerUrl); + connection.socket.addEventListener('message', event => { + const message = JSON.parse(event.data); + if (message.method !== 'Fetch.requestPaused') return; + const { requestId, request } = message.params; + // Fulfill every request locally, including iframe requests; never contact Paycom. + connection.command('Fetch.fulfillRequest', { requestId, responseCode: 200, + responseHeaders: [{ name: 'Content-Type', value: 'text/html' }], + body: Buffer.from(request.url === url ? html : '').toString('base64'), + }).catch(error => failures.push(error)); + }); + await connection.command('Page.enable'); + await connection.command('Fetch.enable', { patterns: [{ urlPattern: '*' }] }); + await connection.command('Page.navigate', { url }); + const initial = await waitForState(connection, 5_000, new Set(['security_questions_required'])); + const submissions = []; + const result = await verifyTimecardApplication(connection, initial, + { pin3: 'fixture-three', pin4: 'fixture-four' }, phase => submissions.push(phase), undefined, value => value, true, browser); + assert.equal(result.state, 'authenticated'); + assert.deepEqual(submissions, ['security_questions']); + assert.deepEqual(await connection.evaluate('window.actions'), ['PIN submit', 'Not Now', 'Confirm', 'Continue']); + + await connection.evaluate(`history.pushState({},'',${JSON.stringify(SECURITY_PROFILE_PATH)});prompt();`); + assert.equal(classify(await connection.evaluate(SNAPSHOT)), 'security_profile_prompt'); + await connection.evaluate(`document.body.insertAdjacentHTML('beforeend','')`); + let snapshot = await connection.evaluate(SNAPSHOT); + assert.equal(snapshot.otpPresent, true); + assert.equal(classify(snapshot), 'manual_verification_required'); + assert.equal(JSON.stringify(snapshot).includes('private-fixture-secret'), false); + await connection.evaluate(`document.querySelector('[autocomplete="one-time-code"]').remove(); + document.body.insertAdjacentHTML('beforeend','')`); + snapshot = await connection.evaluate(SNAPSHOT); + assert.equal(snapshot.captchaPresent, false); + assert.equal(classify(snapshot), 'security_profile_prompt'); + await connection.evaluate(`document.querySelector('iframe').style.display='block'`); + snapshot = await connection.evaluate(SNAPSHOT); + assert.equal(snapshot.captchaPresent, true); + assert.equal(classify(snapshot), 'manual_verification_required'); + assert.equal((await connection.evaluate(securityProfileDismissExpression())).status, 'manual_verification_required'); + assert.deepEqual(failures, []); + } finally { + connection?.close(); + await browser?.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/dsp/runtime/auth-broker/tests/paycom-readiness.test.js b/dsp/runtime/auth-broker/tests/paycom-readiness.test.js new file mode 100644 index 0000000..d56f6c7 --- /dev/null +++ b/dsp/runtime/auth-broker/tests/paycom-readiness.test.js @@ -0,0 +1,115 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { ChromeBrowserRuntime } = require('../src/browser-runtime'); +const { CdpConnection, createTarget } = require('../src/cdp'); +const { + SNAPSHOT, classify, waitForState, submitNativeChallenge, challengeExpression, + SECURITY_QUESTION_PATH, +} = require('../../../plugins/paycom/backend/auth/adapter'); + +const chrome = process.env.DISPATCH_CHROME_EXECUTABLE + || ['/usr/bin/google-chrome', '/usr/bin/chromium', '/usr/bin/chromium-browser'].find(file => fs.existsSync(file)); + +test('PIN submission waits for the provider submit handler to add its request token', { + skip: !chrome && 'Chrome is required for the browser regression', timeout: 30_000, +}, async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'paycom-readiness-')); + const url = `https://www.paycomonline.net${SECURITY_QUESTION_PATH}`; + const credentials = { pin2: 'fixture-two', pin5: 'fixture-five' }; + const challenge = [{ index: 2 }, { index: 5 }]; + let browser, connection, releaseScript, scriptRequested, posted; + const scriptGate = new Promise(resolve => { releaseScript = resolve; }); + const scriptSeen = new Promise(resolve => { scriptRequested = resolve; }); + const submission = new Promise(resolve => { posted = resolve; }); + const failures = []; + const html = ` +
+ + + + + + + +
+ + `; + // Model the observed provider behavior with fake data: the form is visible + // before DOMContentLoaded installs the token-adding submit listener. + const script = `window.addEventListener('DOMContentLoaded', () => { + document.body.addEventListener('submit', event => { + const token = document.createElement('input'); + token.type = 'hidden'; token.name = 'request-token'; token.value = 'fixture-token'; + event.target.appendChild(token); + }); + });`; + async function intercept({ requestId, request }) { + let body = '', mime = 'text/html'; + if (request.url === url && request.method === 'GET') body = html; + else if (request.url === 'https://www.paycomonline.net/fixture-ready.js') { + scriptRequested(); + await scriptGate; + body = script; mime = 'application/javascript'; + } else if (request.url === url && request.method === 'POST') { + posted(new URLSearchParams(request.postData)); + body = 'Fixture accepted'; + } + // Fulfill every request locally; this test never contacts Paycom. + await connection.command('Fetch.fulfillRequest', { requestId, responseCode: 200, + responseHeaders: [{ name: 'Content-Type', value: mime }], body: Buffer.from(body).toString('base64') }); + } + try { + browser = await new ChromeBrowserRuntime({ stateRoot: path.join(root, 'sessions'), + executable: chrome, transport: 'tcp' }).launch({ nativeInput: true }); + const target = await createTarget(browser.endpoint, 'about:blank'); + connection = await CdpConnection.connect(target.webSocketDebuggerUrl); + connection.socket.addEventListener('message', event => { + const message = JSON.parse(event.data); + if (message.method === 'Fetch.requestPaused') intercept(message.params).catch(error => failures.push(error)); + }); + await connection.command('Page.enable'); + await connection.command('Fetch.enable', { patterns: [{ urlPattern: '*' }] }); + await connection.command('Page.navigate', { url }); + await scriptSeen; + // Chrome's preload scanner may request the script before the parser has + // created both inputs. Wait for the fixture layout while keeping the script + // blocked; waiting for DOMContentLoaded would erase the regression scenario. + let loading; + const deadline = Date.now() + 5000; + do { + loading = await connection.evaluate(SNAPSHOT); + if (JSON.stringify(loading.challenge.map(item => item.index)) === '[2,5]') break; + await new Promise(resolve => setTimeout(resolve, 20)); + } while (Date.now() < deadline); + assert.equal(loading.readyState, 'loading'); + assert.deepEqual(loading.challenge.map(item => item.index), [2, 5]); + assert.equal(classify(loading), 'pending'); + await assert.rejects(submitNativeChallenge(connection, credentials, challenge, browser), + error => error.code === 'manual_verification_required'); + assert.equal((await connection.evaluate(challengeExpression(credentials, challenge))).status, 'challenge_layout_changed'); + assert.deepEqual(await connection.evaluate(`Array.from(document.querySelectorAll('input[type="password"]')).map(f => f.value)`), ['', '']); + + const ready = waitForState(connection, 5_000, new Set(['security_questions_required'])); + releaseScript(); + const observed = await ready; + assert.equal(observed.snapshot.readyState, 'complete'); + await submitNativeChallenge(connection, credentials, observed.snapshot.challenge, browser); + const form = await submission; + assert.equal(form.get('firstSecurityQuestion'), credentials.pin2); + assert.equal(form.get('secondSecurityQuestion'), credentials.pin5); + assert.equal(form.get('firstIndex'), '2'); + assert.equal(form.get('secondIndex'), '5'); + assert.equal(form.get('request-token'), 'fixture-token'); + assert.deepEqual(failures, []); + } finally { + releaseScript(); + connection?.close(); + await browser?.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/dsp/runtime/auth-broker/tests/paycom-session-persistence.test.js b/dsp/runtime/auth-broker/tests/paycom-session-persistence.test.js new file mode 100644 index 0000000..8cacaf0 --- /dev/null +++ b/dsp/runtime/auth-broker/tests/paycom-session-persistence.test.js @@ -0,0 +1,95 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { ChromeBrowserRuntime, processGroupAlive } = require('../src/browser-runtime'); +const { CdpConnection, createTarget } = require('../src/cdp'); + +const chrome = process.env.DISPATCH_CHROME_EXECUTABLE + || ['/usr/bin/google-chrome', '/usr/bin/chromium', '/usr/bin/chromium-browser'].find(file => fs.existsSync(file)); + +for (const transport of ['pipe', 'tcp']) { + test(`Paycom ${transport} sessions retain cookies after close and broker reconciliation`, { + skip: !chrome && 'Chrome is required for the browser regression', timeout: 40_000, + }, async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'paycom-cookies-')); + let seeded = false, seedResponses = 0, seedRequests = 0, browser, connection; + const server = http.createServer((request, response) => { + if (request.url === '/seed') seedRequests++; + // Restored tabs must not recreate cookies and conceal lost profile state. + if (request.url === '/seed' && !seeded) { + seeded = true; + seedResponses++; + response.setHeader('Set-Cookie', [ + 'fixture_session=retained; Path=/; HttpOnly', + 'fixture_persistent=retained; Path=/; HttpOnly; Max-Age=3600', + ]); + } + response.end('Local session fixture'); + }); + const options = { stateRoot: path.join(root, 'profiles'), socketRoot: path.join(root, 'run'), executable: chrome, transport }; + try { + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const origin = `http://127.0.0.1:${server.address().port}`; + const cookies = async () => (await connection.command('Network.getCookies', { urls: [origin] })).cookies + .map(cookie => ({ name: cookie.name, value: cookie.value, session: cookie.session })) + .sort((a, b) => a.name.localeCompare(b.name)); + const expected = [ + { name: 'fixture_persistent', value: 'retained', session: false }, + { name: 'fixture_session', value: 'retained', session: true }, + ]; + browser = await new ChromeBrowserRuntime(options).launch({ provider: 'paycom', profile: 'fixture' }); + let target = await createTarget(browser.endpoint, `${origin}/seed`); + connection = await CdpConnection.connect(target.webSocketDebuggerUrl); + for (let i = 0; i < 100 && (await cookies()).length !== 2; i++) await new Promise(resolve => setTimeout(resolve, 50)); + assert.deepEqual(await cookies(), expected); + connection.close(); connection = null; + await browser.close(); browser = null; + + const restartedRuntime = new ChromeBrowserRuntime(options); + await restartedRuntime.reconcile(); + browser = await restartedRuntime.launch({ provider: 'paycom', profile: 'fixture' }); + target = await createTarget(browser.endpoint, 'about:blank'); + connection = await CdpConnection.connect(target.webSocketDebuggerUrl); + assert.deepEqual(await cookies(), expected); + assert.equal(seedResponses, 1); + const browserConnection = await CdpConnection.connect(browser.browserWebSocketUrl); + try { + const targets = await browserConnection.command('Target.getTargets'); + assert.equal(targets.targetInfos.some(target => target.type === 'page' && target.url.startsWith(origin)), false); + } finally { browserConnection.close(); } + assert.equal(seedRequests, 1); + } finally { + connection?.close(); + await browser?.close(); + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + fs.rmSync(root, { recursive: true, force: true }); + } + }); +} + +test('persistent Paycom shutdown still terminates a stalled Chrome process group', { + skip: !chrome && 'Chrome is required for the browser regression', timeout: 25_000, +}, async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'paycom-stalled-')); + let browser; + try { + browser = await new ChromeBrowserRuntime({ stateRoot: path.join(root, 'profiles'), socketRoot: path.join(root, 'run'), + executable: chrome, transport: 'pipe' }).launch({ provider: 'paycom', profile: 'fixture' }); + process.kill(-browser.pid, 'SIGSTOP'); + await browser.close(); + assert.equal(processGroupAlive(browser.pid), false); + await browser.close(); // Cleanup remains idempotent after the forced fallback. + } finally { + if (browser && processGroupAlive(browser.pid)) { + try { process.kill(-browser.pid, 'SIGCONT'); } catch {} + await browser.close(); + } + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/dsp/runtime/auth-broker/tests/paycom-session-start.test.js b/dsp/runtime/auth-broker/tests/paycom-session-start.test.js new file mode 100644 index 0000000..5fa8199 --- /dev/null +++ b/dsp/runtime/auth-broker/tests/paycom-session-start.test.js @@ -0,0 +1,169 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { ChromeBrowserRuntime } = require('../src/browser-runtime'); +const cdp = require('../src/cdp'); +const { CLIENT_LANDING_PATH, LOGIN_URL, LOGIN_ACTION_URL, SECURITY_QUESTION_PATH, + TIMECARD_SEARCH_URL, SNAPSHOT } = require('../../../plugins/paycom/backend/auth/adapter'); + +const chrome = process.env.DISPATCH_CHROME_EXECUTABLE + || ['/usr/bin/google-chrome', '/usr/bin/chromium', '/usr/bin/chromium-browser'].find(file => fs.existsSync(file)); +const landing = `https://www.paycomonline.net${CLIENT_LANDING_PATH}`; +const authenticated = '
MenuLog out'; +const credentials = { clientCode: 'fixture-client', username: 'fixture-user', password: 'fixture-password', + pin3: 'fixture-three', pin4: 'fixture-four' }; +const login = `
+
+ `; + +// Intercept before navigation so every request stays local. The adapter still +// drives its real CDP connection, URL checks, DOM classification and native PIN input. +async function fixture(t, scenario, run, { application = false, nativeInput = scenario === 'expired' } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'paycom-start-')); + const originalCreateTarget = cdp.createTarget; + const modulePath = require.resolve('../../../plugins/paycom/backend/auth/adapter'); + const originalModule = require.cache[modulePath]; + const connections = [], requests = [], failures = [], starts = []; + let browser; + try { + browser = await new ChromeBrowserRuntime({ stateRoot: path.join(root, 'profiles'), socketRoot: path.join(root, 'run'), + executable: chrome, transport: 'pipe' }).launch({ provider: 'paycom', profile: 'fixture', nativeInput }); + t.mock.method(cdp, 'createTarget', async (endpoint, url) => { + starts.push(url); + const target = await originalCreateTarget(endpoint, 'about:blank'); + const connection = await cdp.CdpConnection.connect(target.webSocketDebuggerUrl); + connections.push(connection); + connection.socket.addEventListener('message', event => { + const message = JSON.parse(event.data); + if (message.method !== 'Fetch.requestPaused') return; + const { requestId, request } = message.params; + requests.push({ url: request.url, method: request.method }); + const redirect = scenario === 'expired' && request.url === landing; + let html = ''; + if (request.url === landing) html = scenario === 'captcha' + ? '' + : authenticated; + if (request.url === LOGIN_URL) html = login; + if (request.url === TIMECARD_SEARCH_URL) html = `${authenticated} + Timecard Search

Employee Status Is Active

`; + connection.command('Fetch.fulfillRequest', { requestId, responseCode: redirect ? 302 : 200, + responseHeaders: redirect ? [{ name: 'Location', value: LOGIN_URL }] : [{ name: 'Content-Type', value: 'text/html' }], + body: Buffer.from(redirect ? '' : html).toString('base64'), + }).catch(error => failures.push(error)); + }); + await connection.command('Page.enable'); + await connection.command('Fetch.enable', { patterns: [{ urlPattern: '*' }] }); + await connection.command('Page.navigate', { url }); + return target; + }); + delete require.cache[modulePath]; + const { paycomAdapter } = require(modulePath); + await run(paycomAdapter, browser, connections); + assert.deepEqual(starts, application ? [landing, TIMECARD_SEARCH_URL] : [landing]); + assert.equal(requests[0].url, landing); + if (scenario === 'expired') assert.ok(requests.some(request => request.url === LOGIN_URL)); + else assert.equal(requests.some(request => request.url === LOGIN_URL), false); + assert.deepEqual(failures, []); + } finally { + cdp.createTarget.mock?.restore(); + require.cache[modulePath] = originalModule; + for (const connection of connections) connection.close(); + await browser?.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +} + +for (const operation of ['inspect', 'recover', 'authenticate']) { + test(`Paycom ${operation} reuses a valid landing session without reading or submitting credentials`, { + skip: !chrome && 'Chrome is required for the browser regression', timeout: 30_000, + }, async t => fixture(t, 'valid', async (adapter, browser) => { + const options = { loginOnly: true, onSubmit: () => assert.fail('must not submit credentials') }; + const unreadable = new Proxy({}, { get: () => assert.fail('must not read credentials') }); + const result = operation === 'authenticate' + ? await adapter.authenticate(browser, unreadable, options) : await adapter[operation](browser, options); + assert.equal(result.status || result.state, 'authenticated'); + })); +} + +test('expired Paycom session redirects to login and completes stored credentials plus numbered PINs', { + skip: !chrome && 'Chrome is required for the browser regression', timeout: 30_000, +}, async t => fixture(t, 'expired', async (adapter, browser, connections) => { + const submissions = []; + const result = await adapter.authenticate(browser, credentials, { loginOnly: true, onSubmit: phase => submissions.push(phase) }); + assert.equal(result.status, 'authenticated'); + assert.deepEqual(submissions, ['credentials']); // The durable attempt latch is raised once. + assert.deepEqual(await connections[0].evaluate('window.actions'), ['credentials', 'pins']); +})); + +test('credential-free recovery refuses an expired Paycom session', { + skip: !chrome && 'Chrome is required for the browser regression', timeout: 30_000, +}, async t => fixture(t, 'expired', async (adapter, browser, connections) => { + await assert.rejects(adapter.recover(browser, { loginOnly: true }), { code: 'manual_verification_required' }); + assert.deepEqual(await connections[0].evaluate('window.actions'), []); +})); + +for (const scenario of ['valid', 'expired']) { + test(`${scenario} Paycom session completes automated collector handoff and closes prior tabs`, { + skip: !chrome && 'Chrome is required for the browser regression', timeout: 30_000, + }, async t => fixture(t, scenario, async (adapter, browser, connections) => { + const submissions = []; + const result = await adapter.authenticate(browser, credentials, { onSubmit: phase => submissions.push(phase) }); + assert.equal(result.status, 'authenticated'); + assert.equal(result.replacedCredentialPage, true); + assert.deepEqual(submissions, scenario === 'expired' ? ['credentials'] : []); + const control = await cdp.CdpConnection.connect(browser.browserWebSocketUrl); + try { + const { targetInfos } = await control.command('Target.getTargets'); + assert.deepEqual(targetInfos.filter(target => target.type === 'page') + .map(target => ({ id: target.targetId, url: target.url })), [{ id: result.targetId, url: TIMECARD_SEARCH_URL }]); + const snapshot = await connections.at(-1).evaluate(SNAPSHOT); + assert.equal(snapshot.timecardSearchReady, true); + assert.deepEqual(snapshot.loginPresent, [false, false, false]); + assert.deepEqual(snapshot.challenge, []); + } finally { control.close(); } + }, { application: true })); +} + +test('session-first Paycom CAPTCHA requests a native window before any credential submission', { + skip: !chrome && 'Chrome is required for the browser regression', timeout: 30_000, +}, async t => fixture(t, 'captcha', async (adapter, browser) => { + await assert.rejects(adapter.authenticate(browser, credentials, { + loginOnly: true, onSubmit: () => assert.fail('must not submit through CAPTCHA'), + }), { code: 'browser_interaction_required' }); +})); + + +test('expired session requests a window before reading or submitting credentials', { + skip: !chrome && 'Chrome is required for the browser regression', timeout: 30000, +}, async t => fixture(t, 'expired', async (adapter, browser, connections) => { + const unreadable = new Proxy({}, { get: () => assert.fail('must request a window before reading credentials') }); + await assert.rejects(adapter.authenticate(browser, unreadable, + { loginOnly: true, onSubmit: () => assert.fail('must not submit in headless mode') }), + { code: 'browser_interaction_required' }); + assert.deepEqual(await connections[0].evaluate('window.actions'), []); +}, { nativeInput: false })); diff --git a/dsp/runtime/auth-broker/tests/security.test.js b/dsp/runtime/auth-broker/tests/security.test.js new file mode 100644 index 0000000..d3380c9 --- /dev/null +++ b/dsp/runtime/auth-broker/tests/security.test.js @@ -0,0 +1,186 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const test = require('node:test'); +const { defaultPaths } = require('../src/paths'); +const { CredentialVault, VaultError, MAX_PROFILES } = require('../src/vault'); +const { MAX_RESPONSE_BYTES } = require('dispatch-runtime-kit/auth-broker/src/client'); +const { parseStrictJson, StrictJsonError } = require('dispatch-runtime-kit/auth-broker/src/strict-json'); + +function rootFixture(prefix = 'dispatch-auth-security-') { + const root = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + fs.chmodSync(root, 0o700); + return root; +} + +function pathsFor(root) { + return defaultPaths({ + databaseRoot: path.join(root, 'db'), secretRoot: path.join(root, 'secrets'), + stateRoot: path.join(root, 'state'), runtimeRoot: path.join(root, 'run'), + }); +} + +function basic(index = 0) { + return { username: `user-${index}`, password: `secret-${index}` }; +} + +test('unsafe existing storage mode is rejected rather than silently repaired', () => { + const root = rootFixture(); + const databaseRoot = path.join(root, 'db'); + fs.mkdirSync(databaseRoot, { mode: 0o755 }); + const paths = pathsFor(root); + try { + assert.throws(() => new CredentialVault(paths), error => error instanceof VaultError && error.code === 'unsafe_storage'); + assert.equal(fs.statSync(databaseRoot).mode & 0o777, 0o755); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('symlinked storage root is rejected without chmodding its target', () => { + const root = rootFixture(); + const target = path.join(root, 'target'); + fs.mkdirSync(target, { mode: 0o755 }); + fs.symlinkSync(target, path.join(root, 'db')); + const paths = pathsFor(root); + try { + assert.throws(() => new CredentialVault(paths), error => error instanceof VaultError && error.code === 'unsafe_storage'); + assert.equal(fs.statSync(target).mode & 0o777, 0o755); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('hard-linked keys and databases are rejected', () => { + const root = rootFixture(); + const paths = pathsFor(root); + let vault = new CredentialVault(paths); + vault.close(); + fs.linkSync(paths.key, path.join(paths.secretRoot, 'key-link')); + try { + assert.throws(() => new CredentialVault(paths), error => error instanceof VaultError && error.code === 'unsafe_storage'); + } finally { + fs.unlinkSync(path.join(paths.secretRoot, 'key-link')); + } + fs.linkSync(paths.database, path.join(paths.databaseRoot, 'db-link')); + try { + assert.throws(() => new CredentialVault(paths), error => error instanceof VaultError && error.code === 'unsafe_storage'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('incomplete key/database pairs are rejected without creating replacements', () => { + const root = rootFixture(); + const paths = pathsFor(root); + fs.mkdirSync(paths.databaseRoot, { mode: 0o700 }); + fs.mkdirSync(paths.secretRoot, { mode: 0o700 }); + fs.writeFileSync(paths.key, Buffer.alloc(32, 7), { mode: 0o600 }); + try { + assert.throws(() => new CredentialVault(paths), error => error instanceof VaultError && error.code === 'incomplete_storage'); + assert.equal(fs.existsSync(paths.database), false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('credential database cannot escape its private database root', () => { + const root = rootFixture(); + const paths = defaultPaths({ + databaseRoot: path.join(root, 'db'), + secretRoot: path.join(root, 'secrets'), + stateRoot: path.join(root, 'state'), + runtimeRoot: path.join(root, 'run'), + database: path.join(root, 'outside.sqlite3'), + }); + try { + assert.throws(() => new CredentialVault(paths), error => error instanceof VaultError && error.code === 'unsafe_storage'); + assert.equal(fs.existsSync(paths.database), false); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('AAD detects profile or provider substitution', () => { + const root = rootFixture(); + const paths = pathsFor(root); + const vault = new CredentialVault(paths); + try { + vault.put('site-main', 'basic', basic()); + vault.db.prepare('UPDATE credential_profiles SET profile=? WHERE profile=?').run('site-other', 'site-main'); + assert.throws(() => vault.readForAdapter('site-other'), error => error instanceof VaultError && error.code === 'vault_integrity_failed'); + vault.db.prepare('UPDATE credential_profiles SET profile=? WHERE profile=?').run('site-main', 'site-other'); + vault.db.prepare('UPDATE credential_profiles SET provider=? WHERE profile=?').run('paycom', 'site-main'); + assert.throws(() => vault.readForAdapter('site-main'), error => error instanceof VaultError && error.code === 'vault_integrity_failed'); + } finally { + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('vault enforces a bounded profile count but permits replacement at the limit', () => { + const root = rootFixture(); + const paths = pathsFor(root); + const vault = new CredentialVault(paths); + try { + for (let index = 0; index < MAX_PROFILES; index += 1) vault.put(`p-${String(index).padStart(3, '0')}`, 'basic', basic(index)); + assert.equal(vault.list().length, MAX_PROFILES); + const response = `${JSON.stringify({ ok: true, status: 'found', profiles: vault.list() })}\n`; + assert.ok(Buffer.byteLength(response) <= MAX_RESPONSE_BYTES); + assert.throws(() => vault.put('one-too-many', 'basic', basic(999)), error => error instanceof VaultError && error.code === 'profile_limit'); + assert.equal(vault.list().length, MAX_PROFILES); + vault.put('p-000', 'basic', basic(1000)); + assert.deepEqual(vault.readForAdapter('p-000').credentials, basic(1000)); + } finally { + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('unsupported or expanded database schemas fail closed', () => { + const root = rootFixture(); + const paths = pathsFor(root); + let vault = new CredentialVault(paths); + vault.close(); + let db = new DatabaseSync(paths.database); + db.exec('CREATE TABLE unexpected(value TEXT) STRICT'); + db.close(); + assert.throws(() => new CredentialVault(paths), error => error instanceof VaultError && error.code === 'schema_invalid'); + db = new DatabaseSync(paths.database); + db.exec('DROP TABLE unexpected; PRAGMA user_version=9'); + db.close(); + try { + assert.throws(() => new CredentialVault(paths), error => error instanceof VaultError && error.code === 'schema_invalid'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('tampered public metadata is rejected instead of being returned', () => { + const root = rootFixture(); + const paths = pathsFor(root); + const vault = new CredentialVault(paths); + try { + vault.put('site-main', 'basic', basic()); + vault.db.prepare('UPDATE credential_profiles SET updated_at=? WHERE profile=?').run('not-a-timestamp', 'site-main'); + assert.throws(() => vault.status('site-main'), error => error instanceof VaultError && error.code === 'vault_integrity_failed'); + assert.throws(() => vault.list(), error => error instanceof VaultError && error.code === 'vault_integrity_failed'); + } finally { + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('strict JSON rejects duplicate keys and safely preserves special keys', () => { + assert.throws(() => parseStrictJson('{"action":"health","action":"list"}'), StrictJsonError); + assert.throws(() => parseStrictJson('{"action":"health",}'), StrictJsonError); + const value = parseStrictJson('{"action":"health","__proto__":{"polluted":true}}'); + assert.equal(Object.getPrototypeOf(value), Object.prototype); + assert.equal(Object.hasOwn(value, '__proto__'), true); + assert.equal({}.polluted, undefined); + assert.throws(() => parseStrictJson('\u00a0{"action":"health"}'), StrictJsonError); +}); diff --git a/dsp/runtime/auth-broker/tests/server.test.js b/dsp/runtime/auth-broker/tests/server.test.js new file mode 100644 index 0000000..9819771 --- /dev/null +++ b/dsp/runtime/auth-broker/tests/server.test.js @@ -0,0 +1,393 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const net = require('node:net'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { defaultPaths } = require('../src/paths'); +const { CredentialVault } = require('../src/vault'); +const { AuthBrokerServer, ProtocolError, MAX_REQUEST_BYTES } = require('../src/server'); +const { request } = require('dispatch-runtime-kit/auth-broker/src/client'); +const { acquireMaintenanceLock } = require('../src/maintenance-lock'); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-server-')); + fs.chmodSync(root, 0o700); + const paths = defaultPaths({ + databaseRoot: path.join(root, 'db'), secretRoot: path.join(root, 'secrets'), + stateRoot: path.join(root, 'state'), runtimeRoot: path.join(root, 'run'), + }); + return { root, paths }; +} + +function rawRequest(socketPath, raw) { + return new Promise((resolve, reject) => { + const socket = net.createConnection(socketPath); + const chunks = []; + socket.on('connect', () => socket.end(raw)); + socket.on('data', chunk => chunks.push(chunk)); + socket.on('error', reject); + socket.on('end', () => { + try { resolve(JSON.parse(Buffer.concat(chunks).toString('utf8'))); } catch (error) { reject(error); } + }); + }); +} + +test('broker serves bounded metadata over a private Unix socket', async () => { + const { root, paths } = fixture(); + const vault = new CredentialVault(paths); + vault.put('site-main', 'basic', { username: 'user', password: 'secret' }); + vault.close(); + + const server = new AuthBrokerServer(paths); + try { + await server.start(); + assert.equal(fs.statSync(paths.socket).mode & 0o777, 0o600); + const health = await request(paths.socket, { action: 'health' }); + assert.equal(health.ok, true); + assert.equal(health.status, 'ready'); + assert.equal(health.vault.profiles, 1); + + const status = await request(paths.socket, { action: 'status', profile: 'site-main' }); + assert.equal(status.status, 'configured'); + assert.equal(status.profile.provider, 'basic'); + assert.equal(JSON.stringify(status).includes('secret'), false); + + const providers = await request(paths.socket, { action: 'providers' }); + assert.deepEqual(providers.providers.map(item => item.provider), ['paycom', 'amazon-logistics', 'basic']); + + const locked = await request(paths.socket, { action: 'lock', profile: 'site-main' }); + assert.equal(locked.status, 'locked'); + const after = await request(paths.socket, { action: 'status', profile: 'site-main' }); + assert.equal(after.session, 'locked'); + + const invalid = await request(paths.socket, { action: 'status', profile: 'site-main', extra: true }); + assert.deepEqual(invalid, { ok: false, status: 'invalid_request' }); + } finally { + await server.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('broker rejects duplicate keys, oversized input, and a second live server', async () => { + const { root, paths } = fixture(); + const vault = new CredentialVault(paths); + vault.close(); + const server = new AuthBrokerServer(paths); + try { + await server.start(); + assert.deepEqual(await rawRequest(paths.socket, '{"action":"health","action":"list"}\n'), { ok: false, status: 'invalid_request' }); + assert.deepEqual(await rawRequest(paths.socket, `${'{'.padEnd(MAX_REQUEST_BYTES + 1, 'x')}\n`), { ok: false, status: 'invalid_request' }); + const second = new AuthBrokerServer(paths); + await assert.rejects(() => second.start(), error => error instanceof ProtocolError && error.code === 'already_running'); + assert.equal((await request(paths.socket, { action: 'health' })).status, 'ready'); + } finally { + await server.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('connection deadline is absolute even while a client trickles data', async () => { + const { root, paths } = fixture(); + const vault = new CredentialVault(paths); + vault.close(); + const server = new AuthBrokerServer(paths, { socketTimeoutMs: 80 }); + try { + await server.start(); + const closed = await new Promise((resolve, reject) => { + const socket = net.createConnection(paths.socket); + const interval = setInterval(() => { + if (!socket.destroyed) socket.write(' '); + }, 15); + const timer = setTimeout(() => reject(new Error('connection_deadline_failed')), 500); + socket.on('error', () => {}); + socket.on('close', () => { + clearInterval(interval); + clearTimeout(timer); + resolve(true); + }); + }); + assert.equal(closed, true); + } finally { + await server.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('socket grants and revokes a full browser handoff without returning credentials', async () => { + const { root, paths } = fixture(); + const vault = new CredentialVault(paths); + vault.put('site-main', 'basic', { username: 'ipc-user', password: 'ipc-password' }); + vault.close(); + let browserClosed = false; + const browserRuntime = { + async launch() { + return { endpoint: 'http://127.0.0.1:9666', async close() { browserClosed = true; } }; + }, + }; + const adapters = { + basic: { + provider: 'basic', + async authenticate(_browser, credentials) { + assert.equal(credentials.password, 'ipc-password'); + return { status: 'authenticated' }; + }, + }, + }; + const server = new AuthBrokerServer(paths, { browserRuntime, adapters }); + try { + await server.start(); + const tested = await request(paths.socket, { action: 'test-auth-profile', profile: 'site-main' }); + assert.equal(tested.ok, true); + assert.equal(tested.status, 'authenticated'); + assert.equal(tested.profile.profile, 'site-main'); + assert.equal(tested.profile.provider, 'basic'); + assert.equal(typeof tested.profile.testedAt, 'string'); + assert.equal(JSON.stringify(tested).includes('endpoint'), false); + assert.equal(JSON.stringify(tested).includes('lease'), false); + assert.equal(JSON.stringify(tested).includes('ipc-password'), false); + assert.equal(browserClosed, true); + assert.equal(server.sessions.sessions.size, 0); + assert.equal(server.sessions.byProfile.size, 0); + browserClosed = false; + const acquired = await request(paths.socket, { + action: 'acquire-browser', profile: 'site-main', collector: 'fixture', runId: 'run-ipc', ttlSeconds: 30, + }); + assert.equal(acquired.ok, true); + assert.equal(acquired.session.browser.access, 'full'); + assert.equal(acquired.session.browser.endpoint, 'http://127.0.0.1:9666'); + assert.equal(JSON.stringify(acquired).includes('ipc-password'), false); + assert.equal((await request(paths.socket, { action: 'browser-status', lease: acquired.session.lease })).session.status, 'ready'); + assert.equal((await request(paths.socket, { action: 'renew-browser', lease: acquired.session.lease, ttlSeconds: 60 })).status, 'renewed'); + assert.equal((await request(paths.socket, { action: 'release-browser', lease: acquired.session.lease })).status, 'released'); + assert.equal(browserClosed, true); + } finally { + await server.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('operator inspection returns the last bounded authentication trail after failure', async () => { + const { root, paths } = fixture(); + const vault = new CredentialVault(paths); + vault.put('site-main', 'basic', { username: 'ipc-user', password: 'ipc-password' }); + vault.close(); + const browsers = []; + const browserRuntime = { + async launch() { + const browser = { endpoint: `http://127.0.0.1:${9700 + browsers.length}`, closed: false, async close() { this.closed = true; } }; + browsers.push(browser); + return browser; + }, + }; + const adapters = { basic: { + provider: 'basic', + async authenticate(_browser, _credentials, { onSubmit, onState }) { + onState('credentials_required', { + origin: 'https://www.amazon.com', path: '/ap/signin', queryKeys: ['openid.return_to'], + readyState: 'complete', usernameCount: 1, usernameTypes: ['email'], passwordCount: 1, + passwordTypes: ['password'], formCount: 1, formActionOrigin: 'https://www.amazon.com', + formActionPath: '/ap/signin', formActionQueryKeys: [], formMethod: 'POST', submitIds: ['signInSubmit'], + }); + onSubmit('credentials'); + onState('security_challenge', { + origin: 'https://www.amazon.com', path: '/ap/challenge/approval', queryKeys: [], + readyState: 'complete', otpPresent: false, captchaPresent: false, + }); + throw Object.assign(new Error('manual_verification_required'), { code: 'manual_verification_required' }); + }, + async inspect() { + return { state: 'credentials_required', observedAt: '2026-08-31T00:00:00.000Z', metadata: { path: '/ap/signin' } }; + }, + } }; + const server = new AuthBrokerServer(paths, { browserRuntime, adapters }); + try { + await server.start(); + assert.deepEqual(await request(paths.socket, { action: 'test-auth-profile', profile: 'site-main' }), { + ok: false, status: 'manual_verification_required', + }); + const inspected = await request(paths.socket, { action: 'inspect-auth-profile', profile: 'site-main' }); + assert.equal(inspected.inspection.lastAuthentication.status, 'manual_verification_required'); + assert.deepEqual(inspected.inspection.lastAuthentication.observations.map(item => item.state), [ + 'credentials_required', 'security_challenge', + ]); + assert.equal(inspected.inspection.lastAuthentication.observations[1].metadata.path, '/ap/challenge/approval'); + assert.equal(JSON.stringify(inspected).includes('ipc-password'), false); + assert.equal(JSON.stringify(inspected).includes('endpoint'), false); + assert.equal(browsers.every(browser => browser.closed), true); + } finally { + await server.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('client disconnect cancels an in-progress acquisition and closes its browser', async () => { + const { root, paths } = fixture(); + const vault = new CredentialVault(paths); + vault.put('site-main', 'basic', { username: 'ipc-user', password: 'ipc-password' }); + vault.close(); + let browserClosed = false; + let startedResolve; + const started = new Promise(resolve => { startedResolve = resolve; }); + const browserRuntime = { + async launch() { + return { endpoint: 'http://127.0.0.1:9667', async close() { browserClosed = true; } }; + }, + }; + const adapters = { + basic: { + provider: 'basic', + async authenticate(_browser, _credentials, { signal }) { + startedResolve(); + await new Promise((resolve, reject) => signal.addEventListener('abort', () => reject(Object.assign(new Error('cancelled'), { code: 'acquisition_cancelled' })), { once: true })); + }, + }, + }; + const server = new AuthBrokerServer(paths, { browserRuntime, adapters }); + try { + await server.start(); + const socket = net.createConnection(paths.socket); + await new Promise((resolve, reject) => { + socket.once('connect', resolve); + socket.once('error', reject); + }); + socket.write(`${JSON.stringify({ action: 'acquire-browser', profile: 'site-main', collector: 'fixture', runId: 'run-disconnect', ttlSeconds: 30 })}\n`); + await started; + socket.destroy(); + const deadline = Date.now() + 500; + while ((!browserClosed || server.sessions.pendingProfiles.size !== 0) && Date.now() < deadline) await new Promise(resolve => setTimeout(resolve, 10)); + assert.equal(browserClosed, true); + assert.equal(server.sessions.pendingProfiles.size, 0); + assert.equal(server.sessions.sessions.size, 0); + } finally { + await server.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('maintenance lock excludes broker startup and is reusable after release', async () => { + const { root, paths } = fixture(); + const vault = new CredentialVault(paths); + vault.close(); + const release = acquireMaintenanceLock(paths); + const server = new AuthBrokerServer(paths); + try { + assert.throws(() => acquireMaintenanceLock(paths), error => error.code === 'maintenance_busy'); + await assert.rejects(() => server.start(), error => error.code === 'maintenance_busy'); + assert.equal(fs.existsSync(paths.socket), false); + release(); + await server.start(); + assert.equal((await request(paths.socket, { action: 'health' })).status, 'ready'); + } finally { + try { release(); } catch {} + await server.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('managed enrollment is unavailable in the legacy broker and rejects extra fields', async () => { + const { root, paths } = fixture(); + const vault = new CredentialVault(paths); vault.close(); + const server = new AuthBrokerServer(paths); + const input = { action: 'enroll-paycom', intent: 'create', credentials: { + clientCode: 'fixture', username: 'fixture', password: 'fixture', pin1: '1', pin2: '2', pin3: '3', pin4: '4', pin5: '5', + } }; + try { + await server.start(); + assert.deepEqual(await request(paths.socket, input), { ok: false, status: 'invalid_request' }); + assert.deepEqual(await request(paths.socket, { ...input, profile: 'other-dsp' }), { ok: false, status: 'invalid_request' }); + assert.equal((await request(paths.socket, { action: 'health' })).vault.profiles, 0); + } finally { await server.close(); fs.rmSync(root, { recursive: true, force: true }); } +}); + +async function managedEnrollmentFixture(t) { + const f = fixture(); + const previous = { managed: process.env.DISPATCH_MANAGED_RUNTIME, root: process.env.DISPATCH_PROJECT_ROOT }; + process.env.DISPATCH_MANAGED_RUNTIME = '1'; process.env.DISPATCH_PROJECT_ROOT = '/opt/dispatch'; + const credentials = { clientCode: 'fixture', username: 'account-a', password: 'fixture', pin1: '1', pin2: '2', pin3: '3', pin4: '4', pin5: '5' }; + const vault = new CredentialVault(f.paths); vault.put('paycom-main', 'paycom', credentials); vault.close(); + const server = new AuthBrokerServer(f.paths); + t.after(async () => { + await server.close(); fs.rmSync(f.root, { recursive: true, force: true }); + for (const [key, value] of [['DISPATCH_MANAGED_RUNTIME', previous.managed], ['DISPATCH_PROJECT_ROOT', previous.root]]) { + if (value === undefined) delete process.env[key]; else process.env[key] = value; + } + }); + await server.start(); + const layout = require('../src/browser-runtime').ensurePersistentProfile(f.paths.browserSessions, 'paycom', 'paycom-main'); + fs.writeFileSync(path.join(layout.profileDirectory, 'synthetic-old-account-cookie'), 'account-a', { mode: 0o600 }); + return { ...f, server, credentials, layout }; +} + +test('rejected managed duplicate enrollment leaves the existing profile and browser session unlocked and unchanged', async t => { + const f = await managedEnrollmentFixture(t); + assert.deepEqual(await request(f.paths.socket, { action: 'enroll-paycom', intent: 'create', credentials: f.credentials }), + { ok: false, status: 'profile_exists' }); + assert.equal(f.server.sessions.lockedProfiles.has('paycom-main'), false); + assert.equal(f.server.sessions.attemptGuard.status('paycom-main'), null); + assert.equal(fs.existsSync(f.layout.directory), true); +}); +test('managed replacement removes prior browser authentication before storing the new account', async t => { + const f = await managedEnrollmentFixture(t); + const result = await request(f.paths.socket, { action: 'enroll-paycom', intent: 'replace', credentials: { ...f.credentials, username: 'account-b' } }); + assert.deepEqual(result, { ok: true, status: 'configured' }); + assert.equal(fs.existsSync(f.layout.directory), false); + assert.equal(f.server.vault.readForAdapter('paycom-main').credentials.username, 'account-b'); + assert.equal(f.server.sessions.lockedProfiles.has('paycom-main'), false); +}); +test('managed replacement preserves old vault credentials when browser cleanup cannot be verified', async t => { + const f = await managedEnrollmentFixture(t); + fs.chmodSync(f.layout.directory, 0o755); + const result = await request(f.paths.socket, { action: 'enroll-paycom', intent: 'replace', credentials: { ...f.credentials, username: 'account-b' } }); + assert.equal(result.ok, false); + assert.equal(f.server.vault.readForAdapter('paycom-main').credentials.username, 'account-a'); + assert.equal(fs.existsSync(f.layout.directory), true); +}); + +test('Paycom readiness is credential-free and preserves sanitized challenge evidence across broker restart', async t => { + const { root, paths } = fixture(); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const vault = new CredentialVault(paths); + vault.put('paycom-main', 'paycom', { clientCode: 'fixture-client', username: 'fixture-user', password: 'private-fixture-secret', + pin1: 'one', pin2: 'two', pin3: 'three', pin4: 'four', pin5: 'five' }); + vault.close(); + let launches = 0; + const options = { + browserRuntime: { async launch() { launches++; return { endpoint: 'http://127.0.0.1:9500', async close() {} }; } }, + adapters: { paycom: { provider: 'paycom', async authenticate(_browser, _credentials, { onSubmit, onState }) { + onSubmit('credentials'); + onState('manual_verification_required', { origin: 'https://www.paycomonline.net', + path: '/v4/cl/web.php/security/security-question/login', queryKeys: ['session_nonce'], + readyState: 'complete', loginFormCount: 0, challengeFormCount: 1, challengeIndices: [2, 5], + challengeFormActionPath: '/v4/cl/web.php/security/security-question/login', + diagnostic: { phase: 'security_questions', route: 'security_question', evidence: 'adapter_check', reason: 'additional_verification', raw: 'private-fixture-secret' }, + text: 'private-fixture-secret', title: 'private-fixture-secret', credentials: { password: 'private-fixture-secret' } }); + throw Object.assign(new Error('manual_verification_required'), { code: 'manual_verification_required' }); + } } }, + }; + let server = new AuthBrokerServer(paths, options); + t.after(() => server.close()); + await server.start(); + assert.deepEqual((await request(paths.socket, { action: 'profile-readiness', profile: 'paycom-main' })).readiness, + { state: 'ready', retryAllowed: true, retryAt: null }); + assert.equal(launches, 0); + assert.equal((await request(paths.socket, { action: 'test-auth-profile', profile: 'paycom-main' })).status, 'manual_verification_required'); + await server.close(); + server = new AuthBrokerServer(paths, options); + await server.start(); + const response = await request(paths.socket, { action: 'profile-readiness', profile: 'paycom-main' }); + assert.deepEqual(response.readiness, { state: 'manual', retryAllowed: false, retryAt: null }); + assert.equal(launches, 1, 'Readiness and restart never launch a browser'); + const metadata = response.lastAuthentication.observations[0].metadata; + assert.deepEqual(metadata.challengeIndices, [2, 5]); + assert.equal(metadata.challengeFormCount, 1); + assert.equal(metadata.diagnostic.reason, 'additional_verification'); + const file = path.join(paths.stateRoot, 'authentication-diagnostics.json'); + assert.equal(fs.statSync(file).mode & 0o777, 0o600); + assert.equal(fs.readFileSync(file, 'utf8').includes('private-fixture-secret'), false); + assert.equal(JSON.stringify(response).includes('private-fixture-secret'), false); + assert.deepEqual((await request(paths.socket, { action: 'profile-readiness', profile: 'missing' })).readiness, + { state: 'not_configured', retryAllowed: false, retryAt: null }); +}); diff --git a/dsp/runtime/auth-broker/tests/session-manager.test.js b/dsp/runtime/auth-broker/tests/session-manager.test.js new file mode 100644 index 0000000..5804ba7 --- /dev/null +++ b/dsp/runtime/auth-broker/tests/session-manager.test.js @@ -0,0 +1,586 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { defaultPaths } = require('../src/paths'); +const { CredentialVault } = require('../src/vault'); +const { BrowserSessionManager } = require('../src/session-manager'); +const { AttemptGuard } = require('../src/attempt-guard'); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-session-')); + fs.chmodSync(root, 0o700); + const paths = defaultPaths({ + databaseRoot: path.join(root, 'db'), secretRoot: path.join(root, 'secrets'), + stateRoot: path.join(root, 'state'), runtimeRoot: path.join(root, 'run'), + }); + const vault = new CredentialVault(paths); + vault.put('site-main', 'basic', { username: 'fixture-user', password: 'fixture-password' }); + return { root, paths, vault }; +} + +function fakeRuntime() { + const browsers = []; + return { + browsers, + async launch(options = {}) { + const browser = { + endpoint: `http://127.0.0.1:${9500 + browsers.length}`, + launchOptions: { profile: options.profile, provider: options.provider }, + closed: false, + async close() { this.closed = true; }, + }; + browsers.push(browser); + return browser; + }, + }; +} + +test('profile authentication test destroys the browser and returns metadata only', async () => { + const { root, vault } = fixture(); + const runtime = fakeRuntime(); + const manager = new BrowserSessionManager({ + vault, + browserRuntime: runtime, + adapters: { basic: { provider: 'basic', authenticate: async (browser, credentials, options) => { assert.equal(options.loginOnly, true); return { status: 'authenticated' }; } } }, + clock: () => 1_700_000_000_000, + }); + try { + const result = await manager.testProfile('site-main'); + assert.deepEqual(result, { profile: 'site-main', provider: 'basic', testedAt: '2023-11-14T22:13:20.000Z' }); + assert.equal(runtime.browsers[0].closed, true); + assert.equal(manager.sessions.size, 0); + assert.equal(manager.byProfile.size, 0); + assert.equal(JSON.stringify(result).includes('endpoint'), false); + assert.equal(JSON.stringify(result).includes('lease'), false); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('profile inspection is credential-free, preserves a manual latch, and destroys the browser', async () => { + const { root, paths, vault } = fixture(); + const runtime = fakeRuntime(); + const guard = new AttemptGuard(paths.attempts); + guard.lock('site-main'); + let authenticateCalls = 0; + const manager = new BrowserSessionManager({ + vault, + browserRuntime: runtime, + attemptGuard: guard, + adapters: { basic: { + provider: 'basic', + async inspect() { + return { + state: 'manual_verification_required', + observedAt: '2026-08-30T00:00:00.000Z', + metadata: { path: '/setup', queryKeys: [], title: 'Setup', profileInputNames: [], profileActionLabels: ['Not Now'] }, + }; + }, + async authenticate() { authenticateCalls += 1; return { status: 'authenticated' }; }, + } }, + }); + try { + const result = await manager.inspectProfile('site-main'); + assert.equal(result.state, 'manual_verification_required'); + assert.equal(result.metadata.path, '/setup'); + assert.equal(authenticateCalls, 0); + assert.equal(guard.status('site-main'), 'manual_verification_required'); + assert.equal(runtime.browsers[0].closed, true); + assert.equal(manager.sessions.size, 0); + assert.equal(JSON.stringify(result).includes('fixture-password'), false); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('failed authentication exposes only a bounded sanitized observation trail through inspection', async () => { + const { root, paths, vault } = fixture(); + const runtime = fakeRuntime(); + const guard = new AttemptGuard(paths.attempts); + const manager = new BrowserSessionManager({ + vault, + browserRuntime: runtime, + attemptGuard: guard, + clock: () => 1_700_000_000_000, + adapters: { basic: { + provider: 'basic', + async inspect() { + return { + state: 'credentials_required', observedAt: '2026-08-30T00:00:00.000Z', + metadata: { origin: 'https://example.com', path: '/signin', queryKeys: [], title: 'Sign in' }, + }; + }, + async authenticate(_browser, _credentials, { onSubmit, onState }) { + onSubmit('credentials'); + for (let index = 0; index < 12; index += 1) { + onState(`phase_${index}`, { + origin: 'https://www.amazon.com', path: '/ap/challenge/approval', + queryKeys: ['openid.mode', 'unsafe=value'], readyState: 'complete', + usernameCount: 0, usernameTypes: [], passwordCount: 0, passwordTypes: [], formCount: 1, + formActionOrigin: 'https://www.amazon.com', formActionPath: '/ap/challenge/approval', + formActionQueryKeys: ['openid.mode'], formMethod: 'POST', submitIds: ['continue'], + otpPresent: true, captchaPresent: false, applicationReady: false, + title: 'fixture-password', bodyText: 'fixture-password', endpoint: 'http://127.0.0.1:9999', + }); + } + throw Object.assign(new Error('manual_verification_required'), { code: 'manual_verification_required' }); + }, + } }, + }); + try { + await assert.rejects(() => manager.acquire({ + profile: 'site-main', collector: 'fixture', runId: 'run-diagnostic', ttlSeconds: 30, + }), error => error.code === 'manual_verification_required'); + const result = await manager.inspectProfile('site-main'); + assert.equal(result.lastAuthentication.status, 'manual_verification_required'); + assert.equal(result.lastAuthentication.observations.length, 8); + assert.equal(result.lastAuthentication.observations[0].state, 'phase_4'); + assert.equal(result.lastAuthentication.observations[7].state, 'phase_11'); + assert.deepEqual(result.lastAuthentication.observations[0].metadata.queryKeys, ['openid.mode']); + assert.equal(result.lastAuthentication.observations[0].metadata.otpPresent, true); + assert.equal(guard.status('site-main'), 'manual_verification_required'); + const serialized = JSON.stringify(result.lastAuthentication); + assert.equal(serialized.includes('fixture-password'), false); + assert.equal(serialized.includes('bodyText'), false); + assert.equal(serialized.includes('endpoint'), false); + assert.equal(runtime.browsers.every(browser => browser.closed), true); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('broker authenticates privately then grants full browser access without credentials', async () => { + const { root, vault } = fixture(); + const runtime = fakeRuntime(); + let observed; + const adapter = { + provider: 'basic', + async authenticate(browser, credentials) { + observed = credentials; + assert.equal(browser.endpoint, 'http://127.0.0.1:9500'); + assert.deepEqual(credentials, { username: 'fixture-user', password: 'fixture-password' }); + return { status: 'authenticated' }; + }, + }; + const manager = new BrowserSessionManager({ vault, browserRuntime: runtime, adapters: { basic: adapter } }); + try { + const session = await manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-1', ttlSeconds: 30 }); + assert.deepEqual(runtime.browsers[0].launchOptions, { profile: 'site-main', provider: 'basic' }); + assert.equal(session.browser.access, 'full'); + assert.equal(session.browser.protocol, 'cdp'); + assert.equal(session.browser.endpoint, 'http://127.0.0.1:9500'); + assert.equal(JSON.stringify(session).includes('fixture-password'), false); + assert.deepEqual(observed, { username: '', password: '' }); + assert.equal(manager.profileStatus('site-main'), 'leased'); + const renewed = manager.renew(session.lease, 60); + assert.ok(Date.parse(renewed.expiresAt) > Date.parse(session.expiresAt)); + await assert.rejects(() => manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-2', ttlSeconds: 30 }), error => error.code === 'session_busy'); + const released = await manager.release(session.lease); + assert.equal(released.released, true); + assert.equal(runtime.browsers[0].closed, true); + assert.equal(manager.profileStatus('site-main'), 'not_started'); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('locking a profile revokes its active browser and blocks future handoffs', async () => { + const { root, vault } = fixture(); + const runtime = fakeRuntime(); + const manager = new BrowserSessionManager({ + vault, + browserRuntime: runtime, + adapters: { basic: { provider: 'basic', authenticate: async () => ({ status: 'authenticated' }) } }, + }); + try { + const session = await manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-lock', ttlSeconds: 30 }); + await manager.lock('site-main'); + assert.equal(runtime.browsers[0].closed, true); + assert.equal(manager.profileStatus('site-main'), 'locked'); + assert.throws(() => manager.status(session.lease), error => error.code === 'lease_not_found'); + await assert.rejects(() => manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-after-lock', ttlSeconds: 30 }), error => error.code === 'profile_locked'); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('broker shutdown cancels and drains pending credential-bearing authentication', async () => { + const { root, vault } = fixture(); + const runtime = fakeRuntime(); + let authenticationStarted; + const started = new Promise(resolve => { authenticationStarted = resolve; }); + const manager = new BrowserSessionManager({ + vault, + browserRuntime: runtime, + adapters: { + basic: { + provider: 'basic', + authenticate: async (_browser, _credentials, { signal }) => { + authenticationStarted(); + await new Promise((resolve, reject) => { + signal.addEventListener('abort', () => reject(Object.assign(new Error('cancelled'), { code: 'acquisition_cancelled' })), { once: true }); + }); + }, + }, + }, + }); + try { + const acquisition = manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-pending', ttlSeconds: 30 }); + const rejected = assert.rejects(acquisition, error => error.code === 'acquisition_cancelled'); + await started; + await manager.close(); + await rejected; + assert.equal(runtime.browsers[0].closed, true); + assert.equal(manager.pendingProfiles.size, 0); + assert.equal(manager.sessions.size, 0); + } finally { + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('locking a profile cancels a pending login before reporting locked', async () => { + const { root, vault } = fixture(); + const runtime = fakeRuntime(); + let authenticationStarted; + const started = new Promise(resolve => { authenticationStarted = resolve; }); + const manager = new BrowserSessionManager({ + vault, + browserRuntime: runtime, + adapters: { + basic: { + provider: 'basic', + authenticate: async (_browser, _credentials, { signal }) => { + authenticationStarted(); + await new Promise((resolve, reject) => { + signal.addEventListener('abort', () => reject(Object.assign(new Error('cancelled'), { code: 'acquisition_cancelled' })), { once: true }); + }); + }, + }, + }, + }); + try { + const acquisition = manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-lock-pending', ttlSeconds: 30 }); + const rejected = assert.rejects(acquisition, error => error.code === 'acquisition_cancelled'); + await started; + const locked = await manager.lock('site-main'); + assert.equal(locked.status, 'locked'); + await rejected; + assert.equal(runtime.browsers[0].closed, true); + assert.equal(manager.pendingProfiles.size, 0); + assert.equal(manager.profileStatus('site-main'), 'locked'); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('submitted invalid credentials are latched before a retry can reach the adapter', async () => { + const { root, paths, vault } = fixture(); + const runtime = fakeRuntime(); + const guard = new AttemptGuard(paths.attempts, { clock: () => 1_000_000 }); + let adapterCalls = 0; + const manager = new BrowserSessionManager({ + vault, + browserRuntime: runtime, + attemptGuard: guard, + adapters: { + basic: { + provider: 'basic', + async authenticate(_browser, _credentials, { onSubmit }) { + adapterCalls += 1; + onSubmit('credentials'); + throw Object.assign(new Error('invalid_credentials'), { code: 'invalid_credentials' }); + }, + }, + }, + }); + try { + await assert.rejects(manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-invalid-1', ttlSeconds: 30 }), error => error.code === 'invalid_credentials'); + assert.equal(runtime.browsers[0].closed, true); + await assert.rejects(manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-invalid-2', ttlSeconds: 30 }), error => error.code === 'attempt_cooldown'); + await assert.rejects(manager.testProfile('site-main'), error => error.code === 'attempt_cooldown'); + assert.equal(adapterCalls, 1); + assert.equal(runtime.browsers.length, 1); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('interrupted submission recovers only through credential-free authenticated observation', async () => { + const { root, paths, vault } = fixture(); + const runtime = fakeRuntime(); + const initial = new AttemptGuard(paths.attempts); + initial.submitted('site-main'); + const guard = new AttemptGuard(paths.attempts); + let authenticateCalls = 0; + let recoverCalls = 0; + const manager = new BrowserSessionManager({ + vault, + browserRuntime: runtime, + attemptGuard: guard, + adapters: { + basic: { + provider: 'basic', + async authenticate() { authenticateCalls += 1; return { status: 'authenticated' }; }, + async recover() { recoverCalls += 1; return { status: 'authenticated' }; }, + }, + }, + }); + try { + const session = await manager.acquire({ + profile: 'site-main', collector: 'fixture', runId: 'run-observation-recovery', ttlSeconds: 30, + }); + assert.equal(authenticateCalls, 0); + assert.equal(recoverCalls, 1); + assert.equal(guard.status('site-main'), null); + assert.equal(session.status, 'ready'); + await manager.release(session.lease); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('failed observation leaves the submission guard latched and never reads credentials again', async () => { + const { root, paths, vault } = fixture(); + const runtime = fakeRuntime(); + const initial = new AttemptGuard(paths.attempts); + initial.submitted('site-main'); + const guard = new AttemptGuard(paths.attempts); + let authenticateCalls = 0; + const manager = new BrowserSessionManager({ + vault, + browserRuntime: runtime, + attemptGuard: guard, + adapters: { + basic: { + provider: 'basic', + async authenticate() { authenticateCalls += 1; return { status: 'authenticated' }; }, + async recover() { throw Object.assign(new Error('manual'), { code: 'manual_verification_required' }); }, + }, + }, + }); + try { + await assert.rejects(manager.acquire({ + profile: 'site-main', collector: 'fixture', runId: 'run-observation-failed', ttlSeconds: 30, + }), error => error.code === 'manual_verification_required'); + assert.equal(authenticateCalls, 0); + assert.equal(runtime.browsers[0].closed, true); + assert.equal(guard.status('site-main'), 'manual_verification_required'); + assert.equal(guard.observationRecoverable('site-main'), true); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('profile removal or replacement revokes the next lease operation', async () => { + const { root, vault } = fixture(); + const runtime = fakeRuntime(); + const manager = new BrowserSessionManager({ + vault, + browserRuntime: runtime, + adapters: { basic: { provider: 'basic', authenticate: async () => ({ status: 'authenticated' }) } }, + }); + try { + const session = await manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-profile-change', ttlSeconds: 30 }); + vault.remove('site-main'); + assert.throws(() => manager.renew(session.lease, 30), error => error.code === 'session_revoked'); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(runtime.browsers[0].closed, true); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('lease expiry uses a monotonic deadline when wall time moves backward', async () => { + const { root, vault } = fixture(); + const runtime = fakeRuntime(); + let wall = 1_000_000; + let monotonic = 0; + const manager = new BrowserSessionManager({ + vault, + browserRuntime: runtime, + clock: () => wall, + monotonicClock: () => monotonic, + adapters: { basic: { provider: 'basic', authenticate: async () => ({ status: 'authenticated' }) } }, + }); + try { + const result = await manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-monotonic', ttlSeconds: 30 }); + const session = manager.sessions.get(result.lease); + clearTimeout(session.timer); + wall = 1; + session.deadline = 10; + session.timer = manager._expiryTimer(session); + monotonic = 20; + await new Promise(resolve => setTimeout(resolve, 30)); + assert.equal(runtime.browsers[0].closed, true); + assert.equal(manager.sessions.size, 0); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('failed browser cleanup is retried and reaped without leaving the profile busy', async () => { + const { root, vault } = fixture(); + let closeCalls = 0; + const browser = { + endpoint: 'http://127.0.0.1:9600', + closed: false, + async close() { + closeCalls += 1; + if (closeCalls < 4) throw new Error('fixture cleanup failure'); + this.closed = true; + }, + }; + const manager = new BrowserSessionManager({ + vault, + browserRuntime: { launch: async () => browser }, + cleanupRetryDelaysMs: [1, 1], + cleanupReaperMs: 5, + adapters: { basic: { provider: 'basic', authenticate: async () => ({ status: 'authenticated' }) } }, + }); + try { + const result = await manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-cleanup-reaper', ttlSeconds: 30 }); + await assert.rejects(manager.release(result.lease), error => error.code === 'browser_cleanup_failed'); + assert.equal(manager.profileStatus('site-main'), 'cleanup_failed'); + await new Promise(resolve => setTimeout(resolve, 30)); + assert.equal(closeCalls, 4); + assert.equal(browser.closed, true); + assert.equal(manager.sessions.size, 0); + assert.equal(manager.profileStatus('site-main'), 'not_started'); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('an already-aborted acquisition never launches a browser', async () => { + const { root, vault } = fixture(); + let launches = 0; + const manager = new BrowserSessionManager({ + vault, + browserRuntime: { launch: async () => { launches += 1; throw new Error('must not launch'); } }, + adapters: { basic: { provider: 'basic', authenticate: async () => ({ status: 'authenticated' }) } }, + }); + const controller = new AbortController(); + controller.abort(); + try { + await assert.rejects( + manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-pre-aborted', ttlSeconds: 30 }, { signal: controller.signal }), + error => error.code === 'acquisition_cancelled', + ); + assert.equal(launches, 0); + assert.equal(manager.pendingProfiles.size, 0); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('authentication-error cleanup failures remain tracked until the reaper closes the browser', async () => { + const { root, vault } = fixture(); + let closeCalls = 0; + const browser = { + endpoint: 'http://127.0.0.1:9601', + async close() { + closeCalls += 1; + if (closeCalls < 4) throw new Error('fixture cleanup failure'); + }, + }; + const manager = new BrowserSessionManager({ + vault, + browserRuntime: { launch: async () => browser }, + cleanupRetryDelaysMs: [1, 1], + cleanupReaperMs: 5, + adapters: { + basic: { + provider: 'basic', + authenticate: async () => { throw Object.assign(new Error('invalid_credentials'), { code: 'invalid_credentials' }); }, + }, + }, + }); + try { + await assert.rejects( + manager.acquire({ profile: 'site-main', collector: 'fixture', runId: 'run-auth-cleanup', ttlSeconds: 30 }), + error => error.code === 'browser_cleanup_failed', + ); + assert.equal(manager.profileStatus('site-main'), 'cleanup_failed'); + await new Promise(resolve => setTimeout(resolve, 30)); + assert.equal(closeCalls, 4); + assert.equal(manager.sessions.size, 0); + assert.equal(manager.profileStatus('site-main'), 'not_started'); + } finally { + await manager.close(); + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + + +for (const scenario of ['upgrade', 'reused', 'already_submitted', 'repeat_request', 'launch_failure']) { + test(`native browser transition: ${scenario}`, async () => { + const { root, vault } = fixture(); + const launches = [], browsers = []; + let calls = 0, submissions = 0; + const runtime = { async launch(options) { + launches.push({ provider: options.provider, profile: options.profile, nativeInput: options.nativeInput === true }); + if (options.nativeInput && scenario === 'launch_failure') throw Object.assign(new Error('browser_start_failed'), { code: 'browser_start_failed' }); + const browser = { endpoint: 'http://127.0.0.1:9500', closed: false, + nativeInput: options.nativeInput ? {} : null, async close() { this.closed = true; } }; + browsers.push(browser); return browser; + } }; + const manager = new BrowserSessionManager({ vault, browserRuntime: runtime, + attemptGuard: { check() {}, submitted() { submissions++; }, failed() {}, succeeded() {} }, + adapters: { basic: { provider: 'basic', nativeInteraction: true, + async authenticate(browser, credentials, { onSubmit }) { + calls++; + assert.equal(credentials.password, 'fixture-password'); + if (scenario === 'already_submitted') onSubmit('credentials'); + if (scenario !== 'reused' && (!browser.nativeInput || scenario === 'repeat_request')) { + throw Object.assign(new Error('browser_interaction_required'), { code: 'browser_interaction_required' }); + } + if (browser.nativeInput) { + assert.equal(browsers[0].closed, true); + onSubmit('credentials'); + } + return { status: 'authenticated' }; + } } } }); + try { + if (['upgrade', 'reused'].includes(scenario)) await manager.testProfile('site-main'); + else await assert.rejects(manager.testProfile('site-main')); + assert.equal(launches.length, ['reused', 'already_submitted'].includes(scenario) ? 1 : 2); + assert.equal(calls, ['upgrade', 'repeat_request'].includes(scenario) ? 2 : 1); + assert.equal(submissions, ['upgrade', 'already_submitted'].includes(scenario) ? 1 : 0); + assert.equal(browsers.every(browser => browser.closed), true); + assert.equal(manager.byProfile.size, 0); + if (launches.length === 2) assert.deepEqual(launches[1], { provider: 'basic', profile: 'site-main', nativeInput: true }); + } finally { await manager.close(); vault.close(); fs.rmSync(root, { recursive: true, force: true }); } + }); +} diff --git a/dsp/runtime/auth-broker/tests/vault.test.js b/dsp/runtime/auth-broker/tests/vault.test.js new file mode 100644 index 0000000..dc40ed7 --- /dev/null +++ b/dsp/runtime/auth-broker/tests/vault.test.js @@ -0,0 +1,115 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { defaultPaths } = require('../src/paths'); +const { CredentialVault, VaultError } = require('../src/vault'); +const { ValidationError } = require('../src/providers'); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-vault-')); + fs.chmodSync(root, 0o700); + const paths = defaultPaths({ + databaseRoot: path.join(root, 'db'), secretRoot: path.join(root, 'secrets'), + stateRoot: path.join(root, 'state'), runtimeRoot: path.join(root, 'run'), + }); + return { root, paths }; +} + +const PAYCOM = Object.freeze({ + clientCode: 'TESTCLIENT', + username: 'test.user', + password: 'correct horse battery staple', + pin1: '10101', + pin2: '20202', + pin3: '30303', + pin4: '40404', + pin5: '50505', +}); + +test('vault encrypts credentials and exposes metadata only', () => { + const { root, paths } = fixture(); + const vault = new CredentialVault(paths, { clock: () => new Date('2026-08-25T12:00:00Z') }); + try { + const stored = vault.put('paycom-main', 'paycom', PAYCOM); + assert.equal(stored.configured, true); + assert.equal(stored.provider, 'paycom'); + assert.deepEqual(vault.readForAdapter('paycom-main'), { provider: 'paycom', revision: '2026-08-25T12:00:00.000Z', credentials: PAYCOM }); + assert.deepEqual(vault.list().map(row => Object.keys(row).sort()), [['createdAt', 'profile', 'provider', 'updatedAt']]); + const bytes = fs.readFileSync(paths.database); + for (const secret of Object.values(PAYCOM)) assert.equal(bytes.includes(Buffer.from(secret)), false); + assert.equal(fs.statSync(paths.database).mode & 0o777, 0o600); + assert.equal(fs.statSync(paths.key).mode & 0o777, 0o600); + assert.equal(fs.statSync(paths.databaseRoot).mode & 0o777, 0o700); + assert.deepEqual(vault.verify(), { verified: true, profiles: 1, schemaVersion: 1 }); + } finally { + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('read-only vault inspection cannot create or mutate credential storage', () => { + const { root, paths } = fixture(); + const writable = new CredentialVault(paths); + writable.put('paycom-main', 'paycom', PAYCOM); + writable.close(); + const databaseMtime = fs.statSync(paths.database).mtimeMs; + const keyMtime = fs.statSync(paths.key).mtimeMs; + const readOnly = new CredentialVault(paths, { readOnly: true }); + try { + assert.equal(readOnly.verify().verified, true); + assert.equal(readOnly.status('paycom-main').configured, true); + assert.throws(() => readOnly.put('paycom-other', 'paycom', PAYCOM), error => error.code === 'read_only'); + } finally { + readOnly.close(); + assert.equal(fs.statSync(paths.database).mtimeMs, databaseMtime); + assert.equal(fs.statSync(paths.key).mtimeMs, keyMtime); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('replacement is atomic and invalid credential shapes are rejected', () => { + const { root, paths } = fixture(); + const vault = new CredentialVault(paths); + try { + vault.put('paycom-main', 'paycom', PAYCOM); + const replacement = { ...PAYCOM, password: 'new secret value' }; + vault.put('paycom-main', 'paycom', replacement); + assert.deepEqual(vault.readForAdapter('paycom-main').credentials, replacement); + assert.throws( + () => vault.put('paycom-main', 'paycom', PAYCOM, { operation: 'enroll' }), + error => error instanceof VaultError && error.code === 'profile_exists', + ); + assert.throws( + () => vault.put('missing-profile', 'paycom', PAYCOM, { operation: 'replace' }), + error => error instanceof VaultError && error.code === 'profile_not_configured', + ); + assert.throws(() => vault.put('bad profile', 'paycom', PAYCOM), ValidationError); + assert.throws(() => vault.put('paycom-main', 'paycom', { ...PAYCOM, extra: 'no' }), ValidationError); + assert.throws(() => vault.put('paycom-main', 'paycom', { ...PAYCOM, pin5: PAYCOM.pin1 }), ValidationError); + assert.deepEqual(vault.remove('missing-profile'), { profile: 'missing-profile', removed: false }); + assert.deepEqual(vault.remove('paycom-main'), { profile: 'paycom-main', removed: true }); + } finally { + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('authenticated encryption detects a modified credential record', () => { + const { root, paths } = fixture(); + const vault = new CredentialVault(paths); + try { + vault.put('paycom-main', 'paycom', PAYCOM); + const row = vault.db.prepare('SELECT ciphertext FROM credential_profiles WHERE profile=?').get('paycom-main'); + const changed = Buffer.from(row.ciphertext); + changed[0] ^= 0xff; + vault.db.prepare('UPDATE credential_profiles SET ciphertext=? WHERE profile=?').run(changed, 'paycom-main'); + assert.throws(() => vault.readForAdapter('paycom-main'), error => error instanceof VaultError && error.code === 'vault_integrity_failed'); + } finally { + vault.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/dsp/runtime/cli/package.json b/dsp/runtime/cli/package.json new file mode 100644 index 0000000..7b28071 --- /dev/null +++ b/dsp/runtime/cli/package.json @@ -0,0 +1,16 @@ +{ + "name": "dispatch-cli", + "version": "0.9.0", + "private": true, + "description": "Unified SDK-backed command interface for Dispatch", + "type": "commonjs", + "main": "src/main.js", + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "./scripts/build", + "test": "./scripts/test", + "verify": "./scripts/verify" + } +} diff --git a/dsp/runtime/cli/scripts/build b/dsp/runtime/cli/scripts/build new file mode 100755 index 0000000..5788c86 --- /dev/null +++ b/dsp/runtime/cli/scripts/build @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +for file in "$ROOT"/src/*.js "$ROOT"/src/renderers/*.js "$ROOT"/src/interactions/*.js "$ROOT"/tests/*.js "$ROOT"/../../bin/dispatch; do + node --no-warnings --check "$file" +done +printf '%s\n' '{"ok":true,"status":"built"}' diff --git a/dsp/runtime/cli/scripts/test b/dsp/runtime/cli/scripts/test new file mode 100755 index 0000000..f6e53ac --- /dev/null +++ b/dsp/runtime/cli/scripts/test @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +cd "$ROOT" +node --no-warnings --test tests/*.test.js diff --git a/dsp/runtime/cli/scripts/verify b/dsp/runtime/cli/scripts/verify new file mode 100755 index 0000000..f4b4165 --- /dev/null +++ b/dsp/runtime/cli/scripts/verify @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +node --no-warnings - "$ROOT" <<'NODE' +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const root = process.argv[2]; +const source = path.join(root, 'src'); +const files = []; +function walk(directory) { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const target = path.join(directory, entry.name); + if (entry.isDirectory()) walk(target); + else if (entry.isFile() && target.endsWith('.js')) files.push(target); + } +} +walk(source); +const errors = []; +const forbiddenBuiltins = new Set(['node:fs', 'node:sqlite', 'node:net', 'node:child_process']); +for (const file of files) { + const text = fs.readFileSync(file, 'utf8'); + for (const match of text.matchAll(/require\((['"])([^'"]+)\1\)/g)) { + const value = match[2]; + if (forbiddenBuiltins.has(value)) errors.push(`${path.relative(root, file)} imports ${value}`); + if (value.includes('/auth-broker/') || value.includes('/collection-manager/') || value.includes('/compatibility/providers/')) { + errors.push(`${path.relative(root, file)} imports component internals: ${value}`); + } + if (value.includes('/application/')) errors.push(`${path.relative(root, file)} imports application internals: ${value}`); + if (value.includes('/adapters/local/')) errors.push(`${path.relative(root, file)} imports a local adapter directly: ${value}`); + if (value.includes('/contracts/')) errors.push(`${path.relative(root, file)} bypasses the public SDK contract export: ${value}`); + } +} +const main = fs.readFileSync(path.join(source, 'main.js'), 'utf8'); +if (!main.includes('client.workflows.authSetup.prepare(') || !main.includes('client.workflows.authSetup.run(')) { + errors.push('main.js does not route Auth setup through the public workflow client'); +} +if (errors.length) { + process.stderr.write(`${JSON.stringify({ ok: false, status: 'architecture_failed', errors })}\n`); + process.exit(1); +} +process.stdout.write(`${JSON.stringify({ ok: true, status: 'architecture_verified', filesChecked: files.length })}\n`); +NODE diff --git a/dsp/runtime/cli/src/interactions/line.js b/dsp/runtime/cli/src/interactions/line.js new file mode 100644 index 0000000..1357e97 --- /dev/null +++ b/dsp/runtime/cli/src/interactions/line.js @@ -0,0 +1,89 @@ +'use strict'; + +const readline = require('node:readline/promises'); + +function cancelled() { throw Object.assign(new Error('cancelled'), { code: 'cancelled' }); } +function validText(value) { return typeof value === 'string' && value.length > 0 && value.length <= 160 && !/[\r\n\0]/.test(value); } + +class LineInteraction { + #input; + #output; + #signal; + #rl = null; + + constructor({ input = process.stdin, output = process.stdout, signal = null } = {}) { + this.#input = input; + this.#output = output; + this.#signal = signal; + } + + available() { + return this.#input?.isTTY === true && this.#output?.isTTY === true; + } + + #interface() { + if (!this.available()) throw Object.assign(new Error('interaction_unavailable'), { code: 'interaction_unavailable' }); + if (!this.#rl) this.#rl = readline.createInterface({ input: this.#input, output: this.#output, terminal: false }); + return this.#rl; + } + + write(text = '') { + if (typeof text !== 'string' || text.length > 4096 || text.includes('\0')) throw new TypeError('invalid_interaction_text'); + this.#output.write(`${text}\n`); + } + + async select({ message, options, defaultValue }) { + if (!validText(message) || !Array.isArray(options) || options.length < 1 || options.length > 12 + || options.some(option => !option || !validText(option.label) || !validText(option.value)) + || new Set(options.map(option => option.value)).size !== options.length + || !options.some(option => option.value === defaultValue)) throw new TypeError('invalid_interaction_menu'); + + const rl = this.#interface(); + this.write(`? ${message}`); + options.forEach((option, index) => this.write(` ${index + 1}. ${option.label}${option.value === defaultValue ? ' (default)' : ''}`)); + while (true) { + if (this.#signal?.aborted) cancelled(); + let answer; + let onClose; + const closed = new Promise((unused, reject) => { + onClose = () => reject(Object.assign(new Error('cancelled'), { code: 'cancelled' })); + rl.once('close', onClose); + }); + try { + answer = (await Promise.race([ + rl.question(`Select [1-${options.length}, q to cancel]: `, { signal: this.#signal || undefined }), + closed, + ])).trim(); + } + catch (error) { + if (this.#signal?.aborted || error?.name === 'AbortError' || error?.code === 'cancelled' + || error?.code === 'ERR_USE_AFTER_CLOSE') cancelled(); + throw error; + } finally { rl.removeListener('close', onClose); } + if (answer === '') return defaultValue; + if (['q', 'quit', 'cancel'].includes(answer.toLowerCase())) cancelled(); + const index = Number(answer); + if (Number.isInteger(index) && index >= 1 && index <= options.length) return options[index - 1].value; + this.write(` Enter a number from 1 to ${options.length}, or q to cancel.`); + } + } + + async confirm({ message, defaultValue = false }) { + if (!validText(message) || typeof defaultValue !== 'boolean') throw new TypeError('invalid_interaction_confirmation'); + return (await this.select({ + message, + options: [ + { value: 'yes', label: 'Continue' }, + { value: 'no', label: 'Cancel' }, + ], + defaultValue: defaultValue ? 'yes' : 'no', + })) === 'yes'; + } + + close() { + if (this.#rl) this.#rl.close(); + this.#rl = null; + } +} + +module.exports = { LineInteraction }; diff --git a/dsp/runtime/cli/src/interactions/setup-auth.js b/dsp/runtime/cli/src/interactions/setup-auth.js new file mode 100644 index 0000000..eac732d --- /dev/null +++ b/dsp/runtime/cli/src/interactions/setup-auth.js @@ -0,0 +1,114 @@ +'use strict'; + +const ACTION_LABELS = Object.freeze({ + keep: 'Keep existing credentials', + enroll: 'Enroll credentials', + replace: 'Replace stored credentials', + remove: 'Delete stored credentials', +}); +const PROVIDER_LABELS = Object.freeze({ paycom: 'Paycom', 'amazon-logistics': 'Amazon Logistics' }); + +function shouldInteract(command, interaction) { + return command.format !== 'json' && command.nonInteractive !== true + && interaction && typeof interaction.available === 'function' && interaction.available(); +} + +function commandInput(command, preparation) { + const { target, defaults } = preparation; + return { + provider: target.provider, + profile: target.profile, + credentialAction: command.removeSpecified ? 'remove' : command.replaceSpecified ? 'replace' : defaults.credentialAction, + startBroker: defaults.startBroker, + testAuthentication: command.removeSpecified ? false + : command.testAuthenticationSpecified ? true : defaults.testAuthentication, + }; +} + +function renderState(preparation) { + const { target, state } = preparation; + return [ + 'DISPATCH — Authentication Setup', + '', + 'Current state', + '', + ` Provider ${target.provider}`, + ` Profile ${target.profile}`, + ` Credential profile ${state.profile.status.replaceAll('_', ' ')}`, + ` Vault ${state.vault.status}${state.vault.verified ? ' / verified' : ''}`, + ` Auth Broker ${state.broker.status}${state.broker.managed ? ' / managed' : ''}`, + ].join('\n'); +} + +function renderPlan(input) { + return [ + '', + 'Setup plan', + '', + ` Provider ${input.provider}`, + ` Profile ${input.profile}`, + ` Credential action ${ACTION_LABELS[input.credentialAction].toLowerCase()}`, + ` Start Auth Broker ${input.startBroker ? 'yes' : 'no'}`, + ` Test authentication ${input.testAuthentication ? 'yes' : 'no'}`, + '', + ...(input.credentialAction === 'remove' ? [ + 'This permanently deletes the encrypted profile and clears its authentication-attempt state.', + 'Any active session is revoked when the managed Auth Broker stops.', + ] : [ + 'Credential values are entered only in the Auth Broker protected terminal helper.', + 'Dispatch does not receive those values.', + ]), + ].join('\n'); +} + +async function collectSetupAuthInput(command, interaction, preparation) { + const input = commandInput(command, preparation); + if (!shouldInteract(command, interaction)) { + if (input.credentialAction === 'remove' && !command.confirmed) return { cancelled: true, input, interactive: false }; + return { cancelled: false, input, interactive: false }; + } + + interaction.write(renderState(preparation)); + interaction.write(''); + + if (!command.replaceSpecified && !command.removeSpecified) { + const available = preparation.capabilities.credentialActions.filter(item => item.available); + if (available.length === 0) throw Object.assign(new Error('setup_action_unavailable'), { code: 'setup_action_unavailable' }); + if (available.some(item => !Object.hasOwn(ACTION_LABELS, item.id))) { + throw Object.assign(new Error('setup_action_unavailable'), { code: 'setup_action_unavailable' }); + } + if (available.length === 1) input.credentialAction = available[0].id; + else { + input.credentialAction = await interaction.select({ + message: 'How should Dispatch handle the credential profile?', + options: available.map(item => ({ value: item.id, label: ACTION_LABELS[item.id] })), + defaultValue: available.some(item => item.id === preparation.defaults.credentialAction) + ? preparation.defaults.credentialAction : available[0].id, + }); + } + } + + if (input.credentialAction !== 'remove' && !command.testAuthenticationSpecified && preparation.capabilities.authenticationTest.available) { + input.testAuthentication = (await interaction.select({ + message: `Test ${PROVIDER_LABELS[input.provider] || input.provider} authentication after setup?`, + options: [ + { value: 'skip', label: 'Skip authentication test' }, + { value: 'test', label: 'Test authentication' }, + ], + defaultValue: preparation.defaults.testAuthentication ? 'test' : 'skip', + })) === 'test'; + } + + interaction.write(renderPlan(input)); + const confirmationMessage = input.credentialAction === 'remove' + ? `Permanently delete authentication profile ${input.profile}?` : 'Continue with this setup plan?'; + if (!(await interaction.confirm({ message: confirmationMessage, defaultValue: input.credentialAction !== 'remove' }))) { + return { cancelled: true, input, interactive: true }; + } + interaction.write(''); + return { cancelled: false, input, interactive: true }; +} + +module.exports = { + ACTION_LABELS, PROVIDER_LABELS, collectSetupAuthInput, commandInput, renderPlan, renderState, shouldInteract, +}; diff --git a/dsp/runtime/cli/src/main.js b/dsp/runtime/cli/src/main.js new file mode 100644 index 0000000..573a2c4 --- /dev/null +++ b/dsp/runtime/cli/src/main.js @@ -0,0 +1,148 @@ +'use strict'; + +const { contracts: { failure } } = require('../../sdk/src'); +const { parse } = require('./parse'); +const plain = require('./renderers/plain'); +const json = require('./renderers/json'); +const { LineInteraction } = require('./interactions/line'); +const { collectSetupAuthInput } = require('./interactions/setup-auth'); + +async function main(argv = process.argv.slice(2), { + client = null, + write = chunk => process.stdout.write(chunk), + writeError = chunk => process.stderr.write(chunk), + signal = null, + interaction = null, +} = {}) { + process.umask(0o077); + let command; + try { + command = parse(argv); + } catch { + writeError(`${plain.renderHelp()}\n`); + write(`${json.render(failure('invalid_input'))}\n`); + return 2; + } + if (command.command === 'help') { + write(`${plain.renderHelp()}\n`); + return 0; + } + + let result; + if (command.command === 'setup-auth') { + try { result = await client.workflows.authSetup.prepare({ provider: command.provider, profile: command.profile }); } + catch { result = failure('internal_error'); } + if (!result.ok) { + write(`${command.format === 'json' ? json.render(result) : plain.renderSetupResult(result)}\n`); + return 1; + } + const preparation = result.data; + const menu = interaction || new LineInteraction({ signal }); + let choices; + try { choices = await collectSetupAuthInput(command, menu, preparation); } + catch (error) { + choices = { cancelled: error?.code === 'cancelled', failed: error?.code !== 'cancelled' }; + } finally { + if (typeof menu.close === 'function') menu.close(); + } + if (choices.cancelled || choices.failed) { + result = failure(choices.cancelled ? 'cancelled' : 'internal_error', { + recoverable: choices.cancelled, + data: { + workflow: 'setup_auth', profile: preparation.target.profile, provider: preparation.target.provider, + nextActions: choices.cancelled ? ['retry_setup'] : [], + }, + }); + write(`${command.format === 'json' ? json.render(result) : plain.renderSetupResult(result)}\n`); + return result.ok ? 0 : 1; + } + const events = { + emit(value) { + if (command.format !== 'plain') return; + const output = plain.renderSetupEvent(value, { provider: preparation.target.provider }); + if (output) write(`${output}\n`); + }, + }; + try { + result = await client.workflows.authSetup.run(choices.input, { events, signal }); + } catch { result = failure('internal_error'); } + write(`${command.format === 'json' ? json.render(result) : plain.renderSetupResult(result)}\n`); + return result.ok ? 0 : 1; + } + + if (command.command.startsWith('collect-')) { + const collections = client?.collections; + try { + const request = command.source ? { source: command.source, scope: command.scope, selector: command.selector, mode: command.mode } : null; + if (command.command === 'collect-describe') result = await collections.describe(command.source); + else if (command.command === 'collect-preview') result = await collections.preview(request); + else if (command.command === 'collect-enqueue') result = await collections.enqueue(request, { + ...(command.idempotencyKey ? { idempotencyKey: command.idempotencyKey } : {}), + ...(command.expectedPreviewHash ? { expectedPreviewHash: command.expectedPreviewHash } : {}), + }); + else if (command.command === 'collect-audit') result = await collections.audit(request, { + ...(command.idempotencyKey ? { idempotencyKey: command.idempotencyKey } : {}), + ...(command.expectedPreviewHash ? { expectedPreviewHash: command.expectedPreviewHash } : {}), + }); + else if (command.command === 'collect-batches') result = await collections.batches(); + else if (command.command === 'collect-batch') result = await collections.batchStatus(command.batchId); + else if (command.command === 'collect-cancel') result = await collections.cancelBatch(command.batchId); + else if (command.command === 'collect-retry') result = await collections.retryBatch(command.batchId); + else if (command.command === 'collect-schedules') result = await collections.schedules(); + else if (command.command === 'collect-schedule-create') result = await collections.createSchedule({ + id: command.scheduleId, request, schedule: command.schedule, enabled: true, + }); + else if (command.command === 'collect-schedule-pause') result = await collections.pauseSchedule(command.scheduleId); + else if (command.command === 'collect-schedule-resume') result = await collections.resumeSchedule(command.scheduleId); + else if (command.command === 'collect-schedule-remove') result = await collections.removeSchedule(command.scheduleId); + else if (command.command === 'collect-schedule-run') result = await collections.runScheduleNow(command.scheduleId); + else result = failure('invalid_input'); + } catch { result = failure('internal_error'); } + write(`${command.format === 'json' ? json.render(result) : plain.renderCollection(result)}\n`); + return result.ok ? 0 : 1; + } + + if (command.command.startsWith('sync-')) { + const sync = client?.sync; + try { + if (command.command === 'sync-list') result = await sync.list(); + else if (command.command === 'sync-status') result = await sync.status(command.syncId); + else if (command.command === 'sync-start') result = await sync.start(command.syncId); + else if (command.command === 'sync-stop') result = await sync.stop(command.syncId, { drain: command.drain }); + else if (command.command === 'sync-restart') result = await sync.restart(command.syncId, { drain: command.drain }); + else if (command.command === 'sync-run') result = await sync.runNow(command.syncId); + else if (command.command === 'sync-edit') result = await sync.edit(command.syncId, command.patch, { + ...(command.expectedRevision === undefined ? {} : { expectedRevision: command.expectedRevision }), + applyNow: command.applyNow, + }); + else if (command.command === 'sync-history') result = await sync.history(command.syncId, { + limit: command.limit, offset: command.offset, + }); + else result = failure('invalid_input'); + } catch { result = failure('internal_error'); } + write(`${command.format === 'json' ? json.render(result) : plain.renderSync(result)}\n`); + return result.ok ? 0 : 1; + } + + if (command.command.startsWith('workforce-')) { + const workforce = client?.workforce; + try { + if (command.command === 'workforce-status') result = await workforce.snapshot(); + else if (command.command === 'workforce-employees') result = await workforce.employees(command.query); + else if (command.command === 'workforce-employee') result = await workforce.employee(command.employeeCode); + else if (command.command === 'workforce-timecards') result = await workforce.timecards(command.query); + else if (command.command === 'workforce-punches') result = await workforce.punches(command.query); + else if (command.command === 'workforce-links') result = await workforce.resourceLinks(command.query); + else result = failure('invalid_input'); + } catch { result = failure('internal_error'); } + write(`${command.format === 'json' ? json.render(result) : plain.renderWorkforce(result)}\n`); + return result.ok ? 0 : 1; + } + + try { result = await client.system.status(); } + catch { result = failure('internal_error'); } + write(`${command.format === 'json' ? json.render(result) : plain.renderStatus(result)}\n`); + return result.ok && result.status !== 'failed' ? 0 : 1; +} + +module.exports = { main }; diff --git a/dsp/runtime/cli/src/parse.js b/dsp/runtime/cli/src/parse.js new file mode 100644 index 0000000..f83095d --- /dev/null +++ b/dsp/runtime/cli/src/parse.js @@ -0,0 +1,303 @@ +'use strict'; + +function invalid() { throw Object.assign(new Error('invalid_input'), { code: 'invalid_input' }); } + +function parseOptions(options, allowed) { + if (new Set(options).size !== options.length || options.some(value => !allowed.includes(value)) + || options.includes('--json') && options.includes('--plain')) invalid(); + return { format: options.includes('--json') ? 'json' : 'plain' }; +} + +function flags(values, booleanNames, valueNames) { + const result = {}; + for (let index = 0; index < values.length; index += 1) { + const name = values[index]; + if (Object.hasOwn(result, name) || !booleanNames.includes(name) && !valueNames.includes(name)) invalid(); + if (booleanNames.includes(name)) result[name] = true; + else { + const value = values[++index]; + if (value === undefined || value.startsWith('--')) invalid(); + result[name] = value; + } + } + if (result['--json'] && result['--plain']) invalid(); + return result; +} + +function selectorFrom(options) { + const candidates = [ + options['--current'] && { kind: 'current' }, + options['--latest-complete'] && { kind: 'latest-complete' }, + options['--today'] && { kind: 'relative-date', value: 'today' }, + options['--yesterday'] && { kind: 'relative-date', value: 'yesterday' }, + options['--date'] && { kind: 'date', date: options['--date'] }, + options['--from'] && options['--through'] && { kind: 'date-range', start: options['--from'], end: options['--through'] }, + options['--last-days'] && { kind: 'last-duration', value: Number(options['--last-days']), unit: 'days' }, + options['--target'] && { kind: 'exact-target', key: options['--target'] }, + options['--from-target'] && options['--through-target'] && { + kind: 'target-range', startKey: options['--from-target'], endKey: options['--through-target'], + }, + ].filter(Boolean); + if (candidates.length !== 1 || Boolean(options['--from']) !== Boolean(options['--through']) + || Boolean(options['--from-target']) !== Boolean(options['--through-target'])) invalid(); + return candidates[0]; +} + +const SELECTOR_BOOLEANS = ['--current', '--latest-complete', '--today', '--yesterday']; +const SELECTOR_VALUES = ['--date', '--from', '--through', '--last-days', '--target', '--from-target', '--through-target']; + +function parseCollection(argv) { + const [action, ...args] = argv; + if (action === 'describe' && args.length >= 1) { + const options = flags(args.slice(1), ['--json', '--plain'], []); + return { command: 'collect-describe', source: args[0], format: options['--json'] ? 'json' : 'plain' }; + } + if (action === 'target' && args.length >= 3) { + const options = flags(args.slice(3), ['--json', '--plain'], ['--mode', '--idempotency', '--preview-hash']); + return { + command: 'collect-enqueue', source: args[0], scope: args[1], selector: { kind: 'exact-target', key: args[2] }, + mode: options['--mode'] || 'ensure', idempotencyKey: options['--idempotency'], expectedPreviewHash: options['--preview-hash'], + format: options['--json'] ? 'json' : 'plain', + }; + } + if (action === 'backfill' && args.length >= 4) { + const options = flags(args.slice(4), ['--json', '--plain'], ['--mode', '--idempotency', '--preview-hash']); + return { + command: 'collect-enqueue', source: args[0], scope: args[1], + selector: { kind: 'target-range', startKey: args[2], endKey: args[3] }, + mode: options['--mode'] || 'ensure', idempotencyKey: options['--idempotency'], expectedPreviewHash: options['--preview-hash'], + format: options['--json'] ? 'json' : 'plain', + }; + } + if (['preview', 'enqueue', 'audit'].includes(action) && args.length >= 2) { + const options = flags(args.slice(2), [...SELECTOR_BOOLEANS, '--json', '--plain'], [...SELECTOR_VALUES, '--mode', '--idempotency', '--preview-hash']); + if (options['--json'] && options['--plain'] || action === 'audit' && options['--mode']) invalid(); + return { + command: `collect-${action}`, source: args[0], scope: args[1], selector: selectorFrom(options), + mode: action === 'audit' ? 'verify' : options['--mode'] || 'ensure', idempotencyKey: options['--idempotency'], expectedPreviewHash: options['--preview-hash'], + format: options['--json'] ? 'json' : 'plain', + }; + } + if (action === 'batch' && args.length >= 1) { + const options = flags(args.slice(1), ['--json', '--plain'], []); + return { command: 'collect-batch', batchId: args[0], format: options['--json'] ? 'json' : 'plain' }; + } + if (action === 'batches') { + const options = flags(args, ['--json', '--plain'], []); + return { command: 'collect-batches', format: options['--json'] ? 'json' : 'plain' }; + } + if (action === 'cancel' && args.length >= 1) { + const options = flags(args.slice(1), ['--json', '--plain'], []); + return { command: 'collect-cancel', batchId: args[0], format: options['--json'] ? 'json' : 'plain' }; + } + if (action === 'retry' && args.length >= 1) { + const options = flags(args.slice(1), ['--json', '--plain'], []); + return { command: 'collect-retry', batchId: args[0], format: options['--json'] ? 'json' : 'plain' }; + } + if (action === 'schedules') { + const options = flags(args, ['--json', '--plain'], []); + return { command: 'collect-schedules', format: options['--json'] ? 'json' : 'plain' }; + } + if (action === 'schedule' && ['pause', 'resume', 'remove', 'run'].includes(args[0]) && args.length >= 2) { + const options = flags(args.slice(2), ['--json', '--plain'], []); + return { command: `collect-schedule-${args[0]}`, scheduleId: args[1], format: options['--json'] ? 'json' : 'plain' }; + } + if (action === 'schedule' && args[0] === 'create' && args.length >= 4) { + const options = flags(args.slice(4), [...SELECTOR_BOOLEANS, '--json', '--plain'], [...SELECTOR_VALUES, '--mode', '--interval', '--cron', '--timezone']); + if (options['--json'] && options['--plain'] || Boolean(options['--interval']) === Boolean(options['--cron'])) invalid(); + const schedule = options['--interval'] + ? { type: 'interval', seconds: Number(options['--interval']) } + : { type: 'cron', expression: options['--cron'], timezone: options['--timezone'] }; + if (schedule.type === 'cron' && !schedule.timezone) invalid(); + return { + command: 'collect-schedule-create', scheduleId: args[1], source: args[2], scope: args[3], selector: selectorFrom(options), + mode: options['--mode'] || 'ensure', schedule, format: options['--json'] ? 'json' : 'plain', + }; + } + invalid(); +} + +function parseSyncSetting(value) { + if (typeof value !== 'string') invalid(); + const index = value.indexOf('='); + if (index < 1) invalid(); + const key = value.slice(0, index); + const raw = value.slice(index + 1); + if (!/^[A-Za-z][A-Za-z0-9_-]{0,63}$/.test(key) || raw.length === 0) invalid(); + let parsed = raw; + try { parsed = JSON.parse(raw); } catch {} + if (!['string', 'number', 'boolean'].includes(typeof parsed) || typeof parsed === 'number' && !Number.isFinite(parsed)) invalid(); + return [key, parsed]; +} + +function parseSync(argv) { + const [action, ...args] = argv; + if (action === 'list') { + const options = flags(args, ['--json', '--plain'], []); + return { command: 'sync-list', format: options['--json'] ? 'json' : 'plain' }; + } + if (action === 'history' && args.length >= 1) { + const options = flags(args.slice(1), ['--json', '--plain'], ['--limit', '--offset']); + const limit = options['--limit'] === undefined ? 50 : Number(options['--limit']); + const offset = options['--offset'] === undefined ? 0 : Number(options['--offset']); + if (!Number.isInteger(limit) || limit < 1 || limit > 100 || !Number.isInteger(offset) || offset < 0) invalid(); + return { + command: 'sync-history', syncId: args[0], limit, offset, + format: options['--json'] ? 'json' : 'plain', + }; + } + if (['status', 'start', 'stop', 'restart', 'run'].includes(action) && args.length >= 1) { + const options = flags(args.slice(1), ['--json', '--plain', '--drain'], []); + if (options['--drain'] && !['stop', 'restart'].includes(action)) invalid(); + return { + command: `sync-${action}`, syncId: args[0], drain: Boolean(options['--drain']), + format: options['--json'] ? 'json' : 'plain', + }; + } + if (action === 'edit' && args.length >= 1) { + const syncId = args[0]; + const values = args.slice(1); + const patch = {}; + const settings = {}; + let expectedRevision; + let applyNow = false; + let replaceSettings = false; + let format = 'plain'; + const seenFlags = new Set(); + for (let index = 0; index < values.length; index += 1) { + const name = values[index]; + if (name !== '--set' && seenFlags.has(name)) invalid(); + seenFlags.add(name); + if (name === '--json' || name === '--plain' || name === '--apply-now' || name === '--replace-settings') { + if (name === '--json') format = 'json'; + else if (name === '--plain') format = 'plain'; + else if (name === '--apply-now') applyNow = true; + else { + if (replaceSettings) invalid(); + replaceSettings = true; + } + continue; + } + const value = values[++index]; + if (value === undefined || value.startsWith('--')) invalid(); + if (name === '--interval') patch.intervalSeconds = Number(value); + else if (name === '--jitter') patch.jitterSeconds = Number(value); + else if (name === '--revision') expectedRevision = Number(value); + else if (name === '--set') { + const [key, setting] = parseSyncSetting(value); + if (Object.hasOwn(settings, key)) invalid(); + settings[key] = setting; + } else invalid(); + } + if (values.includes('--json') && values.includes('--plain')) invalid(); + if (replaceSettings && Object.keys(settings).length === 0) invalid(); + if (Object.keys(settings).length) patch.settings = settings; + if (replaceSettings) patch.replaceSettings = true; + if (Object.keys(patch).length === 0) invalid(); + return { command: 'sync-edit', syncId, patch, expectedRevision, applyNow, format }; + } + invalid(); +} + +function parseWorkforceQuery(values) { + const options = flags(values, ['--json', '--plain'], ['--lifecycle', '--limit', '--offset']); + const limit = options['--limit'] === undefined ? 50 : Number(options['--limit']); + const offset = options['--offset'] === undefined ? 0 : Number(options['--offset']); + const lifecycleStatus = options['--lifecycle']; + if (!Number.isInteger(limit) || limit < 1 || limit > 100 || !Number.isInteger(offset) || offset < 0 + || lifecycleStatus !== undefined && !['active', 'inactive', 'unknown'].includes(lifecycleStatus)) invalid(); + return { + query: { limit, offset, ...(lifecycleStatus === undefined ? {} : { lifecycleStatus }) }, + format: options['--json'] ? 'json' : 'plain', + }; +} + +function parseWorkforcePunches(values) { + const options = flags(values, ['--json', '--plain'], [ + '--date', '--kind', '--from-time', '--through-time', '--lifecycle', '--limit', '--offset', + ]); + const date = options['--date']; + const kind = options['--kind']; + const fromTime = options['--from-time']; + const throughTime = options['--through-time']; + const lifecycleStatus = options['--lifecycle']; + const limit = options['--limit'] === undefined ? 50 : Number(options['--limit']); + const offset = options['--offset'] === undefined ? 0 : Number(options['--offset']); + if (typeof date !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(date) + || kind !== undefined && !['in_day', 'out_lunch', 'in_lunch', 'out_day', 'unclassified'].includes(kind) + || fromTime !== undefined && !/^(?:[01]\d|2[0-3]):[0-5]\d$/.test(fromTime) + || throughTime !== undefined && !/^(?:[01]\d|2[0-3]):[0-5]\d$/.test(throughTime) + || fromTime !== undefined && throughTime !== undefined && fromTime > throughTime + || lifecycleStatus !== undefined && !['active', 'inactive', 'unknown'].includes(lifecycleStatus) + || !Number.isInteger(limit) || limit < 1 || limit > 100 || !Number.isInteger(offset) || offset < 0) invalid(); + return { + command: 'workforce-punches', + query: { + date, limit, offset, + ...(kind === undefined ? {} : { kind }), + ...(fromTime === undefined ? {} : { fromTime }), + ...(throughTime === undefined ? {} : { throughTime }), + ...(lifecycleStatus === undefined ? {} : { lifecycleStatus }), + }, + format: options['--json'] ? 'json' : 'plain', + }; +} + +function parseWorkforce(argv) { + const [action, ...args] = argv; + if (action === 'status') { + const options = flags(args, ['--json', '--plain'], []); + return { command: 'workforce-status', format: options['--json'] ? 'json' : 'plain' }; + } + if (action === 'punches') return parseWorkforcePunches(args); + if (['employees', 'timecards', 'links'].includes(action)) { + return { command: `workforce-${action}`, ...parseWorkforceQuery(args) }; + } + if (action === 'employee' && args.length >= 1) { + const options = flags(args.slice(1), ['--json', '--plain'], []); + if (!/^[A-Za-z0-9]{4}$/.test(args[0])) invalid(); + return { command: 'workforce-employee', employeeCode: args[0].toUpperCase(), format: options['--json'] ? 'json' : 'plain' }; + } + invalid(); +} + +function parse(argv) { + if (!Array.isArray(argv)) invalid(); + if (argv.length === 0 || (argv.length === 1 && ['help', '--help', '-h'].includes(argv[0]))) return { command: 'help', format: 'plain' }; + if (argv[0] === 'status') { + const parsed = parseOptions(argv.slice(1), ['--json', '--plain']); + return { command: 'status', ...parsed }; + } + if (argv[0] === 'setup' && argv[1] === 'auth') { + const options = argv.slice(2); + const values = flags(options, + ['--json', '--plain', '--replace', '--remove', '--confirm', '--test-auth', '--no-menu'], + ['--provider', '--profile']); + const format = values['--json'] ? 'json' : 'plain'; + const replaceSpecified = Boolean(values['--replace']); + const removeSpecified = Boolean(values['--remove']); + const confirmed = Boolean(values['--confirm']); + const testAuthentication = Boolean(values['--test-auth']); + const nonInteractive = Boolean(values['--no-menu'] || values['--plain'] || values['--json']); + if (replaceSpecified && removeSpecified || confirmed && !removeSpecified || removeSpecified && testAuthentication + || removeSpecified && nonInteractive && !confirmed) invalid(); + return { + command: 'setup-auth', format, provider: values['--provider'] || 'paycom', profile: values['--profile'] || 'paycom-main', + replaceExisting: replaceSpecified, + replaceSpecified, + removeSpecified, + confirmed, + testAuthentication, + testAuthenticationSpecified: Boolean(values['--test-auth']), + nonInteractive, + }; + } + if (argv[0] === 'collect') return parseCollection(argv.slice(1)); + if (argv[0] === 'sync') return parseSync(argv.slice(1)); + if (argv[0] === 'workforce') return parseWorkforce(argv.slice(1)); + invalid(); +} + +module.exports = { + parse, parseCollection, parseSync, parseWorkforce, parseWorkforceQuery, parseWorkforcePunches, parseSyncSetting, selectorFrom, +}; diff --git a/dsp/runtime/cli/src/renderers/json.js b/dsp/runtime/cli/src/renderers/json.js new file mode 100644 index 0000000..dea5f24 --- /dev/null +++ b/dsp/runtime/cli/src/renderers/json.js @@ -0,0 +1,5 @@ +'use strict'; + +function render(value) { return JSON.stringify(value); } + +module.exports = { render }; diff --git a/dsp/runtime/cli/src/renderers/plain.js b/dsp/runtime/cli/src/renderers/plain.js new file mode 100644 index 0000000..b849a95 --- /dev/null +++ b/dsp/runtime/cli/src/renderers/plain.js @@ -0,0 +1,257 @@ +'use strict'; + +const LABELS = Object.freeze({ auth: 'Auth Broker', collections: 'Collection Manager', paycom: 'Paycom data' }); +const SYMBOLS = Object.freeze({ ready: '✓', stopped: '○', not_initialized: '○', degraded: '!', failed: '×' }); +const STEP_LABELS = Object.freeze({ + preflight: 'Checking Auth Broker and vault state', + initialize_vault: 'Initializing encrypted Auth Broker vault', + capture_profile: 'Opening protected credential entry', + remove_profile: 'Deleting encrypted authentication profile and attempt state', + verify_profile: 'Verifying stored profile metadata', + stop_broker: 'Stopping the verified Dispatch-managed Auth Broker', + start_broker: 'Starting and verifying the Auth Broker', +}); +const PROVIDER_LABELS = Object.freeze({ paycom: 'Paycom', 'amazon-logistics': 'Amazon Logistics' }); + +function renderHelp() { + return [ + 'Dispatch', + '', + 'Usage:', + ' dispatch status [--plain|--json]', + ' dispatch setup auth [--provider paycom|amazon-logistics] [--profile ] [--replace] [--test-auth] [--no-menu] [--plain|--json]', + ' dispatch setup auth --provider --profile --remove [--confirm --no-menu|--plain|--json]', + ' dispatch collect describe [--json]', + ' dispatch collect preview [--mode ensure|refresh|verify] [--json]', + ' dispatch collect enqueue [--mode ensure|refresh|verify] [--idempotency ] [--preview-hash ] [--json]', + ' dispatch collect target [--mode ensure|refresh] [--idempotency ] [--json]', + ' dispatch collect backfill [--mode ensure|refresh] [--idempotency ] [--json]', + ' dispatch collect audit [--idempotency ] [--json]', + ' dispatch collect batches|batch |cancel |retry [--json]', + ' dispatch collect schedules [--json]', + ' dispatch collect schedule create (--interval |--cron --timezone ) [--json]', + ' dispatch collect schedule pause|resume|remove|run [--json]', + ' dispatch sync list [--json]', + ' dispatch sync status|start|stop|restart|run [--json]', + ' dispatch sync stop|restart [--drain] [--json]', + ' dispatch sync edit [--interval ] [--jitter ] [--set ] [--replace-settings] [--revision ] [--apply-now] [--json]', + ' dispatch sync history [--limit <1-100>] [--offset ] [--json]', + ' dispatch workforce status [--json]', + ' dispatch workforce employees|timecards|links [--lifecycle active|inactive|unknown] [--limit ] [--offset ] [--json]', + ' dispatch workforce punches --date YYYY-MM-DD [--kind in_day|out_lunch|in_lunch|out_day|unclassified] [--from-time HH:MM] [--through-time HH:MM] [--lifecycle active|inactive|unknown] [--limit ] [--offset ] [--json]', + ' dispatch workforce employee [--json]', + ' dispatch help', + '', + 'Interactive terminals receive a guided non-secret setup menu.', + 'Use --plain or --no-menu to bypass menus; --json remains machine-readable.', + 'Credential values are accepted only by the Auth Broker protected terminal helper.', + 'Selectors: --current, --latest-complete, --today, --yesterday, --date YYYY-MM-DD,', + ' --from YYYY-MM-DD --through YYYY-MM-DD, --last-days N, --target .', + ].join('\n'); +} + +function renderStatus(result) { + if (!result?.ok) return `× Dispatch status unavailable (${result?.status || 'internal_error'})`; + const lines = [ + '◆ DISPATCH', + '', + ` System status ${result.status.toUpperCase()}`, + '', + ]; + for (const [id, value] of Object.entries(result.data.components)) { + lines.push(` ${SYMBOLS[value.status] || '•'} ${LABELS[id] || id}`.padEnd(25) + value.status); + } + const collections = result.data.components.collections.data; + if (collections?.counts) { + lines.push('', ` Registry ${collections.counts.collectors} collector · ${collections.counts.sources} source · ${collections.counts.plans} plans`); + lines.push(` Runs ${collections.counts.queued} queued · ${collections.counts.running} running · ${collections.counts.failed} failed`); + if (collections.syncAlerts?.total) { + lines.push(` Sync alerts ${collections.syncAlerts.total} active · ${collections.syncAlerts.critical} critical`); + lines.push(...collections.syncAlerts.items.map(alert => ` ${alert.severity.padEnd(8)} ${alert.syncId}: ${alert.code}`)); + if (collections.syncAlerts.hasMore) lines.push(' … additional sync alerts omitted'); + } + } + const paycom = result.data.components.paycom.data; + if (paycom) { + lines.push('', ` Pay periods ${paycom.payPeriods?.verified ? 'verified' : paycom.payPeriods?.code || 'unknown'}`); + lines.push(` Roster ${paycom.roster?.verified ? 'verified' : paycom.roster?.code || 'unknown'}`); + lines.push(` Timecards ${paycom.timecards?.verified ? 'verified' : paycom.timecards?.code || 'unknown'}`); + lines.push(` Resource links ${paycom.resourceLinks?.verified ? 'verified' : paycom.resourceLinks?.code || 'unknown'}`); + } + return lines.join('\n'); +} + +function renderSetupEvent(value, { provider = null } = {}) { + if (value.type === 'workflow_started') return '◆ DISPATCH / AUTH SETUP'; + if (value.type === 'step_started') { + const label = value.data.step === 'test_authentication' + ? `Testing ${PROVIDER_LABELS[provider] || 'provider'} authentication inside the Auth Broker` + : STEP_LABELS[value.data.step] || value.data.step; + return `\n › ${label}`; + } + if (value.type === 'check_completed') return ` ✓ ${value.data.check.replaceAll('_', ' ')}: ${value.data.status}`; + if (value.type === 'credential_capture_started') return ' ◇ Enter values in the protected terminal prompt; Dispatch will not receive them.'; + if (value.type === 'credential_capture_completed') return ' ✓ Encrypted profile stored'; + return null; +} + +function renderSetupResult(result) { + if (!result.ok) { + const action = result.data?.nextActions?.[0]; + return [`\n× Authentication setup stopped: ${result.status}`, ...(action ? [` Next action: ${action.replaceAll('_', ' ')}`] : [])].join('\n'); + } + if (result.data.configured === false) { + return [ + '', + '✓ Authentication profile deleted', + ` Profile ${result.data.profile}`, + ` Provider ${result.data.provider}`, + ` Vault ${result.data.vault}`, + ` Auth Broker ${result.data.broker}`, + ].join('\n'); + } + return [ + '', + '✓ Authentication setup complete', + ` Profile ${result.data.profile}`, + ` Provider ${result.data.provider}`, + ` Vault ${result.data.vault}`, + ` Auth Broker ${result.data.broker}`, + ` Authentication test ${result.data.authenticationTest}`, + ].join('\n'); +} + +function renderCollection(result) { + if (!result?.ok) return `× Collection operation failed (${result?.status || 'internal_error'})`; + const data = result.data; + if (result.status === 'previewed') { + return [ + '◆ DISPATCH / COLLECTION PREVIEW', + ` Source ${data.source}`, + ` Scope ${data.request.scope}`, + ` Target type ${data.targetType}`, + ` Targets ${data.targetCount}`, + ` Tasks ${data.taskCount}`, + ` Preview hash ${data.hash}`, + '', + ...data.targets.map(target => ` • ${target.key} ${target.start} through ${target.end}`), + ].join('\n'); + } + if (data?.id && data?.runCount !== undefined) { + return [ + '◆ DISPATCH / COLLECTION BATCH', + ` Batch ${data.id}`, + ` Status ${data.status}`, + ` Source ${data.source}`, + ` Scope ${data.scope}`, + ` Runs ${data.runCount}`, + ` Preview hash ${data.previewHash}`, + ].join('\n'); + } + return `◆ DISPATCH / COLLECTIONS\n${JSON.stringify(data, null, 2)}`; +} + +function renderSync(result) { + if (!result?.ok) return `× Sync operation failed (${result?.status || 'internal_error'})`; + const data = result.data; + if (Array.isArray(data?.items) && data.items.every(item => item?.desiredState)) { + return [ + '◆ DISPATCH / SYNCS', + ...(data.items.length ? data.items.map(item => ` ${item.id.padEnd(32)} ${item.desiredState.padEnd(8)} ${item.activity}`) : [' No syncs registered']), + ].join('\n'); + } + if (Array.isArray(data?.items) && data.items.every(item => item?.run)) { + return [ + '◆ DISPATCH / SYNC HISTORY', + ...(data.items.length ? data.items.map(item => { + const context = item.businessContext ? ` ${item.businessContext.date} ${item.businessContext.timezone}` : ''; + const changes = item.delta + ? ` timecards Δ${item.delta.timecards.changedCount} punches +${item.delta.punches.addedCount}/~${item.delta.punches.editedCount}/-${item.delta.punches.removedCount}` + : ''; + const attempts = ` attempts ${item.run.attempts.length}/${item.run.attempt}`; + return ` ${item.run.id} ${item.run.status} revision ${item.configRevision}${context}${changes}${attempts}`; + }) : [' No sync runs']), + ].join('\n'); + } + const sync = data?.sync || data; + if (sync?.desiredState) { + return [ + '◆ DISPATCH / SYNC', + ` Sync ${sync.id}`, + ` Desired state ${sync.desiredState}`, + ` Activity ${sync.activity}`, + ` Source ${sync.source}`, + ` Method ${sync.method}`, + ` Interval ${sync.intervalSeconds}s`, + ` Jitter ${sync.jitterSeconds}s`, + ` Revision ${sync.revision}`, + ` Generation ${sync.generation}`, + ` Next due ${sync.nextDueAt === null ? 'not scheduled' : sync.nextDueAt}`, + ` Last success ${sync.lastSucceededAt === null ? 'never' : sync.lastSucceededAt}`, + ` Business date ${sync.businessContext?.date || 'not available'}`, + ` Business timezone ${sync.businessContext?.timezone || 'not available'}`, + ` Active alerts ${sync.alerts.length}`, + ...sync.alerts.map(alert => ` ${alert.severity.padEnd(8)} ${alert.code}${alert.error ? ` (${alert.error})` : ''}`), + ` Last error ${sync.lastError || 'none'}`, + ...(data?.run ? [` Queued run ${data.run.id}`] : []), + ].join('\n'); + } + return `◆ DISPATCH / SYNC\n${JSON.stringify(data, null, 2)}`; +} + +function renderWorkforce(result) { + if (!result?.ok) return `× Workforce read failed (${result?.status || 'internal_error'})`; + const data = result.data; + if (data?.counts && data?.lifecycleCounts) { + return [ + '◆ DISPATCH / WORKFORCE', + ` Target ${data.target}`, + ` Employees ${data.counts.employees}`, + ` Timecards ${data.counts.timecards}`, + ` Resource links ${data.counts.resourceLinks}`, + ` Lifecycle ${data.lifecycleCounts.active} active · ${data.lifecycleCounts.inactive} inactive · ${data.lifecycleCounts.unknown} unknown`, + ` Consistent ${data.consistent ? 'yes' : 'no'}`, + ` Roster collected ${data.collectedAt.roster}`, + ` Timecards collected ${data.collectedAt.timecards}`, + ].join('\n'); + } + if (data?.employee) { + const employee = data.employee; + return [ + '◆ DISPATCH / WORKFORCE EMPLOYEE', + ` Employee ${employee.employeeCode} · ${employee.employeeName}`, + ` Lifecycle ${employee.lifecycleStatus}`, + ` Department ${employee.department.code} · ${employee.department.name}`, + ` Position ${employee.positionTitle}`, + ` Pay class ${employee.payClass}`, + ` Current timecard ${data.timecard ? `${data.timecard.periodTotalHours} hours · ${data.timecard.missingDays} missing days` : 'not published'}`, + ].join('\n'); + } + if (Array.isArray(data?.items)) { + const timecards = data.kind === 'timecards'; + const punches = data.kind === 'punches'; + const links = data.kind === 'resource_links'; + return [ + `◆ DISPATCH / WORKFORCE ${timecards ? 'TIMECARDS' : punches ? 'PUNCHES' : links ? 'RESOURCE LINKS' : 'EMPLOYEES'}`, + ` Target ${data.target}`, + ...(punches ? [ + ` Business date ${data.businessDate}`, + ` Business timezone ${data.businessTimezone}`, + ` Timecards collected ${data.collectedAt}`, + ] : []), + ` Results ${data.items.length} of ${data.total} · offset ${data.offset}`, + '', + ...(data.items.length ? data.items.map(item => timecards + ? ` ${item.employeeCode} ${item.employeeName} ${item.lifecycleStatus} ${item.periodTotalHours}h` + : punches ? ` ${item.time} ${item.kind} ${item.employeeName} ${item.lifecycleStatus}${item.timeBasis === 'displayed' ? ' displayed-time basis' : ''}` + : links ? ` ${item.employeeCode} ${item.employeeName} ${item.lifecycleStatus} ${item.canonicalUrl}` + : ` ${item.employeeCode} ${item.employeeName} ${item.lifecycleStatus} ${item.positionTitle}`) : [' No matching records']), + ].join('\n'); + } + return '◆ DISPATCH / WORKFORCE'; +} + +module.exports = { + renderHelp, renderStatus, renderSetupEvent, renderSetupResult, renderCollection, renderSync, renderWorkforce, + LABELS, SYMBOLS, STEP_LABELS, PROVIDER_LABELS, +}; diff --git a/dsp/runtime/cli/tests/cli.test.js b/dsp/runtime/cli/tests/cli.test.js new file mode 100644 index 0000000..6b7fbdb --- /dev/null +++ b/dsp/runtime/cli/tests/cli.test.js @@ -0,0 +1,426 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { success, event } = require('dispatch-protocol/contracts/src'); +const { parse } = require('../src/parse'); +const { main } = require('../src/main'); + +const STATUS = success('degraded', { + components: { + auth: { healthy: false, status: 'stopped', data: null, error: { code: 'auth_broker_unavailable', recoverable: true } }, + collections: { healthy: true, status: 'stopped', data: { counts: { collectors: 1, sources: 1, plans: 7, queued: 0, running: 0, failed: 0 }, syncAlerts: { total: 0, critical: 0, items: [], hasMore: false } }, error: null }, + paycom: { healthy: true, status: 'ready', data: { payPeriods: { verified: true }, roster: { verified: false, code: 'not_loaded' }, timecards: { verified: false, code: 'not_loaded' } }, error: null }, + }, + summary: { ready: 1, degraded: 2, failed: 0 }, +}); +const SETUP = success('complete', { + workflow: 'setup_auth', provider: 'paycom', profile: 'paycom-main', configured: true, + broker: 'ready', vault: 'verified', authenticationTest: 'authenticated', + nextActions: ['run_collection'], +}); +const REMOVED = success('complete', { + workflow: 'setup_auth', provider: 'paycom', profile: 'paycom-main', configured: false, + broker: 'ready', vault: 'verified', authenticationTest: 'skipped', + nextActions: ['configure_auth_profile'], +}); +const PREPARATION = success('ready', { + workflow: 'setup_auth', + target: { provider: 'paycom', profile: 'paycom-main' }, + state: { + broker: { status: 'stopped', managed: false }, + vault: { status: 'ready', verified: true }, + profile: { status: 'configured', provider: 'paycom' }, + credentialIngress: 'available', + }, + capabilities: { + credentialActions: [ + { id: 'keep', available: true, reason: null }, + { id: 'enroll', available: false, reason: 'profile_exists' }, + { id: 'replace', available: true, reason: null }, + { id: 'remove', available: true, reason: null }, + ], + authenticationTest: { id: 'run', available: true, reason: null }, + }, + defaults: { credentialAction: 'keep', startBroker: true, testAuthentication: false }, +}); + +function fixture() { + let output = ''; + let errors = ''; + let target = { provider: 'paycom', profile: 'paycom-main' }; + const authSetup = { + prepare: async (input = {}) => { + target = { provider: input.provider || 'paycom', profile: input.profile || 'paycom-main' }; + return success('ready', { + ...PREPARATION.data, + target, + state: { ...PREPARATION.data.state, profile: { status: 'configured', provider: target.provider } }, + }); + }, + run: async (input, { events }) => { + await events.emit(event('workflow_started', { workflow: 'setup_auth', state: 'preflight' }, { operationId: 'setup_auth_fixture' })); + assert.equal(input.provider, target.provider); + assert.equal(input.profile, target.profile); + assert.equal(input.startBroker, true); + return success('complete', { ...SETUP.data, provider: target.provider, profile: target.profile }); + }, + }; + const collections = { + describe: async source => success('found', { source, collector: 'paycom', collectorVersion: '0.6.0', targetType: 'pay-period', timezone: 'America/Los_Angeles', selectors: ['date'], scopes: [], limits: { maxTargets: 64, maxRangeDays: 730 } }), + preview: async request => success('previewed', { + id: 'preview_fixture', hash: 'a'.repeat(64), generatedAt: '2026-08-26T00:00:00.000Z', request, + normalizedSelector: request.selector, source: request.source, collector: 'paycom', collectorVersion: '0.6.0', + targetType: 'pay-period', timezone: 'America/Los_Angeles', targetCount: 1, taskCount: 4, + targets: [{ key: '2026-08-22', start: '2026-08-09', end: '2026-08-22' }], tasks: [], + }), + }; + const syncValue = { + id: 'fixture-main-sync', plan: 'fixture-sync-plan', source: 'fixture-main', collector: 'fixture', method: 'fixture.sync', + desiredState: 'stopped', activity: 'idle', intervalSeconds: 60, jitterSeconds: 5, overlap: 'coalesce', + settingsSchema: { type: 'object', properties: { behavior: { type: 'string' } }, required: ['behavior'], additionalProperties: false }, + settings: { behavior: 'no_change' }, revision: 1, generation: 0, nextDueAt: null, + lastStartedAt: null, lastSucceededAt: null, lastError: null, blocked: null, + businessContext: { date: '2026-08-29', timezone: 'America/Los_Angeles' }, alerts: [], + activeRun: null, queuedRunCount: 0, createdAt: 1, updatedAt: 1, + }; + const sync = { + list: async () => success('found', { items: [syncValue], total: 1, limit: 50, offset: 0, hasMore: false }), + status: async () => success('found', syncValue), + start: async () => success('started', { sync: { ...syncValue, desiredState: 'running', activity: 'queued', generation: 1 }, run: null }), + stop: async () => success('stopped', syncValue), + restart: async () => success('restarted', { sync: { ...syncValue, desiredState: 'running', activity: 'queued', generation: 1 }, run: null }), + runNow: async () => success('queued', { sync: { ...syncValue, desiredState: 'running', activity: 'queued' }, run: null }), + edit: async (_id, patch) => success('updated', { sync: { ...syncValue, ...patch, revision: 2 }, run: null }), + history: async (_id, options = {}) => success('found', { + items: [], total: 0, limit: options.limit ?? 50, offset: options.offset ?? 0, hasMore: false, + }), + }; + const workforce = { + snapshot: async () => success('ready', { + target: '2026-09-05', + collectedAt: { roster: '2026-08-29T06:00:00.000Z', timecards: '2026-08-29T06:00:00.000Z', resourceLinks: '2026-08-29T06:00:00.000Z' }, + counts: { employees: 2, timecards: 2, resourceLinks: 2 }, + lifecycleCounts: { active: 1, inactive: 0, unknown: 1 }, consistent: true, + }), + employees: async query => success('found', { + kind: 'employees', target: '2026-09-05', collectedAt: '2026-08-29T06:00:00.000Z', + items: [], total: 0, limit: query.limit, offset: query.offset, hasMore: false, + }), + employee: async code => success('found', { + target: '2026-09-05', collectedAt: '2026-08-29T06:00:00.000Z', + employee: { employeeCode: code, employeeName: 'Fixture Employee', lifecycleStatus: 'active', department: { code: 'D1', name: 'Driver' }, positionTitle: 'Driver', payClass: 'PC' }, + timecard: null, + }), + timecards: async query => success('found', { + kind: 'timecards', target: '2026-09-05', collectedAt: '2026-08-29T06:00:00.000Z', + items: [], total: 0, limit: query.limit, offset: query.offset, hasMore: false, + }), + punches: async query => success('found', { + kind: 'punches', target: '2026-09-05', businessDate: query.date, + businessTimezone: 'America/Los_Angeles', collectedAt: '2026-08-29T06:00:00.000Z', + items: [], total: 0, limit: query.limit, offset: query.offset, hasMore: false, + }), + resourceLinks: async query => success('found', { + kind: 'resource_links', target: '2026-09-05', collectedAt: '2026-08-29T06:00:00.000Z', + items: [], total: 0, limit: query.limit, offset: query.offset, hasMore: false, + }), + }; + return { + client: { system: { status: async () => STATUS }, collections, sync, workforce, workflows: { authSetup } }, + write: chunk => { output += chunk; }, + writeError: chunk => { errors += chunk; }, + values: () => ({ output, errors }), + }; +} + +test('CLI parser keeps the command surface closed', () => { + assert.deepEqual(parse(['status']), { command: 'status', format: 'plain' }); + assert.deepEqual(parse(['status', '--json']), { command: 'status', format: 'json' }); + assert.deepEqual(parse(['setup', 'auth']), { + command: 'setup-auth', format: 'plain', provider: 'paycom', profile: 'paycom-main', replaceExisting: false, replaceSpecified: false, + removeSpecified: false, confirmed: false, + testAuthentication: false, testAuthenticationSpecified: false, nonInteractive: false, + }); + assert.deepEqual(parse(['setup', 'auth', '--replace', '--test-auth', '--json']), { + command: 'setup-auth', format: 'json', provider: 'paycom', profile: 'paycom-main', replaceExisting: true, replaceSpecified: true, + removeSpecified: false, confirmed: false, + testAuthentication: true, testAuthenticationSpecified: true, nonInteractive: true, + }); + assert.equal(parse(['setup', 'auth', '--no-menu']).nonInteractive, true); + assert.deepEqual( + { provider: parse(['setup', 'auth', '--provider', 'amazon-logistics', '--profile', 'amazon-operations']).provider, + profile: parse(['setup', 'auth', '--provider', 'amazon-logistics', '--profile', 'amazon-operations']).profile }, + { provider: 'amazon-logistics', profile: 'amazon-operations' }, + ); + assert.equal(parse(['setup', 'auth', '--plain']).nonInteractive, true); + assert.throws(() => parse(['status', '--json', '--json']), error => error.code === 'invalid_input'); + assert.throws(() => parse(['setup', 'auth', '--replace', '--replace']), error => error.code === 'invalid_input'); + assert.throws(() => parse(['setup', 'auth', '--remove', '--json']), error => error.code === 'invalid_input'); + assert.throws(() => parse(['setup', 'auth', '--remove', '--replace']), error => error.code === 'invalid_input'); + assert.equal(parse(['setup', 'auth', '--remove', '--confirm', '--json']).removeSpecified, true); +}); + +test('collection CLI uses the standard SDK request for arbitrary dates and ranges', async () => { + assert.deepEqual(parse(['collect', 'preview', 'paycom-main', 'full', '--yesterday', '--json']), { + command: 'collect-preview', source: 'paycom-main', scope: 'full', selector: { kind: 'relative-date', value: 'yesterday' }, + mode: 'ensure', idempotencyKey: undefined, expectedPreviewHash: undefined, format: 'json', + }); + const range = parse(['collect', 'enqueue', 'paycom-main', 'links', '--from', '2026-07-01', '--through', '2026-07-31', '--mode', 'refresh']); + assert.deepEqual(range.selector, { kind: 'date-range', start: '2026-07-01', end: '2026-07-31' }); + assert.equal(range.mode, 'refresh'); + const target = parse(['collect', 'target', 'cdf-example', 'cdf', '2026-W34', '--idempotency', 'cdf-W34']); + assert.deepEqual(target.selector, { kind: 'exact-target', key: '2026-W34' }); + assert.equal(target.idempotencyKey, 'cdf-W34'); + const backfill = parse(['collect', 'backfill', 'cdf-example', 'cdf', '2026-W20', '2026-W34', '--mode', 'ensure']); + assert.deepEqual(backfill.selector, { kind: 'target-range', startKey: '2026-W20', endKey: '2026-W34' }); + assert.equal(backfill.command, 'collect-enqueue'); + assert.throws(() => parse(['collect', 'backfill', 'cdf-example', 'cdf', '2026-W20']), error => error.code === 'invalid_input'); + assert.equal(parse(['collect', 'audit', 'paycom-main', 'links', '--date', '2026-08-18']).mode, 'verify'); + assert.equal(parse(['collect', 'retry', 'batch_fixture']).command, 'collect-retry'); + assert.equal(parse(['collect', 'schedule', 'run', 'nightly']).command, 'collect-schedule-run'); + assert.throws(() => parse(['collect', 'audit', 'paycom-main', 'links', '--current', '--mode', 'refresh']), error => error.code === 'invalid_input'); + assert.throws(() => parse(['collect', 'preview', 'paycom-main', 'full', '--current', '--date', '2026-08-18']), error => error.code === 'invalid_input'); + + const io = fixture(); + assert.equal(await main(['collect', 'preview', 'paycom-main', 'full', '--date', '2026-08-18', '--json'], io), 0); + const result = JSON.parse(io.values().output); + assert.equal(result.status, 'previewed'); + assert.equal(result.data.request.selector.date, '2026-08-18'); +}); + +test('sync CLI parses lifecycle and edit commands and calls only the public sync client', async () => { + assert.deepEqual(parse(['sync', 'start', 'fixture-main-sync', '--json']), { + command: 'sync-start', syncId: 'fixture-main-sync', drain: false, format: 'json', + }); + assert.deepEqual(parse(['sync', 'stop', 'fixture-main-sync', '--drain']), { + command: 'sync-stop', syncId: 'fixture-main-sync', drain: true, format: 'plain', + }); + assert.deepEqual(parse(['sync', 'history', 'fixture-main-sync', '--limit', '100', '--offset', '50', '--json']), { + command: 'sync-history', syncId: 'fixture-main-sync', limit: 100, offset: 50, format: 'json', + }); + assert.deepEqual(parse(['sync', 'edit', 'fixture-main-sync', '--interval', '120', '--jitter', '10', + '--set', 'behavior=published', '--replace-settings', '--revision', '1', '--apply-now', '--json']), { + command: 'sync-edit', syncId: 'fixture-main-sync', + patch: { intervalSeconds: 120, jitterSeconds: 10, settings: { behavior: 'published' }, replaceSettings: true }, + expectedRevision: 1, applyNow: true, format: 'json', + }); + assert.throws(() => parse(['sync', 'edit', 'fixture-main-sync', '--replace-settings']), error => error.code === 'invalid_input'); + assert.throws(() => parse(['sync', 'edit', 'fixture-main-sync']), error => error.code === 'invalid_input'); + assert.throws(() => parse(['sync', 'edit', 'fixture-main-sync', '--interval', '60', '--interval', '120']), error => error.code === 'invalid_input'); + assert.throws(() => parse(['sync', 'edit', 'fixture-main-sync', '--apply-now', '--apply-now', '--set', 'behavior=published']), error => error.code === 'invalid_input'); + assert.throws(() => parse(['sync', 'status', 'fixture-main-sync', '--drain']), error => error.code === 'invalid_input'); + + const io = fixture(); + assert.equal(await main(['sync', 'list', '--json'], io), 0); + assert.equal(JSON.parse(io.values().output).data.total, 1); + + const history = fixture(); + assert.equal(await main(['sync', 'history', 'fixture-main-sync', '--limit', '100', '--offset', '50', '--json'], history), 0); + assert.equal(JSON.parse(history.values().output).data.limit, 100); + assert.equal(JSON.parse(history.values().output).data.offset, 50); + + const edited = fixture(); + assert.equal(await main(['sync', 'edit', 'fixture-main-sync', '--interval', '120', '--set', 'behavior=published', '--json'], edited), 0); + const result = JSON.parse(edited.values().output); + assert.equal(result.data.sync.intervalSeconds, 120); + assert.equal(result.data.sync.settings.behavior, 'published'); +}); + +test('workforce CLI exposes only public read operations with closed pagination and lifecycle filters', async () => { + assert.deepEqual(parse(['workforce', 'status', '--json']), { command: 'workforce-status', format: 'json' }); + assert.deepEqual(parse(['workforce', 'employees', '--lifecycle', 'unknown', '--limit', '10', '--offset', '20']), { + command: 'workforce-employees', query: { lifecycleStatus: 'unknown', limit: 10, offset: 20 }, format: 'plain', + }); + assert.deepEqual(parse(['workforce', 'employee', 'a001']), { + command: 'workforce-employee', employeeCode: 'A001', format: 'plain', + }); + assert.equal(parse(['workforce', 'timecards', '--limit', '5']).command, 'workforce-timecards'); + assert.deepEqual(parse([ + 'workforce', 'punches', '--date', '2026-08-30', '--kind', 'in_day', '--from-time', '10:01', '--limit', '10', + ]), { + command: 'workforce-punches', query: { + date: '2026-08-30', kind: 'in_day', fromTime: '10:01', limit: 10, offset: 0, + }, format: 'plain', + }); + assert.equal(parse(['workforce', 'links', '--limit', '5']).command, 'workforce-links'); + assert.throws(() => parse(['workforce', 'employees', '--lifecycle', 'deleted']), error => error.code === 'invalid_input'); + assert.throws(() => parse(['workforce', 'timecards', '--limit', '101']), error => error.code === 'invalid_input'); + assert.throws(() => parse(['workforce', 'punches', '--date', '2026-08-30', '--from-time', '25:00']), error => error.code === 'invalid_input'); + + const status = fixture(); + assert.equal(await main(['workforce', 'status', '--json'], status), 0); + assert.equal(JSON.parse(status.values().output).data.counts.employees, 2); + + const links = fixture(); + assert.equal(await main(['workforce', 'links', '--limit', '5', '--json'], links), 0); + assert.equal(JSON.parse(links.values().output).data.kind, 'resource_links'); + + const punches = fixture(); + assert.equal(await main([ + 'workforce', 'punches', '--date', '2026-08-30', '--kind', 'in_day', '--from-time', '10:01', '--json', + ], punches), 0); + assert.equal(JSON.parse(punches.values().output).data.kind, 'punches'); + + const employees = fixture(); + assert.equal(await main(['workforce', 'employees', '--lifecycle', 'unknown', '--limit', '10', '--json'], employees), 0); + const page = JSON.parse(employees.values().output).data; + assert.equal(page.kind, 'employees'); + assert.equal(page.limit, 10); +}); + +test('dispatch status renders human output from the SDK result', async () => { + const io = fixture(); + assert.equal(await main(['status'], io), 0); + const { output } = io.values(); + assert.equal(output.includes('◆ DISPATCH'), true); + assert.equal(output.includes('Collection Manager'), true); + assert.equal(output.includes('7 plans'), true); +}); + +test('dispatch status --json returns the exact versioned SDK result', async () => { + const io = fixture(); + assert.equal(await main(['status', '--json'], io), 0); + assert.deepEqual(JSON.parse(io.values().output), STATUS); +}); + +test('dispatch setup auth uses only the public workflow client and keeps JSON output machine-readable', async () => { + const io = fixture(); + const original = io.client.workflows.authSetup.run; + io.client.workflows.authSetup.run = async (input, options) => { + assert.equal(input.credentialAction, 'replace'); + assert.equal(input.testAuthentication, true); + return original(input, options); + }; + assert.equal(await main(['setup', 'auth', '--replace', '--test-auth', '--json'], io), 0); + assert.deepEqual(JSON.parse(io.values().output), SETUP); + assert.equal(/password|username|pin\d|cookie|token/i.test(io.values().output), false); +}); + +test('dispatch setup auth targets Amazon Logistics only through explicit provider and profile options', async () => { + const io = fixture(); + assert.equal(await main([ + 'setup', 'auth', '--provider', 'amazon-logistics', '--profile', 'amazon-operations', '--no-menu', '--json', + ], io), 0); + const result = JSON.parse(io.values().output); + assert.equal(result.data.provider, 'amazon-logistics'); + assert.equal(result.data.profile, 'amazon-operations'); + assert.equal(/password|username|cookie|token|endpoint/i.test(io.values().output), false); +}); + +test('dispatch setup auth plain mode renders semantic workflow events', async () => { + const io = fixture(); + assert.equal(await main(['setup', 'auth'], io), 0); + assert.equal(io.values().output.includes('DISPATCH / AUTH SETUP'), true); + assert.equal(io.values().output.includes('Authentication setup complete'), true); +}); + +test('Amazon Logistics setup labels and routes its authentication test without Paycom wording', async () => { + const io = fixture(); + const original = io.client.workflows.authSetup.run; + io.client.workflows.authSetup.run = async (input, options) => { + assert.equal(input.provider, 'amazon-logistics'); + assert.equal(input.profile, 'amazon-operations'); + assert.equal(input.testAuthentication, true); + await options.events.emit(event('step_started', { step: 'test_authentication' }, { operationId: 'setup_auth_fixture' })); + return original(input, options); + }; + assert.equal(await main([ + 'setup', 'auth', '--provider', 'amazon-logistics', '--profile', 'amazon-operations', '--test-auth', '--no-menu', + ], io), 0); + assert.equal(io.values().output.includes('Testing Amazon Logistics authentication'), true); + assert.equal(io.values().output.includes('Testing Paycom authentication'), false); +}); + +test('dispatch setup auth removal requires confirmation and invokes the closed removal action', async () => { + const io = fixture(); + let confirmation; + io.interaction = { + available: () => true, + write: () => {}, + select: async () => { throw new Error('remove must not offer another action'); }, + confirm: async value => { confirmation = value; return true; }, + close: () => {}, + }; + io.client.workflows.authSetup.run = async input => { + assert.equal(input.credentialAction, 'remove'); + assert.equal(input.testAuthentication, false); + return REMOVED; + }; + assert.equal(await main(['setup', 'auth', '--remove'], io), 0); + assert.equal(confirmation.defaultValue, false); + assert.equal(confirmation.message.includes('paycom-main'), true); + assert.equal(io.values().output.includes('Authentication profile deleted'), true); +}); + +test('interactive auth setup derives actions from preparation capabilities', async () => { + const io = fixture(); + const selected = []; + let closed = false; + io.interaction = { + available: () => true, + write: () => {}, + select: async menu => { + selected.push(menu); + return selected.length === 1 ? 'replace' : 'test'; + }, + confirm: async () => true, + close: () => { closed = true; }, + }; + const original = io.client.workflows.authSetup.run; + io.client.workflows.authSetup.run = async (input, options) => { + assert.equal(input.credentialAction, 'replace'); + assert.equal(input.testAuthentication, true); + return original(input, options); + }; + assert.equal(await main(['setup', 'auth'], io), 0); + assert.deepEqual(selected[0].options.map(option => option.value), ['keep', 'replace', 'remove']); + assert.equal(selected.length, 2); + assert.equal(closed, true); +}); + +test('interactive auth setup cancellation does not invoke the workflow', async () => { + const io = fixture(); + let invoked = false; + io.client.workflows.authSetup.run = async () => { invoked = true; return SETUP; }; + io.interaction = { + available: () => true, + write: () => {}, + select: async menu => menu.defaultValue, + confirm: async () => false, + close: () => {}, + }; + assert.equal(await main(['setup', 'auth'], io), 1); + assert.equal(invoked, false); + assert.equal(io.values().output.includes('cancelled'), true); +}); + +test('JSON, plain, and no-menu setup never invoke the non-secret menu adapter', async () => { + for (const argv of [['setup', 'auth', '--json'], ['setup', 'auth', '--plain'], ['setup', 'auth', '--no-menu']]) { + const io = fixture(); + io.interaction = { + available: () => true, + write: () => { throw new Error('menu invoked'); }, + select: async () => { throw new Error('menu invoked'); }, + confirm: async () => { throw new Error('menu invoked'); }, + close: () => {}, + }; + assert.equal(await main(argv, io), 0); + } +}); + +test('invalid CLI arguments return a stable error and help text', async () => { + const io = fixture(); + assert.equal(await main(['status', '--unknown'], io), 2); + assert.equal(JSON.parse(io.values().output).status, 'invalid_input'); + assert.equal(io.values().errors.includes('Usage:'), true); +}); + +test('CLI sanitizes an unexpected SDK failure', async () => { + const io = fixture(); + io.client.system.status = async () => { throw new Error('private fixture detail'); }; + assert.equal(await main(['status', '--json'], io), 1); + const output = JSON.parse(io.values().output); + assert.equal(output.status, 'internal_error'); + assert.equal(JSON.stringify(output).includes('private fixture detail'), false); +}); diff --git a/dsp/runtime/cli/tests/interactions.test.js b/dsp/runtime/cli/tests/interactions.test.js new file mode 100644 index 0000000..851ac65 --- /dev/null +++ b/dsp/runtime/cli/tests/interactions.test.js @@ -0,0 +1,136 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { PassThrough } = require('node:stream'); +const { LineInteraction } = require('../src/interactions/line'); +const { collectSetupAuthInput } = require('../src/interactions/setup-auth'); + +function command(overrides = {}) { + return { + command: 'setup-auth', format: 'plain', replaceExisting: false, replaceSpecified: false, + removeSpecified: false, confirmed: false, + testAuthentication: false, testAuthenticationSpecified: false, nonInteractive: false, + ...overrides, + }; +} +function preparation({ configured = true } = {}) { + return { + workflow: 'setup_auth', target: { provider: 'paycom', profile: 'paycom-main' }, + state: { + broker: { status: 'stopped', managed: false }, + vault: { status: configured ? 'ready' : 'absent', verified: configured }, + profile: { status: configured ? 'configured' : 'not_configured', ...(configured ? { provider: 'paycom' } : {}) }, + credentialIngress: 'available', + }, + capabilities: { + credentialActions: [ + { id: 'keep', available: configured, reason: configured ? null : 'profile_not_configured' }, + { id: 'enroll', available: !configured, reason: configured ? 'profile_exists' : null }, + { id: 'replace', available: configured, reason: configured ? null : 'profile_not_configured' }, + { id: 'remove', available: configured, reason: configured ? null : 'profile_not_configured' }, + ], + authenticationTest: { id: 'run', available: true, reason: null }, + }, + defaults: { credentialAction: configured ? 'keep' : 'enroll', startBroker: true, testAuthentication: false }, + }; +} + +test('line interaction accepts bounded numbered selections and exposes no secret prompt', async () => { + const input = new PassThrough(); + const output = new PassThrough(); + input.isTTY = true; + output.isTTY = true; + let rendered = ''; + output.on('data', chunk => { rendered += chunk.toString('utf8'); }); + const interaction = new LineInteraction({ input, output }); + assert.equal(typeof interaction.secret, 'undefined'); + input.end('2\n'); + const selected = await interaction.select({ + message: 'Choose an option', + options: [ + { value: 'keep', label: 'Keep' }, + { value: 'replace', label: 'Replace' }, + ], + defaultValue: 'keep', + }); + interaction.close(); + assert.equal(selected, 'replace'); + assert.equal(rendered.includes('1. Keep (default)'), true); + assert.equal(rendered.includes('2. Replace'), true); +}); + +test('line interaction treats terminal EOF as cancellation', async () => { + const input = new PassThrough(); + const output = new PassThrough(); + input.isTTY = true; + output.isTTY = true; + const interaction = new LineInteraction({ input, output }); + input.end(); + await assert.rejects(interaction.select({ + message: 'Choose an option', + options: [{ value: 'keep', label: 'Keep' }], + defaultValue: 'keep', + }), error => error.code === 'cancelled'); + interaction.close(); +}); + +test('setup choice collector honors flags and derives remaining choices from preparation', async () => { + let selectCalls = 0; + const interaction = { + available: () => true, + write: () => {}, + select: async menu => { selectCalls += 1; return menu.defaultValue; }, + confirm: async () => true, + }; + const value = await collectSetupAuthInput(command({ + replaceExisting: true, replaceSpecified: true, + }), interaction, preparation()); + assert.equal(value.cancelled, false); + assert.equal(value.input.credentialAction, 'replace'); + assert.equal(value.input.testAuthentication, false); + assert.equal(selectCalls, 1); +}); + +test('setup choice collector offers only core-approved credential actions', async () => { + let options; + const value = await collectSetupAuthInput(command(), { + available: () => true, + write: () => {}, + select: async menu => { if (!options) options = menu.options; return menu.defaultValue; }, + confirm: async () => true, + }, preparation()); + assert.deepEqual(options.map(item => item.value), ['keep', 'replace', 'remove']); + assert.equal(value.input.credentialAction, 'keep'); +}); + +test('setup choice collector fails closed for an unknown available action', async () => { + const prepared = preparation(); + prepared.capabilities.credentialActions.push({ id: 'external_action', available: true, reason: null }); + await assert.rejects(collectSetupAuthInput(command(), { + available: () => true, + write: () => {}, + select: async menu => menu.defaultValue, + confirm: async () => true, + }, prepared), error => error.code === 'setup_action_unavailable'); +}); + +test('setup choice collector falls back without prompting and uses core defaults', async () => { + const interaction = { + available: () => false, + write: () => { throw new Error('unexpected write'); }, + select: async () => { throw new Error('unexpected select'); }, + confirm: async () => { throw new Error('unexpected confirm'); }, + }; + const value = await collectSetupAuthInput(command(), interaction, preparation({ configured: false })); + assert.equal(value.interactive, false); + assert.deepEqual(value.input, { + provider: 'paycom', profile: 'paycom-main', credentialAction: 'enroll', + startBroker: true, testAuthentication: false, + }); + const blockedRemoval = await collectSetupAuthInput(command({ removeSpecified: true }), interaction, preparation()); + assert.equal(blockedRemoval.cancelled, true); + const confirmedRemoval = await collectSetupAuthInput(command({ removeSpecified: true, confirmed: true }), interaction, preparation()); + assert.equal(confirmedRemoval.cancelled, false); + assert.equal(confirmedRemoval.input.credentialAction, 'remove'); +}); diff --git a/dsp/runtime/collection-manager/OVERVIEW.md b/dsp/runtime/collection-manager/OVERVIEW.md new file mode 100644 index 0000000..8ae596d --- /dev/null +++ b/dsp/runtime/collection-manager/OVERVIEW.md @@ -0,0 +1,220 @@ +--- +title: Collection Manager overview +status: current +last_verified: 2026-09-02 +--- + +# Dispatch Collection Manager + +Provider implementations live in [`../providers/`](../../plugins/README.md). + +The Collection Manager is the durable local control plane for every Dispatch collector. It owns registration, source instances, collection methods, schedules, durable runs, dependency gates, retries, resource locks, cancellation, and bounded collector subprocess execution. + +Collectors own domain collection and validation. The manager owns when and how they run. + +## Current capabilities + +- Declarative collector/source/plan specifications. +- Multiple methods per collector and multiple source instances per collector. +- Manual, interval, and five-field cron schedules with IANA timezones. +- Durable SQLite queue and run history. +- Idempotent schedule keys that suppress duplicate interval/cron runs. +- Freshness-bounded plan dependencies. +- Four concurrent workers by default with source and declared resource locks. +- Retry attempts with bounded backoff. +- Pause, resume, cancel, manual retry, and offline drain controls. +- Exact collector executable validation and release version snapshots per run. +- Strict closed method input schemas. +- Collector requests over stdin and one bounded JSON receipt over stdout. +- Sanitized failures; collector stderr is never persisted. +- Secret-bearing configuration keys are rejected. Sources reference auth profiles by identifier only. +- Standard source capability discovery and non-mutating target preview. +- Current, latest-complete, exact-date, yesterday, range, rolling-duration, and exact-target selectors. +- Durable multi-target batches with exact run dependencies and preview-hash fencing. +- Standard relative-selector schedules shared by every capable collector. +- Plugin-registered polling sync definitions backed by existing manual plans. +- Sync start, stop, restart, run-now, edit revisions, history, deterministic jitter, and coalescing. +- Managed-installation first-publication support: exact server-owned Paycom definition attestation, job-bound pay-period/run and workforce-batch idempotency, exact five-plan graph verification, clean database/idle/no-critical-alert readiness evidence, and a private read-only batch-to-publication verifier. Stale activation workers do not cancel shared work. The manager contributes durable run receipts but never writes installation `ready`. + +## Paths + +```text +Component: ./runtime/collection-manager +Database: /collection-manager/collection-manager.sqlite3 +Skill: ./docs/agent-skills/collection-manager/SKILL.md +``` + +`DISPATCH_LOCAL_ROOT` derives external development data and state directories. Without it, new installations use XDG roots. Individual `DISPATCH_DATA_ROOT` and `DISPATCH_STATE_ROOT` values or explicit SDK runtime options may override the defaults. `resolveLocalRuntimePaths()` is authoritative for local mode and rejects mutable roots inside the source worktree. A Provisioner-managed service sets a fixed internal managed-runtime marker; worker launch then requires the complete authority-derived component/provider environment, rejects local-root and Access Control selectors, and never falls back to local/XDG storage. + +The database directory is mode `0700` and the database is mode `0600`. + +## Operator CLI + +```bash +CTL=./runtime/collection-manager/bin/dispatch-collectionctl + +$CTL help +$CTL init +$CTL apply /absolute/path/to/collection-spec.json +$CTL status +$CTL collectors 50 0 +$CTL collector paycom +$CTL methods paycom +$CTL sources 50 0 +$CTL source paycom-main +$CTL plans 50 0 +$CTL plan paycom-current-timecards +$CTL run paycom-current-timecards +$CTL runs 50 0 +$CTL run-status +$CTL pause +$CTL resume +$CTL cancel +$CTL retry +$CTL collection-describe paycom-main +$CTL collection-preview /absolute/path/to/request.json +$CTL collection-enqueue /absolute/path/to/request.json /absolute/path/to/options.json +$CTL batches 50 0 +$CTL batch +$CTL cancel-batch +$CTL retry-batch +$CTL collection-schedules +$CTL put-collection-schedule /absolute/path/to/schedule.json +$CTL run-collection-schedule +$CTL syncs +$CTL sync +$CTL start-sync +$CTL stop-sync +$CTL restart-sync +$CTL run-sync +$CTL edit-sync /absolute/path/to/patch.json +$CTL sync-history +``` + +All commands return one bounded JSON object. List commands, including `runs`, are paginated under `data.items` with `total`, `limit`, `offset`, and `hasMore`. `run-status` uses the run lifecycle state as its envelope status. `apply` is an upsert: it creates or updates declarations but does not delete omitted declarations. + +When the long-running manager is not active, process queued work synchronously: + +```bash +$CTL drain 30000 +``` + +Do not use `drain` while the manager daemon is running; the single-manager lease rejects a second manager. `idle` means the queue is empty, `deferred` returns retry-delayed or dependency-blocked work under `data.pending`, and `drain_timeout` explicitly cancels and reports active run IDs. + +Pausing a plan blocks new scheduled and manual runs. Existing queued runs continue unless cancelled separately; existing running work is not interrupted. + +## Managed polling syncs + +A plugin registers a sync by declaring a normal manual plan plus a top-level `syncs` entry in its manager specification. The manager owns desired state, interval, jitter, `coalesce` overlap, validated non-secret settings, configuration revision, lifecycle generation, scheduling, cancellation, and history. The plugin plan owns source checks, provider cursors, synchronization strategy, validation, reconciliation, and publication. + +Each tick is an ordinary manager run. Starting queues an immediate tick; stopping cancels queued and active ticks and waits for cleanup; `--drain` allows the active tick to finish; restarting starts a new generation; editing increments the immutable configuration revision used by future ticks. A pending tick coalesces additional due windows. + +The public interface is `dispatch.sync` and `./bin/dispatch sync ...`. + +## Service + +```bash +./tooling/start +``` + +The manager heartbeat appears under `status.data.manager`. A running daemon schedules due plans and processes queued runs. SIGINT and SIGTERM cancel active workers, release the manager lease, and close the database. + +A hardened legacy-local user-service template is provided at `integration/systemd/dispatch-collection-manager.service.in`. Render the local Dispatch user-service units for the current checkout and an external local-data root with `core/tooling/render-systemd-units --local-root --output-dir `. Its `KillMode=control-group` ensures collector children are terminated with the manager during service stop or restart. Provisioner-managed DSPs instead receive the fixed server-owned Collection Manager within service-plan version `3`, alongside the Auth Broker, Runtime Gateway, and configured outbound Runtime Agent; the legacy renderer is not used for managed runtimes. + +## Collector process contract + +A collector is one owner-controlled, absolute, regular executable. It receives no command arguments and a single JSON request over stdin: + +```json +{ + "protocolVersion": 1, + "runId": "run_...", + "plan": "paycom-main-timecards", + "source": { + "id": "paycom-main", + "collector": "paycom", + "authProfile": "paycom-main", + "config": {} + }, + "method": "timecards.period", + "input": {}, + "attempt": 1, + "deadline": "2026-08-25T12:00:00.000Z" +} +``` + +It must write exactly one newline-terminated JSON receipt and nothing else to stdout: + +```json +{ + "ok": true, + "status": "published", + "data": { + "rows": 500, + "revision": "..." + }, + "warnings": [] +} +``` + +Successful statuses are `succeeded`, `published`, and `no_change`. Failures use: + +```json +{"ok":false,"status":"failed","error":{"code":"stable_error_code"}} +``` + +Collectors must write logs to their own private operational location, not stdout. The manager discards stderr and stores only bounded receipts or sanitized error codes. + +## Scheduling and dependencies + +Supported schedule shapes: + +```json +{"type":"manual"} +{"type":"interval","seconds":900} +{"type":"cron","expression":"50 15 * * *","timezone":"America/Los_Angeles"} +``` + +A dependency requires a recent successful run of another plan: + +```json +{"plan":"paycom-main-roster","maxAgeSeconds":86400} +``` + +A queued run remains blocked with `blocked: "dependency:"` until all dependencies are fresh. + +Standard batches also persist exact run-to-run dependencies for every resolved target. Their source capability declaration points to a collector-owned resolver and maps scopes to existing registered plans. + +## Concurrency + +Every run automatically locks its source. Methods can declare additional keys: + +```json +[ + "auth:{authProfile}", + "collector:{collector}", + "publish:paycom-timecards" +] +``` + +Supported substitutions are `{source}`, `{authProfile}`, `{collector}`, and `{method}`. This prevents two runs from sharing one browser/auth profile or publication target while allowing unrelated collectors to run concurrently. + +## Security boundary + +- Specifications, source config, method input, receipts, and run history must never contain credentials, cookies, tokens, PINs, or authorization headers. +- `authProfile` is only an identifier. Authenticated browser sessions are acquired through the separate Auth Broker; credentials and browser material are never stored in manager state. +- Collector commands must be absolute owner-controlled executables with no symlinks, extra hard links, or group/other write permissions. +- No collector receives inherited credential environment variables. +- Collector stdout and stderr are bounded; stderr is not retained. +- The manager does not expose arbitrary shell arguments, environment variables, URLs, or commands at run time. + +## Lifecycle + +```bash +./tooling/test +./tooling/build +./tooling/verify +./tooling/health +``` + +The source is canonical. Do not edit generated runtime copies if immutable releases are added later. diff --git a/dsp/runtime/collection-manager/bin/dispatch-collection-manager b/dsp/runtime/collection-manager/bin/dispatch-collection-manager new file mode 100755 index 0000000..522accc --- /dev/null +++ b/dsp/runtime/collection-manager/bin/dispatch-collection-manager @@ -0,0 +1,5 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +process.umask(0o077); +process.env.NODE_NO_WARNINGS = '1'; +require('../src/daemon-cli').main().then(code => { if (Number.isInteger(code)) process.exitCode = code; }); diff --git a/dsp/runtime/collection-manager/bin/dispatch-collectionctl b/dsp/runtime/collection-manager/bin/dispatch-collectionctl new file mode 100755 index 0000000..6367c87 --- /dev/null +++ b/dsp/runtime/collection-manager/bin/dispatch-collectionctl @@ -0,0 +1,5 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +process.umask(0o077); +process.env.NODE_NO_WARNINGS = '1'; +require('../src/control-cli').main().then(code => { process.exitCode = code; }); diff --git a/dsp/runtime/collection-manager/dispatch-plugin.yaml b/dsp/runtime/collection-manager/dispatch-plugin.yaml new file mode 100644 index 0000000..8e55f87 --- /dev/null +++ b/dsp/runtime/collection-manager/dispatch-plugin.yaml @@ -0,0 +1,43 @@ +schema_version: 1 +id: collection-manager +display_name: Dispatch Collection Manager +version: 0.9.0 +summary: Durable collector control plane with collections, managed polling syncs, retries, locks, revisions, and receipts. +owner: + data: collection-manager + team: Dispatch +paths: + source: src + tests: tests + database: /collection-manager + state: /collection-manager +commands: + test: ./tooling/test + build: ./tooling/build + verify: ./tooling/verify + health: ./tooling/health +components: + - id: control-plane + kind: control-plane + source: src + capabilities: + read_local_data: true + mutate_data: true + collect: false + network: false + authentication: false + direct_delivery: false + long_running: true + - id: service + kind: service + source: src + capabilities: + read_local_data: true + mutate_data: true + collect: false + network: false + authentication: false + direct_delivery: false + long_running: true + service_units: + - integration/systemd/dispatch-collection-manager.service.in diff --git a/dsp/runtime/collection-manager/examples/collection-spec.example.json b/dsp/runtime/collection-manager/examples/collection-spec.example.json new file mode 100644 index 0000000..298bc73 --- /dev/null +++ b/dsp/runtime/collection-manager/examples/collection-spec.example.json @@ -0,0 +1,58 @@ +{ + "schemaVersion": 1, + "collectors": [ + { + "id": "example", + "version": "1.0.0", + "description": "Replace this with a real collector", + "command": "/absolute/path/to/owner-controlled-collector", + "sourceSchema": { + "type": "object", + "properties": { + "tenant": { "type": "string", "maxLength": 64 } + }, + "required": ["tenant"], + "additionalProperties": false + }, + "methods": { + "snapshot.full": { + "description": "Collect a complete validated snapshot", + "inputSchema": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": false + }, + "timeoutSeconds": 900, + "maxAttempts": 3, + "backoffSeconds": [60, 300], + "concurrencyKeys": ["auth:{authProfile}", "publish:example"] + } + } + } + ], + "sources": [ + { + "id": "example-main", + "collector": "example", + "authProfile": "example-main", + "config": { "tenant": "main" }, + "enabled": true + } + ], + "plans": [ + { + "id": "example-main-snapshot", + "source": "example-main", + "method": "snapshot.full", + "schedule": { + "type": "cron", + "expression": "0 3 * * *", + "timezone": "America/Los_Angeles" + }, + "input": {}, + "dependsOn": [], + "enabled": true + } + ] +} diff --git a/dsp/runtime/collection-manager/integration/systemd/dispatch-collection-manager.service.in b/dsp/runtime/collection-manager/integration/systemd/dispatch-collection-manager.service.in new file mode 100644 index 0000000..cfc5829 --- /dev/null +++ b/dsp/runtime/collection-manager/integration/systemd/dispatch-collection-manager.service.in @@ -0,0 +1,26 @@ +[Unit] +Description=Dispatch Collection Manager +After=local-fs.target dispatch-auth-broker.service +Wants=dispatch-auth-broker.service + +[Service] +Type=simple +WorkingDirectory=@PROJECT_ROOT@/runtime/collection-manager +Environment=DISPATCH_PROJECT_ROOT=@PROJECT_ROOT@ +Environment=DISPATCH_LOCAL_ROOT=@LOCAL_ROOT@ +Environment=NODE_NO_WARNINGS=1 +Environment=PATH=@COMMAND_PATH@ +ExecStart=@PROJECT_ROOT@/runtime/collection-manager/scripts/start +Restart=always +RestartSec=3 +KillMode=control-group +TimeoutStopSec=30 +UMask=0077 +NoNewPrivileges=true +RestrictSUIDSGID=true +LockPersonality=true +RestrictNamespaces=true +SystemCallArchitectures=native + +[Install] +WantedBy=default.target diff --git a/dsp/runtime/collection-manager/package.json b/dsp/runtime/collection-manager/package.json new file mode 100644 index 0000000..10f19ce --- /dev/null +++ b/dsp/runtime/collection-manager/package.json @@ -0,0 +1,15 @@ +{ + "name": "dispatch-collection-manager", + "version": "0.9.0", + "private": true, + "description": "Durable local control plane for Dispatch collectors", + "type": "commonjs", + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "./scripts/build", + "test": "./scripts/test", + "verify": "./scripts/verify" + } +} diff --git a/dsp/runtime/collection-manager/scripts/build b/dsp/runtime/collection-manager/scripts/build new file mode 100755 index 0000000..d14e226 --- /dev/null +++ b/dsp/runtime/collection-manager/scripts/build @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +shopt -s nullglob +files=("$ROOT"/src/*.js "$ROOT"/tests/*.js "$ROOT"/bin/*) +for file in "${files[@]}"; do + node --check "$file" >/dev/null +done +printf '%s\n' '{"ok":true,"status":"built"}' diff --git a/dsp/runtime/collection-manager/scripts/health b/dsp/runtime/collection-manager/scripts/health new file mode 100755 index 0000000..3ccdadc --- /dev/null +++ b/dsp/runtime/collection-manager/scripts/health @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +exec "$ROOT/bin/dispatch-collectionctl" status diff --git a/dsp/runtime/collection-manager/scripts/start b/dsp/runtime/collection-manager/scripts/start new file mode 100755 index 0000000..168fd46 --- /dev/null +++ b/dsp/runtime/collection-manager/scripts/start @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +export NODE_NO_WARNINGS=1 +exec "$ROOT/bin/dispatch-collection-manager" diff --git a/dsp/runtime/collection-manager/scripts/test b/dsp/runtime/collection-manager/scripts/test new file mode 100755 index 0000000..532f051 --- /dev/null +++ b/dsp/runtime/collection-manager/scripts/test @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +export NODE_NO_WARNINGS=1 +exec node --no-warnings --test --test-concurrency=1 "$ROOT"/tests/*.test.js diff --git a/dsp/runtime/collection-manager/scripts/verify b/dsp/runtime/collection-manager/scripts/verify new file mode 100755 index 0000000..6d94776 --- /dev/null +++ b/dsp/runtime/collection-manager/scripts/verify @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +export NODE_NO_WARNINGS=1 +"$ROOT/tooling/build" >/dev/null +"$ROOT/tooling/test" >/dev/null +exec "$ROOT/bin/dispatch-collectionctl" status diff --git a/dsp/runtime/collection-manager/src/capacity-runner.js b/dsp/runtime/collection-manager/src/capacity-runner.js new file mode 100644 index 0000000..08b4dab --- /dev/null +++ b/dsp/runtime/collection-manager/src/capacity-runner.js @@ -0,0 +1,96 @@ +'use strict'; +const net = require('node:net'); +const crypto = require('node:crypto'); +const { setTimeout: delay } = require('node:timers/promises'); +const { encodeFrame, attachFrameReader } = require('dispatch-protocol/agent/framing'); +const { validateCapacityRequest, validateCapacityResponse } = require('dispatch-protocol/agent/capacity'); +const { runCollector } = require('dispatch-runtime-kit/collection-manager/src/runner'); + +function queryCapacity(socketPath, input) { + return new Promise((resolve, reject) => { + const request = validateCapacityRequest({ type: 'capacity_request', requestId: crypto.randomBytes(16).toString('hex'), ...input }); + const socket = net.createConnection(socketPath); + let settled = false; + const finish = (error, response) => { + if (settled) return; + settled = true; + clearTimeout(timer); + socket.destroy(); + if (error) reject(Error('capacity_unavailable')); else resolve(response); + }; + const timer = setTimeout(() => finish(true), 3000); + socket.on('connect', () => socket.write(encodeFrame(request))); + socket.on('error', () => finish(true)); + socket.on('close', () => finish(true)); + attachFrameReader(socket, { maxFrameBytes: 4096, onError: () => finish(true), onFrame: value => { + try { + const response = validateCapacityResponse(value); + if (response.requestId !== request.requestId) throw Error('capacity_unavailable'); + finish(null, response); + } catch { finish(true); } + } }); + }); +} + +function runWithCapacity(run, { query, execute = runCollector, onState = () => {}, pollMs = 1000, renewMs = 10_000 } = {}) { + const jobId = crypto.createHash('sha256').update(run.id).digest('hex').slice(0, 32); + const workers = Math.max(1, Math.min(6, run.sourceConfig?.maxConcurrency || 1)); + const controller = new AbortController(); + const renewalController = new AbortController(); + let child = null; + let cancelled = false; + let lost = false; + const request = operation => query({ operation, jobId, workers }); + const sleep = (ms, signal) => delay(ms, undefined, { signal }).catch(() => {}); + const promise = (async () => { + let grant; + let outcome; + try { + onState('waiting_for_capacity'); + while (!cancelled) { + if (Number.isInteger(run.retry_deadline) && Date.now() >= run.retry_deadline) { + return { success: false, errorCode: 'polling_window_expired', exitCode: null }; + } + try { grant = await request('acquire'); } + catch { grant = null; } + if (cancelled) break; + if (grant?.status === 'granted') break; + await sleep(pollMs, controller.signal); + } + if (cancelled) return { success: false, cancelled: true, errorCode: 'cancelled', exitCode: null }; + onState(null); + child = execute({ ...run, sourceConfig: { ...run.sourceConfig, maxConcurrency: grant.workers } }); + const renewal = (async () => { + while (!renewalController.signal.aborted) { + await sleep(renewMs, renewalController.signal); + if (renewalController.signal.aborted) break; + try { + const result = await request('renew'); + if (result.status !== 'granted' || result.workers !== grant.workers) throw Error('capacity_lost'); + } catch { lost = true; child.cancel(); break; } + } + })(); + try { outcome = await child.promise; } + finally { renewalController.abort(); await renewal; } + return lost ? { success: false, errorCode: 'capacity_lost', exitCode: outcome.exitCode ?? null } : outcome; + } catch { + return { success: false, errorCode: 'collector_start_failed', exitCode: null }; + } finally { + renewalController.abort(); + // A kill failure retains its reservation until expiry instead of handing + // capacity to another DSP while a child may still be alive. + if (!child || outcome && outcome.errorCode !== 'collector_kill_failed') { try { await request('release'); } catch {} } + } + })(); + return { promise, cancel() { cancelled = true; controller.abort(); child?.cancel(); } }; +} +function coordinatedCollector(run, onState) { + // Core owns admission for installed plugin workers and their browser leases. + // Waiting on the legacy browser budget here would create a second gate. + if (process.env.DISPATCH_PLUGIN_BACKEND === 'core_v1') return runCollector(run); + // Every managed collector uses the host budget, including future plugins. + if (!['native_service_v1', 'directory_service_v1'].includes(process.env.DISPATCH_RUNTIME_BACKEND)) return runCollector(run); + const socket = process.env.DISPATCH_RUNTIME_AGENT_STATUS_SOCKET; + return runWithCapacity(run, { onState, query: input => queryCapacity(socket, input) }); +} +module.exports = { queryCapacity, runWithCapacity, coordinatedCollector }; diff --git a/dsp/runtime/collection-manager/src/control-cli.js b/dsp/runtime/collection-manager/src/control-cli.js new file mode 100644 index 0000000..cf2e988 --- /dev/null +++ b/dsp/runtime/collection-manager/src/control-cli.js @@ -0,0 +1,220 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { defaultPaths } = require('dispatch-runtime-kit/collection-manager/src/paths'); +const { parseStrictJson } = require('dispatch-runtime-kit/collection-manager/src/strict-json'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { CollectionManager } = require('./manager'); +const { StandardCollectionService } = require('dispatch-runtime-kit/collection-manager/src/standard-collections'); +const { SyncService } = require('dispatch-runtime-kit/collection-manager/src/syncs'); + +const SAFE_ERRORS = new Set([ + 'invalid_input', 'invalid_json', 'secret_field_forbidden', 'unsafe_storage', 'unsafe_collector', + 'collector_unavailable', 'collector_not_found', 'source_not_found', 'method_not_found', + 'dependency_not_found', 'dependency_cycle', 'plan_not_found', 'plan_disabled', 'run_not_found', 'run_not_cancellable', + 'run_not_retryable', 'manager_already_running', 'manager_lease_lost', 'drain_timeout', + 'input_not_found', 'input_unreadable', 'database_integrity_failed', 'schema_invalid', + 'collection_capabilities_not_found', 'unsupported_selector', 'unsupported_scope', 'invalid_selector', + 'invalid_collection_request', 'invalid_target_resolution', 'invalid_timezone', 'target_resolution_failed', 'range_too_large', + 'preview_changed', 'idempotency_conflict', 'audit_not_supported', 'batch_not_found', 'batch_not_retryable', 'schedule_not_found', 'invalid_schedule', + 'sync_not_found', 'sync_stopped', 'invalid_sync_state', 'sync_revision_conflict', 'sync_stop_timeout', + 'sync_plan_must_be_manual', 'sync_config_incompatible', 'unsupported_overlap_policy', + 'collection_manager_not_initialized', +]); + +const READ_ONLY_COMMANDS = new Set([ + 'status', 'collectors', 'collector', 'methods', 'sources', 'source', 'plans', 'plan', + 'runs', 'run-status', 'collection-describe', 'collection-preview', 'batches', 'batch', + 'collection-schedules', 'syncs', 'sync', 'sync-history', +]); + +function codedError(code) { + const error = new Error(code); + error.code = code; + return error; +} + +function readJsonFile(file, maxBytes = 262_144) { + const resolved = path.resolve(file); + let info; + try { info = fs.lstatSync(resolved); } + catch (error) { throw codedError(error?.code === 'ENOENT' ? 'input_not_found' : 'input_unreadable'); } + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() || info.size < 2 || info.size > maxBytes + || fs.realpathSync(resolved) !== resolved) throw codedError('invalid_input'); + try { return parseStrictJson(fs.readFileSync(resolved, 'utf8')); } + catch (error) { + if (error?.code === 'invalid_json') throw error; + throw codedError('input_unreadable'); + } +} + +const PROJECT_COMMAND_PREFIX = '${DISPATCH_PROJECT_ROOT}/'; +function materializeSpec(value, projectRoot) { + if (!value || typeof value !== 'object' || Array.isArray(value) || !Array.isArray(value.collectors) + || typeof projectRoot !== 'string' || !path.isAbsolute(projectRoot) || path.resolve(projectRoot) !== projectRoot) { + throw codedError('invalid_input'); + } + for (const collector of value.collectors) { + if (typeof collector?.command !== 'string' || !collector.command.startsWith(PROJECT_COMMAND_PREFIX)) continue; + const relative = collector.command.slice(PROJECT_COMMAND_PREFIX.length); + const resolved = path.resolve(projectRoot, relative); + if (!relative || resolved === projectRoot || !resolved.startsWith(`${projectRoot}${path.sep}`)) throw codedError('invalid_input'); + collector.command = resolved; + } + return value; +} + +function emit(write, ok, status, data = null) { + write(`${JSON.stringify({ ok, status, data })}\n`); +} + +function page(items, args) { + if (args.length > 2) throw codedError('invalid_input'); + const limit = args[0] === undefined ? 50 : Number(args[0]); + const offset = args[1] === undefined ? 0 : Number(args[1]); + if (!Number.isInteger(limit) || limit < 1 || limit > 100 || !Number.isInteger(offset) || offset < 0) throw codedError('invalid_input'); + return { items: items.slice(offset, offset + limit), total: items.length, limit, offset, hasMore: offset + limit < items.length }; +} + +const HELP = Object.freeze({ + commands: [ + 'status', 'collectors [limit] [offset]', 'collector ', 'methods [limit] [offset]', + 'sources [limit] [offset]', 'source ', 'plans [limit] [offset]', 'plan ', + 'runs [limit] [offset]', 'run-status ', 'run [input.json]', + 'pause ', 'resume ', 'cancel ', 'retry ', + 'apply ', 'drain [timeout-ms]', 'init', + 'collection-describe ', 'collection-preview ', + 'collection-enqueue [options.json]', 'batches [limit] [offset]', 'batch ', + 'cancel-batch ', 'retry-batch ', 'collection-schedules', 'put-collection-schedule ', + 'pause-collection-schedule ', 'resume-collection-schedule ', 'remove-collection-schedule ', 'run-collection-schedule ', + 'syncs [limit] [offset]', 'sync ', 'start-sync ', 'stop-sync ', 'restart-sync ', + 'run-sync ', 'edit-sync ', 'sync-history [limit] [offset]', + ], +}); + +async function main(argv = process.argv.slice(2), paths = defaultPaths(), write = chunk => process.stdout.write(chunk)) { + process.umask(0o077); + let store; + let manager; + try { + const [command, ...args] = argv; + if (command === 'help' && args.length === 0) { emit(write, true, 'help', HELP); return 0; } + if (command === 'status' && args.length === 0 && !fs.existsSync(paths.database)) { + emit(write, true, 'not_initialized', { + ok: true, status: 'not_initialized', schemaVersion: null, databaseIntegrity: 'not_initialized', + manager: { running: false, pid: null, heartbeatAt: null }, + counts: { collectors: 0, sources: 0, plans: 0, schedules: 0, batches: 0, syncs: 0, syncing: 0, queued: 0, running: 0, failed: 0 }, + syncAlerts: { total: 0, critical: 0, items: [], hasMore: false }, + }); + return 0; + } + if (READ_ONLY_COMMANDS.has(command) && !fs.existsSync(paths.database)) throw codedError('collection_manager_not_initialized'); + store = new CollectionStore(paths, { readOnly: READ_ONLY_COMMANDS.has(command) }); + if (command === 'init' && args.length === 0) emit(write, true, 'initialized', store.health()); + else if (command === 'apply' && args.length === 1) { + emit(write, true, 'applied', store.applySpec(materializeSpec(readJsonFile(args[0]), paths.projectRoot))); + } + else if (command === 'status' && args.length === 0) { + const health = store.health(); + emit(write, health.ok, health.status, health); + } + else if (command === 'collectors') emit(write, true, 'found', page(store.collectors(), args)); + else if (command === 'collector' && args.length === 1) emit(write, true, 'found', store.collector(args[0])); + else if (command === 'methods' && args.length >= 1 && args.length <= 3) emit(write, true, 'found', page(store.methods(args[0]), args.slice(1))); + else if (command === 'sources') emit(write, true, 'found', page(store.sources(), args)); + else if (command === 'source' && args.length === 1) emit(write, true, 'found', store.source(args[0])); + else if (command === 'plans') emit(write, true, 'found', page(store.plans(), args)); + else if (command === 'plan' && args.length === 1) emit(write, true, 'found', store.plan(args[0])); + else if (command === 'runs' && args.length <= 2) { + const limit = args[0] === undefined ? 50 : Number(args[0]); + const offset = args[1] === undefined ? 0 : Number(args[1]); + const items = store.runs(limit, offset); + const total = store.runCount(); + emit(write, true, 'found', { items, total, limit, offset, hasMore: offset + limit < total }); + } + else if (command === 'collection-describe' && args.length === 1) { + emit(write, true, 'found', new StandardCollectionService(store).describe(args[0])); + } + else if (command === 'collection-preview' && args.length === 1) { + emit(write, true, 'previewed', await new StandardCollectionService(store).preview(readJsonFile(args[0], 65_536))); + } + else if (command === 'collection-enqueue' && (args.length === 1 || args.length === 2)) { + const options = args[1] ? readJsonFile(args[1], 16_384) : {}; + if (!options || typeof options !== 'object' || Array.isArray(options) + || Object.keys(options).some(key => !['expectedPreviewHash', 'idempotencyKey'].includes(key))) throw codedError('invalid_input'); + emit(write, true, 'queued', await new StandardCollectionService(store).enqueue(readJsonFile(args[0], 65_536), options)); + } + else if (command === 'batches' && args.length <= 2) { + const limit = args[0] === undefined ? 50 : Number(args[0]); + const offset = args[1] === undefined ? 0 : Number(args[1]); + const items = store.batches(limit, offset); const total = store.batchCount(); + emit(write, true, 'found', { items, total, limit, offset, hasMore: offset + limit < total }); + } + else if (command === 'batch' && args.length === 1) { const batch = store.batch(args[0]); emit(write, true, batch.status, batch); } + else if (command === 'cancel-batch' && args.length === 1) { const batch = store.cancelBatch(args[0]); emit(write, true, batch.status, batch); } + else if (command === 'retry-batch' && args.length === 1) { const batch = store.retryBatch(args[0]); emit(write, true, batch.status, batch); } + else if (command === 'collection-schedules' && args.length === 0) emit(write, true, 'found', { items: store.collectionSchedules() }); + else if (command === 'put-collection-schedule' && args.length === 1) emit(write, true, 'scheduled', new StandardCollectionService(store).putSchedule(readJsonFile(args[0], 65_536))); + else if (command === 'pause-collection-schedule' && args.length === 1) emit(write, true, 'paused', store.setCollectionScheduleEnabled(args[0], false)); + else if (command === 'resume-collection-schedule' && args.length === 1) emit(write, true, 'resumed', store.setCollectionScheduleEnabled(args[0], true)); + else if (command === 'remove-collection-schedule' && args.length === 1) emit(write, true, 'removed', store.removeCollectionSchedule(args[0])); + else if (command === 'run-collection-schedule' && args.length === 1) emit(write, true, 'queued', await new StandardCollectionService(store).runScheduleNow(args[0])); + else if (command === 'syncs' && args.length <= 2) { + const limit = args[0] === undefined ? 50 : Number(args[0]); + const offset = args[1] === undefined ? 0 : Number(args[1]); + const items = store.syncs(limit, offset); const total = store.syncCount(); + emit(write, true, 'found', { items, total, limit, offset, hasMore: offset + items.length < total }); + } + else if (command === 'sync' && args.length === 1) emit(write, true, 'found', store.sync(args[0])); + else if (command === 'start-sync' && args.length === 1) emit(write, true, 'started', new SyncService(store).start(args[0])); + else if (command === 'stop-sync' && args.length === 1) emit(write, true, 'stopped', await new SyncService(store).stop(args[0])); + else if (command === 'restart-sync' && args.length === 1) emit(write, true, 'restarted', await new SyncService(store).restart(args[0])); + else if (command === 'run-sync' && args.length === 1) emit(write, true, 'queued', new SyncService(store).runNow(args[0])); + else if (command === 'edit-sync' && args.length === 2) emit(write, true, 'updated', await new SyncService(store).edit(args[0], readJsonFile(args[1], 16_384))); + else if (command === 'sync-history' && args.length >= 1 && args.length <= 3) { + const limit = args[1] === undefined ? 50 : Number(args[1]); + const offset = args[2] === undefined ? 0 : Number(args[2]); + emit(write, true, 'found', store.syncHistory(args[0], limit, offset)); + } + else if (command === 'run-status' && args.length === 1) { + const run = store.run(args[0]); + emit(write, true, run.status, run); + } + else if (command === 'run' && (args.length === 1 || args.length === 2)) { + const input = args[1] ? readJsonFile(args[1], 65_536) : {}; + emit(write, true, 'queued', store.enqueuePlan(args[0], { input })); + } else if (command === 'pause' && args.length === 1) emit(write, true, 'paused', store.setPlanEnabled(args[0], false)); + else if (command === 'resume' && args.length === 1) emit(write, true, 'resumed', store.setPlanEnabled(args[0], true)); + else if (command === 'cancel' && args.length === 1) { + const run = store.cancel(args[0]); + emit(write, true, run.status, run); + } + else if (command === 'retry' && args.length === 1) emit(write, true, 'queued', store.retry(args[0])); + else if (command === 'drain' && args.length <= 1) { + const timeoutMs = args[0] === undefined ? 30_000 : Number(args[0]); + if (!Number.isInteger(timeoutMs) || timeoutMs < 100 || timeoutMs > 3_600_000) throw codedError('invalid_input'); + manager = new CollectionManager(store); + await manager.start(); + const result = await manager.runUntilIdle({ timeoutMs }); + const stopped = await manager.stop(); + const data = { ...result, ...stopped, health: store.health() }; + if (result.timedOut) { + emit(write, false, 'drain_timeout', data); + return 1; + } + emit(write, true, result.deferred ? 'deferred' : 'idle', data); + } else throw codedError('invalid_input'); + return 0; + } catch (error) { + try { await manager?.stop(); } catch {} + const code = SAFE_ERRORS.has(error?.code) ? error.code : SAFE_ERRORS.has(error?.message) ? error.message : 'internal_error'; + emit(write, false, code); + return ['invalid_input', 'invalid_json', 'input_not_found', 'input_unreadable'].includes(code) ? 2 : 1; + } finally { + store?.close(); + } +} + +if (require.main === module) main().then(code => { process.exitCode = code; }); +module.exports = { main, readJsonFile, materializeSpec, PROJECT_COMMAND_PREFIX }; diff --git a/dsp/runtime/collection-manager/src/cron.js b/dsp/runtime/collection-manager/src/cron.js new file mode 100644 index 0000000..ef6e087 --- /dev/null +++ b/dsp/runtime/collection-manager/src/cron.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/cron'); diff --git a/dsp/runtime/collection-manager/src/daemon-cli.js b/dsp/runtime/collection-manager/src/daemon-cli.js new file mode 100644 index 0000000..eb4833f --- /dev/null +++ b/dsp/runtime/collection-manager/src/daemon-cli.js @@ -0,0 +1,35 @@ +'use strict'; + +const { defaultPaths } = require('dispatch-runtime-kit/collection-manager/src/paths'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { CollectionManager } = require('./manager'); + +async function main() { + process.umask(0o077); + let store; + let manager; + try { + store = new CollectionStore(defaultPaths()); + manager = new CollectionManager(store); + await manager.start(); + } catch { + try { store?.close(); } catch {} + process.stderr.write('dispatch collection manager: unavailable\n'); + return 1; + } + let stopping = false; + const shutdown = async code => { + if (stopping) return; + stopping = true; + try { await manager.stop(); } finally { store.close(); } + process.exit(code); + }; + process.once('SIGINT', () => shutdown(0)); + process.once('SIGTERM', () => shutdown(0)); + process.once('uncaughtException', () => shutdown(1)); + process.once('unhandledRejection', () => shutdown(1)); + return new Promise(() => {}); +} + +if (require.main === module) main().then(code => { if (Number.isInteger(code)) process.exitCode = code; }); +module.exports = { main }; diff --git a/dsp/runtime/collection-manager/src/execution-control.js b/dsp/runtime/collection-manager/src/execution-control.js new file mode 100644 index 0000000..343ad29 --- /dev/null +++ b/dsp/runtime/collection-manager/src/execution-control.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/execution-control'); diff --git a/dsp/runtime/collection-manager/src/manager.js b/dsp/runtime/collection-manager/src/manager.js new file mode 100644 index 0000000..1e4f25d --- /dev/null +++ b/dsp/runtime/collection-manager/src/manager.js @@ -0,0 +1,303 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { cronMatches, zonedParts } = require('dispatch-runtime-kit/collection-manager/src/cron'); +const { coordinatedCollector } = require('./capacity-runner'); +const { StandardCollectionService } = require('dispatch-runtime-kit/collection-manager/src/standard-collections'); +const { SyncService } = require('dispatch-runtime-kit/collection-manager/src/syncs'); + +function materializeLocks(run) { + const replacements = { + '{source}': run.source_id, + '{authProfile}': run.auth_profile || 'none', + '{collector}': run.collector_id, + '{method}': run.method_id, + }; + const keys = [`source:${run.source_id}`]; + for (const template of JSON.parse(run.concurrency_keys_json)) { + let key = template; + for (const [needle, value] of Object.entries(replacements)) key = key.replaceAll(needle, value); + keys.push(key); + } + return [...new Set(keys)].sort(); +} + +function activePollingWindow(schedule, timestamp) { + const currentMinute = Math.floor(timestamp / 60_000) * 60_000; + const minuteCount = Math.ceil(schedule.windowSeconds / 60); + for (let offset = 0; offset < minuteCount; offset += 1) { + const startedAt = currentMinute - offset * 60_000; + if (!cronMatches(schedule.expression, schedule.timezone, new Date(startedAt))) continue; + const deadline = startedAt + schedule.windowSeconds * 1000; + if (timestamp >= deadline) return null; + return { startedAt, deadline, key: zonedParts(new Date(startedAt), schedule.timezone).key }; + } + return null; +} + +class CollectionManager { + constructor(store, { + maxWorkers = 4, tickMs = 500, leaseMs = 60_000, + collectionService = null, syncService = null, + } = {}) { + if (!Number.isInteger(maxWorkers) || maxWorkers < 1 || maxWorkers > 32) throw new Error('invalid_workers'); + this.store = store; + this.maxWorkers = maxWorkers; + this.tickMs = tickMs; + this.leaseMs = leaseMs; + this.collectionService = collectionService || new StandardCollectionService(store); + this.syncService = syncService || new SyncService(store); + this.instanceId = crypto.randomUUID(); + this.epoch = null; + this.active = new Map(); + this.timer = null; + this.started = false; + this.stopping = false; + this.ticking = false; + this.collectionScheduleWindows = new Map(); + this.collectionPollingStates = new Map(); + this.schedulerController = null; + this.leaseTimer = null; + this.leaseLossPromise = null; + } + + fence() { return { instanceId: this.instanceId, epoch: this.epoch }; } + + schedule(timestamp = Date.now()) { + for (const plan of this.store.schedulablePlans()) { + try { + if (plan.schedule.type === 'interval' && plan.nextDueAt !== null && plan.nextDueAt <= timestamp) { + this.store.enqueuePlan(plan.id, { + trigger: 'interval', timestamp, + logicalKey: `${plan.id}:interval:${plan.nextDueAt}`, + }); + this.store.setNextDue(plan.id, timestamp + plan.schedule.seconds * 1000); + } else if (plan.schedule.type === 'cron' && cronMatches(plan.schedule.expression, plan.schedule.timezone, new Date(timestamp))) { + const minute = zonedParts(new Date(timestamp), plan.schedule.timezone).key; + this.store.enqueuePlan(plan.id, { trigger: 'cron', timestamp, logicalKey: `${plan.id}:cron:${minute}` }); + } + } catch (error) { + if (error?.code !== 'plan_disabled') throw error; + } + } + } + + async scheduleCollections(timestamp = Date.now(), signal = this.schedulerController?.signal || null) { + for (const schedule of this.store.schedulableCollectionSchedules()) { + if (schedule.schedule.type === 'interval' && schedule.nextDueAt !== null && schedule.nextDueAt <= timestamp) { + const due = schedule.nextDueAt; + if (this.collectionScheduleWindows.get(schedule.id) === due) continue; + this.collectionScheduleWindows.set(schedule.id, due); + try { await this.collectionService.fireSchedule(schedule, timestamp, due, { signal }); } + catch { /* A failed resolver must not block unrelated schedules or queued work. */ } + finally { this.store.setCollectionScheduleNextDue(schedule.id, timestamp + schedule.schedule.seconds * 1000); } + } else if (schedule.schedule.type === 'polling-window') { + const checkedMinute = Math.floor(timestamp / 60_000); + const prior = this.collectionPollingStates.get(schedule.id); + if (prior?.done && timestamp < prior.deadline) continue; + if (prior?.checkedMinute === checkedMinute && prior.window === null) continue; + const window = prior?.window && timestamp < prior.window.deadline + ? prior.window : activePollingWindow(schedule.schedule, timestamp); + if (!window) { + this.collectionPollingStates.set(schedule.id, { checkedMinute, window: null }); + continue; + } + if (prior?.window?.key === window.key && prior.nextTryAt > timestamp) continue; + try { + await this.collectionService.fireSchedule(schedule, window.startedAt, window.key, { signal }); + this.collectionPollingStates.set(schedule.id, { + checkedMinute, window, deadline: window.deadline, done: true, + }); + } catch { + this.collectionPollingStates.set(schedule.id, { + checkedMinute, window, deadline: window.deadline, done: false, + nextTryAt: Math.min(window.deadline, + timestamp + schedule.schedule.intervalSeconds * 1000), + }); + } + } else if (schedule.schedule.type === 'cron' + && cronMatches(schedule.schedule.expression, schedule.schedule.timezone, new Date(timestamp))) { + const minute = zonedParts(new Date(timestamp), schedule.schedule.timezone).key; + if (this.collectionScheduleWindows.get(schedule.id) === minute) continue; + this.collectionScheduleWindows.set(schedule.id, minute); + try { await this.collectionService.fireSchedule(schedule, timestamp, minute, { signal }); } + catch { /* A failed resolver is retried only in a later matching window. */ } + } + } + } + + async start() { + if (this.started) return; + const timestamp = Date.now(); + this.epoch = this.store.claimManager(this.instanceId, process.pid, timestamp, this.leaseMs); + this.store.recoverRunning(timestamp); + this.schedulerController = new AbortController(); + this.started = true; + this.leaseTimer = setInterval(() => { + if (!this.started) return; + try { + this.store.renewManager(this.instanceId, process.pid, this.epoch, Date.now(), this.leaseMs); + } catch (error) { + if (error?.code === 'manager_lease_lost' && !this.leaseLossPromise) { + this.leaseLossPromise = this._loseLease().catch(() => {}); + } + } + }, Math.max(5, Math.floor(this.leaseMs / 3))); + this.leaseTimer.unref?.(); + await this.tick(); + if (this.started && !this.stopping) { + this.timer = setInterval(() => { this.tick().catch(() => {}); }, this.tickMs); + } + } + + _completeRun(id, outcome) { + try { + this.store.finishRun(id, outcome, Date.now(), this.fence()); + } catch (error) { + if (!['manager_lease_lost', 'run_not_running'].includes(error?.code)) throw error; + } finally { + this.active.delete(id); + } + } + + async _loseLease() { + clearInterval(this.timer); + clearInterval(this.leaseTimer); + this.stopping = true; + this.schedulerController?.abort(); + for (const task of this.active.values()) task.cancel(); + await Promise.allSettled([...this.active.values()].map(task => task.promise)); + this.active.clear(); + this.started = false; + this.stopping = false; + } + + _queuePages(timestamp, callback) { + let cursor = null; + while (true) { + const page = this.store.queued(timestamp, 100, cursor); + if (page.length === 0) return false; + for (const run of page) { + cursor = run; + if (callback(run) === true) return true; + } + if (page.length < 100) return false; + } + } + + async tick() { + if (!this.started || this.stopping || this.ticking) return; + this.ticking = true; + try { + const timestamp = Date.now(); + this.store.renewManager(this.instanceId, process.pid, this.epoch, timestamp, this.leaseMs); + const execution = require('./execution-control'); + const control = execution.read(this.store.db); + for (const id of this.store.cancelRequestedRuns()) this.active.get(id)?.cancel(); + if (control?.draining) { execution.acknowledge(this.store, control.generation); return; } + const scheduleAt = control ? control.requestedAt : timestamp; + if (!control || scheduleAt !== null && (control.completedAt === null || scheduleAt > control.completedAt)) { + this.schedule(scheduleAt); + await this.scheduleCollections(scheduleAt, this.schedulerController?.signal || null); + this.syncService.schedule(scheduleAt); + if (control) execution.complete(this.store, scheduleAt); + } + if (this.stopping || this.schedulerController?.signal.aborted) return; + this.store.renewManager(this.instanceId, process.pid, this.epoch, Date.now(), this.leaseMs); + // A drain may have arrived while an asynchronous target resolver ran. + const afterScheduling = execution.read(this.store.db); + if (afterScheduling?.draining) { execution.acknowledge(this.store, afterScheduling.generation); return; } + for (const id of this.store.cancelRequestedRuns()) this.active.get(id)?.cancel(); + if (this.active.size >= this.maxWorkers) return; + this._queuePages(timestamp, queued => { + if (this.active.size >= this.maxWorkers) return true; + if (queued.retry_deadline !== null && queued.retry_deadline <= timestamp) { + this.store.expirePollingRun(queued.id, timestamp); + return false; + } + const dependency = this.store.dependencyStatus(queued.plan_id, timestamp, queued.id); + if (!dependency.ready) { + if (dependency.terminal) { + this.store.failDependency(queued.id, timestamp); + return false; + } + this.store.setBlocked(queued.id, dependency.reason); + return false; + } + const plan = this.store.loadPlan(queued.plan_id); + const runForLocks = { ...queued, concurrency_keys_json: plan.concurrency_keys_json }; + let claimed = false; + try { claimed = this.store.claimRun(queued.id, materializeLocks(runForLocks), timestamp, this.fence()); } + catch (error) { if (error?.code !== 'lock_busy') throw error; } + if (!claimed) return false; + const execution = this.store.execution(queued.id); + let task; + try { + task = coordinatedCollector(execution, state => this.store.setCapacityWait(queued.id, state, this.fence())); + } catch (error) { + const code = ['collector_unavailable', 'unsafe_collector'].includes(error?.code) ? error.code : 'collector_start_failed'; + this._completeRun(queued.id, { success: false, errorCode: code, exitCode: null }); + return false; + } + this.active.set(queued.id, task); + task.promise.then( + outcome => this._completeRun(queued.id, outcome), + () => this._completeRun(queued.id, { success: false, errorCode: 'manager_internal_error', exitCode: null }), + ); + return false; + }); + } catch (error) { + if (error?.code === 'manager_lease_lost') await this._loseLease(); + throw error; + } finally { + this.ticking = false; + } + } + + hasRunnableWork(timestamp = Date.now()) { + let found = false; + this._queuePages(timestamp, run => { + if (this.store.dependencyStatus(run.plan_id, timestamp, run.id).ready) { + found = true; + return true; + } + return false; + }); + return found; + } + + async runUntilIdle({ timeoutMs = 30_000 } = {}) { + if (!this.started) await this.start(); + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + await this.tick(); + if (this.active.size === 0 && !this.hasRunnableWork()) { + const pending = this.store.pendingSummary(); + return pending.total === 0 ? { idle: true, pending } : { idle: false, deferred: true, pending }; + } + await new Promise(resolve => setTimeout(resolve, 25)); + } + return { idle: false, timedOut: true, activeRunIds: [...this.active.keys()], pending: this.store.pendingSummary() }; + } + + async stop() { + if (!this.started) return { cancelledRunIds: [] }; + this.stopping = true; + this.schedulerController?.abort(); + clearInterval(this.timer); + const cancelled = new Set(this.active.keys()); + for (const task of this.active.values()) task.cancel(); + while (this.ticking) await new Promise(resolve => setTimeout(resolve, 10)); + for (const id of this.active.keys()) cancelled.add(id); + for (const task of this.active.values()) task.cancel(); + await Promise.allSettled([...this.active.values()].map(task => task.promise)); + clearInterval(this.leaseTimer); + this.store.releaseManager(this.instanceId, this.epoch); + this.active.clear(); + this.started = false; + this.stopping = false; + return { cancelledRunIds: [...cancelled] }; + } +} + +module.exports = { CollectionManager, materializeLocks, activePollingWindow }; diff --git a/dsp/runtime/collection-manager/src/next-wake.js b/dsp/runtime/collection-manager/src/next-wake.js new file mode 100644 index 0000000..73c908c --- /dev/null +++ b/dsp/runtime/collection-manager/src/next-wake.js @@ -0,0 +1,54 @@ +'use strict'; + +const { parseCron } = require('dispatch-runtime-kit/collection-manager/src/cron'); +const cache = new Map(); +async function nextCron(expression, timezone, after) { + const key = `${expression}\n${timezone}`; + const cached = cache.get(key); + if (cached && after >= cached.after && after < cached.next) return cached.next; + const cron = parseCron(expression); + const formatter = new Intl.DateTimeFormat('en-CA', { timeZone: timezone, year: 'numeric', month: '2-digit', day: '2-digit', + hour: '2-digit', minute: '2-digit', weekday: 'short', hourCycle: 'h23' }); + // UTC-minute iteration preserves both sides of DST transitions. Cache only + // the next occurrence, with a bounded catalog; never allocate a timer per DSP. + let next = Math.floor(after / 60000) * 60000 + 60000; + for (let step = 0; step < 366 * 24 * 60 * 8; step++, next += 60000) { + if (step % 1000 === 0) await new Promise(resolve => setImmediate(resolve)); + const p = Object.fromEntries(formatter.formatToParts(new Date(next)).map(part => [part.type, part.value])); + if (!cron.month.has(Number(p.month)) || !cron.hour.has(Number(p.hour)) || !cron.minute.has(Number(p.minute))) continue; + const day = cron.day.has(Number(p.day)), weekday = cron.weekday.has(['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'].indexOf(p.weekday)); + if (!(cron.dayWildcard && cron.weekdayWildcard || cron.dayWildcard && weekday || cron.weekdayWildcard && day + || !cron.dayWildcard && !cron.weekdayWildcard && (day || weekday))) continue; + if (cache.size >= 64) cache.delete(cache.keys().next().value); + cache.set(key, { after, next }); return next; + } + throw new Error('schedule_next_occurrence_unavailable'); +} + +async function nextWake(store, timestamp = Date.now()) { + const times = []; + const add = value => { if (Number.isSafeInteger(value)) times.push(value); }; + const queued = store.db.prepare("SELECT min(run_after) due FROM runs WHERE status='queued' AND cancel_requested=0").get(); add(queued.due); + const scheduling = require('./execution-control').read(store.db); + const after = Math.min(timestamp, scheduling?.completedAt ?? timestamp); + for (const plan of store.schedulablePlans()) { + if (plan.schedule.type === 'interval') add(plan.nextDueAt); + else if (plan.schedule.type === 'cron') add(await nextCron(plan.schedule.expression, plan.schedule.timezone, after)); + } + for (const schedule of store.schedulableCollectionSchedules()) { + if (schedule.schedule.type === 'interval') add(schedule.nextDueAt); + else { + if (schedule.schedule.type === 'polling-window') { + const window = require('./manager').activePollingWindow(schedule.schedule, timestamp); + if (window) add(Math.min(window.deadline, timestamp + schedule.schedule.intervalSeconds * 1000)); + } + add(await nextCron(schedule.schedule.expression, schedule.schedule.timezone, after)); + } + } + // Match the manager's plugin/source/plan gates instead of waking disabled work. + const sync = store.db.prepare(`SELECT min(d.next_due_at) due FROM sync_definitions d + WHERE d.desired_state='running' AND NOT EXISTS(SELECT 1 FROM sync_runs s JOIN runs r ON r.id=s.run_id + WHERE s.sync_id=d.id AND r.status IN ('queued','running'))`).get(); add(sync.due); + return times.length ? Math.min(...times) : null; +} +module.exports = { nextWake, nextCron }; diff --git a/dsp/runtime/collection-manager/src/paths.js b/dsp/runtime/collection-manager/src/paths.js new file mode 100644 index 0000000..21358a5 --- /dev/null +++ b/dsp/runtime/collection-manager/src/paths.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/paths'); diff --git a/dsp/runtime/collection-manager/src/plugin-state.js b/dsp/runtime/collection-manager/src/plugin-state.js new file mode 100644 index 0000000..9e2b85a --- /dev/null +++ b/dsp/runtime/collection-manager/src/plugin-state.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/plugin-state'); diff --git a/dsp/runtime/collection-manager/src/runner.js b/dsp/runtime/collection-manager/src/runner.js new file mode 100644 index 0000000..6117f9d --- /dev/null +++ b/dsp/runtime/collection-manager/src/runner.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/runner'); diff --git a/dsp/runtime/collection-manager/src/schema.js b/dsp/runtime/collection-manager/src/schema.js new file mode 100644 index 0000000..76a94d6 --- /dev/null +++ b/dsp/runtime/collection-manager/src/schema.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/schema'); diff --git a/dsp/runtime/collection-manager/src/standard-collections.js b/dsp/runtime/collection-manager/src/standard-collections.js new file mode 100644 index 0000000..2eb8637 --- /dev/null +++ b/dsp/runtime/collection-manager/src/standard-collections.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/standard-collections'); diff --git a/dsp/runtime/collection-manager/src/store-error.js b/dsp/runtime/collection-manager/src/store-error.js new file mode 100644 index 0000000..5a1d538 --- /dev/null +++ b/dsp/runtime/collection-manager/src/store-error.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/store-error'); diff --git a/dsp/runtime/collection-manager/src/store.js b/dsp/runtime/collection-manager/src/store.js new file mode 100644 index 0000000..21096b4 --- /dev/null +++ b/dsp/runtime/collection-manager/src/store.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/store'); diff --git a/dsp/runtime/collection-manager/src/strict-json.js b/dsp/runtime/collection-manager/src/strict-json.js new file mode 100644 index 0000000..dcd1f64 --- /dev/null +++ b/dsp/runtime/collection-manager/src/strict-json.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/strict-json'); diff --git a/dsp/runtime/collection-manager/src/syncs.js b/dsp/runtime/collection-manager/src/syncs.js new file mode 100644 index 0000000..0835bb0 --- /dev/null +++ b/dsp/runtime/collection-manager/src/syncs.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/syncs'); diff --git a/dsp/runtime/collection-manager/src/targeting.js b/dsp/runtime/collection-manager/src/targeting.js new file mode 100644 index 0000000..3dc9393 --- /dev/null +++ b/dsp/runtime/collection-manager/src/targeting.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/targeting'); diff --git a/dsp/runtime/collection-manager/src/validation.js b/dsp/runtime/collection-manager/src/validation.js new file mode 100644 index 0000000..813df12 --- /dev/null +++ b/dsp/runtime/collection-manager/src/validation.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/collection-manager/src/validation'); diff --git a/dsp/runtime/collection-manager/tests/capacity-runner.test.js b/dsp/runtime/collection-manager/tests/capacity-runner.test.js new file mode 100644 index 0000000..0f10471 --- /dev/null +++ b/dsp/runtime/collection-manager/tests/capacity-runner.test.js @@ -0,0 +1,79 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { runWithCapacity, coordinatedCollector } = require('../src/capacity-runner'); +const { deterministicJitter } = require('dispatch-runtime-kit/collection-manager/src/syncs'); +const run = { id: 'run_fixture', sourceConfig: { maxConcurrency: 6 } }; +const grant = { status: 'granted', workers: 2 }; +test('installed plugin collections reach Core admission without a legacy capacity socket', async t => { + const variables = ['DISPATCH_PLUGIN_BACKEND', 'DISPATCH_RUNTIME_BACKEND', 'DISPATCH_RUNTIME_AGENT_STATUS_SOCKET']; + const previous = Object.fromEntries(variables.map(key => [key, process.env[key]])); + t.after(() => { for (const key of variables) { + if (previous[key] === undefined) delete process.env[key]; else process.env[key] = previous[key]; + } }); + process.env.DISPATCH_PLUGIN_BACKEND = 'core_v1'; + process.env.DISPATCH_RUNTIME_BACKEND = 'directory_service_v1'; + process.env.DISPATCH_RUNTIME_AGENT_STATUS_SOCKET = '/nonexistent/legacy-capacity.sock'; + let requested = false; + t.mock.method(require('dispatch-sdk/runtime'), 'createFrameworkClient', () => ({ + request: async (operation, input) => { + requested = true; + assert.equal(operation, 'plugin.collect'); + assert.equal(input.pluginId, 'paycom'); + assert.equal(input.request.runId, 'run_fixture'); + assert.equal(input.request.source.authProfile, 'paycom-main'); + return { ok: true, status: 'no_change', data: { changeCount: 0 } }; + }, + })); + const states = []; + const task = coordinatedCollector({ ...run, collector_id: 'paycom', source_id: 'paycom-main', + auth_profile: 'paycom-main', plan_id: 'paycom-current-workforce-sync', + method_id: 'sync.current-workforce', input: {}, attempt: 1, timeout_seconds: 5 }, state => states.push(state)); + const timer = setTimeout(() => task.cancel(), 500); + try { + assert.equal((await task.promise).success, true); + assert.equal(requested, true); + assert.deepEqual(states, []); + } finally { clearTimeout(timer); } +}); +test('collection waits, respects the granted worker count, and releases after completion', async () => { + const events = []; + let polls = 0; + const task = runWithCapacity(run, { pollMs: 1, query: async r => { + events.push(r.operation); + return r.operation === 'acquire' && ++polls === 1 ? { status: 'waiting' } : grant; + }, onState: s => events.push(s), execute: selected => { + assert.equal(selected.sourceConfig.maxConcurrency, 2); + events.push('execute'); + return { promise: Promise.resolve({ success: true }), cancel() {} }; + } }); + assert.equal((await task.promise).success, true); + assert.deepEqual(events, ['waiting_for_capacity', 'acquire', 'acquire', null, 'execute', 'release']); +}); +test('cancellation while waiting never starts a collector and removes the queue entry', async () => { + let released = false; + const task = runWithCapacity(run, { pollMs: 1, query: async r => { + released ||= r.operation === 'release'; return { status: 'waiting' }; + }, execute: () => { throw Error('must not execute'); } }); + task.cancel(); + assert.equal((await task.promise).cancelled, true); + assert.equal(released, true); +}); +test('lost renewal cancels the collector and waits for termination before releasing', async () => { + const events = []; + let finish; + const task = runWithCapacity(run, { renewMs: 1, query: async r => { + events.push(r.operation); + return r.operation === 'renew' ? { status: 'lost' } : grant; + }, execute: () => ({ promise: new Promise(resolve => { finish = resolve; }), cancel: () => { + events.push('cancel'); setTimeout(() => { events.push('terminated'); finish({ success: false }); }, 5); + } }) }); + assert.equal((await task.promise).errorCode, 'capacity_lost'); + assert.deepEqual(events, ['acquire', 'renew', 'cancel', 'terminated', 'release']); +}); +test('timing offsets differ across DSPs with identical schedule inputs and are reproducible', () => { + const offsets = Array.from({ length: 20 }, (_, i) => deterministicJitter('paycom-main-workforce', 1, 1000, 60, `dsp-${i}`)); + assert.ok(new Set(offsets).size > 10); + assert.ok(offsets.every(value => value >= 0 && value <= 60)); + assert.equal(offsets[0], deterministicJitter('paycom-main-workforce', 1, 1000, 60, 'dsp-0')); +}); diff --git a/dsp/runtime/collection-manager/tests/cli.test.js b/dsp/runtime/collection-manager/tests/cli.test.js new file mode 100644 index 0000000..7c9fa14 --- /dev/null +++ b/dsp/runtime/collection-manager/tests/cli.test.js @@ -0,0 +1,99 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const { main, materializeSpec } = require('../src/control-cli'); +const { fixture, spec } = require('./helpers'); + +async function call(argv, paths) { + let output = ''; + const code = await main(argv, paths, chunk => { output += String(chunk); }); + return { code, value: JSON.parse(output.trim()) }; +} + +test('status and help do not initialize missing Collection Manager storage', async () => { + const { root, paths } = fixture(); + try { + const status = await call(['status'], paths); + assert.equal(status.code, 0); + assert.equal(status.value.status, 'not_initialized'); + assert.equal(fs.existsSync(paths.databaseRoot), false); + assert.equal((await call(['help'], paths)).value.status, 'help'); + assert.equal(fs.existsSync(paths.databaseRoot), false); + } finally { fs.rmSync(root, { recursive: true, force: true }); } +}); + +test('manager specs materialize trusted project-relative collector commands', () => { + const value = { collectors: [{ command: '${DISPATCH_PROJECT_ROOT}/plugins/paycom/backend/bin/dispatch-paycom-collector' }] }; + assert.equal(materializeSpec(value, '/opt/dispatch').collectors[0].command, + '/opt/dispatch/plugins/paycom/backend/bin/dispatch-paycom-collector'); + assert.throws(() => materializeSpec({ collectors: [{ command: '${DISPATCH_PROJECT_ROOT}/../escape' }] }, '/opt/dispatch'), + error => error.code === 'invalid_input'); +}); + +test('agent CLI applies a spec, queues, drains, and inspects a real run', async () => { + const { root, paths } = fixture(); + const file = path.join(root, 'spec.json'); + fs.writeFileSync(file, JSON.stringify(spec()), { mode: 0o600 }); + try { + const applied = await call(['apply', file], paths); + assert.equal(applied.code, 0); + assert.equal(applied.value.status, 'applied'); + const collectors = await call(['collectors', '1', '0'], paths); + assert.equal(collectors.value.data.items[0].id, 'fixture'); + assert.equal(collectors.value.data.total, 1); + assert.equal(collectors.value.data.hasMore, false); + const plan = await call(['plan', 'fixture-snapshot'], paths); + assert.equal(plan.value.data.method, 'fixture.snapshot'); + const queued = await call(['run', 'fixture-snapshot'], paths); + assert.equal(queued.value.status, 'queued'); + const runId = queued.value.data.id; + const drained = await call(['drain', '5000'], paths); + assert.equal(drained.code, 0); + assert.equal(drained.value.status, 'idle'); + const completed = await call(['run-status', runId], paths); + assert.equal(completed.value.status, 'succeeded'); + assert.equal(completed.value.data.status, 'succeeded'); + const runs = await call(['runs', '1', '0'], paths); + assert.equal(runs.value.data.items[0].id, runId); + assert.equal(runs.value.data.total, 1); + const unknownMethods = await call(['methods', 'missing'], paths); + assert.equal(unknownMethods.value.status, 'collector_not_found'); + const missingInput = await call(['apply', path.join(root, 'missing.json')], paths); + assert.equal(missingInput.code, 2); + assert.equal(missingInput.value.status, 'input_not_found'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('drain timeout explicitly reports and cancels active runs', async () => { + const { root, paths } = fixture(); + const command = path.join(root, 'slow-collector'); + const file = path.join(root, 'slow-spec.json'); + fs.writeFileSync(command, "#!/usr/bin/env node\nprocess.stdin.resume(); setTimeout(() => {}, 30000);\n", { mode: 0o700 }); + const configured = spec(); + configured.collectors[0].command = command; + configured.collectors[0].methods = { + 'fixture.slow': { description: 'Slow', inputSchema: { type: 'object', properties: {}, required: [], additionalProperties: false }, timeoutSeconds: 30, maxAttempts: 1, backoffSeconds: [], concurrencyKeys: [] }, + }; + configured.plans = [ + { id: 'fixture-slow', source: 'fixture-main', method: 'fixture.slow', schedule: { type: 'manual' }, input: {}, dependsOn: [], enabled: true }, + ]; + configured.syncs = []; + fs.writeFileSync(file, JSON.stringify(configured), { mode: 0o600 }); + try { + assert.equal((await call(['apply', file], paths)).code, 0); + const queued = await call(['run', 'fixture-slow'], paths); + const drained = await call(['drain', '100'], paths); + assert.equal(drained.code, 1); + assert.equal(drained.value.status, 'drain_timeout'); + assert.deepEqual(drained.value.data.cancelledRunIds, [queued.value.data.id]); + const cancelled = await call(['run-status', queued.value.data.id], paths); + assert.equal(cancelled.value.status, 'cancelled'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/dsp/runtime/collection-manager/tests/execution.test.js b/dsp/runtime/collection-manager/tests/execution.test.js new file mode 100644 index 0000000..8a50f8a --- /dev/null +++ b/dsp/runtime/collection-manager/tests/execution.test.js @@ -0,0 +1,62 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const { fixture, spec } = require('./helpers'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { CollectionManager } = require('../src/manager'); +const control = require('../src/execution-control'); +const { nextWake, nextCron } = require('../src/next-wake'); + +test('Core owns the schedule clock after adoption, including process restart and replay', async t => { + const c = fixture(); t.after(() => fs.rmSync(c.root, { recursive: true, force: true })); + let store = new CollectionStore(c.paths); store.applySpec(spec()); + store.setNextDue('fixture-interval', Date.now() - 1000); + control.command(store, 'adopt', null); + let manager = new CollectionManager(store, { tickMs: 100000 }); await manager.start(); + assert.equal(store.runCount(), 0, 'startup cannot fire a schedule independently'); + const due = await nextWake(store); + control.command(store, 'tick', due); await manager.tick(); + assert.equal(store.runCount(), 1); + await manager.stop(); store.close(); + store = new CollectionStore(c.paths); manager = new CollectionManager(store, { tickMs: 100000 }); + await manager.start(); + control.command(store, 'tick', due); await manager.tick(); + assert.equal(store.runCount(), 1, 'repeated delivery of the same clock does not duplicate the run'); + assert.equal(control.read(store.db).completedAt, due); + await manager.stop(); store.close(); +}); + +test('drain waits for an asynchronous scheduling resolver and prevents new claims', async t => { + const c = fixture(); t.after(() => fs.rmSync(c.root, { recursive: true, force: true })); + const store = new CollectionStore(c.paths); store.applySpec(spec()); + let release, reached; + const waiting = new Promise(resolve => { reached = resolve; }); + const manager = new CollectionManager(store, { tickMs: 100000 }); + control.command(store, 'adopt', Date.now()); + manager.scheduleCollections = async () => { reached(); await new Promise(resolve => { release = resolve; }); }; + const starting = manager.start(); await waiting; + const draining = control.command(store, 'drain', null); + assert.equal(control.read(store.db).acknowledged, null); + release(); await starting; + assert.equal(control.read(store.db).acknowledged, draining.generation); + store.enqueuePlan('fixture-snapshot'); await manager.tick(); + assert.equal(store.db.prepare("SELECT count(*) n FROM runs WHERE status='running'").get().n, 0); + assert.equal((await nextWake(store)) <= Date.now(), true, 'queued work remains represented in the wake deadline'); + await manager.stop(); store.close(); +}); + +test('cron wake times preserve DST boundaries and weekly calendar rules', async () => { + assert.equal(new Date(await nextCron('30 1 * * *', 'America/Los_Angeles', Date.parse('2026-11-01T08:30:00Z'))).toISOString(), '2026-11-01T09:30:00.000Z'); + assert.equal(new Date(await nextCron('30 2 * * *', 'America/Los_Angeles', Date.parse('2026-03-08T08:00:00Z'))).toISOString(), '2026-03-09T09:30:00.000Z'); + assert.equal(new Date(await nextCron('0 10 * * 1', 'UTC', Date.parse('2026-09-11T00:00:00Z'))).toISOString(), '2026-09-14T10:00:00.000Z'); +}); + +test('damaged external-clock state fails closed instead of reverting to autonomous schedules', t => { + const c = fixture(); t.after(() => fs.rmSync(c.root, { recursive: true, force: true })); + const store = new CollectionStore(c.paths); t.after(() => store.close()); + store.db.prepare('INSERT INTO meta(key,value) VALUES(?,?)').run('core_execution_v1', '{"version":1}'); + assert.throws(() => control.read(store.db), /execution_control_invalid/); + assert.throws(() => control.command(store, 'adopt'), /execution_control_invalid/); + assert.equal(store.runCount(), 0); +}); diff --git a/dsp/runtime/collection-manager/tests/fixture-collector.js b/dsp/runtime/collection-manager/tests/fixture-collector.js new file mode 100755 index 0000000..fe85cc7 --- /dev/null +++ b/dsp/runtime/collection-manager/tests/fixture-collector.js @@ -0,0 +1,54 @@ +#!/usr/bin/env node +'use strict'; + +const chunks = []; +process.stdin.on('data', chunk => chunks.push(chunk)); +process.stdin.on('end', () => { + try { + const request = JSON.parse(Buffer.concat(chunks).toString('utf8')); + if (request.method === 'collection.resolve-targets') { + const input = request.input; + const date = input.date || input.start || input.key; + process.stdout.write(`${JSON.stringify({ ok: true, status: 'succeeded', data: { + targetType: 'day', targets: [{ key: date, start: date, end: input.end || date, values: { label: date } }], + } })}\n`); + return; + } + if (request.method === 'fixture.sync') { + if (request.input.behavior === 'sleep') { + setTimeout(() => process.stdout.write('{"ok":true,"status":"no_change","data":{"checked":true,"changeCount":0}}\n'), 10_000); + return; + } + const status = request.input.behavior === 'published' ? 'published' : 'no_change'; + process.stdout.write(`${JSON.stringify({ ok: true, status, data: { + checked: true, + businessDate: '2026-08-29', + businessTimezone: 'America/Los_Angeles', + changeCount: status === 'published' ? 1 : 0, + label: request.input.label || null, + } })}\n`); + return; + } + if (request.method === 'fixture.unstable' && request.attempt === 1) { + process.stdout.write('{"ok":false,"status":"failed","error":{"code":"temporary_failure"}}\n'); + return; + } + if (request.method === 'fixture.sleep') { + setTimeout(() => process.stdout.write('{"ok":true,"status":"succeeded","data":{"slept":true}}\n'), 10_000); + return; + } + process.stdout.write(`${JSON.stringify({ + ok: true, + status: 'succeeded', + data: { + runId: request.runId, + source: request.source.id, + method: request.method, + attempt: request.attempt, + label: request.input.label || null, + }, + })}\n`); + } catch { + process.stdout.write('{"ok":false,"status":"failed","error":{"code":"invalid_request"}}\n'); + } +}); diff --git a/dsp/runtime/collection-manager/tests/helpers.js b/dsp/runtime/collection-manager/tests/helpers.js new file mode 100644 index 0000000..200c6e6 --- /dev/null +++ b/dsp/runtime/collection-manager/tests/helpers.js @@ -0,0 +1,63 @@ +'use strict'; + +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { defaultPaths } = require('dispatch-runtime-kit/collection-manager/src/paths'); + +const FIXTURE_COLLECTOR = path.resolve(__dirname, "./fixture-collector.js"); +const EMPTY_SCHEMA = Object.freeze({ type: 'object', properties: {}, required: [], additionalProperties: false }); +const LABEL_SCHEMA = Object.freeze({ + type: 'object', + properties: { label: { type: 'string', maxLength: 64 } }, + required: [], + additionalProperties: false, +}); + +const SYNC_SCHEMA = Object.freeze({ + type: 'object', + properties: { + behavior: { type: 'string', enum: ['no_change', 'published', 'sleep'] }, + label: { type: 'string', maxLength: 64 }, + }, + required: ['behavior'], + additionalProperties: false, +}); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-collection-store-')); + fs.chmodSync(root, 0o700); + return { root, paths: defaultPaths({ databaseRoot: path.join(root, 'db'), stateRoot: path.join(root, 'state') }) }; +} + +function spec() { + return { + schemaVersion: 1, + collectors: [{ + id: 'fixture', version: '1.0.0', description: 'Test collector', command: FIXTURE_COLLECTOR, + sourceSchema: { + type: 'object', properties: { tenant: { type: 'string', maxLength: 64 } }, + required: ['tenant'], additionalProperties: false, + }, + methods: { + 'fixture.snapshot': { description: 'Snapshot', inputSchema: LABEL_SCHEMA, timeoutSeconds: 5, maxAttempts: 1, backoffSeconds: [], concurrencyKeys: ['collector:{collector}'] }, + 'fixture.unstable': { description: 'Retry once', inputSchema: EMPTY_SCHEMA, timeoutSeconds: 5, maxAttempts: 2, backoffSeconds: [0], concurrencyKeys: ['collector:{collector}'] }, + 'fixture.sync': { description: 'Sync one fixture pass', inputSchema: SYNC_SCHEMA, timeoutSeconds: 5, maxAttempts: 2, backoffSeconds: [0], concurrencyKeys: ['collector:{collector}'] }, + }, + }], + sources: [{ id: 'fixture-main', collector: 'fixture', authProfile: null, config: { tenant: 'main' }, enabled: true }], + plans: [ + { id: 'fixture-snapshot', source: 'fixture-main', method: 'fixture.snapshot', schedule: { type: 'manual' }, input: {}, dependsOn: [], enabled: true }, + { id: 'fixture-retry', source: 'fixture-main', method: 'fixture.unstable', schedule: { type: 'manual' }, input: {}, dependsOn: [], enabled: true }, + { id: 'fixture-interval', source: 'fixture-main', method: 'fixture.snapshot', schedule: { type: 'interval', seconds: 10 }, input: { label: 'scheduled' }, dependsOn: [], enabled: true }, + { id: 'fixture-sync-plan', source: 'fixture-main', method: 'fixture.sync', schedule: { type: 'manual' }, input: { behavior: 'no_change', label: 'sync' }, dependsOn: [], enabled: true }, + ], + syncs: [{ + id: 'fixture-main-sync', plan: 'fixture-sync-plan', intervalSeconds: 10, jitterSeconds: 0, + overlap: 'coalesce', settingsSchema: SYNC_SCHEMA, + settings: { behavior: 'no_change', label: 'sync' }, desiredState: 'stopped', + }], + }; +} + +module.exports = { fixture, spec, FIXTURE_COLLECTOR }; diff --git a/dsp/runtime/collection-manager/tests/manager.test.js b/dsp/runtime/collection-manager/tests/manager.test.js new file mode 100644 index 0000000..cd3ed58 --- /dev/null +++ b/dsp/runtime/collection-manager/tests/manager.test.js @@ -0,0 +1,180 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { CollectionManager } = require('../src/manager'); +const { cronMatches } = require('dispatch-runtime-kit/collection-manager/src/cron'); +const { fixture, spec } = require('./helpers'); + +test('cron schedules evaluate in the declared timezone', () => { + const instant = new Date('2026-08-25T22:50:00.000Z'); + assert.equal(cronMatches('50 15 * * *', 'America/Los_Angeles', instant), true); + assert.equal(cronMatches('50 14 * * *', 'America/Los_Angeles', instant), false); +}); + +test('manager executes a real collector subprocess and stores a bounded receipt', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + const manager = new CollectionManager(store, { tickMs: 20 }); + try { + store.applySpec(spec()); + const queued = store.enqueuePlan('fixture-snapshot', { input: { label: 'manual' } }); + await manager.start(); + await manager.runUntilIdle({ timeoutMs: 5_000 }); + const run = store.run(queued.id); + assert.equal(run.status, 'succeeded'); + assert.equal(run.attempt, 1); + assert.equal(run.receipt.data.label, 'manual'); + assert.equal(run.receipt.data.source, 'fixture-main'); + } finally { + await manager.stop(); + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('manager retries a transient collector failure and deduplicates scheduled windows', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + const manager = new CollectionManager(store, { tickMs: 20 }); + try { + store.applySpec(spec()); + const retry = store.enqueuePlan('fixture-retry'); + const interval = store.plans().find(plan => plan.id === 'fixture-interval'); + manager.schedule(interval.nextDueAt); + manager.schedule(interval.nextDueAt); + assert.equal(store.runs(20).filter(run => run.plan === 'fixture-interval').length, 1); + await manager.start(); + await manager.runUntilIdle({ timeoutMs: 5_000 }); + const completed = store.run(retry.id); + assert.equal(completed.status, 'succeeded'); + assert.equal(completed.attempt, 2); + assert.deepEqual(completed.attempts.map(value => ({ + attempt: value.attempt, status: value.status, error: value.error, + })), [ + { attempt: 1, status: 'failed', error: 'temporary_failure' }, + { attempt: 2, status: 'succeeded', error: null }, + ]); + assert.equal(completed.attemptHistoryComplete, true); + } finally { + await manager.stop(); + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('a collector removed after registration fails the run without stranding locks', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + const manager = new CollectionManager(store, { tickMs: 10, leaseMs: 2_000 }); + const localCommand = `${root}/collector`; + try { + fs.copyFileSync(spec().collectors[0].command, localCommand); + fs.chmodSync(localCommand, 0o700); + const configured = spec(); + configured.collectors[0].command = localCommand; + store.applySpec(configured); + const queued = store.enqueuePlan('fixture-snapshot'); + fs.rmSync(localCommand); + await manager.start(); + await manager.runUntilIdle({ timeoutMs: 2_000 }); + const completed = store.run(queued.id); + assert.equal(completed.status, 'failed'); + assert.equal(completed.error, 'collector_unavailable'); + } finally { + await manager.stop(); + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('queue scanning reaches runnable work after more than 100 blocked runs', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + const manager = new CollectionManager(store, { tickMs: 10 }); + try { + const configured = spec(); + configured.plans.push( + { id: 'fixture-never', source: 'fixture-main', method: 'fixture.snapshot', schedule: { type: 'manual' }, input: {}, dependsOn: [], enabled: true }, + { id: 'fixture-blocked', source: 'fixture-main', method: 'fixture.snapshot', schedule: { type: 'manual' }, input: {}, dependsOn: [{ plan: 'fixture-never', maxAgeSeconds: 60 }], enabled: true }, + { id: 'fixture-runnable', source: 'fixture-main', method: 'fixture.snapshot', schedule: { type: 'manual' }, input: { label: 'after-blocked' }, dependsOn: [], enabled: true }, + ); + store.applySpec(configured); + for (let index = 0; index < 101; index += 1) store.enqueuePlan('fixture-blocked'); + const runnable = store.enqueuePlan('fixture-runnable'); + await manager.start(); + const result = await manager.runUntilIdle({ timeoutMs: 5_000 }); + assert.equal(store.run(runnable.id).status, 'succeeded'); + assert.equal(result.deferred, true); + assert.equal(result.pending.total, 101); + } finally { + await manager.stop(); + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('manager restart requeues an interrupted run when retry budget remains', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + const manager = new CollectionManager(store, { tickMs: 10 }); + try { + store.applySpec(spec(), 1_000); + const run = store.enqueuePlan('fixture-retry', { timestamp: 1_000 }); + const epoch = store.claimManager('crashed-manager', 333, 1_000, 100); + store.claimRun(run.id, ['source:fixture-main'], 1_000, { instanceId: 'crashed-manager', epoch }); + store.releaseManager('crashed-manager', epoch); + await manager.start(); + await manager.runUntilIdle({ timeoutMs: 5_000 }); + const completed = store.run(run.id); + assert.equal(completed.status, 'succeeded'); + assert.equal(completed.attempt, 2); + assert.deepEqual(completed.attempts.map(value => ({ status: value.status, error: value.error })), [ + { status: 'interrupted', error: 'manager_restarted' }, + { status: 'succeeded', error: null }, + ]); + } finally { + await manager.stop(); + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('manager keeps its lease during scheduler work and stop waits for cancellation', async t => { + t.mock.timers.enable({ apis: ['Date', 'setInterval'], now: Date.now() }); + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + const manager = new CollectionManager(store, { tickMs: 10, leaseMs: 30 }); + let schedulerStarted; + const started = new Promise(resolve => { schedulerStarted = resolve; }); + let schedulerFinished = false; + manager.scheduleCollections = async (_timestamp, signal) => { + schedulerStarted(); + await new Promise((resolve, reject) => { + if (signal.aborted) { reject(Object.assign(new Error('cancelled'), { code: 'cancelled' })); return; } + signal.addEventListener('abort', () => reject(Object.assign(new Error('cancelled'), { code: 'cancelled' })), { once: true }); + }).finally(() => { schedulerFinished = true; }); + }; + try { + store.applySpec(spec()); + const starting = manager.start().catch(error => { + if (error?.code !== 'cancelled') throw error; + }); + await started; + // Advance beyond the original lease while the scheduler is still blocked. + // Each interval must renew it; CI event-loop delays must not expire this fixture. + for (let elapsed = 0; elapsed < 80; elapsed += 10) t.mock.timers.tick(10); + assert.equal(schedulerFinished, false); + assert.equal(store.health().manager.running, true); + const stopped = manager.stop(); + await Promise.all([starting, stopped]); + assert.equal(schedulerFinished, true); + assert.equal(store.health().manager.running, false); + } finally { + await manager.stop(); + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/dsp/runtime/collection-manager/tests/plugins.test.js b/dsp/runtime/collection-manager/tests/plugins.test.js new file mode 100644 index 0000000..e53b1a3 --- /dev/null +++ b/dsp/runtime/collection-manager/tests/plugins.test.js @@ -0,0 +1,53 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const test = require('node:test'); +const { fixture, spec } = require('./helpers'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { installation, applyState } = require('dispatch-runtime-kit/collection-manager/src/plugin-state'); +function command(state, revision) { return { command: 'apply', pluginId: 'paycom', version: '0.18.8', state, revision }; } +function definition() { + const value = spec(); value.collectors[0].id = 'paycom'; value.sources[0].collector = 'paycom'; + value.syncs[0].id = 'paycom-main-workforce'; value.syncs[0].desiredState = 'running'; + return value; +} +test('plugin disable blocks queued work, cancels running work, retains storage and restores schedules', t => { + const f = fixture(); const store = new CollectionStore(f.paths); + t.after(() => { store.close(); fs.rmSync(f.root, { recursive: true, force: true }); }); + const secret = path.join(f.root, 'private-credentials'); fs.writeFileSync(secret, 'synthetic retained ciphertext', { mode: 0o600 }); + assert.equal(installation(store.db, 'paycom').state, 'uninstalled'); + store.applySpec(definition()); + assert.throws(() => store.enqueuePlan('fixture-snapshot'), /plan_disabled/); + applyState(store, command('enabled', 1)); + const first = store.enqueuePlan('fixture-snapshot'); + const second = store.enqueuePlan('fixture-snapshot'); + const runId = first.id || first.run.id; + assert.equal(store.claimRun(runId, ['fixture:running']), true); + applyState(store, command('disabled', 2)); + assert.ok(store.cancelRequestedRuns().includes(runId)); + assert.equal(store.db.prepare('SELECT status FROM runs WHERE id=?').get(second.id || second.run.id).status, 'cancelled'); + assert.throws(() => store.enqueuePlan('fixture-snapshot'), /plan_disabled/); + assert.equal(store.db.prepare('SELECT desired_state FROM sync_definitions').get().desired_state, 'stopped'); + assert.equal(fs.readFileSync(secret, 'utf8'), 'synthetic retained ciphertext'); + assert.equal(applyState(store, command('disabled', 2)).revision, 2); + assert.throws(() => applyState(store, command('enabled', 1)), /plugin_revision_conflict/); + store.finishRun(runId, { success: false, errorCode: 'cancelled', exitCode: null }); + applyState(store, command('enabled', 3)); + assert.equal(store.db.prepare('SELECT desired_state FROM sync_definitions').get().desired_state, 'running'); + assert.equal(store.db.prepare('SELECT COUNT(*) count FROM runs').get().count, 2); + applyState(store, command('uninstalled', 4)); store.close(); + const reopened = new CollectionStore(f.paths); + assert.equal(installation(reopened.db, 'paycom').state, 'uninstalled'); reopened.close(); +}); + +test('runtime schema migration preserves registered collectors without enrolling an empty DSP', t => { + const a = fixture(), b = fixture(); let store = new CollectionStore(a.paths); + t.after(() => { fs.rmSync(a.root, { recursive: true, force: true }); fs.rmSync(b.root, { recursive: true, force: true }); }); + store.applySpec(definition()); + store.db.exec('DROP TABLE plugin_installations; PRAGMA user_version=5'); store.close(); + store = new CollectionStore(a.paths); + assert.equal(installation(store.db, 'paycom').state, 'enabled'); store.close(); + const empty = new CollectionStore(b.paths); + assert.equal(installation(empty.db, 'paycom').state, 'uninstalled'); empty.close(); +}); diff --git a/dsp/runtime/collection-manager/tests/read-snapshots.test.js b/dsp/runtime/collection-manager/tests/read-snapshots.test.js new file mode 100644 index 0000000..8d57c1c --- /dev/null +++ b/dsp/runtime/collection-manager/tests/read-snapshots.test.js @@ -0,0 +1,56 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { SyncService } = require('dispatch-runtime-kit/collection-manager/src/syncs'); +const { runView } = require('dispatch-runtime-kit/sdk/src/collection-client'); +const { actionView, syncView, historyView } = require('dispatch-runtime-kit/sdk/src/sync-client'); +const { fixture, spec } = require('./helpers'); + +for (const operation of ['run', 'sync', 'syncs', 'history', 'manual replay']) { + test(`${operation} reads a consistent snapshot when another connection claims the run`, () => { + const { root, paths } = fixture(); + const writer = new CollectionStore(paths); + let reader; + try { + writer.applySpec(spec()); + const queued = new SyncService(writer).start('fixture-main-sync').run; + reader = new CollectionStore(paths, { readOnly: true }); + const originalAttempts = reader.runAttempts.bind(reader); + let claimed = false; + let reads = 0; + // Commit at the exact boundary between reading a run and reading its attempts. + reader.runAttempts = id => { + if (++reads === (operation === 'manual replay' ? 2 : 1)) { + claimed = true; + assert.equal(writer.claimRun(queued.id, []), true); + } + return originalAttempts(id); + }; + let run; + if (operation === 'run') run = runView(reader.run(queued.id), { attempts: true }); + if (operation === 'sync') run = syncView(reader.sync('fixture-main-sync')).activeRun; + if (operation === 'syncs') run = syncView(reader.syncs()[0]).activeRun; + if (operation === 'history') run = historyView(reader.syncHistory('fixture-main-sync')).items[0].run; + if (operation === 'manual replay') { + run = actionView(new SyncService(reader).runNow('fixture-main-sync', { idempotencyKey: 'same-click' })).run; + } + assert.equal(claimed, true); + assert.equal(run.attemptHistoryComplete, true); + assert.equal(run.attempts.length, run.attempt); + const latest = runView(reader.run(queued.id), { attempts: true }); + assert.equal(latest.status, 'running'); + assert.equal(latest.attempt, 1); + assert.equal(latest.attempts.length, 1); + assert.throws(() => reader.run('missing'), { code: 'run_not_found' }); + assert.equal(reader.run(queued.id).attempt, 1, 'failed reads release their snapshot'); + writer.transaction(() => assert.equal(writer.run(queued.id).attempt, 1)); + } finally { + reader?.close(); + writer.close(); + fs.rmSync(root, { recursive: true, force: true }); + } + }); +} diff --git a/dsp/runtime/collection-manager/tests/runner.test.js b/dsp/runtime/collection-manager/tests/runner.test.js new file mode 100644 index 0000000..9898901 --- /dev/null +++ b/dsp/runtime/collection-manager/tests/runner.test.js @@ -0,0 +1,57 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); +const { runCollector } = require('dispatch-runtime-kit/collection-manager/src/runner'); +const { fixture } = require('./helpers'); + +test('collector timeout settles promptly and kills the collector process group', async () => { + const { root } = fixture(); + const command = `${root}/descendant-collector`; + const pidFile = `${root}/child.pid`; + fs.writeFileSync(command, `#!/bin/sh\nsleep 30 &\nprintf '%s' "$!" > '${pidFile}'\nwait\n`, { mode: 0o700 }); + try { + const started = Date.now(); + const task = runCollector({ + id: 'run_timeout', plan_id: 'timeout-plan', source_id: 'fixture-main', collector_id: 'fixture', + auth_profile: null, sourceConfig: {}, method_id: 'fixture.timeout', input: {}, attempt: 1, + timeout_seconds: 1, command, + }); + const outcome = await task.promise; + assert.equal(outcome.errorCode, 'collector_timeout'); + assert.ok(Date.now() - started < 3_000); + const pid = Number(fs.readFileSync(pidFile, 'utf8')); + assert.throws(() => process.kill(pid, 0), error => error.code === 'ESRCH'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('collector cancellation allows bounded cooperative cleanup before SIGKILL', async () => { + const { root } = fixture(); + const command = `${root}/cleanup-collector`; + const ready = `${root}/ready`; + const cleaned = `${root}/cleaned`; + // Register cleanup before signalling readiness. A shell can fork its sleep + // after the group signal, delaying its TERM trap until after the grace period. + fs.writeFileSync(command, `#!${process.execPath}\nconst fs = require('node:fs');\nprocess.once('SIGTERM', () => setTimeout(() => { fs.writeFileSync(${JSON.stringify(cleaned)}, ''); process.exit(0); }, 2000));\nfs.writeFileSync(${JSON.stringify(ready)}, '');\nsetInterval(() => {}, 30000);\n`, { mode: 0o700 }); + try { + const task = runCollector({ + id: 'run_cancel', plan_id: 'cancel-plan', source_id: 'fixture-main', collector_id: 'fixture', + auth_profile: null, sourceConfig: {}, method_id: 'fixture.cancel', input: {}, attempt: 1, + timeout_seconds: 30, command, + }); + for (let index = 0; index < 100 && !fs.existsSync(ready); index++) await new Promise(resolve => setTimeout(resolve, 10)); + assert.equal(fs.existsSync(ready), true); + const started = Date.now(); + task.cancel(); + const outcome = await task.promise; + assert.equal(outcome.cancelled, true); + assert.equal(fs.existsSync(cleaned), true); + assert.ok(Date.now() - started >= 1_800); + assert.ok(Date.now() - started < 6_000); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/dsp/runtime/collection-manager/tests/standard-collections.test.js b/dsp/runtime/collection-manager/tests/standard-collections.test.js new file mode 100644 index 0000000..f856738 --- /dev/null +++ b/dsp/runtime/collection-manager/tests/standard-collections.test.js @@ -0,0 +1,283 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { CollectionManager } = require('../src/manager'); +const { StandardCollectionService } = require('dispatch-runtime-kit/collection-manager/src/standard-collections'); +const { fixture, spec } = require('./helpers'); + +function configured() { + const value = spec(); + value.collectors[0].sourceSchema.properties.timezone = { type: 'string', maxLength: 64 }; + value.collectors[0].sourceSchema.required.push('timezone'); + value.sources[0].config.timezone = 'UTC'; + value.collectors[0].methods['collection.resolve-targets'] = { + description: 'Resolve dates', + inputSchema: { + type: 'object', + properties: { + selectorKind: { type: 'string', enum: ['date', 'latest-complete', 'date-range', 'exact-target'] }, + date: { type: 'string', maxLength: 10 }, start: { type: 'string', maxLength: 10 }, + end: { type: 'string', maxLength: 10 }, key: { type: 'string', maxLength: 128 }, + }, + required: ['selectorKind'], additionalProperties: false, + }, + timeoutSeconds: 5, maxAttempts: 1, backoffSeconds: [], concurrencyKeys: [], + }; + value.sources[0].collection = { + targetType: 'day', resolverMethod: 'collection.resolve-targets', + selectors: ['current', 'relative-date', 'date', 'date-range', 'exact-target'], targetFields: ['label'], + scopes: { + full: { + description: 'Two ordered fixture tasks', + tasks: [ + { id: 'first', plan: 'fixture-snapshot', input: {}, targetInput: { label: 'label' }, dependsOn: [] }, + { id: 'second', plan: 'fixture-snapshot', input: {}, targetInput: { label: 'label' }, dependsOn: ['first'] }, + ], + auditTasks: [ + { id: 'audit', plan: 'fixture-snapshot', input: {}, targetInput: { label: 'label' }, dependsOn: [] }, + ], + }, + }, + limits: { maxTargets: 10, maxRangeDays: 31 }, + }; + return value; +} + +test('standard preview resolves relative dates and queues a durable dependency batch', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(configured()); + const service = new StandardCollectionService(store, { clock: () => new Date('2026-08-26T12:00:00.000Z') }); + const request = { source: 'fixture-main', scope: 'full', selector: { kind: 'relative-date', value: 'yesterday' }, mode: 'ensure' }; + const controller = new AbortController(); + controller.abort(); + await assert.rejects(() => service.preview(request, { signal: controller.signal }), error => error.code === 'cancelled'); + const preview = await service.preview(request); + assert.equal(preview.normalizedSelector.date, '2026-08-25'); + assert.equal(preview.targetCount, 1); + assert.equal(preview.taskCount, 2); + assert.equal(preview.targets[0].key, '2026-08-25'); + const audit = await service.preview({ ...request, mode: 'verify' }); + assert.equal(audit.taskCount, 1); + assert.equal(audit.tasks[0].taskId, 'audit'); + const first = await service.enqueue(request, { expectedPreviewHash: preview.hash, idempotencyKey: 'fixture-yesterday' }); + const second = await service.enqueue(request, { expectedPreviewHash: preview.hash, idempotencyKey: 'fixture-yesterday' }); + assert.equal(first.id, second.id); + assert.equal(first.runCount, 2); + assert.equal(first.counts.queued, 2); + const summary = store.batches(1, 0)[0]; + assert.equal(Object.hasOwn(summary, 'runs'), false); + assert.equal(Object.hasOwn(summary, 'runPage'), false); + const firstPage = store.batchPage(first.id, 1, 0); + assert.equal(firstPage.runPage.items.length, 1); + assert.equal(firstPage.runPage.total, 2); + assert.equal(firstPage.runPage.hasMore, true); + assert.equal(store.batchPage(first.id, 1, 1).runPage.hasMore, false); + assert.equal(store.cancelBatch(first.id).status, 'cancelled'); + assert.equal(store.retryBatch(first.id).status, 'queued'); + const manager = new CollectionManager(store, { tickMs: 10 }); + await manager.start(); + await manager.runUntilIdle({ timeoutMs: 5_000 }); + await manager.stop(); + const completed = store.batch(first.id); + assert.equal(completed.status, 'succeeded'); + assert.equal(completed.counts.succeeded, 2); + assert.ok(completed.runs[1].run.startedAt >= completed.runs[0].run.finishedAt); + } finally { + store.close(); fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('standard collection schedules are durable and resolve relative selectors when fired', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(configured(), 1_000); + const service = new StandardCollectionService(store); + assert.throws(() => service.putSchedule({ + id: 'unsupported-scope', + request: { source: 'fixture-main', scope: 'missing', selector: { kind: 'current' }, mode: 'ensure' }, + schedule: { type: 'interval', seconds: 10 }, enabled: true, + }, 1_000), error => error.code === 'unsupported_scope'); + const schedule = service.putSchedule({ + id: 'fixture-current', + request: { source: 'fixture-main', scope: 'full', selector: { kind: 'current' }, mode: 'ensure' }, + schedule: { type: 'interval', seconds: 10 }, enabled: true, + }, 1_000); + assert.equal(schedule.nextDueAt, 11_000); + const manager = new CollectionManager(store); + await manager.scheduleCollections(11_000); + assert.equal(store.batchCount(), 1); + assert.equal(store.collectionSchedule('fixture-current').nextDueAt, 21_000); + assert.equal(store.setCollectionScheduleEnabled('fixture-current', false).enabled, false); + assert.equal(store.setCollectionScheduleEnabled('fixture-current', true).enabled, true); + assert.equal(store.removeCollectionSchedule('fixture-current').id, 'fixture-current'); + service.putSchedule({ + id: 'fixture-failing', + request: { source: 'fixture-main', scope: 'full', selector: { kind: 'current' }, mode: 'ensure' }, + schedule: { type: 'interval', seconds: 10 }, enabled: true, + }, 30_000); + const failing = new CollectionManager(store, { collectionService: { fireSchedule: async () => { throw new Error('resolver_failed'); } } }); + await failing.scheduleCollections(40_000); + assert.equal(store.collectionSchedule('fixture-failing').nextDueAt, 50_000); + assert.equal(store.batchCount(), 1); + store.removeCollectionSchedule('fixture-failing'); + assert.equal(store.collectionSchedules().length, 0); + } finally { + store.close(); fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('batch retry preserves succeeded targets and requeues only failed backfill work', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(spec(), 1_000); + const batch = store.createBatch({ + hash: 'a'.repeat(64), + request: { + source: 'fixture-main', scope: 'full', + selector: { kind: 'target-range', startKey: 'target-a', endKey: 'target-c' }, mode: 'ensure', + }, + source: 'fixture-main', scope: 'full', + targets: [ + { key: 'target-a', start: '2026-01-01', end: '2026-01-01', values: {} }, + { key: 'target-b', start: '2026-01-02', end: '2026-01-02', values: {} }, + { key: 'target-c', start: '2026-01-03', end: '2026-01-03', values: {} }, + ], + tasks: [ + { targetKey: 'target-a', taskId: 'collect', plan: 'fixture-snapshot', input: { label: 'a' }, dependsOn: [] }, + { targetKey: 'target-b', taskId: 'collect', plan: 'fixture-snapshot', input: { label: 'b' }, dependsOn: [] }, + { targetKey: 'target-c', taskId: 'collect', plan: 'fixture-snapshot', input: { label: 'c' }, dependsOn: [] }, + ], + }, { timestamp: 1_000 }); + const manager = { instanceId: 'manager-backfill-test', epoch: store.claimManager('manager-backfill-test', process.pid, 1_000, 60_000) }; + const detail = store.batch(batch.id); + const first = detail.runs.find(item => item.targetKey === 'target-a').run; + const second = detail.runs.find(item => item.targetKey === 'target-b').run; + const third = detail.runs.find(item => item.targetKey === 'target-c').run; + store.claimRun(first.id, [], 1_001, manager); + store.finishRun(first.id, { + success: true, receipt: { ok: true, status: 'succeeded', data: { counts: { items: 1 } } }, exitCode: 0, + }, 1_002, manager); + store.claimRun(second.id, [], 1_003, manager); + store.claimRun(third.id, [], 1_003, manager); + store.finishRun(second.id, { success: false, error: 'fixture_failed', exitCode: 1 }, 1_004, manager); + assert.equal(store.batch(batch.id).status, 'running'); + store.finishRun(third.id, { + success: true, receipt: { ok: true, status: 'succeeded', data: { counts: { items: 1 } } }, exitCode: 0, + }, 1_005, manager); + assert.equal(store.batch(batch.id).status, 'failed'); + const retried = store.retryBatch(batch.id); + assert.equal(retried.runs.find(item => item.targetKey === 'target-a').run.status, 'succeeded'); + assert.equal(retried.runs.find(item => item.targetKey === 'target-b').run.status, 'queued'); + assert.equal(retried.runs.find(item => item.targetKey === 'target-c').run.status, 'succeeded'); + } finally { + store.close(); fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('disabled collection schedules can be staged before their source but cannot be enabled early', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + const spec = configured(); + spec.sources[0].enabled = false; + store.applySpec(spec, 1_000); + const definition = { + id: 'fixture-staged-window', + request: { source: 'fixture-main', scope: 'full', selector: { kind: 'current' }, mode: 'ensure' }, + schedule: { + type: 'polling-window', expression: '0 15 * * 2', timezone: 'America/Los_Angeles', + intervalSeconds: 900, windowSeconds: 86_400, retryErrors: ['week_unavailable'], + }, + enabled: false, + }; + assert.equal(new StandardCollectionService(store).putSchedule(definition, 1_000).enabled, false); + assert.throws(() => store.setCollectionScheduleEnabled(definition.id, true), error => error.code === 'plan_disabled'); + assert.equal(store.collectionSchedule(definition.id).enabled, false); + } finally { + store.close(); fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('polling-window resolver failures retry on cadence without hot-looping the window', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(configured(), 1_000); + const definition = { + id: 'fixture-resolver-retry', + request: { source: 'fixture-main', scope: 'full', selector: { kind: 'current' }, mode: 'ensure' }, + schedule: { + type: 'polling-window', expression: '0 15 * * 2', timezone: 'America/Los_Angeles', + intervalSeconds: 900, windowSeconds: 86_400, retryErrors: ['week_unavailable'], + }, + enabled: true, + }; + new StandardCollectionService(store).putSchedule(definition, 1_000); + let calls = 0; + const manager = new CollectionManager(store, { collectionService: { + fireSchedule: async () => { + calls += 1; + if (calls === 1) throw Object.assign(new Error('target_resolution_failed'), { code: 'target_resolution_failed' }); + return { id: 'batch_fixture' }; + }, + } }); + const opening = Date.parse('2026-08-25T22:00:00.000Z'); + await manager.scheduleCollections(opening); + await manager.scheduleCollections(opening + 10_000); + assert.equal(calls, 1); + await manager.scheduleCollections(opening + 900_000); + await manager.scheduleCollections(opening + 1_800_000); + assert.equal(calls, 2); + } finally { + store.close(); fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('polling-window schedules freeze one batch and apply a bounded root-task retry policy', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(configured(), 1_000); + const service = new StandardCollectionService(store); + const definition = { + id: 'fixture-weekly-window', + request: { source: 'fixture-main', scope: 'full', selector: { kind: 'current' }, mode: 'ensure' }, + schedule: { + type: 'polling-window', expression: '0 15 * * 2', timezone: 'America/Los_Angeles', + intervalSeconds: 900, windowSeconds: 86_400, retryErrors: ['week_unavailable'], + }, + enabled: true, + }; + service.putSchedule(definition, 1_000); + assert.throws(() => service.putSchedule({ + ...definition, + id: 'invalid-window', + schedule: { ...definition.schedule, retryErrors: ['week_unavailable', 'week_unavailable'] }, + }), error => error.code === 'invalid_schedule'); + + const manager = new CollectionManager(store); + const opening = Date.parse('2026-08-25T22:00:00.000Z'); + const caughtUpAt = opening + 7 * 60 * 60 * 1000; + await manager.scheduleCollections(caughtUpAt); + await manager.scheduleCollections(caughtUpAt + 10_000); + await new CollectionManager(store).scheduleCollections(caughtUpAt + 60_000); + assert.equal(store.batchCount(), 1); + const batch = store.batches(1)[0]; + const detail = store.batch(batch.id); + const root = detail.runs.find(item => item.taskId === 'first').run; + const dependent = detail.runs.find(item => item.taskId === 'second').run; + assert.equal(root.maxAttempts, 96); + assert.equal(root.retryDeadline, opening + 86_400_000); + assert.deepEqual(root.retryErrors, ['week_unavailable']); + assert.equal(dependent.retryDeadline, null); + } finally { + store.close(); fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/dsp/runtime/collection-manager/tests/store.test.js b/dsp/runtime/collection-manager/tests/store.test.js new file mode 100644 index 0000000..2c79c4b --- /dev/null +++ b/dsp/runtime/collection-manager/tests/store.test.js @@ -0,0 +1,171 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); +const { DatabaseSync } = require('node:sqlite'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { fixture, spec } = require('./helpers'); + +test('spec application creates collectors, sources, plans, and durable queued runs', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + assert.deepEqual(store.applySpec(spec()), { collectors: 1, sources: 1, plans: 4, syncs: 1 }); + assert.equal(store.collectors().length, 1); + assert.equal(store.methods('fixture').length, 3); + assert.equal(store.sources().length, 1); + assert.equal(store.plans().length, 4); + assert.equal(store.syncs().length, 1); + const run = store.enqueuePlan('fixture-snapshot', { input: { label: 'manual' } }); + assert.equal(run.status, 'queued'); + assert.equal(run.method, 'fixture.snapshot'); + assert.equal(fs.statSync(paths.databaseRoot).mode & 0o777, 0o700); + assert.equal(fs.statSync(paths.database).mode & 0o777, 0o600); + assert.equal(store.health().databaseIntegrity, 'ok'); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('spec validation rejects secret-bearing source configuration', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + for (const key of ['password', 'accessToken', 'credential', 'privateKey', 'sessionCookie']) { + const invalid = spec(); + invalid.sources[0].config[key] = 'not-allowed'; + assert.throws(() => store.applySpec(invalid), error => error.code === 'secret_field_forbidden'); + } + assert.equal(store.collectors().length, 0); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('spec application rejects dependency cycles and unsafe executable parents', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + const cyclic = spec(); + cyclic.plans[0].dependsOn = [{ plan: 'fixture-retry', maxAgeSeconds: 60 }]; + cyclic.plans[1].dependsOn = [{ plan: 'fixture-snapshot', maxAgeSeconds: 60 }]; + assert.throws(() => store.applySpec(cyclic), error => error.code === 'dependency_cycle'); + + const unsafeDirectory = `${root}/unsafe`; + const unsafeCommand = `${unsafeDirectory}/collector`; + fs.mkdirSync(unsafeDirectory, { mode: 0o777 }); + fs.copyFileSync(spec().collectors[0].command, unsafeCommand); + fs.chmodSync(unsafeDirectory, 0o777); + fs.chmodSync(unsafeCommand, 0o700); + const unsafe = spec(); + unsafe.collectors[0].command = unsafeCommand; + assert.throws(() => store.applySpec(unsafe), error => error.code === 'unsafe_collector'); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('expired managers are fenced from completing runs', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(spec(), 1_000); + const run = store.enqueuePlan('fixture-snapshot', { timestamp: 1_000 }); + const firstEpoch = store.claimManager('manager-one', 111, 1_000, 100); + assert.equal(store.claimRun(run.id, ['source:fixture-main'], 1_000, + { instanceId: 'manager-one', epoch: firstEpoch }), true); + const secondEpoch = store.claimManager('manager-two', 222, 1_101, 100); + assert.ok(secondEpoch > firstEpoch); + assert.throws(() => store.finishRun(run.id, { success: false, errorCode: 'stale' }, 1_101, + { instanceId: 'manager-one', epoch: firstEpoch }), error => error.code === 'manager_lease_lost'); + assert.equal(store.run(run.id).status, 'running'); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('polling-window runs retry only declared availability errors and stop at the cutoff', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(spec(), 1_000); + const manager = { + instanceId: 'poll-manager', + epoch: store.claimManager('poll-manager', 333, 1_000, 10_000_000), + }; + const policy = { + maxAttempts: 4, backoffSeconds: [900], retryDeadline: 4_000_000, + retryErrors: ['week_unavailable'], + }; + + const unavailable = store.enqueuePlan('fixture-snapshot', { timestamp: 1_000, runPolicy: policy }); + assert.equal(store.claimRun(unavailable.id, ['source:fixture-main'], 1_000, manager), true); + const deferred = store.finishRun(unavailable.id, { + success: false, errorCode: 'week_unavailable', exitCode: 0, + }, 2_000, manager); + assert.equal(deferred.status, 'queued'); + assert.equal(deferred.runAfter, 902_000); + assert.equal(deferred.retryDeadline, 4_000_000); + + const authentication = store.enqueuePlan('fixture-snapshot', { timestamp: 3_000, runPolicy: policy }); + assert.equal(store.claimRun(authentication.id, ['source:fixture-main'], 3_000, manager), true); + const terminal = store.finishRun(authentication.id, { + success: false, errorCode: 'authentication_required', exitCode: 0, + }, 4_000, manager); + assert.equal(terminal.status, 'failed'); + assert.equal(terminal.error, 'authentication_required'); + + const expired = store.enqueuePlan('fixture-snapshot', { + timestamp: 5_000, + runPolicy: { ...policy, retryDeadline: 5_000 }, + }); + assert.equal(store.expirePollingRun(expired.id, 5_000), true); + assert.equal(store.run(expired.id).error, 'polling_window_expired'); + const manualRetry = store.retry(expired.id); + assert.equal(manualRetry.status, 'queued'); + assert.equal(manualRetry.retryDeadline, null); + assert.equal(manualRetry.retryErrors, null); + + const available = store.enqueuePlan('fixture-snapshot', { timestamp: 6_000, runPolicy: policy }); + assert.equal(store.claimRun(available.id, ['source:fixture-main'], 6_000, manager), true); + const published = store.finishRun(available.id, { + success: true, + receipt: { ok: true, status: 'published', data: { checked: true } }, + exitCode: 0, + }, 7_000, manager); + assert.equal(published.status, 'succeeded'); + assert.equal(published.attempt, 1); + assert.equal(published.runAfter, 6_000); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('schema 4 databases migrate polling-window policy columns and plugin state to schema 6', () => { + const { root, paths } = fixture(); + let store = new CollectionStore(paths); + try { + store.close(); + const legacy = new DatabaseSync(paths.database); + legacy.exec(` + ALTER TABLE runs DROP COLUMN retryable_errors_json; + ALTER TABLE runs DROP COLUMN retry_deadline; + PRAGMA user_version=4; + `); + legacy.close(); + store = new CollectionStore(paths); + assert.equal(store.health().schemaVersion, 6); + const columns = store.db.prepare('PRAGMA table_info(runs)').all().map(column => column.name); + assert.equal(columns.includes('retry_deadline'), true); + assert.equal(columns.includes('retryable_errors_json'), true); + } finally { + store?.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/dsp/runtime/collection-manager/tests/syncs.test.js b/dsp/runtime/collection-manager/tests/syncs.test.js new file mode 100644 index 0000000..7a4c8f7 --- /dev/null +++ b/dsp/runtime/collection-manager/tests/syncs.test.js @@ -0,0 +1,415 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { CollectionManager } = require('../src/manager'); +const { SyncService } = require('dispatch-runtime-kit/collection-manager/src/syncs'); +const { fixture, spec } = require('./helpers'); + +async function waitFor(predicate, timeoutMs = 5_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const value = predicate(); + if (value) return value; + await new Promise(resolve => setTimeout(resolve, 20)); + } + throw new Error('wait_timeout'); +} + +test('manual sync runs while automatic sync is paused and repeated clicks coalesce', async t => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + const manager = new CollectionManager(store, { tickMs: 10 }); + t.after(async () => { await manager.stop(); store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + store.applySpec(spec()); + const service = new SyncService(store); + const first = service.runNow('fixture-main-sync', { idempotencyKey: 'click-one' }); + const second = service.runNow('fixture-main-sync', { idempotencyKey: 'click-two' }); + assert.equal(first.run.id, second.run.id); + assert.equal(second.sync.desiredState, 'stopped'); + assert.equal(second.sync.nextDueAt, null); + assert.equal(store.claimRun(first.run.id, []), true); + assert.equal(store.sync('fixture-main-sync').activity, 'syncing'); + store.finishRun(first.run.id, { success: false, errorCode: 'manual_verification_required', exitCode: 0 }); + const retry = service.runNow('fixture-main-sync', { idempotencyKey: 'retry-after-auth' }); + assert.equal(retry.run.trigger, 'sync_manual'); + await manager.start(); await manager.runUntilIdle({ timeoutMs: 5000 }); + const result = store.sync('fixture-main-sync'); + assert.ok(result.lastSucceededAt); + assert.equal(result.desiredState, 'stopped'); assert.equal(result.nextDueAt, null); + assert.equal(store.syncHistory('fixture-main-sync').total, 2); +}); + +test('sync start queues an immediate bounded tick and records no-change history', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + const manager = new CollectionManager(store, { tickMs: 10 }); + try { + const applied = store.applySpec(spec()); + assert.equal(applied.syncs, 1); + const service = new SyncService(store); + const started = service.start('fixture-main-sync'); + assert.equal(started.sync.desiredState, 'running'); + assert.equal(started.sync.generation, 1); + assert.equal(started.run.status, 'queued'); + const repeated = service.start('fixture-main-sync'); + assert.equal(repeated.run, null); + assert.equal(repeated.sync.generation, 1); + assert.equal(repeated.sync.nextDueAt, started.sync.nextDueAt); + await manager.start(); + const drained = await manager.runUntilIdle({ timeoutMs: 5_000 }); + assert.equal(drained.idle, true); + const current = store.sync('fixture-main-sync'); + assert.equal(current.activity, 'idle'); + assert.equal(current.lastSucceededAt !== null, true); + assert.deepEqual(current.businessContext, { date: '2026-08-29', timezone: 'America/Los_Angeles' }); + assert.deepEqual(current.alerts, []); + const history = store.syncHistory('fixture-main-sync'); + assert.equal(history.total, 1); + assert.equal(history.items[0].configRevision, 1); + assert.equal(history.items[0].run.status, 'succeeded'); + assert.equal(history.items[0].run.receipt.status, 'no_change'); + await manager.stop(); + const manual = service.runNow('fixture-main-sync', { idempotencyKey: 'operator-click-1' }); + const repeatedManual = service.runNow('fixture-main-sync', { idempotencyKey: 'operator-click-1' }); + assert.equal(manual.run.id, repeatedManual.run.id); + assert.equal(store.syncHistory('fixture-main-sync').total, 2); + await service.stop('fixture-main-sync'); + assert.equal(store.sync('fixture-main-sync').desiredState, 'stopped'); + } finally { + await manager.stop(); + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('manual sync advances unstarted future work without bypassing retry backoff', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(spec()); + const service = new SyncService(store); + const now = Date.now(), future = now + 3600000; + service.start('fixture-main-sync', { runNow: false, timestamp: now }); + const pending = store.enqueueSync('fixture-main-sync', { + trigger: 'sync_schedule', timestamp: future, windowKey: String(future), + }); + const manual = service.runNow('fixture-main-sync', { timestamp: now, idempotencyKey: 'migration-sync-check' }); + assert.equal(manual.run.id, pending.id); + assert.equal(manual.run.runAfter, now); + assert.equal(store.syncHistory('fixture-main-sync').total, 1); + store.db.prepare('UPDATE runs SET attempt=1,run_after=? WHERE id=?').run(future, pending.id); + const retry = service.runNow('fixture-main-sync', { timestamp: now, idempotencyKey: 'migration-sync-retry' }); + assert.equal(retry.run.id, pending.id); + assert.equal(retry.run.runAfter, future); + assert.equal(store.syncHistory('fixture-main-sync').total, 1); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('sync scheduling coalesces pending work and edits create immutable revisions', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + const manager = new CollectionManager(store, { tickMs: 10 }); + try { + store.applySpec(spec()); + const service = new SyncService(store); + const started = service.start('fixture-main-sync'); + service.schedule(started.sync.nextDueAt + 1); + service.schedule(started.sync.nextDueAt + 1); + assert.equal(store.syncHistory('fixture-main-sync').total, 1); + await service.stop('fixture-main-sync'); + const edited = await service.edit('fixture-main-sync', { + intervalSeconds: 20, + jitterSeconds: 2, + settings: { behavior: 'published' }, + }, { expectedRevision: 1 }); + assert.equal(edited.sync.revision, 2); + assert.equal(edited.sync.intervalSeconds, 20); + assert.equal(edited.sync.settings.behavior, 'published'); + service.start('fixture-main-sync'); + await manager.start(); + await manager.runUntilIdle({ timeoutMs: 5_000 }); + const history = store.syncHistory('fixture-main-sync'); + assert.equal(history.total, 2); + assert.equal(history.items[0].configRevision, 2); + assert.equal(history.items[0].run.receipt.status, 'published'); + assert.equal(history.items[1].run.status, 'cancelled'); + await service.stop('fixture-main-sync'); + } finally { + await manager.stop(); + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('sync settings can be explicitly replaced and declarative schema migrations require stopped state', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(spec()); + const service = new SyncService(store); + const replaced = await service.edit('fixture-main-sync', { + settings: { behavior: 'published' }, replaceSettings: true, + }, { expectedRevision: 1 }); + assert.deepEqual(replaced.sync.settings, { behavior: 'published' }); + assert.equal(replaced.sync.revision, 2); + + const migrated = spec(); + const schema = { + type: 'object', + properties: { behavior: { type: 'string', enum: ['no_change', 'published', 'sleep'] } }, + required: ['behavior'], additionalProperties: false, + }; + migrated.collectors[0].methods['fixture.sync'].inputSchema = schema; + migrated.plans.find(plan => plan.id === 'fixture-sync-plan').input = { behavior: 'no_change' }; + migrated.syncs[0].settingsSchema = schema; + migrated.syncs[0].settings = { behavior: 'no_change' }; + migrated.syncs[0].replaceSettingsOnApply = true; + store.applySpec(migrated); + assert.deepEqual(store.sync('fixture-main-sync').settings, { behavior: 'no_change' }); + assert.equal(store.sync('fixture-main-sync').revision, 3); + + const started = service.start('fixture-main-sync'); + assert.equal(started.sync.desiredState, 'running'); + assert.throws(() => store.applySpec(migrated), error => error.code === 'sync_migration_requires_stopped'); + await service.stop('fixture-main-sync'); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('sync status exposes consecutive failure, integrity, and staleness alerts without private details', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + const configured = spec(); + configured.plans.find(plan => plan.id === 'fixture-sync-plan').maxAttempts = 1; + store.applySpec(configured, 1_000); + const service = new SyncService(store, { clock: () => 1_000 }); + const first = service.start('fixture-main-sync', { timestamp: 1_000 }).run; + assert.equal(store.claimRun(first.id, [], 1_100), true); + store.finishRun(first.id, { success: false, errorCode: 'integrity_failed', exitCode: 1 }, 1_200); + const second = service.runNow('fixture-main-sync', { timestamp: 2_000, idempotencyKey: 'second' }).run; + assert.equal(store.claimRun(second.id, [], 2_100), true); + store.finishRun(second.id, { success: false, errorCode: 'integrity_failed', exitCode: 1 }, 2_200); + + const current = store.sync('fixture-main-sync', 22_001); + assert.deepEqual(current.alerts.map(alert => alert.code), [ + 'consecutive_failures', 'no_success', 'integrity_failure', + ]); + assert.equal(current.alerts[0].count, 2); + assert.equal(current.alerts[0].error, 'integrity_failed'); + assert.equal(JSON.stringify(current.alerts).includes('private'), false); + const history = store.syncHistory('fixture-main-sync'); + assert.deepEqual(history.items[0].run.attempts, [{ + attempt: 1, status: 'failed', category: 'integrity', startedAt: 2_100, finishedAt: 2_200, + error: 'integrity_failed', exitCode: 1, + }]); + assert.equal(history.items[0].run.attemptHistoryComplete, true); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('consecutive failure alerts report the exact streak beyond the history page size', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + const configured = spec(); + configured.plans.find(plan => plan.id === 'fixture-sync-plan').maxAttempts = 1; + store.applySpec(configured, 1_000); + const service = new SyncService(store, { clock: () => 1_000 }); + let run = service.start('fixture-main-sync', { timestamp: 1_000 }).run; + for (let index = 0; index < 23; index += 1) { + const startedAt = 1_100 + index * 10; + assert.equal(store.claimRun(run.id, [], startedAt), true); + store.finishRun(run.id, { success: false, errorCode: 'fixture_failed', exitCode: 1 }, startedAt + 1); + if (index < 22) run = service.runNow('fixture-main-sync', { + timestamp: startedAt + 2, idempotencyKey: `failure-${index}`, + }).run; + } + const alert = store.sync('fixture-main-sync', 2_000).alerts.find(item => item.code === 'consecutive_failures'); + assert.equal(alert.count, 23); + assert.equal(alert.sinceAt, 1_101); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('consecutive failure boundaries are deterministic when terminal timestamps match', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + const configured = spec(); + configured.plans.find(plan => plan.id === 'fixture-sync-plan').maxAttempts = 1; + store.applySpec(configured, 1_000); + const service = new SyncService(store, { clock: () => 1_000 }); + const terminal = []; + let run = service.start('fixture-main-sync', { timestamp: 1_000 }).run; + assert.equal(store.claimRun(run.id, [], 1_100), true); + store.finishRun(run.id, { + success: true, receipt: { ok: true, status: 'no_change', data: {} }, exitCode: 0, + }, 3_000); + terminal.push({ id: run.id, status: 'succeeded' }); + for (let index = 0; index < 4; index += 1) { + run = service.runNow('fixture-main-sync', { + timestamp: 2_000 + index, idempotencyKey: `same-time-${index}`, + }).run; + assert.equal(store.claimRun(run.id, [], 2_100 + index), true); + store.finishRun(run.id, { success: false, errorCode: 'fixture_failed', exitCode: 1 }, 3_000); + terminal.push({ id: run.id, status: 'failed' }); + } + terminal.sort((left, right) => left.id < right.id ? 1 : left.id > right.id ? -1 : 0); + const expected = terminal.findIndex(item => item.status !== 'failed'); + const alert = store.sync('fixture-main-sync', 3_001).alerts.find(item => item.code === 'consecutive_failures'); + if (expected >= 2) assert.equal(alert.count, expected); + else assert.equal(alert, undefined); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('history compaction keeps one daily no-change run and preserves published evidence', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(spec(), 1_000); + const service = new SyncService(store, { clock: () => 1_000 }); + let run = service.start('fixture-main-sync', { timestamp: 1_000 }).run; + const statuses = ['no_change', 'no_change', 'no_change', 'published']; + for (let index = 0; index < statuses.length; index += 1) { + const startedAt = 1_100 + index * 10; + assert.equal(store.claimRun(run.id, [], startedAt), true); + store.finishRun(run.id, { + success: true, + receipt: { ok: true, status: statuses[index], data: { businessDate: '2026-08-30', disposition: statuses[index] } }, + exitCode: 0, + }, startedAt + 1); + if (index < statuses.length - 1) run = service.runNow('fixture-main-sync', { + timestamp: startedAt + 2, idempotencyKey: `history-${index}`, + }).run; + } + const compacted = store.compactHistory(10_000, 100); + assert.equal(compacted.deleted, 2); + const history = store.syncHistory('fixture-main-sync'); + assert.equal(history.total, 2); + assert.deepEqual(history.items.map(item => item.run.receipt.status).sort(), ['no_change', 'published']); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('terminal authentication failure blocks retries, degrades health, and clears after a successful probe', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(spec(), 1_000); + const service = new SyncService(store, { clock: () => 1_000 }); + const run = service.start('fixture-main-sync', { timestamp: 1_000 }).run; + assert.equal(store.claimRun(run.id, [], 1_100), true); + const failed = store.finishRun(run.id, { + success: false, errorCode: 'manual_verification_required', exitCode: 0, + }, 1_200); + assert.equal(failed.status, 'failed'); + assert.equal(failed.attempt, 1); + const blocked = store.sync('fixture-main-sync', 1_200); + assert.equal(blocked.desiredState, 'running'); + assert.equal(blocked.activity, 'blocked'); + assert.equal(blocked.blocked, 'manual_verification_required'); + assert.ok(blocked.nextDueAt >= 3_601_200); + + store.claimManager('fixture-manager', process.pid, 1_200, 1_000_000); + const health = store.health(122_001); + assert.equal(health.status, 'degraded'); + assert.equal(health.syncAlerts.total, 2); + assert.deepEqual(health.syncAlerts.items.map(item => item.code), ['no_success', 'authentication_blocked']); + + const probe = service.runNow('fixture-main-sync', { timestamp: 122_100, idempotencyKey: 'auth-probe' }).run; + assert.equal(store.claimRun(probe.id, [], 122_101), true); + store.finishRun(probe.id, { + success: true, receipt: { ok: true, status: 'no_change', data: {} }, exitCode: 0, + }, 122_102); + const recovered = store.sync('fixture-main-sync', 122_103); + assert.equal(recovered.blocked, null); + assert.equal(recovered.activity, 'idle'); + assert.deepEqual(recovered.alerts, []); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('overdue sync alerts are suppressed during intentional cancellation cleanup', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + store.applySpec(spec(), 1_000); + const service = new SyncService(store, { clock: () => 1_000 }); + const run = service.start('fixture-main-sync', { timestamp: 1_000 }).run; + assert.equal(store.claimRun(run.id, [], 1_100), true); + assert.equal(store.sync('fixture-main-sync', 1_000_000).alerts.some(alert => alert.code === 'run_overdue'), true); + store.cancel(run.id); + assert.equal(store.sync('fixture-main-sync', 1_000_001).alerts.some(alert => alert.code === 'run_overdue'), false); + store.finishRun(run.id, { success: false, cancelled: true, errorCode: 'cancelled', exitCode: null }, 1_000_002); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('sync stop cancels an active worker and returns only after cleanup', async () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + const manager = new CollectionManager(store, { tickMs: 10 }); + try { + store.applySpec(spec()); + const service = new SyncService(store); + await service.edit('fixture-main-sync', { settings: { behavior: 'sleep' } }, { expectedRevision: 1 }); + await manager.start(); + service.start('fixture-main-sync'); + await waitFor(() => store.sync('fixture-main-sync').activity === 'syncing'); + const stopped = await service.stop('fixture-main-sync', { waitMs: 5_000 }); + assert.equal(stopped.desiredState, 'stopped'); + assert.equal(stopped.activity, 'idle'); + assert.equal(stopped.activeRun, null); + const history = store.syncHistory('fixture-main-sync'); + assert.equal(history.items[0].run.status, 'cancelled'); + } finally { + await manager.stop(); + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('capacity waiting is visible and does not consume the collector timeout', () => { + const { root, paths } = fixture(); + const store = new CollectionStore(paths); + try { + const now = Date.now(); + store.applySpec(spec(), now); + const service = new SyncService(store); + const run = service.start('fixture-main-sync').run; + const epoch = store.claimManager('capacity-manager', process.pid, now, 60000); + const fence = { instanceId: 'capacity-manager', epoch }; + assert.equal(store.claimRun(run.id, [], now, fence), true); + store.setCapacityWait(run.id, 'waiting_for_capacity', fence); + const waiting = store.sync('fixture-main-sync', now + 1000000); + assert.equal(waiting.activity, 'waiting_for_capacity'); + assert.equal(waiting.alerts.some(alert => alert.code === 'run_overdue'), false); + assert.throws(() => store.setCapacityWait(run.id, null, { ...fence, epoch: epoch - 1 }), /manager_lease_lost/); + store.setCapacityWait(run.id, null, fence); + assert.equal(store.sync('fixture-main-sync').activity, 'syncing'); + assert.equal(store.sync('fixture-main-sync').alerts.some(alert => alert.code === 'run_overdue'), false); + store.finishRun(run.id, { success: false, cancelled: true, errorCode: 'cancelled', exitCode: null }); + } finally { store.close(); fs.rmSync(root, { recursive: true, force: true }); } +}); diff --git a/dsp/runtime/gateway/OVERVIEW.md b/dsp/runtime/gateway/OVERVIEW.md new file mode 100644 index 0000000..4fd6650 --- /dev/null +++ b/dsp/runtime/gateway/OVERVIEW.md @@ -0,0 +1,131 @@ +--- +title: Runtime Gateway overview +status: current +last_verified: 2026-09-02 +--- + +# Runtime Gateway + +`runtime/runtime-gateway` is the closed owner-private transport for one managed DSP runtime. It keeps the centralized dashboard outside tenant databases and converts only a small allowlist of operational SDK calls into one-request/one-response Unix-socket messages. + +## Request path + +```text +authenticated browser session + -> Access Control reloads active membership and permission + -> Access Control resolves the organization's ready installation + -> dashboard runtime router derives the connector from server configuration + -> runtime gateway verifies protocol version and expected runtime identity + -> tenant-local DispatchClient + -> tenant-local Auth Broker, Collection Manager, and publication stores +``` + +The browser never receives or submits a runtime key, installations root, Unix-socket path, host, port, URL, database path, service name, or command. The gateway is not an HTTP listener and is not exported through the public SDK. + +## Protocol + +Protocol version `1` accepts exactly one newline-terminated JSON request per connection: + +```json +{ + "protocolVersion": 1, + "runtimeKey": "", + "action": "system.status", + "input": {} +} +``` + +`runtimeKey` is internal routing evidence. The gateway compares it with its process-bound runtime identity on every request and never echoes the value in a response. + +Allowed actions are: + +- `health` with `{}`; +- `system.status` with `{}`; +- `workforce.day` with `{ "query": }`; +- `workforce.employees` with `{ "query": }`; +- `workforce.employee` with `{ "code": }`; +- `sync.status` with `{ "id": }`; +- `sync.run_now` with `{ "id": , "options": }`. + +Unknown actions and fields fail closed. Inputs are validated again in the gateway process before they reach the runtime-local SDK. The server returns one closed transport envelope containing a normal sanitized SDK result. Stable transport failures are `invalid_request`, `runtime_identity_mismatch`, `runtime_protocol_mismatch`, and `runtime_gateway_unavailable`. + +Current limits are code-owned: + +- request: 16 KiB; +- response: 300 KiB; +- concurrent connections: 64; +- request/socket deadline: 15 seconds; +- Unix-socket path: at most 107 bytes. + +The strict JSON parser rejects duplicate object keys, non-finite values, trailing content, carriage returns, and multiple messages on one connection. + +## Runtime composition + +`src/managed-runtime.js` reconstructs the managed layout from the complete explicit environment rendered by the Provisioner. Every projected value must equal the deterministic path derived from the process-bound runtime key. The gateway rejects missing roots, fallback roots, symlinks, wrong ownership, wrong modes, replaced directories, wrong devices, and a socket outside `/runtime-gateway.sock`. + +It deliberately does not call `resolveLocalRuntimePaths()` and does not create a tenant-local Access Control store. It constructs a real `DispatchClient` from tenant-local adapters for: + +- Auth Broker metadata and readiness; +- Collection Manager health; +- authenticated synchronization coordination; +- Paycom publication health; +- workforce reads. + +Provider credential setup is not an exposed gateway capability. The Auth Broker and Collection Manager are already supervised by systemd; the gateway checks the existing broker rather than attempting a second lifecycle owner. + +## Socket and process boundary + +The gateway listens only on `/runtime-gateway.sock` with exact mode `0600` inside an exact-mode `0700` runtime directory. Startup removes a stale socket only after validating its type, ownership, mode, canonical identity, and proving no process accepts a connection. Client calls pin and compare socket device/inode identity before and after each exchange. + +Managed service-plan version `3` installs four units per configured runtime in order: + +1. Auth Broker; +2. Collection Manager; +3. Runtime Gateway. +4. outbound Runtime Agent. + +The gateway unit depends on the first two, starts through the same trusted `env --ignore-environment` launcher, uses a closed environment, and is restricted to Unix sockets. The Runtime Agent unit depends on the gateway and has its own owner-private status socket. The Provisioner reads back exact unit bytes, arguments, working directory, cgroup, process environment, socket ownership, and component health. Failure or cancellation uses the existing fenced service compensation and restores the complete prior service state exactly. + +Owner-only Unix modes isolate other operating-system users but do not isolate mutually hostile processes running as the same Unix account. Per-DSP service-account/container isolation remains a separate deployment requirement before a broader pilot. + +## Dashboard routing + +`dashboard/server/runtime-router.js` accepts only the installation object returned by `AccessControlService.runtimeFor()`. It requires: + +- an authenticated session; +- an active current membership with the requested permission; +- an active organization; +- a ready installation whose organization ID matches the selected organization; +- a valid server-owned runtime key; +- pinned private installation and runtime directories. + +`local` continues to resolve to the existing local `DispatchClient`, preserving the reference installation. Managed keys resolve under the optional server-side `DISPATCH_INSTALLATIONS_ROOT`; absence of that root makes managed installations unavailable rather than selecting a fallback. + +Connectors may be cached by server-owned runtime key, but authorization is never cached: Access Control reloads session, membership, role, organization, and installation state on every HTTP request. A suspended membership/organization or changed installation therefore fails before use of a cached connector. + +## Current activation boundary + +Gateway readiness proves private transport identity and access to supervised Auth Broker/Collection Manager health. It does not by itself prove provider credentials, provider authentication, collection completeness, or a first publication, and it never writes installation state. The managed activation controller now uses gateway identity/health as one of nine independent gates; Access Control commits `ready` only after protected provider authentication, exact first-collection completion, publication audits, and final infrastructure read-back also pass. + +## Verification + +Run the component gate: + +```bash +./runtime/gateway/scripts/verify +``` + +Run the dashboard routing gate: + +```bash +npm run build --prefix dashboard +npm test --prefix dashboard +``` + +Run the actual temporary user-systemd gate: + +```bash +npm run verify:systemd --prefix core/provisioner +``` + +The real gate creates two isolated four-service fixture runtimes plus an independent central fixture hub, runs one through the durable seven-stage pipeline, validates cross-runtime identity rejection, exercises Agent-backed and direct gateway status plus `sync.run_now`, verifies worker execution and restart/start-limit behavior, rolls back the fixture units, and confirms the existing reference services retain their PIDs. diff --git a/dsp/runtime/gateway/bin/dispatch-runtime-gateway b/dsp/runtime/gateway/bin/dispatch-runtime-gateway new file mode 100755 index 0000000..d914d53 --- /dev/null +++ b/dsp/runtime/gateway/bin/dispatch-runtime-gateway @@ -0,0 +1,4 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +process.umask(0o077); +require('../src/server-cli').main().then(code => { if (Number.isInteger(code)) process.exitCode = code; }); diff --git a/dsp/runtime/gateway/bin/dispatch-runtime-gatewayctl b/dsp/runtime/gateway/bin/dispatch-runtime-gatewayctl new file mode 100755 index 0000000..5341024 --- /dev/null +++ b/dsp/runtime/gateway/bin/dispatch-runtime-gatewayctl @@ -0,0 +1,4 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +process.umask(0o077); +require('../src/control-cli').main().then(code => { process.exitCode = code; }); diff --git a/dsp/runtime/gateway/examples/two-runtime-gateways.js b/dsp/runtime/gateway/examples/two-runtime-gateways.js new file mode 100644 index 0000000..7c53410 --- /dev/null +++ b/dsp/runtime/gateway/examples/two-runtime-gateways.js @@ -0,0 +1,77 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { success } = require('dispatch-protocol/contracts/src'); +const { RuntimeGatewayServer, createRuntimeGatewayDispatchClient } = require('../src'); + +function fixtureClient(value) { + return { + system: { status: async () => success('ready', { + value, + components: { + auth: { healthy: true, ready: true }, + collections: { healthy: true, ready: true, status: 'ready', data: { manager: { running: true } } }, + }, + summary: { ready: 3, degraded: 0, failed: 0 }, + }) }, + workforce: { day: async query => success('found', { value, date: query.date }) }, + sync: { + status: async id => success('found', { value, id }), + runNow: async id => success('queued', { value, id }), + start: async id => success('started', { value, id }), + stop: async id => success('stopped', { value, id }), + }, + collections: { health: async () => success('ready', { value, counts: { queued: 0, running: 0 } }) }, + }; +} + +async function main() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-gateway-exercise-')); + fs.chmodSync(root, 0o700); + const values = [ + { key: 'fixture_alpha', marker: 'alpha' }, + { key: 'fixture_bravo', marker: 'bravo' }, + ]; + const servers = []; + try { + for (const value of values) { + const runtimeRoot = path.join(root, value.marker); + fs.mkdirSync(runtimeRoot, { mode: 0o700 }); + const socketPath = path.join(runtimeRoot, 'runtime-gateway.sock'); + const server = new RuntimeGatewayServer({ + socketPath, + runtimeKey: value.key, + client: fixtureClient(value.marker), + }); + await server.start(); + servers.push({ server, socketPath, ...value }); + } + const clients = servers.map(value => createRuntimeGatewayDispatchClient({ + socketPath: value.socketPath, + runtimeKey: value.key, + })); + const results = await Promise.all(clients.map(client => client.system.status())); + assert.deepEqual(results.map(result => result.data.value), ['alpha', 'bravo']); + const crossed = createRuntimeGatewayDispatchClient({ + socketPath: servers[1].socketPath, + runtimeKey: servers[0].key, + }); + assert.equal((await crossed.system.status()).status, 'runtime_identity_mismatch'); + process.stdout.write(`${JSON.stringify({ + ok: true, + status: 'verified', + gatewayProtocolVersion: 1, + fixtures: 2, + isolated: true, + crossRouteRejected: true, + })}\n`); + } finally { + await Promise.all(servers.map(value => value.server.close())); + fs.rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(() => { process.stderr.write('runtime gateway exercise failed\n'); process.exitCode = 1; }); diff --git a/dsp/runtime/gateway/package.json b/dsp/runtime/gateway/package.json new file mode 100644 index 0000000..7e305c6 --- /dev/null +++ b/dsp/runtime/gateway/package.json @@ -0,0 +1,16 @@ +{ + "name": "dispatch-runtime-gateway", + "version": "0.1.0", + "private": true, + "description": "Closed private transport for one isolated Dispatch runtime", + "type": "commonjs", + "main": "src/index.js", + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "./scripts/build", + "test": "./scripts/test", + "verify": "./scripts/verify" + } +} diff --git a/dsp/runtime/gateway/scripts/build b/dsp/runtime/gateway/scripts/build new file mode 100755 index 0000000..41e7e4c --- /dev/null +++ b/dsp/runtime/gateway/scripts/build @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +for file in "$ROOT"/src/*.js "$ROOT"/tests/*.js "$ROOT"/examples/*.js "$ROOT"/bin/*; do + node --no-warnings --check "$file" +done +test -x "$ROOT/bin/dispatch-runtime-gateway" +test -x "$ROOT/bin/dispatch-runtime-gatewayctl" +node --no-warnings -e "require('$ROOT/src')" +printf '%s\n' '{"ok":true,"status":"built"}' diff --git a/dsp/runtime/gateway/scripts/test b/dsp/runtime/gateway/scripts/test new file mode 100755 index 0000000..e4fe2b6 --- /dev/null +++ b/dsp/runtime/gateway/scripts/test @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +export NODE_NO_WARNINGS=1 +exec node --test "$ROOT"/tests/*.test.js diff --git a/dsp/runtime/gateway/scripts/verify b/dsp/runtime/gateway/scripts/verify new file mode 100755 index 0000000..86cb16d --- /dev/null +++ b/dsp/runtime/gateway/scripts/verify @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +"$ROOT/tooling/build" +"$ROOT/tooling/test" +node --no-warnings "$ROOT/examples/two-runtime-gateways.js" diff --git a/dsp/runtime/gateway/src/control-cli.js b/dsp/runtime/gateway/src/control-cli.js new file mode 100644 index 0000000..91f71b8 --- /dev/null +++ b/dsp/runtime/gateway/src/control-cli.js @@ -0,0 +1,29 @@ +'use strict'; + +const { failure } = require('dispatch-protocol/contracts/src'); +const { managedRuntimeConfiguration } = require('./managed-runtime'); +const { createRuntimeGatewayDispatchClient } = require('dispatch-protocol/gateway/client'); + +async function main(argv = process.argv.slice(2), environment = process.env, write = chunk => process.stdout.write(chunk)) { + process.umask(0o077); + if (argv.length !== 1 || argv[0] !== 'health') { + write(`${JSON.stringify(failure('invalid_input'))}\n`); + return 2; + } + try { + const configuration = managedRuntimeConfiguration(environment); + const client = createRuntimeGatewayDispatchClient({ + socketPath: configuration.gatewaySocket, + runtimeKey: configuration.runtimeKey, + }); + const result = await client.health(); + write(`${JSON.stringify(result)}\n`); + return result.ok && result.status === 'ready' ? 0 : 1; + } catch { + write(`${JSON.stringify(failure('runtime_gateway_unavailable', { recoverable: true }))}\n`); + return 1; + } +} + +if (require.main === module) main().then(code => { process.exitCode = code; }); +module.exports = { main }; diff --git a/dsp/runtime/gateway/src/index.js b/dsp/runtime/gateway/src/index.js new file mode 100644 index 0000000..54f1060 --- /dev/null +++ b/dsp/runtime/gateway/src/index.js @@ -0,0 +1,8 @@ +'use strict'; + +const protocol = require('dispatch-protocol/gateway/protocol'); +const server = require('./server'); +const client = require('dispatch-protocol/gateway/client'); +const managed = require('./managed-runtime'); + +module.exports = Object.freeze({ ...protocol, ...server, ...client, ...managed }); diff --git a/dsp/runtime/gateway/src/managed-runtime.js b/dsp/runtime/gateway/src/managed-runtime.js new file mode 100644 index 0000000..fe99cd9 --- /dev/null +++ b/dsp/runtime/gateway/src/managed-runtime.js @@ -0,0 +1,152 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const { + MANAGED_INSTALLATION_LAYOUT_VERSION, + MANAGED_INSTALLATION_LAYOUT_TEMPLATE, + MANAGED_INSTALLATION_DIRECTORY_FIELDS, + resolveManagedInstallationRuntimePaths, + managedInstallationRuntimeEnvironment, +} = require('dispatch-protocol/paths/runtime-paths'); +const { INSTALLATION_IDENTIFIER_RE, failure } = require('dispatch-protocol/contracts/src'); + +const MANAGED_ENVIRONMENT_KEYS = Object.freeze([ + 'DISPATCH_PROJECT_ROOT', + 'DISPATCH_DATA_ROOT', + 'DISPATCH_SECRETS_ROOT', + 'DISPATCH_STATE_ROOT', + 'DISPATCH_RUNTIME_ROOT', + 'DISPATCH_STAGING_ROOT', + 'DISPATCH_AUTH_DATABASE_ROOT', + 'DISPATCH_AUTH_SECRET_ROOT', + 'DISPATCH_AUTH_STATE_ROOT', + 'DISPATCH_AUTH_SOCKET', + 'DISPATCH_COLLECTION_DATABASE_ROOT', + 'DISPATCH_COLLECTION_STATE_ROOT', + 'DISPATCH_PAYCOM_DATA_ROOT', + 'DISPATCH_PAYCOM_STAGING_ROOT', + 'DISPATCH_CDF_DATA_ROOT', + 'DISPATCH_CDF_STAGING_ROOT', +]); + +function fail(code = 'runtime_boundary_violation') { + throw Object.assign(new Error(code), { code }); +} + +function absolute(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value || /[\0\r\n]/.test(value)) fail(); + return value; +} + +function privateDirectory(target, expectedDevice = null) { + const selected = absolute(target); + let info; + try { info = fs.lstatSync(selected); } catch { fail(); } + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== process.geteuid() + || (info.mode & 0o7777) !== 0o700 || expectedDevice !== null && info.dev !== expectedDevice + || fs.realpathSync(selected) !== selected) fail(); + return Object.freeze({ dev: info.dev, ino: info.ino }); +} + +function managedRuntimeConfiguration(environment = process.env) { + if (!environment || typeof environment !== 'object' || Array.isArray(environment)) fail(); + const runtimeKey = environment.DISPATCH_RUNTIME_KEY; + if (typeof runtimeKey !== 'string' || !INSTALLATION_IDENTIFIER_RE.test(runtimeKey)) fail(); + const projectRoot = absolute(environment.DISPATCH_PROJECT_ROOT); + const installationRoot = path.dirname(absolute(environment.DISPATCH_DATA_ROOT)); + if (path.basename(installationRoot) !== runtimeKey) fail('runtime_identity_mismatch'); + const directories = Object.fromEntries(Object.entries(MANAGED_INSTALLATION_DIRECTORY_FIELDS) + .map(([field, relative]) => [field, path.join(installationRoot, relative)])); + const layout = Object.freeze({ + layoutVersion: MANAGED_INSTALLATION_LAYOUT_VERSION, + templateId: MANAGED_INSTALLATION_LAYOUT_TEMPLATE, + runtimeKey, + projectRoot, + installationRoot, + directories: Object.freeze(directories), + }); + const paths = resolveManagedInstallationRuntimePaths(layout); + const expected = managedInstallationRuntimeEnvironment(layout); + for (const key of MANAGED_ENVIRONMENT_KEYS) { + if (environment[key] !== expected[key]) fail('runtime_identity_mismatch'); + } + const gatewaySocket = absolute(environment.DISPATCH_RUNTIME_GATEWAY_SOCKET); + if (gatewaySocket !== path.join(paths.runtimeRoot, 'runtime-gateway.sock')) fail('runtime_identity_mismatch'); + const root = privateDirectory(paths.installationRoot); + const protectedAuth = environment.DISPATCH_PLUGIN_BACKEND === 'core_v1' + ? new Set([directories.authDataRoot, directories.authSecretsRoot, directories.authStateRoot]) : new Set(); + for (const directory of protectedAuth) { + let accessible = false; + try { fs.accessSync(directory, fs.constants.R_OK | fs.constants.W_OK); accessible = true; } catch {} + if (accessible) fail(); + } + for (const directory of Object.values(paths).filter(value => typeof value === 'string' && value.startsWith(`${paths.installationRoot}${path.sep}`))) { + if (!protectedAuth.has(directory) && fs.existsSync(directory) && fs.lstatSync(directory).isDirectory()) privateDirectory(directory, root.dev); + } + for (const directory of Object.values(layout.directories)) if (!protectedAuth.has(directory)) privateDirectory(directory, root.dev); + return Object.freeze({ runtimeKey, gatewaySocket, layout, paths }); +} + +class SupervisedAuthBrokerServicePort { + constructor(auth) { this.auth = auth; } + + async start() { + const result = await this.auth.health(); + if (!result?.ok || result.status !== 'ready') fail('auth_broker_start_failed'); + return Object.freeze({ status: 'ready', managed: true, started: false }); + } +} + +function createManagedRuntimeDispatchClient(configuration) { + // The agent uses only configuration validation; keep client composition and + // provider implementations out of that otherwise small process. + const { AuthClient } = require('../../sdk/src/auth-client'); + const { CollectionClient } = require('dispatch-runtime-kit/sdk/src/collection-client'); + const { SyncClient } = require('dispatch-runtime-kit/sdk/src/sync-client'); + const { PaycomClient } = require('../../sdk/src/paycom-client'); + const { WorkforceClient } = require('../../sdk/src/workforce-client'); + const { DispatchClient } = require('../../sdk/src/dispatch-client'); + const { AuthenticatedSyncPort } = require('../../application/sync/authenticated-sync-port'); + const { LocalAuthBrokerPort } = require('../../adapters/local/auth-broker-port'); + const { LocalCollectionManagerPort } = require('../../adapters/local/collection-manager-port'); + const { LocalSyncManagerPort } = require('dispatch-runtime-kit/adapters/local/sync-manager-port'); + const { contributions, unavailablePort } = require('../../plugin-host/contributions'); + if (!configuration || typeof configuration !== 'object' || !configuration.paths) fail(); + const { paths } = configuration; + const auth = new AuthClient({ port: new LocalAuthBrokerPort({ socketPath: paths.auth.socket }) }); + const collectionPort = new LocalCollectionManagerPort({ paths: paths.collection }); + const collections = new CollectionClient({ port: collectionPort }); + const syncPort = new LocalSyncManagerPort({ paths: paths.collection }); + const sync = new SyncClient({ + port: new AuthenticatedSyncPort({ + sync: syncPort, + auth, + authService: new SupervisedAuthBrokerServicePort(auth), + }), + }); + const ports = contributions('createClientPorts', { paths, collectionPort }); + const paycom = new PaycomClient({ port: ports.paycom || unavailablePort }); + const workforce = new WorkforceClient({ port: ports.workforce || unavailablePort }); + const unavailableSetup = Object.freeze({ + prepare: async () => failure('installation_not_ready', { recoverable: true }), + run: async () => failure('installation_not_ready', { recoverable: true }), + }); + return new DispatchClient({ + auth, + connections: new (require('../../sdk/src/connections-client').ConnectionsClient)({ socketPath: paths.auth.socket }), + collections, + sync, + paycom, + workforce, + authSetup: unavailableSetup, + transport: 'injected', + }); +} + +module.exports = { + MANAGED_ENVIRONMENT_KEYS, + managedRuntimeConfiguration, + createManagedRuntimeDispatchClient, + SupervisedAuthBrokerServicePort, +}; diff --git a/dsp/runtime/gateway/src/server-cli.js b/dsp/runtime/gateway/src/server-cli.js new file mode 100644 index 0000000..9332e7c --- /dev/null +++ b/dsp/runtime/gateway/src/server-cli.js @@ -0,0 +1,52 @@ +'use strict'; + +const { RuntimeGatewayServer } = require('./server'); +const { managedRuntimeConfiguration, createManagedRuntimeDispatchClient } = require('./managed-runtime'); +const { RUNTIME_GATEWAY_PROTOCOL_VERSION } = require('dispatch-protocol/gateway/protocol'); + +async function main(environment = process.env, write = chunk => process.stdout.write(chunk)) { + process.umask(0o077); + let server; + try { + const configuration = managedRuntimeConfiguration(environment); + const client = createManagedRuntimeDispatchClient(configuration); + const plugins = require('../../plugin-host/index').createRuntimePlugins(configuration, client); + client.system.includePaycom = () => plugins.enabled('paycom'); + const selectedClient = environment.DISPATCH_PROJECT_ROOT === '/opt/dispatch' + ? Object.assign(Object.create(client), { paycomSetup: plugins.setup, pluginsManage: plugins.manage, pluginsInvoke: plugins.invoke, authorizePlugin: plugins.authorize, + connectionsManage: require('dispatch-runtime-kit/supervisor/src/connections').createRuntimeConnections(configuration), + diagnosticsSeed: input => require('../../supervisor/src/diagnostics-seed').createDiagnosticsSeed(configuration)(input) }) : Object.create(client); + server = new RuntimeGatewayServer({ + socketPath: configuration.gatewaySocket, + runtimeKey: configuration.runtimeKey, + client: selectedClient, + }); + selectedClient.runtimeExecution = require('../../supervisor/src/execution').createExecution({ + configuration, client: selectedClient, plugins, activeRequests: () => server.activeRequests, + }); + await server.start(); + } catch { + try { await server?.close(); } catch {} + process.stderr.write('dispatch runtime gateway: unavailable\n'); + return 1; + } + write(`${JSON.stringify({ + ok: true, + status: 'ready', + protocolVersion: RUNTIME_GATEWAY_PROTOCOL_VERSION, + })}\n`); + let stopping = false; + const shutdown = async code => { + if (stopping) return; + stopping = true; + try { await server.close(); } finally { process.exit(code); } + }; + process.once('SIGINT', () => shutdown(0)); + process.once('SIGTERM', () => shutdown(0)); + process.once('uncaughtException', () => shutdown(1)); + process.once('unhandledRejection', () => shutdown(1)); + return new Promise(() => {}); +} + +if (require.main === module) main().then(code => { if (Number.isInteger(code)) process.exitCode = code; }); +module.exports = { main }; diff --git a/dsp/runtime/gateway/src/server.js b/dsp/runtime/gateway/src/server.js new file mode 100644 index 0000000..8c00bba --- /dev/null +++ b/dsp/runtime/gateway/src/server.js @@ -0,0 +1,219 @@ +'use strict'; + +const fs = require('node:fs'); +const net = require('node:net'); +const path = require('node:path'); +const { INSTALLATION_IDENTIFIER_RE, success, failure } = require('dispatch-protocol/contracts/src'); +const { parseStrictJson } = require('dispatch-protocol/gateway/strict-json'); +const { + RUNTIME_GATEWAY_PROTOCOL_VERSION, + validateGatewayRequest, + gatewaySuccess, + gatewayFailure, +} = require('dispatch-protocol/gateway/protocol'); + +const MAX_GATEWAY_REQUEST_BYTES = require('dispatch-protocol/contracts/src/connections').CONNECTION_REQUEST_MAX_BYTES; +const MAX_GATEWAY_CONNECTIONS = 64; +const GATEWAY_SOCKET_TIMEOUT_MS = 15_000; +const { privateDirectory, socketIdentity, sameIdentity, probeUnixSocket: probe, MAX_UNIX_SOCKET_PATH_BYTES } = require('dispatch-protocol/transport/unix-socket'); + +function fail(code = 'runtime_gateway_unavailable') { + throw Object.assign(new Error(code), { code }); +} + +function validateServerOptions(options) { + if (!options || typeof options !== 'object' || Array.isArray(options) + || Object.getPrototypeOf(options) !== Object.prototype + || Object.keys(options).some(key => !['socketPath', 'runtimeKey', 'client', 'socketTimeoutMs'].includes(key)) + || typeof options.socketPath !== 'string' || path.resolve(options.socketPath) !== options.socketPath + || path.basename(options.socketPath) !== 'runtime-gateway.sock' + || Buffer.byteLength(options.socketPath, 'utf8') > MAX_UNIX_SOCKET_PATH_BYTES + || typeof options.runtimeKey !== 'string' || !INSTALLATION_IDENTIFIER_RE.test(options.runtimeKey) + || !options.client?.workforce || typeof options.client.workforce.day !== 'function' + || !options.client?.sync || typeof options.client.sync.status !== 'function' + || typeof options.client.sync.runNow !== 'function' || typeof options.client.sync.start !== 'function' + || typeof options.client.sync.stop !== 'function' + || !options.client?.collections || typeof options.client.collections.health !== 'function' + || !options.client?.system || typeof options.client.system.status !== 'function') fail(); + const socketTimeoutMs = options.socketTimeoutMs === undefined ? GATEWAY_SOCKET_TIMEOUT_MS : options.socketTimeoutMs; + if (!Number.isInteger(socketTimeoutMs) || socketTimeoutMs < 100 || socketTimeoutMs > 60_000) fail(); + return Object.freeze({ ...options, socketTimeoutMs }); +} + +async function dispatchRuntimeRequest(client, value, expectedRuntimeKey, transport = 'unix') { + if (!['unix', 'outbound_agent'].includes(transport)) fail('invalid_request'); + const request = validateGatewayRequest(value, expectedRuntimeKey); + let result; + if (request.action === 'runtime.execution') return typeof client.runtimeExecution === 'function' + ? client.runtimeExecution(request.input) : failure('execution_unavailable'); + if (request.action === 'plugins.invoke') return typeof client.pluginsInvoke === 'function' + ? client.pluginsInvoke(request.input) : failure('plugin_unavailable'); + if (request.action === 'plugins.manage') return typeof client.pluginsManage === 'function' + ? client.pluginsManage(request.input) : failure('plugin_unavailable'); + if (client.authorizePlugin) { + const denied = await client.authorizePlugin(request.action, request.input); + if (denied) return denied; + } + if (request.action === 'diagnostics.seed') { + if (typeof client.diagnosticsSeed !== 'function') return failure('installation_not_ready'); + return client.diagnosticsSeed(request.input); + } + if (request.action === 'connections.manage') { + if (typeof client.connectionsManage !== 'function') return failure('installation_not_ready'); + return client.connectionsManage(request.input); + } + if (request.action === 'paycom.setup') { + if (typeof client.paycomSetup !== 'function') return failure('installation_not_ready'); + return client.paycomSetup(request.input); + } + if (request.action === 'health') { + const system = await client.system.status(); + if (!system?.ok) return system; + const components = system.data?.components; + if (components?.auth?.ready !== true || components?.collections?.healthy !== true + || !['ready', 'degraded'].includes(components.collections.status) + || components.collections.data?.manager?.running !== true) fail(); + result = success('ready', { + gatewayProtocolVersion: RUNTIME_GATEWAY_PROTOCOL_VERSION, + transport, + runtimeIdentity: 'matched', + }); + } else if (request.action === 'system.status') result = await client.system.status(); + else if (request.action === 'workforce.day') result = await client.workforce.day(request.input.query); + else if (request.action === 'workforce.employees') result = typeof client.workforce.employees === 'function' ? await client.workforce.employees(request.input.query) : failure('workforce_unavailable'); + else if (request.action === 'workforce.employee') result = typeof client.workforce.employee === 'function' ? await client.workforce.employee(request.input.code) : failure('workforce_unavailable'); + else if (request.action === 'sync.status') result = await client.sync.status(request.input.id); + else if (request.action === 'sync.run_now') result = await client.sync.runNow(request.input.id, request.input.options); + else if (request.action === 'sync.start') result = await client.sync.start(request.input.id); + else if (request.action === 'sync.stop') result = await client.sync.stop(request.input.id, request.input.options); + else if (request.action === 'collections.health') result = await client.collections.health(); + else fail('invalid_request'); + return result; +} + +class RuntimeGatewayServer { + constructor(options) { + const selected = validateServerOptions(options); + this.socketPath = selected.socketPath; + this.runtimeKey = selected.runtimeKey; + this.client = selected.client; + this.socketTimeoutMs = selected.socketTimeoutMs; + this.server = null; + this.identity = null; + this.rootIdentity = null; + this.connections = new Set(); + this.activeRequests = 0; + } + + async handle(value) { + if (value?.action === 'runtime.execution') return dispatchRuntimeRequest(this.client, value, this.runtimeKey); + this.activeRequests++; + try { return await dispatchRuntimeRequest(this.client, value, this.runtimeKey); } + finally { this.activeRequests--; } + } + + async start() { + const runtimeRoot = path.dirname(this.socketPath); + this.rootIdentity = privateDirectory(runtimeRoot); + if (fs.existsSync(this.socketPath)) { + const before = socketIdentity(this.socketPath); + if (await probe(this.socketPath)) fail('runtime_gateway_unavailable'); + const after = socketIdentity(this.socketPath); + if (!sameIdentity(before, after) || !sameIdentity(this.rootIdentity, privateDirectory(runtimeRoot))) fail(); + fs.unlinkSync(this.socketPath); + } + this.server = net.createServer({ allowHalfOpen: true }, socket => { + this.connections.add(socket); + let chunks = []; + let size = 0; + let handling = false; + let responded = false; + const timer = setTimeout(() => socket.destroy(), this.socketTimeoutMs); + const send = value => { + if (responded || socket.destroyed) return; + responded = true; + socket.end(`${JSON.stringify(value)}\n`); + }; + socket.on('data', chunk => { + if (responded || handling) return socket.destroy(); + size += chunk.length; + if (size > MAX_GATEWAY_REQUEST_BYTES) return send(gatewayFailure('invalid_request')); + chunks.push(chunk); + const raw = Buffer.concat(chunks).toString('utf8'); + if (!raw.includes('\n')) return; + chunks = []; + try { + if (!raw.endsWith('\n') || raw.includes('\r') || raw.slice(0, -1).includes('\n')) fail('invalid_request'); + const request = parseStrictJson(raw.slice(0, -1)); + handling = true; + Promise.resolve(this.handle(request)).then( + result => { + try { + send(gatewaySuccess(result)); + } catch (error) { + send(gatewayFailure(error?.code)); + } + }, + error => send(gatewayFailure(error?.code)), + ); + } catch (error) { + send(gatewayFailure(error?.code)); + } + }); + socket.on('end', () => { + if (!responded && !handling && size > 0) send(gatewayFailure('invalid_request')); + }); + socket.on('error', () => socket.destroy()); + socket.on('close', () => { + clearTimeout(timer); + this.connections.delete(socket); + }); + }); + this.server.maxConnections = MAX_GATEWAY_CONNECTIONS; + try { + await new Promise((resolve, reject) => { + this.server.once('error', reject); + this.server.listen(this.socketPath, () => { + this.server.off('error', reject); + resolve(); + }); + }); + if (!sameIdentity(this.rootIdentity, privateDirectory(runtimeRoot))) fail(); + this.identity = socketIdentity(this.socketPath, { requireMode: false }); + fs.chmodSync(this.socketPath, 0o600); + if (!sameIdentity(this.identity, socketIdentity(this.socketPath))) fail(); + return this; + } catch (error) { + await this.close(); + throw error; + } + } + + async close() { + for (const socket of this.connections) socket.destroy(); + this.connections.clear(); + if (this.server) { + if (this.server.listening) await new Promise(resolve => this.server.close(() => resolve())); + this.server = null; + } + try { + const current = socketIdentity(this.socketPath); + if (sameIdentity(current, this.identity)) fs.unlinkSync(this.socketPath); + } catch {} + this.identity = null; + this.rootIdentity = null; + } +} + +module.exports = { + RuntimeGatewayServer, + dispatchRuntimeRequest, + MAX_GATEWAY_REQUEST_BYTES, + MAX_GATEWAY_CONNECTIONS, + GATEWAY_SOCKET_TIMEOUT_MS, + MAX_UNIX_SOCKET_PATH_BYTES, + privateDirectory, + socketIdentity, + sameIdentity, + probeUnixSocket: probe, +}; diff --git a/dsp/runtime/gateway/tests/gateway.test.js b/dsp/runtime/gateway/tests/gateway.test.js new file mode 100644 index 0000000..2836592 --- /dev/null +++ b/dsp/runtime/gateway/tests/gateway.test.js @@ -0,0 +1,230 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { success } = require('dispatch-protocol/contracts/src'); +const { + RUNTIME_GATEWAY_PROTOCOL_VERSION, + validateGatewayRequest, + gatewaySuccess, + validateGatewayResponse, + RuntimeGatewayServer, + gatewayRequest, + createRuntimeGatewayDispatchClient, +} = require('../src'); +const { parseStrictJson } = require('dispatch-protocol/gateway/strict-json'); + +function isCode(code) { return error => error?.code === code; } + +function fixtureClient(label, calls = []) { + return { + system: { status: async () => { + calls.push(['system.status']); + return success('ready', { + label, + components: { + auth: { healthy: true, ready: true }, + collections: { healthy: true, ready: true, status: 'ready', data: { manager: { running: true } } }, + }, + summary: { ready: 3, degraded: 0, failed: 0 }, + }); + } }, + workforce: { day: async query => { + calls.push(['workforce.day', query]); + return success('found', { label, businessDate: query.date }); + } }, + sync: { + status: async id => { + calls.push(['sync.status', id]); + return success('found', { label, id }); + }, + runNow: async (id, options) => { + calls.push(['sync.run_now', id, options]); + return success('queued', { label, id, replayed: false }); + }, + start: async id => success('started', { label, id }), + stop: async (id, options) => success('stopped', { label, id, options }), + }, + collections: { health: async () => success('ready', { label }) }, + }; +} + +function request(runtimeKey, action, input) { + return { protocolVersion: RUNTIME_GATEWAY_PROTOCOL_VERSION, runtimeKey, action, input }; +} + +test('outbound agent forwards plugin lifecycle and actions through the Unix gateway to persistent DSP state', async t => { + const { fixture } = require('../../collection-manager/tests/helpers'); + const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); + const { createRuntimePlugins } = require('../../plugin-host/index'); + const { dispatchRuntimeRequest } = require('../src/server'); + const f = fixture(); + const socketPath = path.join(f.root, 'runtime-gateway.sock'); + const client = fixtureClient('alpha'); + const plugins = createRuntimePlugins({ paths: { collection: f.paths } }, client, { + createStore: () => new CollectionStore(f.paths), + load: () => ({ invoke: (_action, input) => client.workforce.day(input.query) }), + }); + Object.assign(client, { pluginsManage: plugins.manage, pluginsInvoke: plugins.invoke, authorizePlugin: plugins.authorize }); + const server = new RuntimeGatewayServer({ socketPath, runtimeKey: 'fixture_alpha', client }); + await server.start(); + t.after(async () => { await server.close(); fs.rmSync(f.root, { recursive: true, force: true }); }); + const proxy = createRuntimeGatewayDispatchClient({ socketPath, runtimeKey: 'fixture_alpha' }); + const forward = (action, input) => dispatchRuntimeRequest(proxy, request('fixture_alpha', action, input), 'fixture_alpha', 'outbound_agent'); + const invoke = { pluginId: 'paycom', action: 'workforce.day', input: { query: { date: '2026-09-02' } } }; + assert.equal((await forward('plugins.invoke', invoke)).status, 'plugin_disabled'); + const state = { command: 'apply', pluginId: 'paycom', version: '0.18.8', state: 'enabled', revision: 1 }; + assert.equal((await forward('plugins.manage', state)).status, 'applied'); + assert.equal((await forward('plugins.invoke', invoke)).data.label, 'alpha'); + assert.equal((await forward('plugins.manage', { ...state, state: 'disabled', revision: 2 })).status, 'applied'); + assert.equal((await forward('plugins.manage', { command: 'status' })).data.items[0].state, 'disabled'); + assert.equal((await forward('plugins.invoke', invoke)).status, 'plugin_disabled'); + assert.equal(proxy.capabilities().data.actions.includes('plugins.manage'), false); +}); + +test('gateway protocol is closed, versioned, target-bound, and strict', () => { + const selected = validateGatewayRequest(request('fixture_alpha', 'workforce.day', { + query: { date: '2026-09-02', limit: 10, offset: 0 }, + }), 'fixture_alpha'); + assert.equal(selected.input.query.date, '2026-09-02'); + assert.throws(() => validateGatewayRequest({ ...request('fixture_alpha', 'health', {}), extra: true }), isCode('invalid_request')); + assert.throws(() => validateGatewayRequest(request('fixture_alpha', 'unknown', {})), isCode('invalid_request')); + assert.throws(() => validateGatewayRequest(request('fixture_bravo', 'health', {}), 'fixture_alpha'), isCode('runtime_identity_mismatch')); + assert.throws(() => validateGatewayRequest({ ...request('fixture_alpha', 'health', {}), protocolVersion: 2 }), isCode('runtime_protocol_mismatch')); + assert.throws(() => parseStrictJson('{"action":"health","action":"sync.status"}'), isCode('invalid_json')); + const result = success('ready', { checked: true }); + assert.equal(validateGatewayResponse(gatewaySuccess(result)), result); + assert.throws(() => gatewaySuccess({ + contractVersion: 1, ok: true, status: 'ready', data: { credential: 'fixture' }, + }), isCode('runtime_gateway_unavailable')); + assert.throws(() => validateGatewayResponse({ ...gatewaySuccess(result), extra: true }), + isCode('invalid_request')); +}); + +test('gateway health requires a ready Auth Broker and running Collection Manager', async () => { + const server = new RuntimeGatewayServer({ + socketPath: '/tmp/runtime-gateway.sock', + runtimeKey: 'fixture_alpha', + client: { + ...fixtureClient('alpha'), + system: { status: async () => success('degraded', { + components: { + auth: { healthy: true, ready: true }, + collections: { healthy: true, ready: false, status: 'stopped', data: { manager: { running: false } } }, + }, + summary: { ready: 1, degraded: 2, failed: 0 }, + }) }, + }, + }); + await assert.rejects(server.handle(request('fixture_alpha', 'health', {})), + isCode('runtime_gateway_unavailable')); +}); + +test('invalid upstream results are sanitized without terminating the gateway', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'drg-invalid-result-')); + fs.chmodSync(root, 0o700); + const socketPath = path.join(root, 'runtime-gateway.sock'); + const upstream = fixtureClient('alpha'); + let valid = false; + upstream.system.status = async () => valid + ? success('ready', { label: 'alpha' }) + : { ok: true, status: 'ready', data: {} }; + const server = new RuntimeGatewayServer({ + socketPath, + runtimeKey: 'fixture_alpha', + client: upstream, + socketTimeoutMs: 200, + }); + await server.start(); + let unhandled = 0; + const recordUnhandled = () => { unhandled += 1; }; + process.on('unhandledRejection', recordUnhandled); + t.after(async () => { + process.off('unhandledRejection', recordUnhandled); + await server.close(); + fs.rmSync(root, { recursive: true, force: true }); + }); + + const client = createRuntimeGatewayDispatchClient({ + socketPath, + runtimeKey: 'fixture_alpha', + requestImpl: (selectedPath, payload) => gatewayRequest(selectedPath, payload, { timeoutMs: 500 }), + }); + const rejected = await client.system.status(); + assert.equal(rejected.ok, false); + assert.equal(rejected.status, 'runtime_gateway_unavailable'); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(unhandled, 0); + + valid = true; + const recovered = await client.system.status(); + assert.equal(recovered.ok, true); + assert.equal(recovered.data.label, 'alpha'); +}); + +test('real Unix gateway routes only its bound runtime and cleans up exactly', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'drg-')); + fs.chmodSync(root, 0o700); + const socketPath = path.join(root, 'runtime-gateway.sock'); + const calls = []; + const server = new RuntimeGatewayServer({ + socketPath, + runtimeKey: 'fixture_alpha', + client: fixtureClient('alpha', calls), + }); + await server.start(); + t.after(async () => { + await server.close(); + fs.rmSync(root, { recursive: true, force: true }); + }); + assert.equal(fs.lstatSync(socketPath).mode & 0o7777, 0o600); + + const client = createRuntimeGatewayDispatchClient({ socketPath, runtimeKey: 'fixture_alpha' }); + assert.equal((await client.health()).status, 'ready'); + assert.equal((await client.system.status()).data.label, 'alpha'); + assert.equal((await client.workforce.day({ date: '2026-09-02', limit: 10, offset: 0 })).data.label, 'alpha'); + assert.equal((await client.sync.status('paycom-main-workforce')).data.label, 'alpha'); + assert.equal((await client.sync.runNow('paycom-main-workforce', { + idempotencyKey: 'gateway:fixture-request-0001', + })).data.label, 'alpha'); + assert.equal(calls.filter(call => call[0] === 'workforce.day').length, 1); + + const wrong = createRuntimeGatewayDispatchClient({ socketPath, runtimeKey: 'fixture_bravo' }); + const blocked = await wrong.workforce.day({ date: '2026-09-02', limit: 10, offset: 0 }); + assert.equal(blocked.status, 'runtime_identity_mismatch'); + assert.equal(blocked.ok, false); + assert.equal(calls.filter(call => call[0] === 'workforce.day').length, 1); + + const raw = await gatewayRequest(socketPath, request('fixture_alpha', 'health', {})); + assert.equal(raw.status, 'ready'); + await server.close(); + assert.equal(fs.existsSync(socketPath), false); +}); + +test('two real gateway sockets cannot be confused across runtimes', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'drg2-')); + fs.chmodSync(root, 0o700); + const roots = ['a', 'b'].map(name => { + const value = path.join(root, name); + fs.mkdirSync(value, { mode: 0o700 }); + return value; + }); + const servers = [ + new RuntimeGatewayServer({ socketPath: path.join(roots[0], 'runtime-gateway.sock'), runtimeKey: 'fixture_alpha', client: fixtureClient('alpha') }), + new RuntimeGatewayServer({ socketPath: path.join(roots[1], 'runtime-gateway.sock'), runtimeKey: 'fixture_bravo', client: fixtureClient('bravo') }), + ]; + await Promise.all(servers.map(server => server.start())); + t.after(async () => { + await Promise.all(servers.map(server => server.close())); + fs.rmSync(root, { recursive: true, force: true }); + }); + const alpha = createRuntimeGatewayDispatchClient({ socketPath: path.join(roots[0], 'runtime-gateway.sock'), runtimeKey: 'fixture_alpha' }); + const bravo = createRuntimeGatewayDispatchClient({ socketPath: path.join(roots[1], 'runtime-gateway.sock'), runtimeKey: 'fixture_bravo' }); + assert.equal((await alpha.system.status()).data.label, 'alpha'); + assert.equal((await bravo.system.status()).data.label, 'bravo'); + const crossed = createRuntimeGatewayDispatchClient({ socketPath: path.join(roots[1], 'runtime-gateway.sock'), runtimeKey: 'fixture_alpha' }); + assert.equal((await crossed.system.status()).status, 'runtime_identity_mismatch'); +}); diff --git a/dsp/runtime/gateway/tests/managed-runtime.test.js b/dsp/runtime/gateway/tests/managed-runtime.test.js new file mode 100644 index 0000000..0f0e77f --- /dev/null +++ b/dsp/runtime/gateway/tests/managed-runtime.test.js @@ -0,0 +1,88 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { createInstallationLayoutManager } = require('dispatch-core/core/installations/src/index.js'); +const { DispatchClient } = require('../../sdk/src/dispatch-client'); +const { + managedRuntimeConfiguration, + createManagedRuntimeDispatchClient, + MANAGED_ENVIRONMENT_KEYS, +} = require('../src'); + +function manifest() { + return { + manifestVersion: 1, + revision: 1, + organization: { id: 'org_fixture_gateway', stationCode: 'TST1', timezone: 'America/Los_Angeles' }, + runtime: { key: 'fixture_gateway', templateId: 'isolated_dsp_v1', releaseId: 'dispatch_fixture_1' }, + }; +} + +function authority(value) { + return { + revision: value.revision, + organization: { ...value.organization }, + runtime: { ...value.runtime }, + }; +} + +function isBoundary(error) { return ['runtime_boundary_violation', 'runtime_identity_mismatch'].includes(error?.code); } + +function fileInventory(root, base = root, found = []) { + for (const entry of fs.readdirSync(root, { withFileTypes: true })) { + const target = path.join(root, entry.name); + if (entry.isDirectory()) fileInventory(target, base, found); + else found.push(path.relative(base, target)); + } + return found.sort(); +} + +test('managed gateway composition requires the complete explicit runtime projection', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dgc-')); + fs.chmodSync(root, 0o700); + const installationsRoot = path.join(root, 'i'); + fs.mkdirSync(installationsRoot, { mode: 0o700 }); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + + const selected = manifest(); + const manager = createInstallationLayoutManager({ installationsRoot }); + manager.materialize(selected, authority(selected)); + const projected = manager.runtimeEnvironment(selected, authority(selected)); + const environment = { + ...projected, + DISPATCH_RUNTIME_KEY: selected.runtime.key, + DISPATCH_RUNTIME_GATEWAY_SOCKET: path.join(projected.DISPATCH_RUNTIME_ROOT, 'runtime-gateway.sock'), + }; + const configuration = managedRuntimeConfiguration(environment); + assert.equal(configuration.runtimeKey, 'fixture_gateway'); + assert.equal(configuration.gatewaySocket, path.join(configuration.paths.runtimeRoot, 'runtime-gateway.sock')); + assert.equal(Object.hasOwn(configuration.paths, 'accessControl'), false); + assert.equal(MANAGED_ENVIRONMENT_KEYS.every(key => environment[key] === projected[key]), true); + + const filesBefore = fileInventory(configuration.paths.installationRoot); + const client = createManagedRuntimeDispatchClient(configuration); + assert.equal(client instanceof DispatchClient, true); + const status = await client.system.status(); + assert.equal(status.ok, true); + assert.equal(status.status, 'degraded'); + assert.equal(status.data.components.collections.status, 'not_initialized'); + assert.equal(status.data.components.paycom.status, 'not_initialized'); + assert.equal((await client.workforce.day({ date: '2026-09-02', limit: 10, offset: 0 })).status, 'not_initialized'); + assert.deepEqual(fileInventory(configuration.paths.installationRoot), filesBefore); + + const missing = { ...environment }; + delete missing.DISPATCH_COLLECTION_STATE_ROOT; + assert.throws(() => managedRuntimeConfiguration(missing), isBoundary); + assert.throws(() => managedRuntimeConfiguration({ + ...environment, + DISPATCH_RUNTIME_KEY: 'fixture_other', + }), isBoundary); + assert.throws(() => managedRuntimeConfiguration({ + ...environment, + DISPATCH_RUNTIME_GATEWAY_SOCKET: path.join(root, 'other.sock'), + }), isBoundary); +}); diff --git a/dsp/runtime/package.json b/dsp/runtime/package.json new file mode 100644 index 0000000..2614829 --- /dev/null +++ b/dsp/runtime/package.json @@ -0,0 +1,10 @@ +{ + "name": "dispatch-dsp-container", + "private": true, + "type": "commonjs", + "engines": { "node": ">=22" }, + "scripts": { + "build": "./scripts/build", + "verify": "./scripts/verify" + } +} diff --git a/dsp/runtime/plugin-host/README.md b/dsp/runtime/plugin-host/README.md new file mode 100644 index 0000000..18fd583 --- /dev/null +++ b/dsp/runtime/plugin-host/README.md @@ -0,0 +1,15 @@ +# Runtime plugin host + +`installed.js` verifies a DSP-installed package, creates its scoped SDK client +and loads its declared runtime entrypoint lazily. Actions require a manifest +declaration and current authorization both before and after execution. + +`storage.js` supplies plugin-scoped database and file handles with private paths, +bounded SQLite caches, at most eight open databases, bounded file reads and +atomic file writes. Filesystem namespaces remain the enforcement boundary for +executable plugin code; SDK path validation alone is not a sandbox. + +`index.js` remains the compatibility runtime until package installation and the +scoped worker launcher are connected. It has not silently switched existing DSPs +to installed code. Installed-package tests exercise the new loader independently +with copied packages and injected SDK services. diff --git a/dsp/runtime/plugin-host/availability.js b/dsp/runtime/plugin-host/availability.js new file mode 100644 index 0000000..77940e4 --- /dev/null +++ b/dsp/runtime/plugin-host/availability.js @@ -0,0 +1,15 @@ +'use strict'; + +// Read-only, trusted runtime paths. Missing/uninstalled/disabled plugins cannot +// start host assistance, including a connection check started before disabling. +function pluginEnabled(pluginId) { + const { resolveLocalRuntimePaths } = require('dispatch-protocol/paths/runtime-paths'); + const { DatabaseSync } = require('node:sqlite'); + let database; + try { + database = new DatabaseSync(resolveLocalRuntimePaths().collection.database, { readOnly: true }); + return database.prepare('SELECT state FROM plugin_installations WHERE plugin_id=?').get(pluginId)?.state === 'enabled'; + } catch { return false; } + finally { database?.close(); } +} +module.exports = { pluginEnabled }; diff --git a/dsp/runtime/plugin-host/contributions.js b/dsp/runtime/plugin-host/contributions.js new file mode 100644 index 0000000..d0dd6af --- /dev/null +++ b/dsp/runtime/plugin-host/contributions.js @@ -0,0 +1,44 @@ +'use strict'; + +const { catalog, pluginEntry, ROOT } = require('dispatch-protocol/plugin-sdk/catalog'); + +function contributions(method, context) { + if (process.env.DISPATCH_PLUGIN_BACKEND === 'core_v1') { + if (method !== 'createClientPorts') return Object.freeze({}); + const sdk = require('dispatch-sdk/runtime').createFrameworkClient(); + const call = async (view, query) => { + const response = await sdk.request('plugin.read', { pluginId: 'paycom', request: { view, query: query || {} } }); + if (!response.ok) { + if (response.status === 'not_initialized') return null; + throw Object.assign(new Error(response.status), { code: response.status }); + } + return response.data; + }; + const workforce = Object.fromEntries(['snapshot', 'employees', 'timecards', 'punches', 'day', 'resourceLinks'] + .map(view => [view, query => call(view, query)])); + workforce.employee = code => call('employee', { code }); + workforce.health = () => call('snapshot', {}); + return Object.freeze({ workforce, paycom: { health: () => sdk.request('plugin.inspect', { pluginId: 'paycom', request: {} }) } }); + } + const result = Object.create(null); + for (const definition of catalog()) { + if (!definition.runtime) continue; + const runtime = require(pluginEntry(ROOT, definition, 'runtime')); + if (runtime[method] === undefined) continue; + if (typeof runtime[method] !== 'function') throw new Error('plugin_contribution_invalid'); + const values = runtime[method](context); + if (!values || Object.getPrototypeOf(values) !== Object.prototype) throw new Error('plugin_contribution_invalid'); + for (const [name, value] of Object.entries(values)) { + if (!/^[a-z][a-z0-9-]{0,63}$/.test(name) || Object.hasOwn(result, name)) throw new Error('plugin_contribution_conflict'); + result[name] = value; + } + } + return Object.freeze(result); +} + +// Preserve existing SDK response shapes when an optional provider is absent. +const unavailablePort = Object.freeze(Object.fromEntries( + ['health', 'snapshot', 'employees', 'employee', 'timecards', 'punches', 'day', 'resourceLinks'] + .map(name => [name, async () => null]))); + +module.exports = { contributions, unavailablePort }; diff --git a/dsp/runtime/plugin-host/index.js b/dsp/runtime/plugin-host/index.js new file mode 100644 index 0000000..18f4484 --- /dev/null +++ b/dsp/runtime/plugin-host/index.js @@ -0,0 +1,95 @@ +'use strict'; + +const { catalog, plugin, pluginEntry, ROOT, gatewayPlugin } = require('dispatch-protocol/plugin-sdk/catalog'); +const { pluginRequest, pluginInvocation } = require('dispatch-protocol/plugin-sdk/contract'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { installation, applyState } = require('dispatch-runtime-kit/collection-manager/src/plugin-state'); +const { success, failure } = require('dispatch-protocol/contracts/src/result'); + +function createRuntimePlugins(configuration, client, { createStore = () => new CollectionStore(configuration.paths.collection), + load = definition => process.env.DISPATCH_PLUGIN_BACKEND === 'core_v1' ? { + invoke: (action, input) => require('dispatch-sdk/runtime').createFrameworkClient().request('plugin.invoke', + { pluginId: definition.id, request: { action, input } }), + setup: require('./paycom-setup').createSetup(configuration, client), + } : definition.runtime + ? require(pluginEntry(ROOT, definition, 'runtime')).createPlugin({ configuration, client }) : {} } = {}) { + const instances = new Map(); + const applying = new Map(); + function withStore(callback) { const store = createStore(); try { return callback(store); } finally { store.close(); } } + function enabled(id) { return Boolean(plugin(id)) && withStore(store => installation(store.db, id).state === 'enabled'); } + function instance(definition) { + if (!instances.has(definition.id)) instances.set(definition.id, load(definition)); + return instances.get(definition.id); + } + async function manage(value) { + try { + const input = pluginRequest(value); + if (input.command === 'status') { + if (process.env.DISPATCH_PLUGIN_BACKEND === 'core_v1') return success('found', { + items: withStore(store => catalog().map(item => installation(store.db, item.id))), + }); + for (const definition of catalog()) { + const before = withStore(store => installation(store.db, definition.id)); + if (before.revision !== 0 || before.state !== 'uninstalled' || !definition.legacyProfile) continue; + const profile = await client.auth.profileStatus(definition.legacyProfile); + if (!profile?.ok) return failure('plugin_unavailable', { recoverable: true }); + if (profile.data?.profile?.configured) withStore(store => store.db.prepare( + "UPDATE plugin_installations SET state='enabled' WHERE plugin_id=? AND revision=0 AND state='uninstalled'" + ).run(definition.id)); + } + return success('found', { items: withStore(store => catalog().map(item => installation(store.db, item.id))) }); + } + if (applying.has(input.pluginId)) return failure('plugin_busy', { recoverable: true }); + const definition = plugin(input.pluginId); + const hooks = instance(definition); + if (hooks.busy?.()) return failure('plugin_busy', { recoverable: true }); + applying.set(input.pluginId, true); + try { + // Persist the access gate and cancellation requests before awaiting any + // provider cleanup. Replaying the same revision resumes interrupted work. + const before = withStore(store => installation(store.db, input.pluginId)); + if (before.revision > input.revision || before.revision === input.revision && before.state !== input.state) { + throw Object.assign(new Error('plugin_revision_conflict'), { code: 'plugin_revision_conflict' }); + } + if (input.state === 'enabled' && before.state !== 'enabled') await hooks.enable?.(); + const current = withStore(store => applyState(store, input)); + if (input.state !== 'enabled') { + await hooks.disable?.(); + const deadline = Date.now() + 8000; + for (;;) { + const busy = withStore(store => definition.collectors.some(id => + store.db.prepare("SELECT 1 FROM runs WHERE collector_id=? AND status='running'").get(id))); + if (!busy) break; + if (Date.now() >= deadline) return failure('plugin_busy', { recoverable: true }); + await new Promise(resolve => setTimeout(resolve, 100)); + } + } + return success('applied', current); + } finally { applying.delete(input.pluginId); } + } catch (error) { + return failure(['plugin_revision_conflict', 'invalid_request'].includes(error?.code) ? error.code : 'plugin_unavailable', { recoverable: true }); + } + } + async function authorize(action, input) { + const owner = gatewayPlugin(action, input); + if (owner && (!enabled(owner.id) || applying.has(owner.id))) return failure('plugin_disabled'); + return null; + } + async function setup(input) { + const owner = gatewayPlugin('paycom.setup'); + if (!owner || !enabled(owner.id) || applying.has(owner.id)) return failure('plugin_disabled'); + return instance(owner).setup(input); + } + async function invoke(value) { + try { + const input = pluginInvocation(value); + if (!enabled(input.pluginId) || applying.has(input.pluginId)) return failure('plugin_disabled'); + const result = await instance(plugin(input.pluginId)).invoke(input.action, input.input); + if (!enabled(input.pluginId) || applying.has(input.pluginId)) return failure('plugin_disabled'); + return result; + } catch (error) { return failure(error?.code === 'invalid_request' || error?.code === 'invalid_input' ? 'invalid_input' : 'plugin_unavailable'); } + } + return { manage, authorize, setup, enabled, invoke, + busy: () => applying.size > 0 || [...instances.values()].some(value => value.busy?.()) }; +} +module.exports = { createRuntimePlugins }; diff --git a/dsp/runtime/plugin-host/installed.js b/dsp/runtime/plugin-host/installed.js new file mode 100644 index 0000000..d77df8e --- /dev/null +++ b/dsp/runtime/plugin-host/installed.js @@ -0,0 +1,30 @@ +'use strict'; +const path = require('node:path'); +const { verifyPackage } = require('dispatch-protocol/plugin-sdk/package-files'); +const { createDispatchClient } = require('dispatch-sdk'); + +// Package selection is supplied by the authenticated worker launcher. A plugin +// invocation cannot choose a module, package version, digest or filesystem root. +function loadInstalledPlugin({ packageRoot, digest, pluginId, transport, storage, authorize }) { + if (typeof authorize !== 'function' || typeof transport?.request !== 'function') throw new TypeError('plugin_authority_required'); + const manifest = verifyPackage(packageRoot, digest); + if (manifest.plugin.id !== pluginId || !manifest.plugin.runtime) throw new Error('plugin_identity_mismatch'); + const dispatch = createDispatchClient({ transport, ...(storage ? { storage } : {}) }); + let instance; + async function invoke(action, input, options = {}) { + if (!manifest.plugin.actions.some(item => item.id === action) || !await authorize(action) || options.signal?.aborted) { + throw new Error('plugin_action_denied'); + } + if (!instance) { + const implementation = require(path.join(packageRoot, manifest.plugin.runtime)); + if (typeof implementation.createPlugin !== 'function') throw new Error('plugin_entrypoint_invalid'); + instance = implementation.createPlugin({ dispatch }); + if (typeof instance?.invoke !== 'function') throw new Error('plugin_entrypoint_invalid'); + } + const result = await instance.invoke(action, input, options); + if (!await authorize(action) || options.signal?.aborted) throw new Error('plugin_action_denied'); + return result; + } + return Object.freeze({ manifest, invoke }); +} +module.exports = { loadInstalledPlugin }; diff --git a/dsp/runtime/plugin-host/paycom-activation.js b/dsp/runtime/plugin-host/paycom-activation.js new file mode 100644 index 0000000..f00307d --- /dev/null +++ b/dsp/runtime/plugin-host/paycom-activation.js @@ -0,0 +1,396 @@ +'use strict'; + +const path = require('node:path'); +const { + isResult, + serverInstallationManifest, +} = require('dispatch-protocol/contracts/src'); +const { + managedPaycomDefinition, + managedPaycomFirstPublicationRequest, + PAYCOM_FIRST_PUBLICATION_TASKS, + PAYCOM_PROFILE_ID, + PAYCOM_SYNC_ID, +} = require('./paycom-definition'); + +const DEFAULT_PUBLICATION_TIMEOUT_MS = 2 * 60 * 60 * 1000; +const DEFAULT_PUBLICATION_POLL_MS = 1000; +const MAX_PAY_PERIOD_PREPARATION_MS = 5 * 60 * 1000; +const PAYCOM_PERIODS_PLAN = 'paycom-periods'; +const EXPECTED_FIRST_PUBLICATION_PLANS = Object.freeze(Object.fromEntries( + Object.entries(PAYCOM_FIRST_PUBLICATION_TASKS).map(([plan, definition]) => [plan, definition.method]), +)); + +function fail(code) { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, fields, code = 'runtime_boundary_violation') { + if (!plain(value) || Object.keys(value).sort().join(',') !== [...fields].sort().join(',')) fail(code); + return value; +} + +function same(left, right) { + return JSON.stringify(left) === JSON.stringify(right); +} + +function successful(result, statuses, code) { + if (!isResult(result) || !result.ok || !statuses.includes(result.status)) fail(code); + return result; +} + +function createManagedPaycomActivationRuntime(options) { + const optionFields = [ + 'manifest', 'manifestAuthority', 'layout', 'serviceManager', 'supervisor', 'client', + 'collectionAdmin', 'gateway', 'evidenceVerifier', 'clock', 'delay', 'publicationTimeoutMs', 'publicationPollMs', + 'projectRoot', 'infrastructureVerifier', + ]; + if (!plain(options) || Object.keys(options).some(key => !optionFields.includes(key)) + || !['manifest', 'manifestAuthority', 'client', + 'collectionAdmin', 'gateway', 'evidenceVerifier'].every(key => Object.hasOwn(options, key))) { + fail('runtime_boundary_violation'); + } + const manifest = serverInstallationManifest(options.manifest, options.manifestAuthority); + const layout = options.layout; + const serviceManager = options.serviceManager; + const supervisor = options.supervisor; + const client = options.client; + const collectionAdmin = options.collectionAdmin; + const gateway = options.gateway; + const evidenceVerifier = options.evidenceVerifier; + if ((typeof options.infrastructureVerifier !== 'function' && (!layout || typeof layout.inspect !== 'function' + || !serviceManager || typeof serviceManager.plan !== 'function' || typeof serviceManager.inspectInstalled !== 'function' + || !supervisor || typeof supervisor.inspect !== 'function' || typeof supervisor.health !== 'function')) + || !client?.auth || !client?.collections || !client?.sync || !client?.paycom + || !collectionAdmin || !['preview', 'apply', 'inspect', 'attest'] + .every(method => typeof collectionAdmin[method] === 'function') + || !gateway || typeof gateway.health !== 'function' + || !evidenceVerifier || typeof evidenceVerifier.verify !== 'function') fail('runtime_boundary_violation'); + const clock = options.clock === undefined ? Date.now : options.clock; + const delay = options.delay === undefined ? ms => new Promise(resolve => setTimeout(resolve, ms)) : options.delay; + const publicationTimeoutMs = options.publicationTimeoutMs === undefined + ? DEFAULT_PUBLICATION_TIMEOUT_MS : options.publicationTimeoutMs; + const publicationPollMs = options.publicationPollMs === undefined + ? DEFAULT_PUBLICATION_POLL_MS : options.publicationPollMs; + if (typeof clock !== 'function' || typeof delay !== 'function' + || !Number.isSafeInteger(publicationTimeoutMs) || publicationTimeoutMs < 1000 || publicationTimeoutMs > DEFAULT_PUBLICATION_TIMEOUT_MS + || !Number.isSafeInteger(publicationPollMs) || publicationPollMs < 10 || publicationPollMs > 10_000) { + fail('runtime_boundary_violation'); + } + const projectRoot = options.projectRoot; + const definitionOptions = projectRoot === undefined ? {} : { projectRoot }; + const expectedDefinition = () => managedPaycomDefinition(manifest, options.manifestAuthority, definitionOptions); + + function selectedManifest(value) { + const selected = serverInstallationManifest(value, options.manifestAuthority); + if (!same(selected, manifest)) fail('runtime_identity_mismatch'); + return selected; + } + + async function verifyInfrastructure(manifestValue) { + const selected = selectedManifest(manifestValue); + if (options.infrastructureVerifier) return options.infrastructureVerifier(selected); + const selectedLayout = layout.inspect(selected, options.manifestAuthority); + const plan = serviceManager.plan(selected, options.manifestAuthority, selectedLayout); + serviceManager.inspectInstalled(plan); + supervisor.inspect(plan); + supervisor.health(plan); + const auth = successful(await client.auth.health(), ['ready'], 'runtime_health_failed'); + const collections = successful(await client.collections.health(), ['ready'], 'runtime_health_failed'); + if (auth.data?.vault?.verified !== true || collections.data?.manager?.running !== true + || collections.data?.databaseIntegrity !== 'ok' + || collections.data?.syncAlerts?.critical !== 0) fail('runtime_health_failed'); + const gatewayHealth = successful(await gateway.health(), ['ready'], 'runtime_health_failed'); + if (gatewayHealth.data?.runtimeIdentity !== 'matched') fail('runtime_identity_mismatch'); + return Object.freeze({ + runtimeKey: manifest.runtime.key, + runtime_layout: true, + service_supervision: true, + auth_broker: true, + collection_manager: true, + runtime_gateway: true, + }); + } + + async function configure(definition) { + const expected = expectedDefinition(); + if (!definition || definition.digest !== expected.digest + || !same(definition.specification, expected.specification)) fail('runtime_boundary_violation'); + const preview = collectionAdmin.preview(expected.specification); + if (!plain(preview) || preview.valid !== true) fail('runtime_health_failed'); + const applied = collectionAdmin.apply(expected.specification); + const expectedCounts = { collectors: 1, sources: 1, plans: 15, syncs: 1 }; + if (!same(applied, expectedCounts)) fail('runtime_health_failed'); + const inspected = collectionAdmin.inspect(); + if (inspected?.initialized !== true || !same(inspected.counts, expectedCounts)) fail('runtime_health_failed'); + const attested = collectionAdmin.attest(expected.specification); + if (!plain(attested) || Object.keys(attested).length !== 1 || attested.matched !== true) { + fail('runtime_health_failed'); + } + const [source, sync] = await Promise.all([ + client.collections.source('paycom-main'), + client.sync.status('paycom-main-workforce'), + ]); + successful(source, ['found'], 'runtime_health_failed'); + successful(sync, ['found'], 'runtime_health_failed'); + if (source.data?.authProfile !== PAYCOM_PROFILE_ID || sync.data?.desiredState !== 'stopped') { + fail('runtime_health_failed'); + } + return Object.freeze({ digest: expected.digest, ...expectedCounts }); + } + + async function startWorkforceSync() { + const expected = expectedDefinition(); + let current = await client.sync.status(PAYCOM_SYNC_ID); + if (!current.ok && current.status === 'sync_not_found') { + await configure(expected); + current = await client.sync.status(PAYCOM_SYNC_ID); + } + successful(current, ['found'], 'runtime_health_failed'); + if (!['running', 'stopped'].includes(current.data?.desiredState)) fail('runtime_health_failed'); + // Reconnection must not reapply definitions or reset an existing hourly window. + const spec = structuredClone(expected.specification); + const sync = spec.syncs.find(item => item.id === PAYCOM_SYNC_ID); + sync.desiredState = current.data.desiredState; + sync.intervalSeconds = current.data.intervalSeconds; + sync.jitterSeconds = current.data.jitterSeconds; + if (collectionAdmin.attest(spec)?.matched !== true) fail('runtime_health_failed'); + if (current.data.intervalSeconds !== 3600 || current.data.jitterSeconds !== 0) { + successful(await client.sync.edit(PAYCOM_SYNC_ID, { intervalSeconds: 3600, jitterSeconds: 0 }), ['updated'], 'runtime_health_failed'); + } + successful(await client.sync.start(PAYCOM_SYNC_ID), ['started'], 'runtime_health_failed'); + return Object.freeze({ syncId: PAYCOM_SYNC_ID, intervalSeconds: 3600, desiredState: 'running' }); + } + + async function testProvider(profileId) { + if (profileId !== PAYCOM_PROFILE_ID) fail('provider_auth_required'); + const status = successful(await client.auth.profileStatus(profileId), ['configured'], 'provider_auth_required'); + if (status.data?.profile?.configured !== true || status.data.profile.provider !== 'paycom') { + fail('provider_auth_required'); + } + const response = await client.auth.testProfile(profileId); + if (!response?.ok) fail(require('dispatch-protocol/contracts/src/paycom-setup').setupFailure(response?.status)); + const tested = successful(response, ['authenticated'], 'provider_auth_required'); + if (tested.data?.profile !== profileId || tested.data?.provider !== 'paycom' + || typeof tested.data?.testedAt !== 'string' || Number.isNaN(Date.parse(tested.data.testedAt))) { + fail('provider_auth_required'); + } + return Object.freeze({ + profileId, + provider: 'paycom', + status: 'authenticated', + testedAt: tested.data.testedAt, + }); + } + + async function batch(batchId) { + return successful(await client.collections.batchStatus(batchId, { limit: 50, offset: 0 }), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + } + + function publicationFromBatch(result) { + const value = result.data; + if (!value || typeof value.id !== 'string' || !value.counts || value.runPage?.hasMore !== false + || value.runCount !== value.runPage.total || value.runCount !== value.runPage.items.length + || value.runCount !== Object.keys(EXPECTED_FIRST_PUBLICATION_PLANS).length) { + fail('first_publication_failed'); + } + const plans = value.runPage.items.map(item => item?.run?.plan).sort(); + if (!same(plans, Object.keys(EXPECTED_FIRST_PUBLICATION_PLANS).sort()) + || value.runPage.items.some(item => item?.run?.source !== 'paycom-main' + || item.run.method !== EXPECTED_FIRST_PUBLICATION_PLANS[item.run.plan] + || item.taskId !== PAYCOM_FIRST_PUBLICATION_TASKS[item.run.plan]?.taskId + || item.targetKey !== value.runPage.items[0]?.targetKey)) fail('first_publication_failed'); + return Object.freeze({ + batchId: value.id, + status: value.status, + runCount: value.runCount, + succeededRuns: value.counts.succeeded, + failedRuns: value.counts.failed, + cancelledRuns: value.counts.cancelled, + }); + } + + async function cancelAndDrain(batchId) { + let result; + try { + result = successful(await client.collections.cancelBatch(batchId, { limit: 50, offset: 0 }), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + } catch { + fail('installation_operation_in_progress'); + } + const deadline = clock() + Math.min(60_000, publicationTimeoutMs); + while (['queued', 'running'].includes(result.status)) { + if (clock() >= deadline) fail('installation_operation_in_progress'); + await delay(publicationPollMs); + try { result = await batch(batchId); } + catch { fail('installation_operation_in_progress'); } + } + return result; + } + + async function cancelRunAndDrain(runId) { + let result; + try { + result = successful(await client.collections.cancelRun(runId), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + } catch { + fail('installation_operation_in_progress'); + } + const deadline = clock() + Math.min(60_000, publicationTimeoutMs); + while (['queued', 'running'].includes(result.status)) { + if (clock() >= deadline) fail('installation_operation_in_progress'); + await delay(publicationPollMs); + try { + result = successful(await client.collections.runStatus(runId), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + } catch { fail('installation_operation_in_progress'); } + } + return result; + } + + async function ensurePayPeriodBaseline(operationOptions) { + const idempotencyKey = `${operationOptions.idempotencyKey}:periods`; + let result = successful(await client.collections.startRun( + PAYCOM_PERIODS_PLAN, {}, { idempotencyKey }, + ), ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + const runId = result.data?.id; + if (typeof runId !== 'string') fail('first_publication_failed'); + const deadline = clock() + Math.min(MAX_PAY_PERIOD_PREPARATION_MS, publicationTimeoutMs); + for (;;) { + await operationOptions.heartbeat(); + if (result.status === 'succeeded') return runId; + if (['failed', 'cancelled'].includes(result.status)) fail('first_publication_failed'); + if (clock() >= deadline) { + await cancelRunAndDrain(runId); + fail('first_publication_failed'); + } + await delay(publicationPollMs); + result = successful(await client.collections.runStatus(runId), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + } + } + + async function publishFirst(request, operationOptions) { + if (!same(request, managedPaycomFirstPublicationRequest())) fail('runtime_boundary_violation'); + exact(operationOptions, ['idempotencyKey', 'heartbeat']); + if (typeof operationOptions.idempotencyKey !== 'string' + || !/^activation:[a-z][a-z0-9_-]{2,95}$/.test(operationOptions.idempotencyKey) + || typeof operationOptions.heartbeat !== 'function') { + fail('runtime_boundary_violation'); + } + const preparationRunId = await ensurePayPeriodBaseline(operationOptions); + let result = successful(await client.collections.enqueue(request, { idempotencyKey: operationOptions.idempotencyKey }), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + const batchId = result.data?.id; + if (typeof batchId !== 'string') fail('first_publication_failed'); + const deadline = clock() + publicationTimeoutMs; + for (;;) { + await operationOptions.heartbeat(); + result = await batch(batchId); + if (['succeeded', 'failed', 'cancelled'].includes(result.status)) { + return Object.freeze({ ...publicationFromBatch(result), preparationRunId }); + } + if (clock() >= deadline) { + await cancelAndDrain(batchId); + fail('first_publication_failed'); + } + await delay(publicationPollMs); + } + } + + async function verifyPublication(batchId, preparationRunId) { + if (typeof preparationRunId !== 'string') fail('first_publication_failed'); + const batchResult = await batch(batchId); + const terminal = publicationFromBatch(batchResult); + if (terminal.status !== 'succeeded' || terminal.succeededRuns !== terminal.runCount + || terminal.failedRuns !== 0 || terminal.cancelledRuns !== 0) fail('first_publication_failed'); + const manager = successful(await client.collections.health(), ['ready'], 'first_publication_failed'); + if (manager.data?.manager?.running !== true || manager.data?.databaseIntegrity !== 'ok' + || manager.data?.counts?.queued !== 0 + || manager.data?.counts?.running !== 0 + || manager.data?.syncAlerts?.critical !== 0) fail('first_publication_failed'); + const sync = successful(await client.sync.status(PAYCOM_SYNC_ID), ['found'], 'first_publication_failed'); + if (sync.data?.desiredState !== 'stopped' || sync.data?.activity !== 'idle' + || sync.data?.activeRun !== null || sync.data?.queuedRunCount !== 0) { + fail('first_publication_failed'); + } + const health = successful(await client.paycom.health(), ['ready'], 'first_publication_failed'); + const data = health.data; + if (data?.ready !== true || data.publicationStatus !== 'ready' + || data.payPeriods?.projectionValid !== true + || ![data.payPeriods, data.roster, data.timecards, data.resourceLinks] + .every(value => value?.verified === true)) fail('first_publication_failed'); + const target = data.roster.target; + if (typeof target !== 'string' || data.timecards.target !== target || data.resourceLinks.target !== target) { + fail('first_publication_failed'); + } + if (batchResult.data.runPage.items[0].targetKey !== target) fail('first_publication_failed'); + const evidence = await evidenceVerifier.verify({ + batchId, + preparationRunId, + definitionDigest: expectedDefinition().digest, + }); + exact(evidence, [ + 'definitionDigest', 'requestDigest', 'previewDigest', 'batchId', 'preparationRunId', 'target', 'runs', + 'publications', 'capturedAt', + ], 'first_publication_failed'); + if (evidence.batchId !== batchId || evidence.preparationRunId !== preparationRunId + || evidence.target !== target) fail('first_publication_failed'); + return Object.freeze({ ...evidence }); + } + + async function inspectSchedule() { + const sync = successful(await client.sync.status(PAYCOM_SYNC_ID), ['found'], 'runtime_health_failed'); + if (!['running', 'stopped'].includes(sync.data?.desiredState)) fail('runtime_health_failed'); + return Object.freeze({ syncWasRunning: sync.data.desiredState === 'running' }); + } + + async function quiesceSchedule(syncWasRunning) { + if (typeof syncWasRunning !== 'boolean') fail('runtime_boundary_violation'); + const before = successful(await client.sync.status(PAYCOM_SYNC_ID), ['found'], 'runtime_health_failed'); + if (before.data?.desiredState === 'running') { + const stopped = successful(await client.sync.stop(PAYCOM_SYNC_ID, { drain: true }), ['stopped'], 'runtime_health_failed'); + if (stopped.data?.desiredState !== 'stopped' || stopped.data?.activity !== 'idle' + || stopped.data?.activeRun !== null || stopped.data?.queuedRunCount !== 0) fail('runtime_health_failed'); + } else if (before.data?.desiredState !== 'stopped') fail('runtime_health_failed'); + const manager = successful(await client.collections.health(), ['ready'], 'runtime_health_failed'); + if (manager.data?.counts?.queued !== 0 || manager.data?.counts?.running !== 0) fail('runtime_health_failed'); + return Object.freeze({ syncWasRunning }); + } + + async function restoreSchedule(syncWasRunning) { + if (typeof syncWasRunning !== 'boolean') fail('runtime_boundary_violation'); + const before = successful(await client.sync.status(PAYCOM_SYNC_ID), ['found'], 'runtime_health_failed'); + if (syncWasRunning && before.data?.desiredState === 'stopped') { + const started = successful(await client.sync.start(PAYCOM_SYNC_ID), ['started'], 'runtime_health_failed'); + if (started.data?.sync?.desiredState !== 'running') fail('runtime_health_failed'); + } else if (!syncWasRunning && before.data?.desiredState === 'running') { + const stopped = successful(await client.sync.stop(PAYCOM_SYNC_ID, { drain: true }), ['stopped'], 'runtime_health_failed'); + if (stopped.data?.desiredState !== 'stopped' || stopped.data?.activity !== 'idle' + || stopped.data?.activeRun !== null || stopped.data?.queuedRunCount !== 0) fail('runtime_health_failed'); + } else if (before.data?.desiredState !== (syncWasRunning ? 'running' : 'stopped')) { + fail('runtime_health_failed'); + } + return Object.freeze({ syncWasRunning }); + } + + return Object.freeze({ + verifyInfrastructure, + configure, + startWorkforceSync, + testProvider, + publishFirst, + verifyPublication, + inspectSchedule, + quiesceSchedule, + restoreSchedule, + }); +} + +module.exports = { DEFAULT_PUBLICATION_TIMEOUT_MS, DEFAULT_PUBLICATION_POLL_MS, MAX_PAY_PERIOD_PREPARATION_MS, PAYCOM_PERIODS_PLAN, EXPECTED_FIRST_PUBLICATION_PLANS, createManagedPaycomActivationRuntime }; diff --git a/dsp/runtime/plugin-host/paycom-definition.js b/dsp/runtime/plugin-host/paycom-definition.js new file mode 100644 index 0000000..8e297f4 --- /dev/null +++ b/dsp/runtime/plugin-host/paycom-definition.js @@ -0,0 +1,28 @@ +'use strict'; +const path = require('node:path'); +const crypto = require('node:crypto'); +const { installedPackage, installationReceipt } = require('dispatch-protocol/plugin-sdk/installed'); +const { read } = require('dispatch-protocol/plugin-sdk/package-files'); +const { serverInstallationManifest } = require('dispatch-protocol/contracts/src'); +const shared = require('dispatch-protocol/paycom-activation'); +const PAYCOM_FIRST_PUBLICATION_TASKS = Object.freeze({ + 'paycom-period-roster': { taskId: 'roster', method: 'roster.period', publication: 'roster' }, + 'paycom-period-timecards-from-roster': { taskId: 'timecards', method: 'timecards.from-published-roster', publication: 'timecards' }, + 'paycom-period-timecards-audit': { taskId: 'timecards-audit', method: 'timecards.audit', publication: null }, + 'paycom-period-resource-links': { taskId: 'links', method: 'resource-links.period', publication: 'resourceLinks' }, + 'paycom-period-resource-links-audit': { taskId: 'links-audit', method: 'resource-links.audit', publication: null }, +}); +function managedPaycomDefinition(value, authority) { + const manifest = serverInstallationManifest(value, authority); + const dspRoot = path.dirname(process.env.DISPATCH_DATA_ROOT || ''); + if (dspRoot !== `/var/lib/dispatch/${manifest.runtime.key}`) throw new Error('runtime_boundary_violation'); + const receipt = installationReceipt(dspRoot, 'paycom'); + const installed = installedPackage({ dspRoot, pluginId: 'paycom', revision: receipt.revision }); + const specification = JSON.parse(read(installed.directory, 'migrations/collections.json', 1024 * 1024)); + for (const collector of specification.collectors) collector.command = '/opt/dispatch/bin/dispatch-plugin-collector'; + for (const source of specification.sources) source.config.timezone = manifest.organization.timezone; + require('dispatch-runtime-kit/collection-manager/src/validation').validateSpec(specification); + return { profileId: shared.PAYCOM_PROFILE_ID, sourceId: shared.PAYCOM_SOURCE_ID, syncId: shared.PAYCOM_SYNC_ID, + specification, digest: crypto.createHash('sha256').update(JSON.stringify(specification)).digest('hex') }; +} +module.exports = { ...shared, PAYCOM_FIRST_PUBLICATION_TASKS, managedPaycomDefinition }; diff --git a/dsp/runtime/plugin-host/paycom-setup.js b/dsp/runtime/plugin-host/paycom-setup.js new file mode 100644 index 0000000..39bb5c4 --- /dev/null +++ b/dsp/runtime/plugin-host/paycom-setup.js @@ -0,0 +1,135 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { success, failure } = require('dispatch-protocol/contracts/src/result'); +const { setupRequest, paycomReadiness } = require('dispatch-protocol/contracts/src/paycom-setup'); +const { request: brokerRequest } = require('dispatch-runtime-kit/auth-broker/src/client'); +const { ensurePrivateDirectory } = require('../auth-broker/src/vault'); +const { managedInstallationRuntimeEnvironment } = require('dispatch-protocol/paths/runtime-paths'); +const { createManagedPaycomActivationRuntime } = require('./paycom-activation'); +const { managedPaycomDefinition, managedPaycomFirstPublicationRequest } = require('./paycom-definition'); +const { LocalCollectionAdminPort } = require('../adapters/local/collection-admin-port'); +const { createFrameworkClient } = require('dispatch-sdk/runtime'); +const { configuration, assertMountBoundary } = require('../supervisor/src/supervisor'); + +function fail(code = 'runtime_boundary_violation') { throw Object.assign(new Error(code), { code }); } +const SAFE_ERRORS = new Set(['provider_auth_required', 'first_publication_failed', 'runtime_health_failed', + 'profile_exists', 'profile_not_configured', 'invalid_input', 'setup_interrupted', 'setup_busy', + 'mfa_required', 'captcha_required', 'account_locked', 'invalid_credentials', 'primary_credentials_rejected', + 'security_answers_rejected', 'manual_verification_required', 'attempt_cooldown', 'profile_locked']); +function errorCode(error) { return SAFE_ERRORS.has(error?.code) ? error.code : 'runtime_boundary_violation'; } + +function createContainerPaycomSetup(config, client) { + const root = require('dispatch-protocol/paths/feature-paths').featurePaths(config.paths, 'paycom').stateRoot; + // Fresh DSPs have no feature state yet. Validate/create each private level + // rather than bypassing the storage boundary with recursive directory creation. + ensurePrivateDirectory(path.dirname(root)); + ensurePrivateDirectory(root); + const running = new Map(); + function file(key) { return path.join(root, `${key}.json`); } + function read(key) { + ensurePrivateDirectory(root); + let info; + try { info = fs.lstatSync(file(key)); } catch (error) { if (error.code === 'ENOENT') return null; throw error; } + if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || info.uid !== process.geteuid() + || (info.mode & 0o7777) !== 0o600 || info.size > 128 * 1024) fail(); + const value = JSON.parse(fs.readFileSync(file(key), 'utf8')); + if (!value || !['running', 'succeeded', 'failed'].includes(value.status) + || Object.keys(value).sort().join(',') !== 'data,error,status') fail(); + return value; + } + function write(key, value) { + read(key); + if (fs.readdirSync(root).length > 256) fail(); + const candidate = `${file(key)}.${crypto.randomBytes(8).toString('hex')}.tmp`; + const fd = fs.openSync(candidate, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, 0o600); + try { fs.writeFileSync(fd, `${JSON.stringify(value)}\n`); fs.fsyncSync(fd); } finally { fs.closeSync(fd); } + fs.renameSync(candidate, file(key)); + const parent = fs.openSync(root, fs.constants.O_RDONLY | fs.constants.O_DIRECTORY | fs.constants.O_NOFOLLOW); + try { fs.fsyncSync(parent); } finally { fs.closeSync(parent); } + } + function runtime(input) { + const environment = managedInstallationRuntimeEnvironment(config.layout); + return createManagedPaycomActivationRuntime({ + manifest: input.manifest, manifestAuthority: input.manifestAuthority, client, + projectRoot: '/opt/dispatch', + gateway: { health: async () => success('ready', { runtimeIdentity: 'matched' }) }, + collectionAdmin: new LocalCollectionAdminPort({ paths: config.paths.collection }), + evidenceVerifier: { verify: request => createFrameworkClient().request('plugin.evidence', { pluginId: 'paycom', request }) }, + infrastructureVerifier: async manifest => { + assertMountBoundary(configuration()); + const [auth, manager] = await Promise.all([client.auth.health(), client.collections.health()]); + if (!auth.ok || auth.data?.vault?.verified !== true || !manager.ok + || manager.data?.manager?.running !== true || manager.data?.databaseIntegrity !== 'ok') fail('runtime_health_failed'); + return { runtimeKey: manifest.runtime.key, runtime_layout: true, service_supervision: true, + auth_broker: true, collection_manager: true, runtime_gateway: true }; + }, + }); + } + async function execute(input) { + if (input.command === 'enroll') { + if (input.expiresAt < Date.now() || input.expiresAt > Date.now() + 60_000) fail('invalid_input'); + let result = await brokerRequest(config.paths.auth.socket, { + action: 'enroll-paycom', credentials: input.credentials, intent: input.intent, + }); + // A failed first delivery may leave nothing to replace. Only the broker's + // explicit missing-profile response permits saving these credentials anew. + if (!result.ok && result.status === 'profile_not_configured' && input.intent === 'replace') { + result = await brokerRequest(config.paths.auth.socket, { + action: 'enroll-paycom', credentials: input.credentials, intent: 'create', + }); + } + if (!result.ok) fail(result.status); + return { configured: true }; + } + const selected = runtime(input); + if (input.step === 'test') return selected.testProvider('paycom-main'); + await selected.verifyInfrastructure(input.manifest); + if (input.step === 'sync') return selected.startWorkforceSync(); + if (input.step === 'configure') return selected.configure(managedPaycomDefinition(input.manifest, input.manifestAuthority, { projectRoot: '/opt/dispatch' })); + if (input.step === 'publish') return selected.publishFirst(managedPaycomFirstPublicationRequest(), { + idempotencyKey: `activation:${input.parameters.jobId}`, heartbeat: async () => {}, + }); + if (input.step === 'verify') return selected.verifyPublication(input.parameters.batchId, input.parameters.preparationRunId); + fail('invalid_input'); + } + const handle = async value => { + try { + const input = setupRequest(value, config.runtimeKey); + if (input.step === 'readiness') { + // Always read the broker's current guard, never a cached setup receipt. + const response = await brokerRequest(config.paths.auth.socket, { action: 'profile-readiness', profile: 'paycom-main' }); + if (!response.ok || response.status !== 'found') fail('runtime_health_failed'); + return success('succeeded', paycomReadiness(response.readiness)); + } + if (input.step === 'infrastructure') return success('succeeded', await runtime(input).verifyInfrastructure(input.manifest)); + const enrollment = input.command === 'enroll'; + const identity = enrollment ? { requestId: input.requestId, intent: input.intent } + : { requestId: input.requestId, step: input.step, manifest: input.manifest, parameters: input.parameters }; + const key = crypto.createHash('sha256').update(JSON.stringify(identity)).digest('hex'); + let saved = read(key); + if (saved?.status === 'running' && !running.has(key)) { + saved = { status: 'failed', data: null, error: 'setup_interrupted' }; write(key, saved); + } + if (!saved && input.command === 'status') return success('not_started', null); + if (!saved) { + if (running.size) return failure('setup_busy'); + saved = { status: 'running', data: null, error: null }; write(key, saved); + const work = Promise.resolve().then(() => execute(input)).then( + data => write(key, { status: 'succeeded', data, error: null }), + error => write(key, { status: 'failed', data: null, error: errorCode(error) }), + ).finally(() => running.delete(key)); + running.set(key, work); + // Enrollment is short and never survives in the control-plane database. + if (enrollment) { await work; saved = read(key); } + else work.catch(() => {}); + } + return saved.status === 'failed' ? failure(saved.error) : success(saved.status, saved.data); + } catch (error) { return failure(errorCode(error)); } + }; + handle.busy = () => running.size > 0; + return handle; +} +module.exports = { createSetup: createContainerPaycomSetup }; diff --git a/dsp/runtime/plugin-host/storage.js b/dsp/runtime/plugin-host/storage.js new file mode 100644 index 0000000..c5c0180 --- /dev/null +++ b/dsp/runtime/plugin-host/storage.js @@ -0,0 +1,16 @@ +'use strict'; +const path = require('node:path'); +const { featurePaths } = require('dispatch-protocol/paths/feature-paths'); +const { privateDirectory } = require('dispatch-protocol/paths/private-directory'); +const { createLocalStorage } = require('dispatch-sdk/runtime/storage'); + +function createPluginStorage({ roots, pluginId }) { + featurePaths(roots, pluginId); + return createLocalStorage(Object.fromEntries(Object.entries({ + database: path.join(roots.dataRoot, 'db', pluginId), + files: path.join(roots.dataRoot, 'files', pluginId), + state: path.join(roots.stateRoot, 'plugins', pluginId), + staging: path.join(roots.stagingRoot, 'plugins', pluginId), + }).map(([key, value]) => [key, privateDirectory(value)]))); +} +module.exports = { createPluginStorage }; diff --git a/dsp/runtime/plugin-host/tests/storage.test.js b/dsp/runtime/plugin-host/tests/storage.test.js new file mode 100644 index 0000000..e0d74e7 --- /dev/null +++ b/dsp/runtime/plugin-host/tests/storage.test.js @@ -0,0 +1,35 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { createPluginStorage } = require('../storage'); + +test('plugin storage keeps databases and files separate across DSPs and plugins', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-plugin-storage-')); + const stores = []; + t.after(() => { for (const store of stores) store.close(); fs.rmSync(root, { recursive: true, force: true }); }); + function scoped(dsp, plugin) { + const base = path.join(root, dsp); + const roots = { projectRoot: path.resolve(__dirname, '../../..'), dataRoot: path.join(base, 'data'), + stateRoot: path.join(base, 'state'), stagingRoot: path.join(base, 'staging') }; + for (const field of ['dataRoot', 'stateRoot', 'stagingRoot']) fs.mkdirSync(roots[field], { recursive: true, mode: 0o700 }); + const store = createPluginStorage({ roots, pluginId: plugin }); stores.push(store); return store; + } + const first = scoped('dsp-a', 'sample'), second = scoped('dsp-b', 'sample'), sibling = scoped('dsp-a', 'other'); + const db = first.database('records'); db.exec("CREATE TABLE record(value TEXT); INSERT INTO record VALUES('private');"); + first.files('exports').write('report.csv', 'first DSP'); + assert.equal(first.files('exports').read('report.csv').toString(), 'first DSP'); + assert.throws(() => second.files('exports').read('report.csv'), { code: 'ENOENT' }); + assert.throws(() => sibling.files('exports').read('report.csv'), { code: 'ENOENT' }); + assert.equal(second.database('records').prepare("SELECT count(*) n FROM sqlite_master WHERE name='record'").get().n, 0); + assert.throws(() => first.database('../other/records'), { code: 'plugin_storage_name_invalid' }); + assert.throws(() => first.files('exports').read('../../other/report.csv'), { code: 'plugin_storage_name_invalid' }); + const outside = path.join(root, 'unrelated.txt'); fs.writeFileSync(outside, 'private'); + fs.symlinkSync(outside, path.join(root, 'dsp-a/data/files/sample/exports/link.txt')); + assert.throws(() => first.files('exports').read('link.txt'), { code: 'plugin_storage_unsafe' }); + assert.throws(() => first.files('exports').write('link.txt', 'overwrite'), { code: 'plugin_storage_unsafe' }); + assert.equal(fs.readFileSync(outside, 'utf8'), 'private'); + first.close(); assert.throws(() => first.database('records'), { code: 'plugin_storage_closed' }); +}); diff --git a/dsp/runtime/scripts/build b/dsp/runtime/scripts/build new file mode 100755 index 0000000..c1dd5e1 --- /dev/null +++ b/dsp/runtime/scripts/build @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" +if test "$#" -ne 1; then + echo 'Usage: runtime/tooling/build /absolute/new-output-directory' >&2 + exit 2 +fi +exec node --no-warnings "$ROOT/runtime/supervisor/examples/build-native-fixture.js" "$1" diff --git a/dsp/runtime/scripts/verify b/dsp/runtime/scripts/verify new file mode 100755 index 0000000..4fab5a5 --- /dev/null +++ b/dsp/runtime/scripts/verify @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd)" +cd "$ROOT" +PACKAGE_PARENT="$(mktemp -d /tmp/dispatch-native-verify-XXXXXX)" +trap 'rm -rf -- "$PACKAGE_PARENT"' EXIT +node --no-warnings runtime/supervisor/examples/build-native-fixture.js "$PACKAGE_PARENT/package" +node --no-warnings runtime/supervisor/examples/native-host-fixture.js "$PACKAGE_PARENT/package" diff --git a/dsp/runtime/scripts/verify-image b/dsp/runtime/scripts/verify-image new file mode 100755 index 0000000..4698871 --- /dev/null +++ b/dsp/runtime/scripts/verify-image @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail +IMAGE=localhost/dispatch-runtime:dev +inspect="$(podman image inspect "$IMAGE")" +node --no-warnings -e ' +const image = JSON.parse(process.argv[1])[0]; +const config = image.Config; +if (image.Os !== "linux" || image.Architecture !== "amd64") throw new Error("invalid_platform"); +if (config.User !== "10001:10001") throw new Error("invalid_image_user"); +if (JSON.stringify(config.Entrypoint) !== JSON.stringify(["/usr/bin/tini","--","/usr/local/bin/node","--no-warnings","/opt/dispatch/runtime/supervisor/src/supervisor.js"])) throw new Error("invalid_entrypoint"); +if (!image.Healthcheck || !Array.isArray(image.Healthcheck.Test) || image.Healthcheck.Test.includes("NONE")) throw new Error("invalid_healthcheck"); +if (!Array.isArray(config.Env) || config.Env.some(value => /^DISPATCH_(?:RUNTIME_KEY|ACCESS_CONTROL)/.test(value))) throw new Error("unsafe_image_environment"); +' "$inspect" +podman run --rm --pull=never --network none --read-only --entrypoint /usr/local/bin/node "$IMAGE" --no-warnings -e ' +const fs = require("node:fs"); +const path = require("node:path"); +const releaseIdentity = require("/opt/dispatch/runtime/supervisor/src/create-release-manifest.js"); +const manifest = JSON.parse(fs.readFileSync("/opt/dispatch/runtime-release-manifest.json", "utf8")); +if (manifest.schemaVersion !== 1 || !/^sha256:[a-f0-9]{64}$/.test(manifest.codeTreeDigest) || manifest.sourceFileCount < 1) throw new Error("invalid_runtime_manifest"); +const packageNames = Object.keys(manifest.osPackages || {}).sort(); +if (JSON.stringify(packageNames) !== JSON.stringify(["caCertificates","chromium","chromiumSandbox","fontsLiberation","tini","utilLinux"].sort()) + || !Object.values(manifest.osPackages).every(value => /^\d+[A-Za-z0-9.+:~_-]*$/.test(value))) throw new Error("invalid_os_packages"); +for (const forbidden of ["dispatch-core", "interfaces", "compatibility"]) if (fs.existsSync(`/opt/dispatch/${forbidden}`)) throw new Error("central_component_in_image"); +for (const forbidden of ["hub.js", "client.js", "control.js"]) if (fs.existsSync(`/opt/dispatch/runtime/agent/src/${forbidden}`)) throw new Error("central_agent_component_in_image"); +for (const forbidden of ["/usr/local/bin/npm", "/usr/local/bin/npx", "/usr/local/bin/corepack", "/usr/bin/gcc", "/usr/bin/g++", "/usr/bin/make"]) if (fs.existsSync(forbidden)) throw new Error("build_tool_in_image"); +for (const plugin of ["cdf", "paycom"]) { + if (!fs.statSync(`/opt/dispatch/compatibility/providers/${plugin}/scripts/health`).isFile()) throw new Error("missing_provider_health_command"); + for (const name of ["build", "test", "verify"]) if (fs.existsSync(`/opt/dispatch/compatibility/providers/${plugin}/scripts/${name}`)) throw new Error("development_script_in_image"); + const bin = fs.readdirSync(`/opt/dispatch/compatibility/providers/${plugin}/bin`).map(name => `/opt/dispatch/compatibility/providers/${plugin}/bin/${name}`); + if (bin.some(file => fs.lstatSync(file).uid !== 10001 || (fs.lstatSync(file).mode & 0o022) !== 0)) throw new Error("unsafe_collector_ownership"); + const config = fs.readdirSync(`/opt/dispatch/compatibility/providers/${plugin}/config`).map(name => `/opt/dispatch/compatibility/providers/${plugin}/config/${name}`); + if (config.some(file => fs.lstatSync(file).uid !== 10001 || (fs.lstatSync(file).mode & 0o022) !== 0)) throw new Error("unsafe_definition_ownership"); +} +let sourceFiles = 0; +function visit(target) { + const stat = fs.lstatSync(target); + if (stat.isSymbolicLink() || (stat.mode & 0o022) !== 0) throw new Error("mutable_or_linked_runtime_source"); + if (stat.isDirectory()) { + if (["tests", "test", "examples", "docs"].includes(path.basename(target))) throw new Error("non_runtime_source_in_image"); + for (const child of fs.readdirSync(target)) visit(path.join(target, child)); + return; + } + if (!stat.isFile() || stat.nlink !== 1) throw new Error("unsafe_runtime_source"); + if (target !== "/opt/dispatch/runtime-release-manifest.json") sourceFiles += 1; + const tenantOwned = /^\/opt\/dispatch\/(?:runtime\/providers\/cdf|plugins\/paycom\/backend)\/(?:bin|config)\//.test(target); + const legacyEntrypoint = /^\/opt\/dispatch\/plugins\/(?:cdf|paycom)\/bin\//.test(target); + if (stat.uid !== (tenantOwned || legacyEntrypoint ? 10001 : 0) || stat.gid !== (tenantOwned || legacyEntrypoint ? 10001 : 0)) throw new Error("invalid_runtime_source_owner"); +} +visit("/opt/dispatch"); +if (sourceFiles !== manifest.sourceFileCount) throw new Error("runtime_manifest_file_count_mismatch"); +if (releaseIdentity.treeDigest(releaseIdentity.sourceFiles()) !== manifest.codeTreeDigest) throw new Error("runtime_manifest_digest_mismatch"); +console.log(JSON.stringify({ ok: true, status: "runtime_manifest_verified", sourceFileCount: manifest.sourceFileCount })); +' +for provider in paycom cdf; do + output="$(printf '{}\n' | podman run --rm -i --pull=never --network none --read-only \ + --entrypoint /usr/local/bin/node "$IMAGE" --no-warnings \ + "/opt/dispatch/plugins/$provider/bin/dispatch-$provider-collector")" + node -e 'const value = JSON.parse(process.argv[1]); if (value.ok !== false) throw new Error("legacy_entrypoint_response_invalid");' "$output" +done +podman run --rm --pull=never --network none --read-only --entrypoint /usr/local/bin/node "$IMAGE" --version +printf '%s\n' '{"ok":true,"status":"runtime_image_verified","nonRoot":true,"fixedEntrypoint":true,"healthcheck":true}' diff --git a/dsp/runtime/sdk/CONNECTIONS.md b/dsp/runtime/sdk/CONNECTIONS.md new file mode 100644 index 0000000..77f0b35 --- /dev/null +++ b/dsp/runtime/sdk/CONNECTIONS.md @@ -0,0 +1,71 @@ +# Shared service connections + +DSP owners manage **Cortex** and **Paycom** under **Settings → Connections**. +Each service has one credential set per DSP. Internal identities are fixed in +`shared/contracts/src/connections.js`: Cortex uses `amazon-operations` and +Paycom uses `paycom-main`. Existing profiles under those names are discovered +without copying or reentering credentials. Existing collector definitions with +other explicitly selected profiles retain their legacy behavior. + +The dashboard routes management requests through the authenticated, selected DSP +runtime. It never accepts a runtime key, profile name, provider name, endpoint, +or website URL from the credential form. Both the selected membership's Owner +role and permission are required. Platform support viewing does not grant +credential management. The DSP broker stores provider credentials in its +existing encrypted vault; Core stores audit metadata only. + +Save, test, and disconnect return bounded status views. A save carries a short +expiry and credentials travel in the live transport only. Login checks run in +the broker and survive page navigation. After a broker restart an unfinished +check is reported as unavailable, never as authenticated. Last-check timestamps +describe a past verification, not a guarantee that the provider's session will +remain valid. Current sessions are checked by the adapter when acquired. + +Paycom first enrollment still uses the existing onboarding worker to verify +login and start workforce collection. Later credential changes use the same +fixed vault profile. Connection management refuses changes while the profile is +in use; try again after collection finishes. Replacement removes retained browser +authentication before saving the new account. Disconnect removes credentials and +browser state and prevents new acquisitions; previously collected data remains. + +## Feature access inside a DSP runtime + +Use the SDK's connection client from an authorized feature worker: + +```js +const { createLocalDispatchClient } = require('./src'); +const dispatch = createLocalDispatchClient({ runtime: trustedRuntimePaths }); + +const result = await dispatch.connections.withSession( + { service: 'cortex', feature: 'delivery-reports', runId, signal }, + async ({ endpoint, protocol, access, signal }) => { + return collectReports({ endpoint, protocol, access, signal }); + }, +); +``` + +The callback receives CDP access to an authenticated browser, never vault +credentials. The helper renews its lease, propagates cancellation and renewal +failure, and releases the browser on success or failure. Callbacks must honor +the signal. The broker allows one operation per profile; `session_busy` means +the caller should queue/retry through its scheduler, not start another login. +The owning DSP runtime provides isolation. Feature authorization belongs in the +feature's existing API/worker boundary; the feature label is attribution, not +an independent authorization credential. Browser endpoints and leases must stay +inside the runtime, not in dashboard responses. + +CDF and Paycom's default browser acquisition paths use this same service mapping. +For low-level workers the shared `acquireServiceBrowser` helper returns a lease +that the worker must renew and release itself. + +## Additional services and verification + +Add a service definition with fixed provider/profile identities and bounded +credential fields, register its authentication adapter, and extend contract and +integration coverage. The settings form is generated from the service registry. +Changing credential limits also requires checking transport byte limits. + +MFA, CAPTCHA, and unfamiliar security challenges are human intervention states. +This change reports them and supports retry after operator recovery; it does not +provide an interactive challenge browser or collect verification codes in the +dashboard. Unsupported adapter flows remain blocked rather than bypassed. diff --git a/dsp/runtime/sdk/examples/auth-setup.js b/dsp/runtime/sdk/examples/auth-setup.js new file mode 100644 index 0000000..b39fba2 --- /dev/null +++ b/dsp/runtime/sdk/examples/auth-setup.js @@ -0,0 +1,32 @@ +'use strict'; + +const { createLocalDispatchClient } = require('../src'); + +async function inspectAuthSetup(dispatch = createLocalDispatchClient()) { + return dispatch.workflows.authSetup.prepare(); +} + +async function executeAuthSetup(dispatch, preparation, { + credentialAction = preparation.data.defaults.credentialAction, + testAuthentication = preparation.data.defaults.testAuthentication, + events = { emit() {} }, + signal = null, +} = {}) { + if (!preparation?.ok) return preparation; + return dispatch.workflows.authSetup.run({ + provider: preparation.data.target.provider, + profile: preparation.data.target.profile, + credentialAction, + startBroker: preparation.data.defaults.startBroker, + testAuthentication, + }, { events, signal }); +} + +if (require.main === module) { + inspectAuthSetup().then(result => { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + process.exitCode = result.ok ? 0 : 1; + }).catch(() => { process.exitCode = 1; }); +} + +module.exports = { inspectAuthSetup, executeAuthSetup }; diff --git a/dsp/runtime/sdk/package.json b/dsp/runtime/sdk/package.json new file mode 100644 index 0000000..a9b59a2 --- /dev/null +++ b/dsp/runtime/sdk/package.json @@ -0,0 +1,20 @@ +{ + "name": "dispatch-dsp-runtime-client", + "version": "0.9.0", + "private": true, + "description": "Internal DSP runtime adapters; the public dispatch-sdk package is supplied by Core", + "type": "commonjs", + "main": "src/index.js", + "exports": { + ".": "./src/index.js", + "./local": "./src/local.js" + }, + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "./scripts/build", + "test": "./scripts/test", + "verify": "./scripts/verify" + } +} diff --git a/dsp/runtime/sdk/scripts/build b/dsp/runtime/sdk/scripts/build new file mode 100755 index 0000000..040a7a1 --- /dev/null +++ b/dsp/runtime/sdk/scripts/build @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +for file in "$ROOT"/../../shared/contracts/src/*.js "$ROOT"/src/*.js "$ROOT"/../application/*/*.js "$ROOT"/../adapters/local/*.js "$ROOT"/examples/*.js "$ROOT"/tests/*.js; do + node --no-warnings --check "$file" +done +node --no-warnings -e "require('$ROOT/src'); require('$ROOT/src/local'); require('$ROOT/src/public-contracts')" +printf '%s\n' '{"ok":true,"status":"built"}' diff --git a/dsp/runtime/sdk/scripts/test b/dsp/runtime/sdk/scripts/test new file mode 100755 index 0000000..f6e53ac --- /dev/null +++ b/dsp/runtime/sdk/scripts/test @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +cd "$ROOT" +node --no-warnings --test tests/*.test.js diff --git a/dsp/runtime/sdk/scripts/verify b/dsp/runtime/sdk/scripts/verify new file mode 100755 index 0000000..c55354f --- /dev/null +++ b/dsp/runtime/sdk/scripts/verify @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +cd "$ROOT" +node --no-warnings - <<'NODE' +const fs = require('node:fs'); +const path = require('node:path'); +const sdk = require('./src'); + +(async () => { + const expected = [ + 'AuthClient', 'AuthSetupWorkflowClient', 'CollectionAdminClient', 'CollectionClient', 'DispatchClient', + 'PaycomClient', 'SyncClient', 'SystemClient', 'WorkforceClient', 'contracts', + 'createLocalDispatchClient', 'resolveLocalRuntimePaths', + ].sort(); + if (JSON.stringify(Object.keys(sdk).sort()) !== JSON.stringify(expected)) process.exit(1); + const paths = sdk.resolveLocalRuntimePaths(); + if (![paths.projectRoot, paths.dataRoot, paths.stateRoot, paths.runtimeRoot, paths.auth.database, + paths.collection.database, paths.paycom.database, paths.paycom.collectorCommand].every(path.isAbsolute)) process.exit(1); + if (!fs.existsSync(paths.paycom.collectorCommand)) process.exit(1); + + const client = sdk.createLocalDispatchClient(); + if (Object.hasOwn(client, 'admin')) process.exit(1); + const capabilities = client.capabilities(); + if (!capabilities.ok || capabilities.status !== 'found' || capabilities.data.transport !== 'local' + || capabilities.data.operator.collectionAdmin !== false + || capabilities.data.contractVersion !== sdk.contracts.CONTRACT_VERSION) process.exit(1); + const result = await client.system.status(); + if (!result.ok || !['ready', 'degraded'].includes(result.status)) process.exit(1); + const prepared = await client.workflows.authSetup.prepare(); + if (!prepared.ok || prepared.status !== 'ready') process.exit(1); + process.stdout.write(`${JSON.stringify({ + ok: true, status: 'verified', contractVersion: result.contractVersion, + sdkVersion: capabilities.data.sdkVersion, transport: capabilities.data.transport, + systemStatus: result.status, authSetupStatus: prepared.status, + })}\n`); +})().catch(() => process.exit(1)); +NODE diff --git a/dsp/runtime/sdk/src/auth-client.js b/dsp/runtime/sdk/src/auth-client.js new file mode 100644 index 0000000..d467639 --- /dev/null +++ b/dsp/runtime/sdk/src/auth-client.js @@ -0,0 +1,202 @@ +'use strict'; + +const { success, failure } = require('dispatch-protocol/contracts/src'); +const { + AUTH_PROTOCOL_VERSION, AUTH_PROFILE_RE, AUTH_PROVIDER_RE, + AUTH_PROFILE_SESSION_STATES, AUTH_SUCCESS_STATUSES, +} = require('dispatch-protocol/contracts/src/auth'); + +const PROVIDER_RE = AUTH_PROVIDER_RE; +const FIELD_RE = /^[A-Za-z][A-Za-z0-9_-]{0,63}$/; +const SESSION_STATES = new Set(AUTH_PROFILE_SESSION_STATES); +const INSPECTION_STATES = new Set([ + 'logged_out', 'authenticated', 'timecard_application', + 'primary_rejected', 'security_questions', 'security_answers_rejected', + 'security_profile_prompt', 'security_profile_confirmation', + 'mfa_required', 'captcha_required', 'security_challenge', 'account_locked', + 'manual_verification_required', +]); +const RECOVERABLE = new Set([ + 'auth_broker_unavailable', 'profile_not_configured', 'profile_locked', 'session_busy', 'browser_profile_busy', 'attempt_cooldown', + 'primary_credentials_rejected', 'security_answers_rejected', 'invalid_credentials', 'account_locked', + 'mfa_required', 'captcha_required', 'security_challenge', 'manual_verification_required', 'authentication_timeout', + 'acquisition_cancelled', +]); +const SAFE_CODES = new Set([ + ...RECOVERABLE, + 'invalid_request', 'invalid_input', 'vault_integrity_failed', 'unsafe_storage', 'incomplete_storage', + 'adapter_unavailable', 'browser_unavailable', 'unsafe_browser', 'browser_start_failed', 'browser_profile_busy', + 'browser_protocol_failed', 'browser_timeout', 'authentication_failed', 'browser_cleanup_failed', 'broker_closing', 'session_revoked', + 'acquisition_cancelled', 'attempt_state_invalid', +]); + +function safeProfile(value) { + if (typeof value !== 'string' || !AUTH_PROFILE_RE.test(value)) throw Object.assign(new Error('invalid_input'), { code: 'invalid_input' }); + return value; +} +function safeProvider(value) { + if (typeof value !== 'string' || !PROVIDER_RE.test(value)) throw new Error('invalid_component_response'); + return value; +} +function validTimestamp(value) { return typeof value === 'string' && !Number.isNaN(Date.parse(value)); } +function metadata(item) { + if (!item || typeof item !== 'object') throw new Error('invalid_component_response'); + safeProfile(item.profile); + safeProvider(item.provider); + if (!validTimestamp(item.createdAt) || !validTimestamp(item.updatedAt)) throw new Error('invalid_component_response'); + return { profile: item.profile, provider: item.provider, createdAt: item.createdAt, updatedAt: item.updatedAt }; +} +function healthView(response) { + if (response.protocolVersion !== AUTH_PROTOCOL_VERSION || typeof response.vault?.verified !== 'boolean' + || !Number.isInteger(response.vault?.profiles) || response.vault.profiles < 0 + || !Number.isInteger(response.vault?.schemaVersion) || response.vault.schemaVersion < 1) { + throw new Error('invalid_component_response'); + } + return { protocolVersion: AUTH_PROTOCOL_VERSION, vault: { + verified: response.vault.verified, + profiles: response.vault.profiles, + schemaVersion: response.vault.schemaVersion, + } }; +} + +function inspectionMetadata(value) { + const keys = new Set([ + 'origin', 'path', 'queryKeys', 'title', 'readyState', 'loginFormCount', 'challengeFormCount', + 'profileInputNames', 'profileActionLabels', 'challengeIndices', 'challengeFormActionPath', 'diagnostic', + ]); + if (!value || typeof value !== 'object' || Array.isArray(value) + || Object.keys(value).some(key => !keys.has(key)) + || value.origin !== 'https://www.paycomonline.net' + || typeof value.path !== 'string' || value.path.length > 256 || !value.path.startsWith('/') + || !Array.isArray(value.queryKeys) || value.queryKeys.length > 32 + || value.queryKeys.some(key => typeof key !== 'string' || !FIELD_RE.test(key)) + || value.title !== null && (typeof value.title !== 'string' || value.title.length > 120) + || ![null, 'loading', 'interactive', 'complete'].includes(value.readyState) + || ![value.loginFormCount, value.challengeFormCount].every(count => count === null || Number.isInteger(count) && count >= 0 && count <= 32) + || !Array.isArray(value.profileInputNames) || value.profileInputNames.length > 16 + || value.profileInputNames.some(name => typeof name !== 'string' || name.length > 64) + || !Array.isArray(value.profileActionLabels) || value.profileActionLabels.length > 16 + || value.profileActionLabels.some(label => typeof label !== 'string' || label.length > 64) + || !Array.isArray(value.challengeIndices) || value.challengeIndices.length > 5 + || value.challengeIndices.some(index => !Number.isInteger(index) || index < 1 || index > 5) + || value.challengeFormActionPath !== null + && (typeof value.challengeFormActionPath !== 'string' || !value.challengeFormActionPath.startsWith('/') || value.challengeFormActionPath.length > 256) + || value.diagnostic !== null) throw new Error('invalid_component_response'); + return { + origin: value.origin, + path: value.path, + queryKeys: [...value.queryKeys], + title: value.title, + readyState: value.readyState, + loginFormCount: value.loginFormCount, + challengeFormCount: value.challengeFormCount, + profileInputNames: [...value.profileInputNames], + profileActionLabels: [...value.profileActionLabels], + challengeIndices: [...value.challengeIndices], + challengeFormActionPath: value.challengeFormActionPath, + }; +} + +class AuthClient { + #port; + + constructor({ port } = {}) { + if (!port || typeof port.request !== 'function') throw new TypeError('auth_port_required'); + this.#port = port; + } + + async #request(payload, expectedStatuses, map, options = {}) { + let response; + try { response = await this.#port.request(payload, options); } + catch (error) { + const code = SAFE_CODES.has(error?.code) ? error.code : 'auth_broker_unavailable'; + return failure(code, { recoverable: RECOVERABLE.has(code) || code === 'auth_broker_unavailable' }); + } + if (!response || response.ok !== true) { + const code = SAFE_CODES.has(response?.status) ? response.status : 'invalid_component_response'; + return failure(code, { recoverable: RECOVERABLE.has(code) }); + } + if (!expectedStatuses.includes(response.status)) return failure('invalid_component_response'); + try { return success(response.status, map(response)); } + catch { return failure('invalid_component_response'); } + } + + health() { return this.#request({ action: 'health' }, AUTH_SUCCESS_STATUSES.health, healthView); } + + providers() { + return this.#request({ action: 'providers' }, AUTH_SUCCESS_STATUSES.providers, response => { + if (!Array.isArray(response.providers) || response.providers.length > 32) throw new Error('invalid_component_response'); + return { items: response.providers.map(item => { + safeProvider(item?.provider); + if (!Array.isArray(item.fields) || item.fields.length > 32 || item.fields.some(field => typeof field !== 'string' || !FIELD_RE.test(field))) { + throw new Error('invalid_component_response'); + } + return { provider: item.provider, fields: [...item.fields] }; + }) }; + }); + } + + profiles() { + return this.#request({ action: 'list' }, AUTH_SUCCESS_STATUSES.list, response => { + if (!Array.isArray(response.profiles) || response.profiles.length > 128) throw new Error('invalid_component_response'); + return { items: response.profiles.map(metadata) }; + }); + } + + profileStatus(profile) { + try { profile = safeProfile(profile); } catch { return Promise.resolve(failure('invalid_input')); } + return this.#request({ action: 'status', profile }, AUTH_SUCCESS_STATUSES.status, response => { + if (typeof response.profile?.configured !== 'boolean' || response.profile.profile !== profile + || response.profile.configured !== (response.status === 'configured') + || !SESSION_STATES.has(response.session)) throw new Error('invalid_component_response'); + const view = response.profile.configured + ? { configured: true, ...metadata(response.profile) } + : { configured: false, profile }; + return { profile: view, session: response.session }; + }); + } + + lockProfile(profile) { + try { profile = safeProfile(profile); } catch { return Promise.resolve(failure('invalid_input')); } + return this.#request({ action: 'lock', profile }, AUTH_SUCCESS_STATUSES.lock, response => { + if (response.profile !== profile) throw new Error('invalid_component_response'); + return { profile }; + }); + } + + unlockProfile(profile) { + try { profile = safeProfile(profile); } catch { return Promise.resolve(failure('invalid_input')); } + return this.#request({ action: 'unlock', profile }, AUTH_SUCCESS_STATUSES.unlock, response => { + if (response.profile !== profile) throw new Error('invalid_component_response'); + return { profile }; + }); + } + + inspectProfile(profile, { signal = null } = {}) { + try { profile = safeProfile(profile); } catch { return Promise.resolve(failure('invalid_input')); } + return this.#request({ action: 'inspect-auth-profile', profile }, AUTH_SUCCESS_STATUSES.inspectProfile, response => { + const inspection = response.inspection; + if (!inspection || inspection.profile !== profile || !INSPECTION_STATES.has(inspection.state) + || !validTimestamp(inspection.observedAt)) throw new Error('invalid_component_response'); + return { + profile, + provider: safeProvider(inspection.provider), + state: inspection.state, + observedAt: inspection.observedAt, + metadata: inspectionMetadata(inspection.metadata), + }; + }, { timeoutMs: require('dispatch-protocol/browser-assistance/protocol').AUTH_REQUEST_MS + 10_000, signal }); + } + + testProfile(profile, { signal = null } = {}) { + try { profile = safeProfile(profile); } catch { return Promise.resolve(failure('invalid_input')); } + return this.#request({ action: 'test-auth-profile', profile }, AUTH_SUCCESS_STATUSES.testProfile, response => { + if (response.status !== 'authenticated' || response.profile?.profile !== profile) throw new Error('invalid_component_response'); + safeProvider(response.profile.provider); + if (!validTimestamp(response.profile.testedAt)) throw new Error('invalid_component_response'); + return { profile, provider: response.profile.provider, testedAt: response.profile.testedAt }; + }, { timeoutMs: require('dispatch-protocol/browser-assistance/protocol').AUTH_REQUEST_MS + 10_000, signal }); + } +} + +module.exports = { AuthClient, SAFE_CODES, RECOVERABLE, AUTH_PROFILE_RE, SESSION_STATES, INSPECTION_STATES, inspectionMetadata }; diff --git a/dsp/runtime/sdk/src/auth-setup-client.js b/dsp/runtime/sdk/src/auth-setup-client.js new file mode 100644 index 0000000..48629bd --- /dev/null +++ b/dsp/runtime/sdk/src/auth-setup-client.js @@ -0,0 +1,147 @@ +'use strict'; + +const { success, failure, isResult, exactObject, STATUS_RE } = require('dispatch-protocol/contracts/src'); +const { SAFE_FAILURES } = require('../../application/auth/setup-auth'); + +const PROFILE_RE = /^[a-z][a-z0-9_-]{0,47}$/; +const OPERATION_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/; +const SETUP_PROVIDERS = new Set(['paycom', 'amazon-logistics']); +const CREDENTIAL_ACTIONS = new Set(['auto', 'keep', 'enroll', 'replace', 'remove']); +const BROKER_STATES = new Set(['ready', 'starting', 'stopped']); +const VAULT_STATES = new Set(['ready', 'absent']); +const PUBLIC_FAILURES = new Set([...SAFE_FAILURES, 'setup_auth_failed']); + +function invalid() { throw Object.assign(new Error('invalid_input'), { code: 'invalid_input' }); } +function plain(value) { return value && typeof value === 'object' && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype; } +function exact(value, keys) { return plain(value) && Object.keys(value).sort().join(',') === [...keys].sort().join(','); } +function safeCode(value) { return value === null || typeof value === 'string' && STATUS_RE.test(value); } + +function validateTarget(value) { + if (!exact(value, ['provider', 'profile']) || !SETUP_PROVIDERS.has(value.provider) + || typeof value.profile !== 'string' || !PROFILE_RE.test(value.profile)) throw new Error('invalid_component_response'); + return { provider: value.provider, profile: value.profile }; +} + +function validateAction(value, expectedId) { + if (!exact(value, ['id', 'available', 'reason']) || value.id !== expectedId + || typeof value.available !== 'boolean' || !safeCode(value.reason) + || (value.available && value.reason !== null) || (!value.available && value.reason === null)) throw new Error('invalid_component_response'); + return { id: value.id, available: value.available, reason: value.reason }; +} + +function preparationData(value) { + if (!exact(value, ['workflow', 'target', 'state', 'capabilities', 'defaults']) || value.workflow !== 'setup_auth') throw new Error('invalid_component_response'); + const target = validateTarget(value.target); + if (!exact(value.state, ['broker', 'vault', 'profile', 'credentialIngress']) + || !exact(value.state.broker, ['status', 'managed']) || !BROKER_STATES.has(value.state.broker.status) + || typeof value.state.broker.managed !== 'boolean' + || !exact(value.state.vault, ['status', 'verified']) || !VAULT_STATES.has(value.state.vault.status) + || typeof value.state.vault.verified !== 'boolean' + || !['available', 'unavailable'].includes(value.state.credentialIngress) + || !plain(value.state.profile)) throw new Error('invalid_component_response'); + const profileKeys = value.state.profile.status === 'configured' ? ['status', 'provider'] : ['status']; + if (!exact(value.state.profile, profileKeys) || !['configured', 'not_configured'].includes(value.state.profile.status) + || value.state.profile.status === 'configured' && value.state.profile.provider !== target.provider) throw new Error('invalid_component_response'); + if (!exact(value.capabilities, ['credentialActions', 'authenticationTest']) + || !Array.isArray(value.capabilities.credentialActions) || value.capabilities.credentialActions.length !== 4) throw new Error('invalid_component_response'); + const actions = ['keep', 'enroll', 'replace', 'remove'].map((id, index) => validateAction(value.capabilities.credentialActions[index], id)); + const authenticationTest = validateAction(value.capabilities.authenticationTest, 'run'); + if (!exact(value.defaults, ['credentialAction', 'startBroker', 'testAuthentication']) + || !CREDENTIAL_ACTIONS.has(value.defaults.credentialAction) || value.defaults.credentialAction === 'auto' + || typeof value.defaults.startBroker !== 'boolean' || typeof value.defaults.testAuthentication !== 'boolean') throw new Error('invalid_component_response'); + return { + workflow: 'setup_auth', target, + state: { + broker: { status: value.state.broker.status, managed: value.state.broker.managed }, + vault: { status: value.state.vault.status, verified: value.state.vault.verified }, + profile: { status: value.state.profile.status, ...(value.state.profile.status === 'configured' ? { provider: value.state.profile.provider } : {}) }, + credentialIngress: value.state.credentialIngress, + }, + capabilities: { credentialActions: actions, authenticationTest }, + defaults: { ...value.defaults }, + }; +} + +function validateRunInput(input) { + exactObject(input, ['provider', 'profile', 'credentialAction', 'startBroker', 'testAuthentication', 'operationId']); + const value = { + provider: input.provider === undefined ? 'paycom' : input.provider, + profile: input.profile === undefined ? 'paycom-main' : input.profile, + credentialAction: input.credentialAction === undefined ? 'auto' : input.credentialAction, + startBroker: input.startBroker === undefined ? true : input.startBroker, + testAuthentication: input.testAuthentication === undefined ? false : input.testAuthentication, + ...(input.operationId === undefined ? {} : { operationId: input.operationId }), + }; + if (!SETUP_PROVIDERS.has(value.provider) || typeof value.profile !== 'string' || !PROFILE_RE.test(value.profile) + || !CREDENTIAL_ACTIONS.has(value.credentialAction) || value.credentialAction === 'remove' && value.testAuthentication + || typeof value.startBroker !== 'boolean' || typeof value.testAuthentication !== 'boolean' + || value.operationId !== undefined && (typeof value.operationId !== 'string' || !OPERATION_RE.test(value.operationId))) invalid(); + return value; +} + +function safeResult(value, kind) { + if (!isResult(value)) return failure('invalid_component_response'); + if (!value.ok) { + if (!PUBLIC_FAILURES.has(value.status)) return failure('invalid_component_response'); + if (value.data !== null && (!exact(value.data, ['workflow', 'profile', 'provider', 'nextActions', 'cause', 'state']) + || value.data.workflow !== 'setup_auth' || !SETUP_PROVIDERS.has(value.data.provider) + || !PROFILE_RE.test(value.data.profile) || !Array.isArray(value.data.nextActions) + || value.data.nextActions.length > 8 || value.data.nextActions.some(item => typeof item !== 'string' || !STATUS_RE.test(item)) + || value.data.cause !== null && (typeof value.data.cause !== 'string' || !STATUS_RE.test(value.data.cause)) + || !exact(value.data.state, ['broker', 'profile', 'mutation', 'recovery']) + || !['ready', 'stopped', 'unknown'].includes(value.data.state.broker) + || !['configured', 'not_configured', 'unknown'].includes(value.data.state.profile) + || !['none', 'enrolled', 'replaced', 'removed'].includes(value.data.state.mutation) + || !['not_needed', 'restored', 'failed'].includes(value.data.state.recovery))) { + return failure('invalid_component_response'); + } + return failure(value.status, { recoverable: value.error.recoverable, data: value.data }); + } + try { + if (kind === 'prepare') { + if (value.status !== 'ready') throw new Error('invalid_component_response'); + return success('ready', preparationData(value.data)); + } + if (value.status !== 'complete' || !exact(value.data, ['workflow', 'provider', 'profile', 'configured', 'broker', 'vault', 'authenticationTest', 'nextActions']) + || value.data.workflow !== 'setup_auth' || !SETUP_PROVIDERS.has(value.data.provider) || !PROFILE_RE.test(value.data.profile) + || typeof value.data.configured !== 'boolean' || !['ready', 'stopped'].includes(value.data.broker) || value.data.vault !== 'verified' + || !['skipped', 'authenticated'].includes(value.data.authenticationTest) || !Array.isArray(value.data.nextActions) + || !value.data.configured && value.data.authenticationTest !== 'skipped' + || value.data.nextActions.length > 8 || value.data.nextActions.some(item => typeof item !== 'string' || !STATUS_RE.test(item))) throw new Error('invalid_component_response'); + return success('complete', value.data); + } catch { return failure('invalid_component_response'); } +} + +class AuthSetupWorkflowClient { + #port; + + constructor({ port } = {}) { + if (!port || typeof port.prepare !== 'function' || typeof port.run !== 'function') throw new TypeError('auth_setup_port_required'); + this.#port = port; + } + + async prepare(input = {}) { + let value; + try { + exactObject(input, ['provider', 'profile']); + value = await this.#port.prepare(input); + } catch (error) { + return failure(error?.code === 'invalid_input' ? 'invalid_input' : 'setup_auth_failed'); + } + return safeResult(value, 'prepare'); + } + + async run(input = {}, { events = { emit() {} }, signal = null } = {}) { + let values; + try { + values = validateRunInput(input); + if (!events || typeof events.emit !== 'function') invalid(); + } catch { return failure('invalid_input'); } + let value; + try { value = await this.#port.run(values, { events, signal }); } + catch { return failure('setup_auth_failed'); } + return safeResult(value, 'run'); + } +} + +module.exports = { AuthSetupWorkflowClient, validateRunInput, preparationData, safeResult }; diff --git a/dsp/runtime/sdk/src/collection-admin-client.js b/dsp/runtime/sdk/src/collection-admin-client.js new file mode 100644 index 0000000..ffcff2c --- /dev/null +++ b/dsp/runtime/sdk/src/collection-admin-client.js @@ -0,0 +1,81 @@ +'use strict'; + +const { success, failure, jsonValue } = require('dispatch-protocol/contracts/src'); + +const SAFE_CODES = new Set([ + 'invalid_input', 'secret_field_forbidden', 'unsafe_storage', 'unsafe_collector', 'schema_invalid', + 'dependency_not_found', 'dependency_cycle', 'collection_manager_not_initialized', +]); + +function invalid() { throw Object.assign(new Error('invalid_component_response'), { code: 'invalid_component_response' }); } +function count(value) { if (!Number.isInteger(value) || value < 0) invalid(); return value; } +function counts(value) { + if (!value || typeof value !== 'object') invalid(); + return Object.fromEntries(['collectors', 'sources', 'plans', 'syncs'].map(key => [key, count(value[key])])); +} +function inspectView(value) { + if (!value || typeof value.initialized !== 'boolean') invalid(); + return { + initialized: value.initialized, + schemaVersion: value.schemaVersion === null ? null : count(value.schemaVersion), + counts: counts(value.counts), + }; +} +function previewView(value) { + if (!value || value.valid !== true || !value.changes) invalid(); + const changes = Object.fromEntries(['collectors', 'sources', 'plans', 'syncs'].map(key => { + const item = value.changes[key]; + if (!item) invalid(); + return [key, { create: count(item.create), update: count(item.update) }]; + })); + return { valid: true, incoming: counts(value.incoming), changes }; +} +function spec(value) { + const normalized = jsonValue(value); + if (!normalized || typeof normalized !== 'object' || Array.isArray(normalized) + || Buffer.byteLength(JSON.stringify(normalized)) > 262_144) throw new Error('invalid_input'); + return normalized; +} + +class CollectionAdminClient { + #port; + + constructor({ port } = {}) { + if (!port || ['inspect', 'initialize', 'preview', 'apply'].some(method => typeof port[method] !== 'function')) { + throw new TypeError('collection_admin_port_required'); + } + this.#port = port; + } + + async #call(operation) { + try { return await operation(); } + catch (error) { + const code = error?.code === 'invalid_component_response' ? 'invalid_component_response' + : SAFE_CODES.has(error?.code) ? error.code : SAFE_CODES.has(error?.message) ? error.message : 'collection_admin_unavailable'; + return failure(code, { recoverable: ['collection_manager_not_initialized', 'collection_admin_unavailable'].includes(code) }); + } + } + + inspect() { + return this.#call(async () => { + const value = inspectView(await this.#port.inspect()); + return success(value.initialized ? 'ready' : 'not_initialized', value); + }); + } + + initialize() { return this.#call(async () => success('initialized', inspectView(await this.#port.initialize()))); } + + preview(value) { + let normalized; + try { normalized = spec(value); } catch { return Promise.resolve(failure('invalid_input')); } + return this.#call(async () => success('previewed', previewView(await this.#port.preview(normalized)))); + } + + apply(value) { + let normalized; + try { normalized = spec(value); } catch { return Promise.resolve(failure('invalid_input')); } + return this.#call(async () => success('applied', counts(await this.#port.apply(normalized)))); + } +} + +module.exports = { CollectionAdminClient }; diff --git a/dsp/runtime/sdk/src/collection-client.js b/dsp/runtime/sdk/src/collection-client.js new file mode 100644 index 0000000..f3b4118 --- /dev/null +++ b/dsp/runtime/sdk/src/collection-client.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/sdk/src/collection-client'); diff --git a/dsp/runtime/sdk/src/connections-client.js b/dsp/runtime/sdk/src/connections-client.js new file mode 100644 index 0000000..7bcfe54 --- /dev/null +++ b/dsp/runtime/sdk/src/connections-client.js @@ -0,0 +1,15 @@ +'use strict'; + +const { acquireServiceBrowser } = require('../../auth-broker/src/service-client'); +const { withLease } = require('dispatch-sdk/runtime/session'); + +class ConnectionsClient { + constructor({ socketPath, acquire = acquireServiceBrowser } = {}) { + this.socketPath = socketPath; this.acquire = acquire; + } + withSession({ service, feature, runId, ttlSeconds = 180, signal = null }, useSession) { + return withLease(acquireSignal => this.acquire({ service, feature, runId, ttlSeconds, + socketPath: this.socketPath, signal: acquireSignal }), useSession, { signal, ttlMs: ttlSeconds * 1000 }); + } +} +module.exports = { ConnectionsClient }; diff --git a/dsp/runtime/sdk/src/dispatch-client.js b/dsp/runtime/sdk/src/dispatch-client.js new file mode 100644 index 0000000..5a4ef89 --- /dev/null +++ b/dsp/runtime/sdk/src/dispatch-client.js @@ -0,0 +1,56 @@ +'use strict'; + +const { success, CONTRACT_VERSION } = require('dispatch-protocol/contracts/src'); +const { AUTH_PROTOCOL_VERSION } = require('dispatch-protocol/contracts/src/auth'); +const { version: SDK_VERSION } = require('../package.json'); +const { SystemClient } = require('./system-client'); + +const NAMESPACE_CAPABILITIES = Object.freeze({ + connections: Object.freeze(['with-session']), + auth: Object.freeze(['health', 'providers', 'profiles', 'profile-status', 'lock-profile', 'unlock-profile', 'test-profile']), + collections: Object.freeze([ + 'health', 'collectors', 'collector', 'methods', 'method', 'sources', 'source', 'plans', 'plan', + 'runs', 'run-status', 'start-run', 'cancel-run', 'retry-run', 'describe', 'preview', 'enqueue', + 'audit', 'batches', 'batch-status', 'cancel-batch', 'retry-batch', 'schedules', 'schedule', + ]), + sync: Object.freeze(['list', 'status', 'start', 'stop', 'restart', 'run-now', 'edit', 'history']), + paycom: Object.freeze(['health']), + workforce: Object.freeze(['snapshot', 'employees', 'employee', 'timecards', 'resource-links']), + system: Object.freeze(['status']), + workflows: Object.freeze(['auth-setup']), +}); + +class DispatchClient { + constructor({ auth, connections = null, collections, sync, paycom, workforce, authSetup, collectionAdmin = null, transport = 'injected' } = {}) { + if (!auth || !collections || !sync || !paycom || !workforce || !authSetup) throw new TypeError('dispatch_clients_required'); + if (!['injected', 'local'].includes(transport) || collectionAdmin !== null && typeof collectionAdmin.inspect !== 'function' + || connections !== null && typeof connections.withSession !== 'function') { + throw new TypeError('dispatch_capabilities_invalid'); + } + this.auth = auth; + this.connections = connections; + this.collections = collections; + this.sync = sync; + this.paycom = paycom; + this.workforce = workforce; + this.system = new SystemClient({ auth, collections, paycom }); + this.workflows = Object.freeze({ authSetup }); + if (collectionAdmin) this.admin = Object.freeze({ collections: collectionAdmin }); + Object.defineProperty(this, '_transport', { value: transport, enumerable: false }); + Object.freeze(this); + } + + capabilities() { + return success('found', { + sdkVersion: SDK_VERSION, + contractVersion: CONTRACT_VERSION, + authProtocolVersion: AUTH_PROTOCOL_VERSION, + transport: this._transport, + namespaces: this.connections ? NAMESPACE_CAPABILITIES + : Object.fromEntries(Object.entries(NAMESPACE_CAPABILITIES).filter(([name]) => name !== 'connections')), + operator: { collectionAdmin: Boolean(this.admin?.collections) }, + }); + } +} + +module.exports = { DispatchClient, SDK_VERSION, NAMESPACE_CAPABILITIES }; diff --git a/dsp/runtime/sdk/src/index.js b/dsp/runtime/sdk/src/index.js new file mode 100644 index 0000000..2fcc61e --- /dev/null +++ b/dsp/runtime/sdk/src/index.js @@ -0,0 +1,30 @@ +'use strict'; + +const { ConnectionsClient } = require('./connections-client'); +const { AuthClient } = require('./auth-client'); +const { AuthSetupWorkflowClient } = require('./auth-setup-client'); +const { CollectionClient } = require('dispatch-runtime-kit/sdk/src/collection-client'); +const { CollectionAdminClient } = require('./collection-admin-client'); +const { SyncClient } = require('dispatch-runtime-kit/sdk/src/sync-client'); +const { PaycomClient } = require('./paycom-client'); +const { WorkforceClient } = require('./workforce-client'); +const { SystemClient } = require('./system-client'); +const { DispatchClient } = require('./dispatch-client'); +const { createLocalDispatchClient, resolveLocalRuntimePaths } = require('./local'); +const contracts = require('./public-contracts'); + +module.exports = Object.freeze({ + AuthClient, + ConnectionsClient, + AuthSetupWorkflowClient, + CollectionClient, + CollectionAdminClient, + SyncClient, + PaycomClient, + WorkforceClient, + SystemClient, + DispatchClient, + createLocalDispatchClient, + resolveLocalRuntimePaths, + contracts, +}); diff --git a/dsp/runtime/sdk/src/local.js b/dsp/runtime/sdk/src/local.js new file mode 100644 index 0000000..fec9f14 --- /dev/null +++ b/dsp/runtime/sdk/src/local.js @@ -0,0 +1,10 @@ +'use strict'; + +const { resolveLocalRuntimePaths } = require('dispatch-protocol/paths/runtime-paths'); + +function createLocalDispatchClient(options) { + const local = require('../../adapters/local/create-local-dispatch-client'); + return local.createLocalDispatchClient(options); +} + +module.exports = { createLocalDispatchClient, resolveLocalRuntimePaths }; diff --git a/dsp/runtime/sdk/src/paycom-client.js b/dsp/runtime/sdk/src/paycom-client.js new file mode 100644 index 0000000..abd00a0 --- /dev/null +++ b/dsp/runtime/sdk/src/paycom-client.js @@ -0,0 +1,64 @@ +'use strict'; + +const { success, failure } = require('dispatch-protocol/contracts/src'); + +const SAFE_CODES = new Set(['unsafe_storage', 'schema_invalid', 'publication_verification_failed', 'candidate_invalid', 'not_initialized']); +const KINDS = new Set(['pay_periods', 'roster', 'timecards', 'resource_links']); +const CODE_RE = /^[a-z][a-z0-9_]{0,63}$/; + +function invalidComponent() { throw Object.assign(new Error('invalid_component_response'), { code: 'invalid_component_response' }); } +function auditView(value, expectedKind) { + if (!value || typeof value !== 'object' || typeof value.verified !== 'boolean' || value.kind !== expectedKind + || typeof value.code !== 'string' || !CODE_RE.test(value.code) + || value.target !== null && (typeof value.target !== 'string' || value.target.length > 128)) invalidComponent(); + const result = { verified: value.verified, code: value.code, kind: value.kind, target: value.target }; + if (value.verified) { + if (!Number.isInteger(value.rowCount) || value.rowCount < 0 || typeof value.collectedAt !== 'string' || Number.isNaN(Date.parse(value.collectedAt))) invalidComponent(); + result.rowCount = value.rowCount; + result.collectedAt = value.collectedAt; + if (expectedKind === 'pay_periods') result.projectionValid = value.projectionValid === true; + } + return result; +} +function unloaded(kind) { return { verified: false, code: 'not_loaded', kind, target: null }; } + +class PaycomClient { + #port; + + constructor({ port } = {}) { + if (!port || typeof port.health !== 'function') throw new TypeError('paycom_port_required'); + this.#port = port; + } + + async health() { + try { + const value = await this.#port.health(); + if (value === null) return success('not_initialized', { + database: 'missing', storageStatus: 'missing', publicationStatus: 'not_initialized', ready: false, + payPeriods: unloaded('pay_periods'), roster: unloaded('roster'), timecards: unloaded('timecards'), + resourceLinks: unloaded('resource_links'), + }); + const audits = { + payPeriods: auditView(value.payPeriods, 'pay_periods'), + roster: auditView(value.roster, 'roster'), + timecards: auditView(value.timecards, 'timecards'), + resourceLinks: auditView(value.resourceLinks, 'resource_links'), + }; + const ready = Object.values(audits).every(item => item.verified); + const publicationStatus = ready ? 'ready' + : Object.values(audits).some(item => item.verified) ? 'degraded' : 'not_loaded'; + return success(ready ? 'ready' : 'degraded', { + database: 'ready', storageStatus: 'ready', publicationStatus, ready, ...audits, + }); + } catch (error) { + if (['invalid_component_response', 'invalid_contract', 'unsafe_contract'].includes(error?.code) + || ['invalid_component_response', 'invalid_contract', 'unsafe_contract'].includes(error?.message)) { + return failure('invalid_component_response'); + } + const code = SAFE_CODES.has(error?.code) ? error.code : SAFE_CODES.has(error?.message) ? error.message : 'paycom_unavailable'; + return failure(code, { recoverable: code === 'paycom_unavailable' || code === 'not_initialized' }); + } + } +} + +module.exports = { PaycomClient, SAFE_CODES, KINDS, auditView }; diff --git a/dsp/runtime/sdk/src/public-contracts.js b/dsp/runtime/sdk/src/public-contracts.js new file mode 100644 index 0000000..a498b4b --- /dev/null +++ b/dsp/runtime/sdk/src/public-contracts.js @@ -0,0 +1,25 @@ +'use strict'; + +const source = require('dispatch-protocol/contracts/src'); + +module.exports = Object.freeze({ + CONTRACT_VERSION: source.CONTRACT_VERSION, + success: source.success, + failure: source.failure, + isResult: source.isResult, + event: source.event, + pagination: source.pagination, + identifier: source.identifier, + collectionSelector: source.collectionSelector, + collectionRequest: source.collectionRequest, + collectionEnqueueOptions: source.collectionEnqueueOptions, + collectionSchedule: source.collectionSchedule, + syncEditPatch: source.syncEditPatch, + syncEditOptions: source.syncEditOptions, + syncStopOptions: source.syncStopOptions, + syncRunOptions: source.syncRunOptions, + workforceQuery: source.workforceQuery, + workforceEmployeeCode: source.workforceEmployeeCode, + AUTH_PROTOCOL_VERSION: source.AUTH_PROTOCOL_VERSION, + AUTH_PROFILE_SESSION_STATES: source.AUTH_PROFILE_SESSION_STATES, +}); diff --git a/dsp/runtime/sdk/src/sync-client.js b/dsp/runtime/sdk/src/sync-client.js new file mode 100644 index 0000000..3402860 --- /dev/null +++ b/dsp/runtime/sdk/src/sync-client.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/sdk/src/sync-client'); diff --git a/dsp/runtime/sdk/src/system-client.js b/dsp/runtime/sdk/src/system-client.js new file mode 100644 index 0000000..4737b35 --- /dev/null +++ b/dsp/runtime/sdk/src/system-client.js @@ -0,0 +1,18 @@ +'use strict'; + +const { getSystemStatus } = require('../../application/system/get-status'); + +class SystemClient { + constructor({ auth, collections, paycom }) { + this.auth = auth; + this.collections = collections; + this.paycom = paycom; + this.includePaycom = () => true; + } + + status() { + return getSystemStatus({ auth: this.auth, collections: this.collections, paycom: this.includePaycom() ? this.paycom : null }); + } +} + +module.exports = { SystemClient }; diff --git a/dsp/runtime/sdk/src/workforce-client.js b/dsp/runtime/sdk/src/workforce-client.js new file mode 100644 index 0000000..d8479ad --- /dev/null +++ b/dsp/runtime/sdk/src/workforce-client.js @@ -0,0 +1,5 @@ +'use strict'; + +// The validated workforce read contract is shared by isolated runtime clients +// and Core's published-data reader. Provider execution remains in the runtime. +module.exports = require('dispatch-protocol/contracts/src/workforce-client'); diff --git a/dsp/runtime/sdk/tests/connections.test.js b/dsp/runtime/sdk/tests/connections.test.js new file mode 100644 index 0000000..ded1114 --- /dev/null +++ b/dsp/runtime/sdk/tests/connections.test.js @@ -0,0 +1,44 @@ +'use strict'; + +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { ConnectionsClient } = require('../src/connections-client'); + +test('features request a service, receive browser access only, and always release it', async () => { + const calls = []; let released = 0; + const client = new ConnectionsClient({ socketPath: '/runtime-a/auth-broker.sock', acquire: async options => { + calls.push(options); + return { endpoint: 'http://127.0.0.1:43210', protocol: 'cdp', access: 'full', + renew: async () => {}, release: async () => { released++; } }; + } }); + const options = { service: 'cortex', feature: 'cdf', runId: 'run-1' }; + assert.equal(await client.withSession(options, async session => { + assert.deepEqual(Object.keys(session).sort(), ['access', 'endpoint', 'protocol', 'signal']); + return 'collected'; + }), 'collected'); + assert.equal(calls[0].service, 'cortex'); + assert.equal(calls[0].socketPath, '/runtime-a/auth-broker.sock'); + assert.equal(released, 1); + await assert.rejects(client.withSession(options, async () => { throw new Error('collection failed'); }), /collection failed/); + assert.equal(released, 2); +}); + +test('a renewal failure racing callback completion rejects the operation and releases its lease', async t => { + t.mock.timers.enable({ apis: ['setInterval'] }); + let finishCallback; let rejectRenewal; let signal; let released = false; + const callbackReady = new Promise(resolve => { finishCallback = resolve; }); + const client = new ConnectionsClient({ acquire: async () => ({ endpoint: 'http://127.0.0.1:43210', + protocol: 'cdp', access: 'full', renew: () => new Promise((_resolve, reject) => { rejectRenewal = reject; }), + release: async () => { released = true; } }) }); + const result = client.withSession({ service: 'cortex', feature: 'cdf', runId: 'renewal-race', ttlSeconds: 3 }, + async session => { signal = session.signal; return callbackReady; }); + await Promise.resolve(); + t.mock.timers.tick(1000); + await Promise.resolve(); + finishCallback('collected'); + await Promise.resolve(); + rejectRenewal(new Error('lease_expired')); + await assert.rejects(result, /lease_expired/); + assert.equal(signal.aborted, true); + assert.equal(released, true); +}); diff --git a/dsp/runtime/sdk/tests/local-sync-port.test.js b/dsp/runtime/sdk/tests/local-sync-port.test.js new file mode 100644 index 0000000..d38858d --- /dev/null +++ b/dsp/runtime/sdk/tests/local-sync-port.test.js @@ -0,0 +1,112 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { success } = require('dispatch-protocol/contracts/src'); +const { AuthenticatedSyncPort } = require('../../application/sync/authenticated-sync-port'); +const { LocalSyncManagerPort } = require('dispatch-runtime-kit/adapters/local/sync-manager-port'); + +function fixture({ configured = true, authProfile = 'paycom-main', profileProvider = 'paycom', session = 'not_started' } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-local-sync-')); + const database = path.join(root, 'collection.sqlite3'); + fs.writeFileSync(database, 'fixture', { mode: 0o600 }); + const events = []; + const storeFactory = () => ({ + sync: () => ({ id: 'paycom-main-workforce', source: 'paycom-main' }), + source: () => ({ id: 'paycom-main', collector: 'paycom', authProfile }), + close: () => events.push('store.close'), + }); + const serviceFactory = () => ({ + start: () => { events.push('sync.start'); return { sync: {}, run: {} }; }, + stop: async () => { events.push('sync.stop'); return {}; }, + restart: async () => { events.push('sync.restart'); return { sync: {}, run: {} }; }, + runNow: () => { events.push('sync.run'); return { sync: {}, run: {} }; }, + edit: async () => { events.push('sync.edit'); return { sync: {}, run: null }; }, + }); + const manager = new LocalSyncManagerPort({ paths: { database }, storeFactory, serviceFactory }); + const authService = { start: async () => { events.push('auth.start'); return { status: 'ready' }; } }; + const auth = { + profileStatus: async profile => { + events.push(`auth.status:${profile}`); + return success(configured ? 'configured' : 'not_configured', configured + ? { + profile: { + configured: true, profile, provider: profileProvider, + createdAt: '2026-08-29T00:00:00.000Z', updatedAt: '2026-08-29T00:00:00.000Z', + }, + session, + } + : { profile: { configured: false, profile }, session }); + }, + }; + return { + root, events, manager, + port: new AuthenticatedSyncPort({ sync: manager, auth, authService }), + }; +} + +function cleanup(value) { fs.rmSync(value.root, { recursive: true, force: true }); } + +test('application sync coordination ensures Auth before authenticated mutations', async () => { + const value = fixture(); + try { + await value.port.start('paycom-main-workforce'); + await value.port.restart('paycom-main-workforce', {}); + await value.port.runNow('paycom-main-workforce'); + await value.port.edit('paycom-main-workforce', { settings: { mode: 'publish' } }, { applyNow: true }); + assert.deepEqual(value.events.filter(event => event !== 'store.close'), [ + 'auth.start', 'auth.status:paycom-main', 'sync.start', + 'auth.start', 'auth.status:paycom-main', 'sync.restart', + 'auth.start', 'auth.status:paycom-main', 'sync.run', + 'auth.start', 'auth.status:paycom-main', 'sync.edit', + ]); + } finally { cleanup(value); } +}); + +test('recoverable manual session state can queue a tick while an operator lock cannot', async () => { + const recovering = fixture({ session: 'manual_verification_required' }); + try { + await recovering.port.start('paycom-main-workforce'); + assert.equal(recovering.events.includes('sync.start'), true); + } finally { cleanup(recovering); } + + const locked = fixture({ session: 'locked' }); + try { + await assert.rejects(locked.port.start('paycom-main-workforce'), error => error.code === 'profile_locked'); + assert.equal(locked.events.includes('sync.start'), false); + } finally { cleanup(locked); } +}); + +test('application sync coordination fails before state mutation for missing or mismatched profiles', async () => { + const missing = fixture({ configured: false }); + try { + await assert.rejects(missing.port.start('paycom-main-workforce'), error => error.code === 'profile_not_configured'); + assert.equal(missing.events.includes('sync.start'), false); + } finally { cleanup(missing); } + + const mismatched = fixture({ profileProvider: 'other' }); + try { + await assert.rejects(mismatched.port.start('paycom-main-workforce'), error => error.code === 'profile_provider_mismatch'); + assert.equal(mismatched.events.includes('sync.start'), false); + } finally { cleanup(mismatched); } +}); + +test('sync without an Auth profile starts without touching the broker', async () => { + const value = fixture({ authProfile: null }); + try { + await value.port.start('paycom-main-workforce'); + assert.deepEqual(value.events.filter(event => event !== 'store.close'), ['sync.start']); + } finally { cleanup(value); } +}); + +test('local sync manager port exposes only a sanitized authentication requirement', () => { + const value = fixture(); + try { + assert.deepEqual(value.manager.authentication('paycom-main-workforce'), { + required: true, profile: 'paycom-main', provider: 'paycom', + }); + } finally { cleanup(value); } +}); diff --git a/dsp/runtime/sdk/tests/sdk.test.js b/dsp/runtime/sdk/tests/sdk.test.js new file mode 100644 index 0000000..f6b9790 --- /dev/null +++ b/dsp/runtime/sdk/tests/sdk.test.js @@ -0,0 +1,525 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { success, failure, isResult } = require('dispatch-protocol/contracts/src'); +const { AuthClient } = require('../src/auth-client'); +const { CollectionClient } = require('dispatch-runtime-kit/sdk/src/collection-client'); +const { SyncClient } = require('dispatch-runtime-kit/sdk/src/sync-client'); +const { PaycomClient } = require('../src/paycom-client'); +const { LocalCollectionManagerPort } = require('../../adapters/local/collection-manager-port'); +const { LocalPaycomPublicationPort } = require('../../../plugins/paycom/backend/adapters/publication'); +const { getSystemStatus } = require('../../application/system/get-status'); +const publicSdk = require('../src'); + +const COLLECTION_HEALTH = { + schemaVersion: 1, + databaseIntegrity: 'ok', + manager: { running: false, pid: null, heartbeatAt: null }, + counts: { collectors: 1, sources: 1, plans: 7, queued: 0, running: 0, failed: 0 }, + syncAlerts: { total: 0, critical: 0, items: [], hasMore: false }, +}; +const PAYCOM_HEALTH = { + database: 'ready', + storageStatus: 'ready', + publicationStatus: 'degraded', + ready: false, + payPeriods: { verified: true, code: 'verified', kind: 'pay_periods', target: '2026-08-25', rowCount: 3, collectedAt: '2026-08-25T00:00:00.000Z', projectionValid: true }, + roster: { verified: false, code: 'not_loaded', kind: 'roster', target: null }, + timecards: { verified: false, code: 'not_loaded', kind: 'timecards', target: null }, + resourceLinks: { verified: false, code: 'not_loaded', kind: 'resource_links', target: null }, +}; +function run(id = 'run_fixture_1') { + return { + id, plan: 'paycom-roster', source: 'paycom-main', collector: 'paycom', method: 'roster.snapshot', + trigger: 'manual', logicalKey: 'sdk:paycom-roster:button-click-1', status: 'queued', attempt: 0, + maxAttempts: 2, runAfter: 1, startedAt: null, finishedAt: null, error: null, blocked: null, + attempts: [], attemptHistoryComplete: true, + cancelRequested: false, collectorVersion: '0.3.0', + }; +} + +test('SDK root exposes only the deliberate public surface', () => { + assert.deepEqual(Object.keys(publicSdk).sort(), [ + 'AuthClient', 'AuthSetupWorkflowClient', 'ConnectionsClient', 'CollectionAdminClient', 'CollectionClient', 'DispatchClient', 'PaycomClient', + 'SyncClient', 'SystemClient', 'WorkforceClient', 'contracts', 'createLocalDispatchClient', 'resolveLocalRuntimePaths', + ].sort()); + assert.equal(Object.hasOwn(publicSdk, 'SAFE_CODES'), false); + assert.equal(Object.hasOwn(publicSdk, 'RecordingEventSink'), false); + assert.equal(Object.hasOwn(publicSdk.contracts, 'jsonValue'), false); + assert.equal(Object.isFrozen(publicSdk), true); + assert.equal(Object.isFrozen(publicSdk.contracts), true); +}); + +test('local runtime roots are explicit, absolute, and source-tree independent', () => { + const roots = publicSdk.resolveLocalRuntimePaths({ + projectRoot: '/opt/dispatch', dataRoot: '/var/lib/dispatch', stateRoot: '/var/lib/dispatch-state', + secretsRoot: '/etc/dispatch-secrets', runtimeRoot: '/run/user/1000/dispatch', + stagingRoot: '/var/lib/dispatch-state/staging', + }); + assert.equal(roots.auth.database, '/var/lib/dispatch/auth-broker/credentials.sqlite3'); + assert.equal(roots.auth.key, '/etc/dispatch-secrets/auth-broker/master.key'); + assert.equal(roots.auth.socket, '/run/user/1000/dispatch/auth-broker.sock'); + assert.equal(roots.collection.database, '/var/lib/dispatch/collection-manager/collection-manager.sqlite3'); + assert.equal(roots.accessControl.database, '/var/lib/dispatch/access-control/access-control.sqlite3'); + assert.equal(roots.cdf.database, '/var/lib/dispatch/db/cdf/cdf.sqlite3'); + assert.equal(roots.paycom.collectorCommand, path.join('/opt/dispatch', 'plugins/paycom/backend/bin/dispatch-paycom-collector')); + assert.throws(() => publicSdk.resolveLocalRuntimePaths({ dataRoot: 'relative' }), error => error.code === 'unsafe_runtime_config'); + assert.throws(() => publicSdk.resolveLocalRuntimePaths({ + projectRoot: '/opt/dispatch', dataRoot: '/opt/dispatch/db', + }), error => error.code === 'unsafe_runtime_config'); + assert.throws(() => publicSdk.resolveLocalRuntimePaths({ + projectRoot: '/opt/dispatch', dataRoot: path.join(__dirname, "../../../unsafe-data"), + }), error => error.code === 'unsafe_runtime_config'); + assert.throws(() => publicSdk.resolveLocalRuntimePaths({ + dataRoot: '/var/lib/dispatch', secretsRoot: '/var/lib/dispatch/secrets', + }), error => error.code === 'unsafe_runtime_config'); + assert.throws(() => publicSdk.resolveLocalRuntimePaths({ unknown: '/tmp/value' }), error => error.code === 'unsafe_runtime_config'); +}); + +test('one external local root derives the development storage layout', () => { + const roots = publicSdk.resolveLocalRuntimePaths({ projectRoot: '/opt/dispatch', localRoot: '/srv/dispatch-local' }); + assert.equal(roots.dataRoot, '/srv/dispatch-local/data'); + assert.equal(roots.auth.key, '/srv/dispatch-local/secrets/auth-broker/master.key'); + assert.equal(roots.auth.browserSessions, '/srv/dispatch-local/state/auth-broker/browser-sessions'); + assert.equal(roots.auth.socket, '/srv/dispatch-local/run/auth-broker.sock'); + assert.equal(roots.paycom.stagingRoot, '/srv/dispatch-local/staging/plugins/paycom'); +}); + +test('explicit runtime roots and provider imports work without a Linux home directory', () => { + const sourceRoot = path.resolve(__dirname, "../../.."); + const environment = Object.fromEntries(Object.entries(process.env).filter(([key]) => + key !== 'HOME' && !key.startsWith('XDG_') && !key.startsWith('DISPATCH_'))); + for (const configuredRoots of [ + { DISPATCH_LOCAL_ROOT: '/srv/dispatch-fixture' }, + { DISPATCH_DATA_ROOT: '/srv/dispatch-fixture/data', DISPATCH_SECRETS_ROOT: '/srv/dispatch-fixture/secrets', + DISPATCH_STATE_ROOT: '/srv/dispatch-fixture/state', DISPATCH_RUNTIME_ROOT: '/srv/dispatch-fixture/run', + DISPATCH_STAGING_ROOT: '/srv/dispatch-fixture/staging' }, + ]) { + const result = spawnSync(process.execPath, ['--no-warnings', '-e', ` + require('node:os').homedir = () => { throw new Error('no_passwd_entry'); }; + const roots = require('dispatch-protocol/paths/runtime-paths').resolveLocalRuntimePaths(); + const provider = require('./plugins/paycom/backend/src/paths'); + require('./plugins/paycom/backend/src/collector'); + process.stdout.write(JSON.stringify({ data: roots.dataRoot, database: provider.DATABASE })); + `], { cwd: sourceRoot, env: { ...environment, ...configuredRoots }, encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout), { + data: '/srv/dispatch-fixture/data', database: '/srv/dispatch-fixture/data/db/paycom/paycom.sqlite3', + }); + } +}); + +test('runtime roots reject symlink aliases into the source worktree', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-runtime-boundary-')); + fs.chmodSync(root, 0o700); + const alias = path.join(root, 'worktree-alias'); + fs.symlinkSync(path.resolve(__dirname, "../../.."), alias, 'dir'); + try { + assert.throws(() => publicSdk.resolveLocalRuntimePaths({ + dataRoot: path.join(alias, 'data'), + }), error => error.code === 'unsafe_runtime_config'); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('an explicit Auth state override does not relocate its runtime socket', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-paths-')); + fs.chmodSync(root, 0o700); + try { + const stateRoot = path.join(root, 'state'); + const paths = require('../../auth-broker/src/paths').defaultPaths({ stateRoot }); + const resolved = publicSdk.resolveLocalRuntimePaths(); + assert.equal(paths.stateRoot, stateRoot); + assert.equal(paths.runtimeRoot, resolved.auth.runtimeRoot); + assert.equal(path.dirname(paths.socket), resolved.auth.runtimeRoot); + assert.notEqual(paths.runtimeRoot, paths.stateRoot); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('component-specific runtime overrides cannot bypass the worktree boundary', () => { + const projectRoot = path.resolve(__dirname, "../../.."); + const baseEnvironment = Object.fromEntries( + Object.entries(process.env).filter(([key]) => !key.startsWith('DISPATCH_')), + ); + const cases = [ + ['runtime/auth-broker/src/paths.js', 'defaultPaths', 'DISPATCH_AUTH_DATABASE_ROOT'], + ['runtime/auth-broker/src/paths.js', 'defaultPaths', 'DISPATCH_AUTH_SECRET_ROOT'], + ['runtime/auth-broker/src/paths.js', 'defaultPaths', 'DISPATCH_AUTH_STATE_ROOT'], + ['runtime/auth-broker/src/paths.js', 'defaultPaths', 'DISPATCH_AUTH_SOCKET'], + ['runtime/collection-manager/src/paths.js', 'defaultPaths', 'DISPATCH_COLLECTION_DATABASE_ROOT'], + ['runtime/collection-manager/src/paths.js', 'defaultPaths', 'DISPATCH_COLLECTION_STATE_ROOT'], + ['plugins/paycom/backend/src/paths.js', null, 'DISPATCH_PAYCOM_DATA_ROOT'], + ['plugins/paycom/backend/src/paths.js', null, 'DISPATCH_PAYCOM_STAGING_ROOT'], + ['plugins/paycom/backend/src/paths.js', null, 'DISPATCH_AUTH_SOCKET'], + ['compatibility/cdf/src/paths.js', null, 'DISPATCH_CDF_DATA_ROOT'], + ['compatibility/cdf/src/paths.js', null, 'DISPATCH_CDF_STAGING_ROOT'], + ]; + for (const [relativeModule, method, environmentName] of cases) { + const modulePath = path.join(projectRoot, relativeModule); + const invocation = method + ? `require(${JSON.stringify(modulePath)}).${method}()` + : `require(${JSON.stringify(modulePath)})`; + const child = spawnSync(process.execPath, ['--no-warnings', '-e', invocation], { + encoding: 'utf8', + env: { ...baseEnvironment, [environmentName]: path.join(projectRoot, `unsafe-${environmentName.toLowerCase()}`) }, + }); + assert.notEqual(child.status, 0, `${relativeModule} accepted ${environmentName} inside the worktree`); + assert.match(`${child.stdout}${child.stderr}`, /unsafe_runtime_config/); + } +}); + +test('capability discovery is explicit and collection administration is opt-in', async () => { + const admin = new publicSdk.CollectionAdminClient({ port: { + inspect: async () => ({ initialized: false, schemaVersion: null, counts: { collectors: 0, sources: 0, plans: 0, syncs: 0 } }), + initialize: async () => ({ initialized: true, schemaVersion: 1, counts: { collectors: 0, sources: 0, plans: 0, syncs: 0 } }), + preview: async () => ({ + valid: true, incoming: { collectors: 1, sources: 1, plans: 1, syncs: 0 }, + changes: { + collectors: { create: 1, update: 0 }, sources: { create: 1, update: 0 }, + plans: { create: 1, update: 0 }, syncs: { create: 0, update: 0 }, + }, + }), + apply: async () => ({ collectors: 1, sources: 1, plans: 1, syncs: 0 }), + } }); + const inert = {}; + const dispatch = new publicSdk.DispatchClient({ + auth: inert, collections: inert, sync: inert, paycom: inert, workforce: inert, authSetup: inert, + collectionAdmin: admin, transport: 'local', + }); + const capabilities = dispatch.capabilities(); + assert.equal(capabilities.data.transport, 'local'); + assert.equal(capabilities.data.operator.collectionAdmin, true); + assert.equal((await admin.inspect()).status, 'not_initialized'); + assert.equal((await admin.initialize()).status, 'initialized'); +}); + +test('AuthClient maps broker metadata without exposing transport or raw responses', async () => { + const calls = []; + const options = []; + const client = new AuthClient({ port: { request: async (payload, requestOptions) => { + calls.push(payload); + options.push(requestOptions); + if (payload.action === 'health') return { ok: true, status: 'ready', protocolVersion: 5, vault: { verified: true, profiles: 1, schemaVersion: 1 }, ignored: 'not forwarded' }; + if (payload.action === 'inspect-auth-profile') return { + ok: true, status: 'inspected', inspection: { + profile: payload.profile, provider: 'paycom', state: 'security_profile_prompt', observedAt: '2026-08-25T22:59:00.000Z', + metadata: { + origin: 'https://www.paycomonline.net', path: '/v4/cl/web.php/security-profile', queryKeys: [], title: 'Setup Your Security Profile', + readyState: 'complete', loginFormCount: 0, challengeFormCount: 0, + profileInputNames: [], profileActionLabels: ['Not Now'], challengeIndices: [], challengeFormActionPath: null, diagnostic: null, + }, + }, ignored: 'not forwarded', + }; + if (payload.action === 'test-auth-profile') return { ok: true, status: 'authenticated', profile: { profile: payload.profile, provider: 'paycom', testedAt: '2026-08-25T23:00:00.000Z' }, ignored: 'not forwarded' }; + return { ok: true, status: 'configured', profile: { configured: true, profile: payload.profile, provider: 'paycom', createdAt: '2026-08-25T00:00:00.000Z', updatedAt: '2026-08-25T00:00:00.000Z' }, session: 'not_started', endpoint: 'not-forwarded' }; + } } }); + const health = await client.health(); + const profile = await client.profileStatus('paycom-main'); + const inspected = await client.inspectProfile('paycom-main'); + const tested = await client.testProfile('paycom-main'); + assert.equal(Object.hasOwn(client, 'port'), false); + assert.deepEqual(health.data, { protocolVersion: 5, vault: { verified: true, profiles: 1, schemaVersion: 1 } }); + assert.equal(JSON.stringify(health).includes('ignored'), false); + assert.equal(JSON.stringify(profile).includes('endpoint'), false); + assert.equal(profile.data.profile.provider, 'paycom'); + assert.equal(inspected.data.state, 'security_profile_prompt'); + assert.deepEqual(inspected.data.metadata.profileActionLabels, ['Not Now']); + assert.equal(JSON.stringify(inspected).includes('ignored'), false); + assert.deepEqual(tested.data, { profile: 'paycom-main', provider: 'paycom', testedAt: '2026-08-25T23:00:00.000Z' }); + assert.equal(JSON.stringify(tested).includes('endpoint'), false); + assert.equal(JSON.stringify(tested).includes('lease'), false); + assert.deepEqual(calls.map(value => value.action), ['health', 'status', 'inspect-auth-profile', 'test-auth-profile']); + const authDeadline = require('dispatch-protocol/browser-assistance/protocol').AUTH_REQUEST_MS; + assert.equal(options[2].timeoutMs, authDeadline + 10_000); + assert.equal(options[3].timeoutMs, authDeadline + 10_000); +}); + +test('AuthClient returns stable failures for transport and malformed component responses', async () => { + const client = new AuthClient({ port: { request: async () => { throw new Error('fixture transport details'); } } }); + const result = await client.health(); + assert.equal(result.status, 'auth_broker_unavailable'); + assert.equal(result.error.recoverable, true); + assert.equal(JSON.stringify(result).includes('fixture transport details'), false); + const malformed = new AuthClient({ port: { request: async () => ({ ok: true, status: 'ready', protocolVersion: 5, vault: { verified: true } }) } }); + assert.equal((await malformed.health()).status, 'invalid_component_response'); + assert.equal((await malformed.profileStatus('BAD PROFILE')).status, 'invalid_input'); + + const cleanup = new AuthClient({ port: { request: async () => ({ + ok: true, status: 'configured', + profile: { configured: true, profile: 'paycom-main', provider: 'paycom', createdAt: '2026-08-25T00:00:00.000Z', updatedAt: '2026-08-25T00:00:00.000Z' }, + session: 'cleanup_failed', + }) } }); + assert.equal((await cleanup.profileStatus('paycom-main')).data.session, 'cleanup_failed'); + + const wrongStatus = new AuthClient({ port: { request: async () => ({ + ok: true, status: 'unexpected_success', protocolVersion: 5, + vault: { verified: true, profiles: 1, schemaVersion: 1 }, + }) } }); + assert.equal((await wrongStatus.health()).status, 'invalid_component_response'); + + for (const code of ['mfa_required', 'captcha_required', 'security_challenge']) { + const challenge = new AuthClient({ port: { request: async () => ({ ok: false, status: code }) } }); + const result = await challenge.testProfile('amazon-operations'); + assert.equal(result.status, code); + assert.equal(result.error.recoverable, true); + } +}); + +test('CollectionClient returns closed view models and propagates a durable idempotency key', async () => { + const runsByKey = new Map(); + const enqueued = []; + const port = { + health: async () => ({ ok: true, status: 'stopped', ...COLLECTION_HEALTH }), + collectors: async () => [ + { id: 'a', version: '1.0.0', description: 'A', command: '/private/a', sourceSchema: {}, enabled: true, updatedAt: 1 }, + { id: 'b', version: '1.0.0', description: 'B', command: '/private/b', sourceSchema: {}, enabled: true, updatedAt: 2 }, + ], + startRun: async (_plan, _input, logicalKey) => { + if (runsByKey.has(logicalKey)) return runsByKey.get(logicalKey); + const value = run(`run_fixture_${enqueued.length + 1}`); + value.logicalKey = logicalKey; + enqueued.push(value); + runsByKey.set(logicalKey, value); + return value; + }, + }; + const client = new CollectionClient({ port }); + const result = await client.collectors({ limit: 1, offset: 1 }); + assert.deepEqual(result.data.items[0], { id: 'b', version: '1.0.0', description: 'B', enabled: true, updatedAt: '1970-01-01T00:00:00.002Z' }); + assert.equal(JSON.stringify(result).includes('/private/b'), false); + assert.equal((await client.collectors({ limit: 1, extra: true })).status, 'invalid_input'); + const first = await client.startRun('paycom-roster', {}, { idempotencyKey: 'button-click-1' }); + const second = await client.startRun('paycom-roster', {}, { idempotencyKey: 'button-click-1' }); + assert.equal(first.data.id, second.data.id); + assert.equal(enqueued.length, 1); + assert.equal(enqueued[0].logicalKey, 'sdk:paycom-roster:button-click-1'); + assert.equal((await client.startRun('paycom-roster', { passwordHash: 'blocked' })).status, 'invalid_input'); +}); + +test('CollectionClient exposes a bounded run receipt summary without publication internals', async () => { + const detailed = { + ...run('run_receipt_fixture'), status: 'succeeded', attempt: 1, startedAt: 2, finishedAt: 3, + attempts: [{ attempt: 1, status: 'succeeded', category: 'success', startedAt: 2, finishedAt: 3, error: null, exitCode: 0 }], + receipt: { + ok: true, status: 'published', + data: { rowCount: 12, checked: true, publicationId: 'private-publication', contentSha256: 'private-hash' }, + warnings: ['fixture warning'], + }, + }; + const client = new CollectionClient({ port: { + health: async () => ({ ok: true, status: 'stopped', ...COLLECTION_HEALTH }), + run: async () => detailed, + } }); + const result = await client.runStatus(detailed.id); + assert.equal(result.status, 'succeeded'); + assert.deepEqual(result.data.receipt.metrics, { rowCount: 12, checked: true }); + assert.equal(JSON.stringify(result).includes('private-publication'), false); + assert.equal(JSON.stringify(result).includes('private-hash'), false); +}); + +test('CollectionClient exposes standard preview, batch, and schedule operations without private task input', async () => { + const batchSummary = { + id: 'batch_fixture', source: 'paycom-main', scope: 'full', + request: { source: 'paycom-main', scope: 'full', selector: { kind: 'date', date: '2026-08-18' }, mode: 'ensure' }, + previewHash: 'a'.repeat(64), logicalKey: null, status: 'queued', + counts: { queued: 1, running: 0, succeeded: 0, failed: 0, cancelled: 0 }, runCount: 1, + createdAt: 1, + }; + const batch = { + ...batchSummary, + runPage: { items: [{ targetKey: '2026-08-22', taskId: 'roster', run: run() }], total: 1, limit: 50, offset: 0, hasMore: false }, + }; + const schedule = { + id: 'cdf-weekly-window', request: batch.request, + schedule: { + type: 'polling-window', expression: '0 15 * * 2', timezone: 'America/Los_Angeles', + intervalSeconds: 900, windowSeconds: 86_400, retryErrors: ['week_unavailable'], + }, + enabled: true, nextDueAt: null, createdAt: 1, updatedAt: 1, + }; + const port = { + health: async () => ({ ok: true, status: 'stopped', ...COLLECTION_HEALTH }), + describeCollection: async () => ({ + source: 'paycom-main', collector: 'paycom', collectorVersion: '0.6.0', targetType: 'pay-period', + timezone: 'America/Los_Angeles', selectors: ['date'], scopes: [{ id: 'full', description: 'Everything', taskCount: 4, auditSupported: true, auditTaskCount: 1 }], + limits: { maxTargets: 64, maxRangeDays: 730 }, privateTasks: ['not exposed'], + }), + previewCollection: async request => ({ + id: 'preview_fixture', hash: 'a'.repeat(64), generatedAt: '2026-08-26T00:00:00.000Z', + request, normalizedSelector: request.selector, source: request.source, collector: 'paycom', collectorVersion: '0.6.0', + targetType: 'pay-period', timezone: 'America/Los_Angeles', targetCount: 1, taskCount: 1, + targets: [{ key: '2026-08-22', start: '2026-08-09', end: '2026-08-22', values: { periodEnd: 'private' } }], + tasks: [{ targetKey: '2026-08-22', taskId: 'roster', plan: 'paycom-period-roster', input: { private: true }, dependsOn: [] }], + }), + enqueueCollection: async () => batch, + batches: async () => ({ items: [batchSummary], total: 1 }), batch: async () => batch, + cancelBatch: async () => ({ ...batch, status: 'cancelled', counts: { ...batch.counts, queued: 0, cancelled: 1 } }), + retryBatch: async () => batch, + collectionSchedules: async () => [schedule], putCollectionSchedule: async () => schedule, + setCollectionScheduleEnabled: async (_id, enabled) => ({ ...schedule, enabled }), removeCollectionSchedule: async () => schedule, + runCollectionSchedule: async () => batch, + }; + const client = new CollectionClient({ port }); + const request = batch.request; + const described = await client.describe('paycom-main'); + assert.equal(JSON.stringify(described).includes('privateTasks'), false); + const previewed = await client.preview(request); + assert.equal(previewed.status, 'previewed'); + assert.equal(JSON.stringify(previewed).includes('periodEnd'), false); + assert.equal(JSON.stringify(previewed).includes('"private"'), false); + assert.equal((await client.enqueue(request, { expectedPreviewHash: 'a'.repeat(64) })).data.id, 'batch_fixture'); + assert.equal((await client.batches()).data.total, 1); + assert.equal((await client.batchStatus('batch_fixture')).status, 'queued'); + assert.equal((await client.cancelBatch('batch_fixture')).status, 'cancelled'); + assert.equal((await client.retryBatch('batch_fixture')).status, 'queued'); + assert.equal((await client.audit(request)).status, 'queued'); + assert.equal((await client.createSchedule({ id: schedule.id, request, schedule: schedule.schedule })).status, 'scheduled'); + assert.equal((await client.pauseSchedule(schedule.id)).data.enabled, false); + assert.equal((await client.resumeSchedule(schedule.id)).data.enabled, true); + assert.equal((await client.runScheduleNow(schedule.id)).status, 'queued'); + assert.equal((await client.removeSchedule(schedule.id)).status, 'removed'); +}); + +test('SyncClient exposes closed lifecycle, edit, and history operations', async () => { + const sync = { + id: 'fixture-main-sync', plan: 'fixture-sync-plan', source: 'fixture-main', collector: 'fixture', method: 'fixture.sync', + desiredState: 'stopped', activity: 'idle', intervalSeconds: 60, jitterSeconds: 5, overlap: 'coalesce', + settingsSchema: { type: 'object', properties: { behavior: { type: 'string' } }, required: ['behavior'], additionalProperties: false }, + settings: { behavior: 'no_change' }, revision: 1, generation: 0, nextDueAt: null, + lastStartedAt: null, lastSucceededAt: null, lastError: null, blocked: null, + businessContext: { date: '2026-08-29', timezone: 'America/Los_Angeles' }, alerts: [], + activeRun: null, queuedRunCount: 0, createdAt: 1, updatedAt: 1, + }; + const queued = { ...run('run_sync_fixture'), plan: sync.plan, method: sync.method, trigger: 'sync_start' }; + const completed = { + ...queued, status: 'succeeded', attempt: 2, startedAt: 10, finishedAt: 20, + attempts: [ + { attempt: 1, status: 'failed', category: 'provider', startedAt: 2, finishedAt: 3, error: 'paycom_timeout', exitCode: 0 }, + { attempt: 2, status: 'succeeded', category: 'success', startedAt: 10, finishedAt: 20, error: null, exitCode: 0 }, + ], + receipt: { ok: true, status: 'published', data: { + businessDate: '2026-08-29', businessTimezone: 'America/Los_Angeles', + delta: { + roster: { addedCount: 0, profileChangedCount: 0, summaryChangedCount: 0, recordChangedCount: 0, becameUnknownCount: 0, returnedFromUnknownCount: 0 }, + timecards: { addedCount: 0, changedCount: 2, unchangedCount: 98, removedCount: 0 }, + days: { addedCount: 0, changedCount: 2, removedCount: 0, missingPunchAddedCount: 0, missingPunchResolvedCount: 1, unresolvedSlotAddedCount: 0, unresolvedSlotResolvedCount: 1, commentSectionsChangedCount: 0, totalSectionsChangedCount: 1 }, + punches: { addedCount: 7, editedCount: 0, removedCount: 0, kindChangedCount: 0, addedByKind: { inDayCount: 0, outLunchCount: 0, inLunchCount: 0, outDayCount: 7, unclassifiedCount: 0 }, removedByKind: { inDayCount: 0, outLunchCount: 0, inLunchCount: 0, outDayCount: 0, unclassifiedCount: 0 } }, + details: { additionalRowSectionsChangedCount: 0, approvalSectionsChangedCount: 1, attestationSectionsChangedCount: 0, mealWaiverSectionsChangedCount: 0 }, + }, + persistence: { verified: true, code: 'verified', date: '2026-08-29', timecardCount: 100, dateRowCount: 100, selectedTimecardCount: 100, persistedSelectedTimecardCount: 100, selectedMismatchCount: 0, punchCount: 117, inDayPunchCount: 38, inDayTimecardCount: 38, outLunchPunchCount: 37, inLunchPunchCount: 35, outDayPunchCount: 7, unclassifiedPunchCount: 0 }, + } }, + }; + const port = { + syncs: async () => ({ items: [sync], total: 1 }), + sync: async () => sync, + start: async () => ({ sync: { ...sync, desiredState: 'running', activity: 'queued', generation: 1, activeRun: queued, queuedRunCount: 1 }, run: queued }), + stop: async () => sync, + restart: async () => ({ sync: { ...sync, desiredState: 'running', activity: 'queued', generation: 2, activeRun: queued, queuedRunCount: 1 }, run: queued }), + runNow: async () => ({ sync: { ...sync, desiredState: 'running', activity: 'queued', activeRun: queued, queuedRunCount: 1 }, run: queued }), + edit: async (_id, patch) => ({ sync: { ...sync, ...patch, revision: 2 }, run: null }), + history: async () => ({ items: [{ generation: 1, configRevision: 1, windowKey: 'start:1', trigger: 'sync_start', run: completed }], total: 1, limit: 50, offset: 0, hasMore: false }), + }; + const client = new SyncClient({ port }); + assert.equal((await client.list()).data.total, 1); + const status = await client.status(sync.id); + assert.equal(status.data.settings.behavior, 'no_change'); + assert.deepEqual(status.data.businessContext, { date: '2026-08-29', timezone: 'America/Los_Angeles' }); + assert.deepEqual(status.data.alerts, []); + assert.equal((await client.start(sync.id)).status, 'started'); + assert.equal((await client.edit(sync.id, { intervalSeconds: 120 }, { expectedRevision: 1 })).data.sync.revision, 2); + assert.equal((await client.restart(sync.id)).status, 'restarted'); + assert.equal((await client.runNow(sync.id)).status, 'queued'); + const history = await client.history(sync.id); + assert.equal(history.data.items[0].run.id, queued.id); + assert.equal(history.data.items[0].businessContext.date, '2026-08-29'); + assert.equal(history.data.items[0].delta.punches.addedByKind.outDayCount, 7); + assert.equal(history.data.items[0].delta.details.approvalSectionsChangedCount, 1); + assert.equal(history.data.items[0].persistence.selectedMismatchCount, 0); + assert.equal(history.data.items[0].run.attempts[0].error, 'paycom_timeout'); + assert.equal((await client.stop(sync.id)).status, 'stopped'); + assert.equal((await client.edit(sync.id, {})).status, 'invalid_input'); +}); + +test('local Collection Manager port uses read-only queries and always closes stores', () => { + const options = []; + let closed = 0; + const store = { + health: () => ({ ok: true, status: 'stopped', ...COLLECTION_HEALTH }), + collectors: () => [], + close: () => { closed += 1; }, + }; + const port = new LocalCollectionManagerPort({ + paths: { database: __filename }, + storeFactory: (_paths, value) => { options.push(value); return store; }, + }); + assert.equal(port.health().status, 'stopped'); + assert.deepEqual(port.collectors(), []); + assert.deepEqual(options, [{ readOnly: true }, { readOnly: true }]); + assert.equal(closed, 2); + + const missingDatabase = `/tmp/dispatch-sdk-uninitialized-${process.pid}.sqlite3`; + const missingManager = new LocalCollectionManagerPort({ paths: { database: missingDatabase } }); + const missingPaycom = new LocalPaycomPublicationPort({ database: missingDatabase }); + assert.equal(missingManager.health().status, 'not_initialized'); + assert.equal(missingPaycom.health(), null); + assert.equal(fs.existsSync(missingDatabase), false); +}); + +test('PaycomClient returns a closed publication-health DTO and does not initialize missing storage', async () => { + const missing = new PaycomClient({ port: { health: async () => null } }); + assert.equal((await missing.health()).status, 'not_initialized'); + const client = new PaycomClient({ port: { health: async () => ({ + payPeriods: { ...PAYCOM_HEALTH.payPeriods, publicationId: 'private', contentSha256: 'private', quickCheck: 'ok' }, + roster: PAYCOM_HEALTH.roster, + timecards: PAYCOM_HEALTH.timecards, + resourceLinks: PAYCOM_HEALTH.resourceLinks, + }) } }); + const result = await client.health(); + assert.equal(result.status, 'degraded'); + assert.equal(result.data.ready, false); + assert.equal(result.data.storageStatus, 'ready'); + assert.equal(JSON.stringify(result).includes('publicationId'), false); + assert.equal(JSON.stringify(result).includes('contentSha256'), false); +}); + +test('system status validates component DTOs and sanitizes rejected or malicious results', async () => { + const result = await getSystemStatus({ + auth: { health: async () => failure('auth_broker_unavailable', { recoverable: true }) }, + collections: { health: async () => success('stopped', COLLECTION_HEALTH) }, + paycom: { health: async () => success('degraded', PAYCOM_HEALTH) }, + }); + assert.equal(isResult(result), true); + assert.equal(result.status, 'degraded'); + assert.equal(result.data.components.auth.status, 'stopped'); + assert.deepEqual(result.data.summary, { ready: 0, degraded: 2, failed: 0 }); + + const failed = await getSystemStatus({ + auth: { health: async () => { throw new Error('private fixture detail'); } }, + collections: { health: async () => success('ready', COLLECTION_HEALTH) }, + paycom: { health: async () => ({ contractVersion: 1, ok: true, status: 'ready', data: { password: 'fixture' } }) }, + }); + assert.equal(failed.status, 'failed'); + assert.equal(failed.data.components.auth.error.code, 'auth_client_failed'); + assert.equal(failed.data.components.paycom.error.code, 'invalid_component_response'); + assert.equal(JSON.stringify(failed).includes('private fixture detail'), false); + assert.equal(JSON.stringify(failed).includes('fixture'), false); +}); + +test('component DTO defects remain invalid responses rather than availability failures', async () => { + const collections = new CollectionClient({ port: { health: async () => ({ ok: true, status: 'ready', manager: {}, counts: {} }) } }); + assert.equal((await collections.health()).status, 'invalid_component_response'); + + const sync = new SyncClient({ port: { syncs: async () => ({ items: [{}], total: 1 }) } }); + assert.equal((await sync.list()).status, 'invalid_component_response'); + + const paycom = new PaycomClient({ port: { health: async () => ({ payPeriods: {}, roster: {}, timecards: {} }) } }); + assert.equal((await paycom.health()).status, 'invalid_component_response'); +}); diff --git a/dsp/runtime/sdk/tests/setup-auth.test.js b/dsp/runtime/sdk/tests/setup-auth.test.js new file mode 100644 index 0000000..cbc5e1a --- /dev/null +++ b/dsp/runtime/sdk/tests/setup-auth.test.js @@ -0,0 +1,522 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { EventEmitter } = require('node:events'); +const test = require('node:test'); +const { AuthSetupWorkflowClient } = require('../src'); +const { runSetupAuth, RecordingEventSink } = require('../../application/auth/setup-auth'); +const { prepareAuthSetup } = require('../../application/auth/prepare-auth-setup'); +const { success, failure } = require('dispatch-protocol/contracts/src'); +const { runJson } = require('../../adapters/local/process-helper'); +const { PAYCOM_HELPER, GENERIC_HELPER, credentialHelper } = require('../../adapters/local/credential-ingress'); +const { LocalAuthSetupPort } = require('../../adapters/local/auth-setup-port'); +const { LocalAuthBrokerServicePort } = require('../../adapters/local/auth-broker-service-port'); +const { defaultPaths } = require('../../auth-broker/src/paths'); + +function setupFixture({ broker = 'stopped', vault = 'absent', configured = false, managed = true, + provider = 'paycom', profile = 'paycom-main' } = {}) { + const state = { broker, vault, configured, managed, provider, profile, initialized: 0, captured: 0, removed: 0, started: 0, stopped: 0, tested: 0, testedProfile: null }; + return { + state, + setup: { + inspect: async profile => ({ + broker: state.broker, + vault: { state: state.vault, verified: state.vault === 'ready', schemaVersion: state.vault === 'ready' ? 1 : null, profiles: state.configured ? 1 : 0 }, + profile: { configured: state.configured, profile, ...(state.configured ? { provider: state.provider } : {}) }, + }), + initialize: async () => { state.initialized += 1; state.vault = 'ready'; return { verified: true, schemaVersion: 1, profiles: 0 }; }, + remove: async profile => { state.removed += 1; state.configured = false; return { profile, removed: true }; }, + }, + ingress: { + available: () => true, + capture: async ({ operation, provider, profile }) => { + assert.equal(operation, state.configured ? 'replace' : 'enroll'); + assert.equal(provider, state.provider); + assert.equal(profile, state.profile); + state.captured += 1; + state.configured = true; + return { stored: true, provider, profile }; + }, + }, + service: { + status: async () => ({ status: state.broker === 'ready' ? 'ready' : 'stopped', managed: state.managed }), + start: async () => { state.started += 1; state.broker = 'ready'; return { status: 'ready', managed: true, started: true }; }, + stop: async () => { state.stopped += 1; state.broker = 'stopped'; return { status: 'stopped', managed: true, stopped: true }; }, + }, + authentication: { + testProfile: async profile => { + assert.equal(profile, state.profile); + state.tested += 1; + state.testedProfile = profile; + return success('authenticated', { profile, provider: state.provider, testedAt: '2026-08-25T23:00:00.000Z' }); + }, + }, + }; +} + +test('auth setup preparation returns sanitized state and closed action capabilities', async () => { + const missing = setupFixture(); + const prepared = await prepareAuthSetup({ setup: missing.setup, service: missing.service, ingress: missing.ingress }); + assert.equal(prepared.ok, true); + assert.deepEqual(prepared.data.target, { provider: 'paycom', profile: 'paycom-main' }); + assert.deepEqual(prepared.data.capabilities.credentialActions.map(item => [item.id, item.available]), [ + ['keep', false], ['enroll', true], ['replace', false], ['remove', false], + ]); + assert.equal(prepared.data.defaults.credentialAction, 'enroll'); + assert.equal(/password|username|pin\d|cookie|endpoint|lease|token/i.test(JSON.stringify(prepared)), false); + + const unmanaged = setupFixture({ broker: 'ready', vault: 'ready', configured: true, managed: false }); + const blocked = await prepareAuthSetup({ setup: unmanaged.setup, service: unmanaged.service, ingress: unmanaged.ingress }); + assert.equal(blocked.data.capabilities.credentialActions[2].available, false); + assert.equal(blocked.data.capabilities.credentialActions[2].reason, 'auth_broker_unmanaged'); + assert.equal(blocked.data.capabilities.credentialActions[3].reason, 'auth_broker_unmanaged'); + assert.equal(blocked.data.capabilities.authenticationTest.available, true); +}); + +test('Amazon Logistics auth setup uses the same protected workflow with an explicit profile', async () => { + const fixture = setupFixture({ provider: 'amazon-logistics', profile: 'amazon-operations' }); + const prepared = await prepareAuthSetup({ + setup: fixture.setup, service: fixture.service, ingress: fixture.ingress, + }, { provider: 'amazon-logistics', profile: 'amazon-operations' }); + assert.equal(prepared.ok, true); + assert.deepEqual(prepared.data.target, { provider: 'amazon-logistics', profile: 'amazon-operations' }); + const result = await runSetupAuth({ + setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, authentication: fixture.authentication, + events: new RecordingEventSink(), + }, { + provider: 'amazon-logistics', profile: 'amazon-operations', credentialAction: 'enroll', + startBroker: true, testAuthentication: true, + }); + assert.equal(result.ok, true, JSON.stringify(result)); + assert.equal(result.data.provider, 'amazon-logistics'); + assert.equal(result.data.profile, 'amazon-operations'); + assert.equal(fixture.state.captured, 1); + assert.equal(fixture.state.tested, 1); + assert.equal(fixture.state.testedProfile, 'amazon-operations'); + assert.deepEqual(credentialHelper('amazon-logistics', 'enroll', 'amazon-operations'), { + executable: GENERIC_HELPER, args: ['enroll', 'amazon-operations', 'amazon-logistics'], + }); + assert.deepEqual(credentialHelper('paycom', 'replace', 'paycom-main'), { + executable: PAYCOM_HELPER, args: ['replace', 'paycom-main'], + }); +}); + +test('public auth setup workflow client validates preparation and explicit credential intent', async () => { + const fixture = setupFixture(); + const port = { + prepare: input => prepareAuthSetup({ setup: fixture.setup, service: fixture.service, ingress: fixture.ingress }, input), + run: (input, options) => runSetupAuth({ + setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, + authentication: fixture.authentication, events: options.events, signal: options.signal, + }, input), + }; + const client = new AuthSetupWorkflowClient({ port }); + const prepared = await client.prepare(); + assert.equal(prepared.status, 'ready'); + const result = await client.run({ credentialAction: 'enroll' }); + assert.equal(result.status, 'complete'); + assert.equal(fixture.state.captured, 1); + assert.equal((await client.run({ credentialAction: 'enroll' })).status, 'profile_exists'); + const removed = await client.run({ credentialAction: 'remove' }); + assert.equal(removed.status, 'complete'); + assert.equal(removed.data.configured, false); + assert.equal((await client.run({ credentialAction: 'remove' })).status, 'profile_not_configured'); + assert.equal((await client.run({ credentialAction: 'unknown' })).status, 'invalid_input'); + + const malformed = new AuthSetupWorkflowClient({ port: { + prepare: async () => ({ contractVersion: 1, ok: true, status: 'ready', data: { private: 'value' } }), + run: async () => ({ contractVersion: 1, ok: true, status: 'complete', data: {} }), + } }); + assert.equal((await malformed.prepare()).status, 'invalid_component_response'); + assert.equal((await malformed.run()).status, 'invalid_component_response'); +}); + +test('auth setup enrolls through the dedicated ingress and emits only semantic events', async () => { + const fixture = setupFixture(); + const events = new RecordingEventSink(); + const result = await runSetupAuth({ setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, authentication: fixture.authentication, events }); + assert.equal(result.ok, true); + assert.equal(result.status, 'complete'); + assert.equal(result.data.profile, 'paycom-main'); + assert.equal(fixture.state.initialized, 1); + assert.equal(fixture.state.captured, 1); + assert.deepEqual(events.events.map(value => value.type), [ + 'workflow_started', 'step_started', 'check_completed', 'check_completed', 'check_completed', + 'step_started', 'check_completed', 'step_started', 'credential_capture_started', + 'credential_capture_completed', 'step_started', 'check_completed', 'step_started', 'check_completed', + 'workflow_completed', + ]); + const serialized = JSON.stringify(events.events); + assert.equal(/password|username|pin\d|cookie|endpoint|lease|token/i.test(serialized), false); +}); + +test('auth setup is idempotent for an existing profile and replacement is explicit', async () => { + const fixture = setupFixture({ vault: 'ready', configured: true }); + fixture.ingress.available = () => { throw new Error('must not request terminal'); }; + fixture.ingress.capture = () => { throw new Error('must not capture'); }; + const result = await runSetupAuth({ setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, authentication: fixture.authentication, events: new RecordingEventSink() }); + assert.equal(result.status, 'complete'); + assert.equal(fixture.state.initialized, 0); + assert.equal(fixture.state.captured, 0); + + const running = setupFixture({ broker: 'ready', vault: 'ready', configured: true, managed: false }); + const blocked = await runSetupAuth( + { setup: running.setup, ingress: running.ingress, service: running.service, authentication: running.authentication, events: new RecordingEventSink() }, + { replaceExisting: true }, + ); + assert.equal(blocked.status, 'auth_broker_unmanaged'); + assert.equal(blocked.error.recoverable, true); + assert.deepEqual(blocked.data.nextActions, ['stop_auth_broker_manually', 'retry_setup']); + assert.equal(running.state.captured, 0); +}); + +test('auth setup safely restarts a managed broker and optionally tests authentication', async () => { + const replacement = setupFixture({ broker: 'ready', vault: 'ready', configured: true, managed: true }); + const replaced = await runSetupAuth( + { setup: replacement.setup, ingress: replacement.ingress, service: replacement.service, authentication: replacement.authentication, events: new RecordingEventSink() }, + { replaceExisting: true, testAuthentication: true }, + ); + assert.equal(replaced.status, 'complete'); + assert.equal(replaced.data.broker, 'ready'); + assert.equal(replaced.data.authenticationTest, 'authenticated'); + assert.equal(replacement.state.stopped, 1); + assert.equal(replacement.state.captured, 1); + assert.equal(replacement.state.started, 1); + assert.equal(replacement.state.tested, 1); + assert.equal(/endpoint|lease|cookie|token/i.test(JSON.stringify(replaced)), false); +}); + +test('auth setup preserves closed Amazon and browser statuses instead of reporting an invalid component', async () => { + const fixture = setupFixture({ + broker: 'ready', vault: 'ready', configured: true, + provider: 'amazon-logistics', profile: 'amazon-operations', + }); + fixture.authentication.testProfile = async profile => { + assert.equal(profile, 'amazon-operations'); + return failure('mfa_required', { recoverable: true }); + }; + const result = await runSetupAuth( + { setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, + authentication: fixture.authentication, events: new RecordingEventSink() }, + { provider: 'amazon-logistics', profile: 'amazon-operations', credentialAction: 'keep', testAuthentication: true }, + ); + assert.equal(result.status, 'mfa_required'); + assert.equal(result.error.recoverable, true); + assert.equal(result.data.provider, 'amazon-logistics'); + assert.equal(result.data.profile, 'amazon-operations'); + + fixture.authentication.testProfile = async () => failure('browser_protocol_failed', { recoverable: true }); + const browserFailure = await runSetupAuth( + { setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, + authentication: fixture.authentication, events: new RecordingEventSink() }, + { provider: 'amazon-logistics', profile: 'amazon-operations', credentialAction: 'keep', testAuthentication: true }, + ); + assert.equal(browserFailure.status, 'browser_protocol_failed'); + assert.equal(browserFailure.error.recoverable, true); +}); + +test('auth setup deletes a configured profile, clears it before restart, and returns metadata only', async () => { + const fixture = setupFixture({ broker: 'ready', vault: 'ready', configured: true, managed: true }); + fixture.ingress.available = () => { throw new Error('remove must not require credential ingress'); }; + fixture.ingress.capture = () => { throw new Error('remove must not capture credentials'); }; + const result = await runSetupAuth( + { setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, authentication: fixture.authentication, events: new RecordingEventSink() }, + { credentialAction: 'remove' }, + ); + assert.equal(result.status, 'complete'); + assert.equal(result.data.configured, false); + assert.equal(result.data.authenticationTest, 'skipped'); + assert.equal(fixture.state.stopped, 1); + assert.equal(fixture.state.removed, 1); + assert.equal(fixture.state.started, 1); + assert.equal(fixture.state.configured, false); + assert.equal((await runSetupAuth( + { setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, authentication: fixture.authentication, events: new RecordingEventSink() }, + { credentialAction: 'remove' }, + )).status, 'profile_not_configured'); +}); + +test('auth setup fails before mutation without an interactive terminal', async () => { + const fixture = setupFixture(); + fixture.ingress.available = () => false; + const result = await runSetupAuth({ setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, authentication: fixture.authentication, events: new RecordingEventSink() }); + assert.equal(result.status, 'interactive_terminal_required'); + assert.equal(result.error.recoverable, true); + assert.equal(fixture.state.initialized, 0); + assert.equal(fixture.state.captured, 0); +}); + +test('auth setup cancellation fails closed before component access', async () => { + const fixture = setupFixture(); + fixture.setup.inspect = async () => { throw new Error('must not inspect'); }; + const controller = new AbortController(); + controller.abort(); + const result = await runSetupAuth({ + setup: fixture.setup, + ingress: fixture.ingress, + service: fixture.service, + authentication: fixture.authentication, + events: new RecordingEventSink(), + signal: controller.signal, + }); + assert.equal(result.status, 'cancelled'); + assert.equal(result.error.recoverable, true); + assert.deepEqual(result.data.nextActions, ['retry_setup']); +}); + +test('auth setup restores the managed broker and reports partial state after post-stop failure', async () => { + const fixture = setupFixture({ broker: 'ready', vault: 'ready', configured: true, managed: true }); + fixture.ingress.capture = async () => { throw new Error('fixture helper failure'); }; + const result = await runSetupAuth( + { setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, authentication: fixture.authentication, events: new RecordingEventSink() }, + { credentialAction: 'replace' }, + ); + assert.equal(result.status, 'setup_auth_failed'); + assert.equal(fixture.state.broker, 'ready'); + assert.equal(fixture.state.stopped, 1); + assert.equal(fixture.state.started, 1); + assert.deepEqual(result.data.state, { + broker: 'ready', profile: 'configured', mutation: 'none', recovery: 'restored', + }); +}); + +test('auth setup does not restart a broker stopped by a concurrent setup', async () => { + const fixture = setupFixture({ broker: 'ready', vault: 'ready', configured: true, managed: true }); + fixture.service.stop = async () => { + fixture.state.stopped += 1; + fixture.state.broker = 'stopped'; + return { status: 'stopped', managed: true, stopped: false }; + }; + fixture.ingress.capture = async () => { throw new Error('concurrent maintenance lock'); }; + const result = await runSetupAuth( + { setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, + authentication: fixture.authentication, events: new RecordingEventSink() }, + { credentialAction: 'replace' }, + ); + assert.equal(result.status, 'setup_auth_failed'); + assert.equal(fixture.state.stopped, 1); + assert.equal(fixture.state.started, 0); + assert.equal(fixture.state.broker, 'stopped'); + assert.equal(result.data.state.recovery, 'not_needed'); +}); + +test('auth setup keeps presentation event failures outside domain execution', async () => { + const fixture = setupFixture({ broker: 'ready', vault: 'ready', configured: true, managed: true }); + const result = await runSetupAuth( + { setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, authentication: fixture.authentication, + events: { emit: async () => { throw new Error('fixture renderer failure'); } } }, + { credentialAction: 'replace' }, + ); + assert.equal(result.status, 'complete'); + assert.equal(fixture.state.broker, 'ready'); + assert.equal(fixture.state.stopped, 1); + assert.equal(fixture.state.started, 1); +}); + +test('auth setup reports recovery failure when it cannot restore the original broker state', async () => { + const fixture = setupFixture({ broker: 'ready', vault: 'ready', configured: true, managed: true }); + fixture.ingress.capture = async () => { throw new Error('fixture helper failure'); }; + fixture.service.start = async () => { fixture.state.started += 1; throw new Error('fixture start failure'); }; + const result = await runSetupAuth( + { setup: fixture.setup, ingress: fixture.ingress, service: fixture.service, authentication: fixture.authentication, events: new RecordingEventSink() }, + { credentialAction: 'replace' }, + ); + assert.equal(result.status, 'setup_recovery_failed'); + assert.equal(result.data.cause, 'setup_auth_failed'); + assert.equal(result.data.state.recovery, 'failed'); + assert.equal(result.data.state.broker, 'stopped'); + assert.deepEqual(result.data.nextActions, ['start_auth_broker', 'run_status', 'retry_setup']); +}); + +test('auth setup validates port DTOs before they reach events or results', async () => { + const events = new RecordingEventSink(); + const result = await runSetupAuth({ + setup: { + inspect: async () => ({ + broker: 'private component detail', + vault: { state: 'absent', verified: false, schemaVersion: null, profiles: 0 }, + profile: { configured: false, profile: 'paycom-main' }, + }), + initialize: async () => ({}), + remove: async () => ({}), + }, + ingress: { available: () => true, capture: async () => ({}) }, + service: { status: async () => ({}), start: async () => ({}), stop: async () => ({}) }, + authentication: { testProfile: async () => ({}) }, + events, + }); + assert.equal(result.status, 'invalid_component_response'); + assert.equal(JSON.stringify(result).includes('private component detail'), false); + assert.equal(JSON.stringify(events.events).includes('private component detail'), false); +}); + +test('local process helper uses a fixed environment, no shell, and bounded JSON', () => { + let invocation; + const spawn = (executable, args, options) => { + invocation = { executable, args, options }; + return { status: 0, signal: null, stdout: '{"ok":true,"status":"ok"}\n', stderr: '' }; + }; + const result = runJson(PAYCOM_HELPER, ['enroll', 'paycom-main'], { spawn, stdinFd: 7 }); + assert.equal(result.value.ok, true); + assert.equal(invocation.executable, PAYCOM_HELPER); + assert.deepEqual(invocation.args, ['enroll', 'paycom-main']); + assert.equal(invocation.options.shell, false); + assert.equal(invocation.options.stdio[0], 7); + assert.deepEqual(Object.keys(invocation.options.env).sort(), ['HOME', 'LANG', 'LC_ALL', 'PATH']); + assert.equal(invocation.options.env.HOME, os.homedir()); + assert.equal(JSON.stringify(invocation.options.env).match(/password|username|pin|token|secret/i), null); + + runJson(PAYCOM_HELPER, ['status'], { spawn, interpreter: 'node' }); + assert.equal(invocation.executable, process.execPath); + assert.deepEqual(invocation.args, ['--no-warnings', PAYCOM_HELPER, 'status']); + assert.throws(() => runJson(PAYCOM_HELPER, [], { spawn, interpreter: 'python' }), + error => error.code === 'invalid_input'); + + const environment = Object.fromEntries(Array.from({ length: 16 }, (_value, index) => [ + `DISPATCH_PATH_${String(index).padStart(2, '0')}`, `/private/path/${index}`, + ])); + environment.DISPATCH_MANAGED_RUNTIME = '1'; + runJson(PAYCOM_HELPER, [], { spawn, input: '{"batch":"fixture"}', environment }); + assert.equal(invocation.options.stdio[0], 'pipe'); + assert.equal(invocation.options.input, '{"batch":"fixture"}\n'); + assert.equal(invocation.options.env.DISPATCH_MANAGED_RUNTIME, '1'); + assert.throws(() => runJson(PAYCOM_HELPER, [], { + spawn, input: '{}', environment: { DISPATCH_MANAGED_RUNTIME: '0' }, + }), error => error.code === 'invalid_input'); + + assert.throws(() => runJson(PAYCOM_HELPER, ['enroll', 'paycom-main'], { + spawn: () => ({ status: 1, signal: null, stdout: '{"ok":true,"status":"ok"}\n', stderr: '' }), + }), error => error.code === 'invalid_helper_response'); + assert.throws(() => runJson(PAYCOM_HELPER, ['enroll', 'paycom-main'], { + spawn: () => ({ status: 0, signal: null, stdout: '{"ok":true,"ok":false,"status":"ok"}\n', stderr: '' }), + }), error => error.code === 'invalid_helper_response'); + assert.throws(() => runJson(PAYCOM_HELPER, ['enroll', 'paycom-main'], { + spawn: () => ({ status: 1, signal: null, stdout: '{"ok":false,"status":"helper_failed","extra":"blocked"}\n', stderr: '' }), + }), error => error.code === 'invalid_helper_response'); +}); + +test('local Auth setup inspection does not create missing storage and rejects malformed live responses', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-setup-port-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const paths = { + database: path.join(root, 'vault', 'credentials.sqlite3'), + key: path.join(root, 'vault', 'master.key'), + socket: path.join(root, 'state', 'auth-broker.sock'), + }; + const unavailable = Object.assign(new Error('connect ENOENT'), { code: 'ENOENT' }); + const port = new LocalAuthSetupPort({ paths, requestImpl: async () => { throw unavailable; } }); + const state = await port.inspect('paycom-main'); + assert.equal(state.vault.state, 'absent'); + assert.equal(state.profile.configured, false); + assert.equal(fs.existsSync(path.dirname(paths.database)), false); + assert.equal(fs.existsSync(path.dirname(paths.socket)), false); + + const malformed = new LocalAuthSetupPort({ + paths, + requestImpl: async (_socket, request) => request.action === 'health' + ? { ok: true, protocolVersion: 2, vault: { verified: true } } + : { ok: true, profile: { profile: 'paycom-main', configured: false } }, + }); + await assert.rejects(() => malformed.inspect('paycom-main'), error => error.code === 'invalid_component_response'); + const failedHealth = new LocalAuthSetupPort({ + paths, + requestImpl: async () => ({ ok: false, status: 'vault_integrity_failed' }), + }); + await assert.rejects(() => failedHealth.inspect('paycom-main'), error => error.code === 'vault_integrity_failed'); +}); + +test('local Auth Broker lifecycle stops only the recorded verified process identity, including an older protocol', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-service-')); + fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const paths = defaultPaths({ + databaseRoot: path.join(root, 'db'), secretRoot: path.join(root, 'secrets'), + stateRoot: path.join(root, 'state'), runtimeRoot: path.join(root, 'run'), + }); + const identity = { bootId: '11111111-1111-1111-1111-111111111111', startTicks: '42' }; + let alive = false; + let ready = false; + let protocolVersion = 5; + const signals = []; + const spawnImpl = () => { + alive = true; + ready = true; + const child = new EventEmitter(); + child.pid = 43210; + child.unref = () => {}; + child.kill = signal => { signals.push(signal); alive = false; ready = false; }; + return child; + }; + const killImpl = (_pid, signal) => { signals.push(signal); alive = false; ready = false; }; + const requestImpl = async () => { + if (!ready) throw Object.assign(new Error('missing'), { code: 'ENOENT' }); + return { ok: true, status: 'ready', protocolVersion, vault: { verified: true } }; + }; + let currentIdentity = identity; + const port = new LocalAuthBrokerServicePort({ + paths, spawnImpl, killImpl, requestImpl, + identityImpl: () => alive ? currentIdentity : null, + delayImpl: async () => {}, + clock: () => 1_700_000_000_000, + }); + + assert.deepEqual(await port.start(), { status: 'ready', managed: true, started: true }); + const record = path.join(paths.stateRoot, 'auth-broker-service.json'); + assert.equal(fs.statSync(record).mode & 0o777, 0o600); + assert.deepEqual(await port.status(), { status: 'ready', managed: true }); + protocolVersion = 4; + await assert.rejects(() => port.status(), error => error.code === 'invalid_component_response'); + assert.deepEqual(await port.stop(), { status: 'stopped', managed: true, stopped: true }); + assert.deepEqual(signals, ['SIGTERM']); + assert.equal(fs.existsSync(record), false); + + protocolVersion = 5; + await port.start(); + currentIdentity = { ...identity, startTicks: '99' }; + const signalCount = signals.length; + await assert.rejects(() => port.stop(), error => error.code === 'auth_broker_unmanaged'); + assert.equal(signals.length, signalCount); + assert.equal(fs.existsSync(record), false); +}); + +test('local Auth Broker lifecycle cleans up a spawned process after malformed readiness', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-service-failure-')); + fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const paths = defaultPaths({ + databaseRoot: path.join(root, 'db'), secretRoot: path.join(root, 'secrets'), + stateRoot: path.join(root, 'state'), runtimeRoot: path.join(root, 'run'), + }); + const identity = { bootId: '22222222-2222-2222-2222-222222222222', startTicks: '51' }; + let alive = false; + let requests = 0; + const signals = []; + const port = new LocalAuthBrokerServicePort({ + paths, + requestImpl: async () => { + requests += 1; + if (requests === 1) throw Object.assign(new Error('missing'), { code: 'ENOENT' }); + return { ok: true, status: 'ready', protocolVersion: 999, vault: { verified: true } }; + }, + spawnImpl: () => { + alive = true; + const child = new EventEmitter(); + child.pid = 43211; + child.unref = () => {}; + child.kill = signal => { signals.push(signal); alive = false; }; + return child; + }, + killImpl: (_pid, signal) => { signals.push(signal); alive = false; }, + identityImpl: () => alive ? identity : null, + delayImpl: async () => {}, + clock: () => 1_700_000_000_000, + }); + + await assert.rejects(() => port.start(), error => error.code === 'invalid_component_response'); + assert.deepEqual(signals, ['SIGTERM']); + assert.equal(alive, false); + assert.equal(fs.existsSync(path.join(paths.stateRoot, 'auth-broker-service.json')), false); +}); diff --git a/dsp/runtime/sdk/tests/workforce.test.js b/dsp/runtime/sdk/tests/workforce.test.js new file mode 100644 index 0000000..75b3a58 --- /dev/null +++ b/dsp/runtime/sdk/tests/workforce.test.js @@ -0,0 +1,416 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const test = require('node:test'); + +const { WorkforceClient } = require('../src/workforce-client'); +const { LocalPaycomWorkforcePort, dailyRowOrder } = require('../../../plugins/paycom/backend/adapters/workforce'); + +const TARGET = '2026-09-05'; +const COLLECTED = '2026-08-29T06:00:00.000Z'; +const BASE_URL = 'https://www.paycomonline.net/v4/cl/web.php/timecard/index'; +function canonicalUrl(code) { + const value = new URL(BASE_URL); + value.searchParams.set('firstrefno', code); + value.searchParams.set('perioddates', `2026-08-23_${TARGET}`); + value.searchParams.set('formtype', 'SUMMARY'); + return value.href; +} + +function employee(code = 'A001', lifecycleStatus = 'active') { + return { + employeeCode: code, + employeeName: `Employee ${code}`, + lifecycleStatus, + isActive: lifecycleStatus !== 'inactive', + isDriverDepartment: true, + isDriverPosition: true, + departmentCode: 'D1', + departmentDesc: 'Driver', + deliveryStationCode: 'S1', + deliveryStationDesc: 'Station', + positionTitle: 'Driver', + payClass: 'PC', + payType: 'Hourly', + primarySupervisor: 'Supervisor', + }; +} + +function rawWorkforce() { + const employees = [employee('A001'), employee('A002', 'unknown')]; + const rosterPublication = { + id: 'roster-private', target: TARGET, content_sha256: 'a'.repeat(64), collected_at: COLLECTED, + }; + return { + roster: { publication: rosterPublication, employees }, + timecards: { + publication: { + id: 'timecards-private', target: TARGET, collected_at: COLLECTED, + metadata_json: JSON.stringify({ rosterPublicationId: rosterPublication.id, rosterContentSha256: rosterPublication.content_sha256 }), + }, + rows: employees.map(row => ({ + employeeCode: row.employeeCode, + employeeName: row.employeeName, + observedAt: COLLECTED, + sourceSha256: 'private', + record: { + employeeCode: row.employeeCode, + periodStart: '2026-08-23', + periodEnd: TARGET, + periodTotalHours: 8, + days: [{ + date: '2026-08-30', + missingPunch: row.employeeCode === 'A002', + punches: [{ + kind: row.employeeCode === 'A001' ? 'IN DAY' : 'OUT DAY', + displayTime: row.employeeCode === 'A001' ? '10:01 AM' : '05:00 PM', + actualTime: row.employeeCode === 'A001' ? '10:01 AM' : '05:00 PM', + provenanceAvailable: true, + }], + }], + }, + })), + }, + resourceLinks: { + publication: { + id: 'links-private', target: TARGET, collected_at: COLLECTED, + roster_publication_id: rosterPublication.id, roster_content_sha256: rosterPublication.content_sha256, + resource_type: 'paycom.timecard.summary', period_key: `2026-08-23_${TARGET}`, + }, + rows: employees.map(row => ({ employeeCode: row.employeeCode, canonicalUrl: canonicalUrl(row.employeeCode) })), + }, + }; +} + +function portFixture() { + const calls = []; + const views = rawWorkforce(); + const port = { + snapshot: async () => ({ + target: TARGET, + collectedAt: { roster: COLLECTED, timecards: COLLECTED, resourceLinks: COLLECTED }, + counts: { employees: 2, timecards: 2, resourceLinks: 2 }, + lifecycleCounts: { active: 1, inactive: 0, unknown: 1 }, + consistent: true, + publicationId: 'not-forwarded', + }), + employees: async query => { + calls.push(['employees', query]); + const items = [employeeView('A002', 'unknown')]; + return { target: TARGET, collectedAt: COLLECTED, items, total: 1, limit: query.limit, offset: query.offset, hasMore: false, private: true }; + }, + employee: async code => { + calls.push(['employee', code]); + return { + target: TARGET, collectedAt: COLLECTED, + employee: employeeView(code, 'active'), + timecard: timecardView(code, 'active'), + private: true, + }; + }, + timecards: async query => { + calls.push(['timecards', query]); + return { target: TARGET, collectedAt: COLLECTED, items: [timecardView('A001', 'active')], total: 1, limit: query.limit, offset: query.offset, hasMore: false }; + }, + punches: async query => { + calls.push(['punches', query]); + return { + target: TARGET, businessDate: query.date, businessTimezone: 'America/Los_Angeles', collectedAt: COLLECTED, + items: [{ employeeName: 'Employee A001', lifecycleStatus: 'active', date: query.date, kind: 'in_day', time: '10:01', timeBasis: 'actual', observedAt: COLLECTED }], + total: 1, limit: query.limit, offset: query.offset, hasMore: false, + }; + }, + day: async query => { + calls.push(['day', query]); + return { + target: TARGET, businessDate: query.date, businessTimezone: 'America/Los_Angeles', + periodStart: '2026-08-23', periodEnd: TARGET, available: true, collectedAt: COLLECTED, + summary: { + employees: 2, activeEmployees: 1, inDayPunches: 1, completeTimecards: 0, + needsReview: 1, noActivity: 0, missingOutDay: 1, incompleteLunch: 0, unclassifiedPunches: 0, + }, + items: [{ + employeeCode: 'A001', employeeName: 'Employee A001', lifecycleStatus: 'active', isDriver: true, + department: { code: 'D1', name: 'Driver' }, deliveryStation: { code: 'S1', name: 'Station' }, + businessDate: query.date, condition: 'incomplete', missingPunch: false, totalHours: '8', + punchCount: 1, unresolvedSlotCount: 0, + punches: { inDay: [{ time: '10:01', timeBasis: 'actual' }], outLunch: [], inLunch: [], outDay: [], unclassified: [] }, + observedAt: COLLECTED, + }], + total: 1, limit: query.limit, offset: query.offset, hasMore: false, + }; + }, + resourceLinks: async query => { + calls.push(['resourceLinks', query]); + return { target: TARGET, collectedAt: COLLECTED, items: [resourceLinkView('A001', 'active')], total: 1, limit: query.limit, offset: query.offset, hasMore: false }; + }, + }; + return { port, calls, views }; +} + +function employeeView(code, lifecycleStatus) { + return { + employeeCode: code, employeeName: `Employee ${code}`, lifecycleStatus, lastExplicitActive: true, + department: { code: 'D1', name: 'Driver' }, deliveryStation: { code: 'S1', name: 'Station' }, + positionTitle: 'Driver', payClass: 'PC', payType: 'Hourly', primarySupervisor: 'Supervisor', isDriver: true, + }; +} + +function timecardView(code, lifecycleStatus) { + return { + employeeCode: code, employeeName: `Employee ${code}`, lifecycleStatus, + periodStart: '2026-08-23', periodEnd: TARGET, periodTotalHours: '8', missingDays: 0, + observedAt: COLLECTED, canonicalUrl: canonicalUrl(code), + }; +} + +function resourceLinkView(code, lifecycleStatus) { + return { + employeeCode: code, employeeName: `Employee ${code}`, lifecycleStatus, + resourceType: 'paycom.timecard.summary', periodStart: '2026-08-23', periodEnd: TARGET, + canonicalUrl: canonicalUrl(code), + }; +} + +test('daily workforce ordering keeps no-activity employees at the bottom', () => { + const rows = [ + { condition: 'no_activity', employeeName: 'Alpha', employeeCode: 'A001' }, + { condition: 'complete', employeeName: 'Zulu', employeeCode: 'A004' }, + { condition: 'incomplete', employeeName: 'Bravo', employeeCode: 'A002' }, + { condition: 'no_activity', employeeName: 'Charlie', employeeCode: 'A003' }, + ]; + rows.sort(dailyRowOrder); + assert.deepEqual(rows.map(row => [row.condition, row.employeeName]), [ + ['incomplete', 'Bravo'], + ['complete', 'Zulu'], + ['no_activity', 'Alpha'], + ['no_activity', 'Charlie'], + ]); +}); + +test('WorkforceClient exposes closed snapshot, roster, employee, and timecard views', async () => { + const fixture = portFixture(); + const client = new WorkforceClient({ port: fixture.port }); + const snapshot = await client.snapshot(); + assert.equal(snapshot.status, 'ready'); + assert.deepEqual(snapshot.data.lifecycleCounts, { active: 1, inactive: 0, unknown: 1 }); + assert.equal(JSON.stringify(snapshot).includes('publicationId'), false); + + const employees = await client.employees({ lifecycleStatus: 'unknown', limit: 10, offset: 0 }); + assert.equal(employees.data.kind, 'employees'); + assert.equal(employees.data.items[0].lifecycleStatus, 'unknown'); + assert.deepEqual(fixture.calls[0], ['employees', { lifecycleStatus: 'unknown', limit: 10, offset: 0 }]); + assert.equal(JSON.stringify(employees).includes('private'), false); + + const detail = await client.employee('a001'); + assert.equal(detail.data.employee.employeeCode, 'A001'); + assert.equal(detail.data.timecard.periodTotalHours, '8'); + assert.deepEqual(fixture.calls[1], ['employee', 'A001']); + + const timecards = await client.timecards({ limit: 5 }); + assert.equal(timecards.data.kind, 'timecards'); + assert.equal(timecards.data.items[0].canonicalUrl.startsWith('https://'), true); + assert.deepEqual(fixture.calls[2], ['timecards', { lifecycleStatus: null, limit: 5, offset: 0 }]); + + const punches = await client.punches({ date: '2026-08-30', kind: 'in_day', fromTime: '10:01', limit: 5 }); + assert.equal(punches.data.kind, 'punches'); + assert.equal(punches.data.items[0].employeeName, 'Employee A001'); + assert.equal(punches.data.items[0].time, '10:01'); + assert.equal(Object.hasOwn(punches.data.items[0], 'employeeCode'), false); + assert.deepEqual(fixture.calls[3], ['punches', { + date: '2026-08-30', kind: 'in_day', fromTime: '10:01', throughTime: null, + lifecycleStatus: null, limit: 5, offset: 0, + }]); + + const day = await client.day({ date: '2026-08-30', search: 'Employee', attention: 'incomplete', limit: 10 }); + assert.equal(day.data.kind, 'workforce_day'); + assert.equal(day.data.available, true); + assert.equal(day.data.items[0].employeeCode, 'A001'); + assert.equal(day.data.items[0].punches.inDay[0].time, '10:01'); + assert.equal(day.data.summary.missingOutDay, 1); + assert.deepEqual(fixture.calls[4], ['day', { + date: '2026-08-30', search: 'Employee', attention: 'incomplete', lifecycleStatus: null, + limit: 10, offset: 0, + }]); + + const links = await client.resourceLinks({ limit: 5 }); + assert.equal(links.data.kind, 'resource_links'); + assert.equal(links.data.items[0].resourceType, 'paycom.timecard.summary'); + assert.deepEqual(fixture.calls[5], ['resourceLinks', { lifecycleStatus: null, limit: 5, offset: 0 }]); + + assert.equal((await client.employees({ extra: true })).status, 'invalid_input'); + assert.equal((await client.day({ date: 'bad' })).status, 'invalid_input'); + assert.equal((await client.employee('BAD')).status, 'invalid_input'); + const missing = new WorkforceClient({ port: { + snapshot: async () => null, employees: async () => null, employee: async () => null, + timecards: async () => null, punches: async () => null, day: async () => null, resourceLinks: async () => null, + } }); + assert.equal((await missing.snapshot()).status, 'not_initialized'); +}); + +test('WorkforceClient rejects malformed component data', async () => { + const fixture = portFixture(); + fixture.port.employees = async query => ({ + target: TARGET, collectedAt: COLLECTED, total: 1, limit: query.limit, offset: query.offset, + hasMore: false, items: [{ ...employeeView('A001', 'active'), employeeCode: 'BAD' }], + }); + const result = await new WorkforceClient({ port: fixture.port }).employees(); + assert.equal(result.status, 'invalid_component_response'); +}); + +test('LocalPaycomWorkforcePort uses one read-only store per operation and closes it', () => { + const options = []; + let closed = 0; + const port = new LocalPaycomWorkforcePort({ + database: __filename, + storeFactory: (_file, value) => { + options.push(value); + return { activeWorkforce: rawWorkforce, close: () => { closed += 1; } }; + }, + }); + const snapshot = port.snapshot(); + assert.deepEqual(snapshot.lifecycleCounts, { active: 1, inactive: 0, unknown: 1 }); + const unknown = port.employees({ lifecycleStatus: 'unknown', limit: 10, offset: 0 }); + assert.equal(unknown.items.length, 1); + assert.equal(unknown.items[0].employeeCode, 'A002'); + const detail = port.employee('A001'); + assert.equal(detail.timecard.employeeCode, 'A001'); + assert.equal(detail.timecard.periodTotalHours, '8'); + const cards = port.timecards({ lifecycleStatus: null, limit: 1, offset: 1 }); + assert.equal(cards.items[0].employeeCode, 'A002'); + const punches = port.punches({ + date: '2026-08-30', kind: 'in_day', fromTime: '10:01', throughTime: null, + lifecycleStatus: null, limit: 10, offset: 0, + }); + assert.equal(punches.items.length, 1); + assert.deepEqual(punches.items[0], { + employeeName: 'Employee A001', lifecycleStatus: 'active', date: '2026-08-30', + kind: 'in_day', time: '10:01', timeBasis: 'actual', observedAt: COLLECTED, + }); + const day = port.day({ + date: '2026-08-30', search: null, attention: null, lifecycleStatus: null, limit: 10, offset: 0, + }); + assert.equal(day.available, true); + assert.equal(day.items.length, 2); + assert.equal(day.items[0].punchCount, 1); + assert.equal(day.summary.needsReview, 1); + assert.equal(day.summary.missingOutDay, 1); + const unavailable = port.day({ + date: '2026-08-01', search: null, attention: null, lifecycleStatus: null, limit: 10, offset: 0, + }); + assert.equal(unavailable.available, false); + assert.equal(unavailable.total, 0); + const links = port.resourceLinks({ lifecycleStatus: null, limit: 1, offset: 0 }); + assert.equal(links.items[0].resourceType, 'paycom.timecard.summary'); + assert.deepEqual(options, Array.from({ length: 8 }, () => ({ readOnly: true }))); + assert.equal(closed, 8); +}); + +test('daily sorts span the entire roster before pagination and leave empty values last', async () => { + const { workforceDayQuery } = require('dispatch-protocol/contracts/src/workforce'); + const { compareDailyRows } = require('../../../plugins/paycom/backend/adapters/workforce'); + const raw = rawWorkforce(); + raw.roster.employees[0].employeeName = 'Zulu'; + raw.roster.employees[1].employeeName = 'Alpha'; + const client = new WorkforceClient({ port: new LocalPaycomWorkforcePort({ database: __filename, + storeFactory: () => ({ activeWorkforce: () => raw, close() {} }) }) }); + const asc = await client.day({ date: '2026-08-30', sort: 'employeeName', direction: 'asc', limit: 1 }); + assert.equal(asc.ok, true); + assert.equal(asc.data.items[0].employeeName, 'Alpha'); + assert.equal(asc.data.hasMore, true); + const desc = await client.day({ date: '2026-08-30', sort: 'employeeName', direction: 'desc', limit: 1 }); + assert.equal(desc.data.items[0].employeeName, 'Zulu'); + const base = asc.data.items[0]; + const rows = [ + { ...base, employeeName: 'Empty', totalHours: null, punches: { ...base.punches, inDay: [] } }, + { ...base, employeeName: 'Ten', totalHours: '10', punches: { ...base.punches, inDay: [{ time: '10:00' }] } }, + { ...base, employeeName: 'Two', totalHours: '2', punches: { ...base.punches, inDay: [{ time: '02:00' }] } }, + ]; + for (const key of ['totalHours', 'inDay']) { + assert.deepEqual([...rows].sort((a,b) => compareDailyRows(a,b,key,'asc')).map(r=>r.employeeName), ['Two','Ten','Empty']); + assert.deepEqual([...rows].sort((a,b) => compareDailyRows(a,b,key,'desc')).map(r=>r.employeeName), ['Ten','Two','Empty']); + } + for (const query of [{ date: '2026-02-30' }, { date: '2026-99-01' }, { date: '2026-08-30', sort: 'sourceUrl' }, { date: '2026-08-30', direction: 'random' }]) { + assert.throws(() => workforceDayQuery(query), { code: 'invalid_input' }); + } +}); + +test('workforce dates use the current configured timezone and reject missing configuration', async () => { + let timezone = 'UTC'; + const port = new LocalPaycomWorkforcePort({ database: __filename, timezone: () => timezone, + storeFactory: () => ({ activeWorkforce: rawWorkforce, close() {} }) }); + const client = new WorkforceClient({ port }); + const query = { date: '2026-08-30', limit: 10, offset: 0 }; + assert.equal((await client.day(query)).data.businessTimezone, 'UTC'); + assert.equal(port.employee('A001').businessTimezone, 'UTC'); + assert.equal((await client.punches(query)).data.businessTimezone, 'UTC'); + timezone = 'America/New_York'; + assert.equal((await client.day(query)).data.businessTimezone, timezone); + assert.equal(port.employee('A001').businessTimezone, timezone); + timezone = undefined; + assert.equal((await client.day(query)).status, 'workforce_inconsistent'); + timezone = 'invalid/timezone'; + assert.equal((await client.day(query)).status, 'workforce_inconsistent'); +}); + +test('daily history selects the saved period and never substitutes latest-period employee rows', async () => { + const { workforceDayQuery } = require('dispatch-protocol/contracts/src/workforce'); + const latest = rawWorkforce(); + const historical = JSON.parse(JSON.stringify(latest).replaceAll('2026-09-05','2026-08-22').replaceAll('2026-08-23','2026-08-09').replaceAll('2026-08-30','2026-08-16')); + historical.roster.employees[0].employeeName = 'Historical employee'; + const targets = []; let closes = 0; + const port = new LocalPaycomWorkforcePort({ database: __filename, storeFactory: () => ({ + active: (_kind, target) => target === null || target === '2026-08-22', + activeWorkforce: target => { targets.push(target); return target === '2026-08-22' ? historical : latest; }, + close() { closes++; }, + }) }); + const day = port.day(workforceDayQuery({ date: '2026-08-16' })); + assert.equal(day.available, true); + assert.equal(day.items.find(row => row.employeeCode === 'A001').employeeName, 'Historical employee'); + assert.deepEqual(targets, [null,'2026-08-22']); + assert.equal(closes, 1); + const missing = port.day(workforceDayQuery({ date: '2026-07-01' })); + assert.equal(missing.available, false); + assert.deepEqual(missing.items, []); +}); + +test('employee timecard days cross the SDK only through the protected daily projection', async () => { + const raw = rawWorkforce(); + raw.timecards.rows[0].record.days[0].comments = ['private comment']; + const client = new WorkforceClient({ port: new LocalPaycomWorkforcePort({ database: __filename, + storeFactory: () => ({ activeWorkforce: () => raw, close() {} }) }) }); + const result = await client.employee('A001'); + assert.equal(result.ok, true); + assert.equal(result.data.days.length, 1); + assert.equal(result.data.days[0].punches.inDay[0].time, '10:01'); + assert.equal(result.data.days[0].employeeCode, 'A001'); + assert.doesNotMatch(JSON.stringify(result), /private comment|sourceSha256|record_json|publicationId|clockName/); +}); + +test('an inactive employee without an active-only timecard does not break the daily page', async () => { + const raw = rawWorkforce(); raw.roster.employees.push(employee('A003', 'inactive')); + const client = new WorkforceClient({ port: new LocalPaycomWorkforcePort({ database: __filename, + storeFactory: () => ({ activeWorkforce: () => raw, close() {} }) }) }); + const day = await client.day({ date: '2026-08-30', sort: 'employeeName' }); + assert.equal(day.ok, true); + assert.equal(day.data.items.length, 2); + const detail = await client.employee('A003'); + assert.equal(detail.ok, true); + assert.equal(detail.data.employee.lifecycleStatus, 'inactive'); + assert.equal(detail.data.timecard, null); + assert.deepEqual(detail.data.days, []); +}); + +test('an initialized empty database remains a first-collection state until publication', async t => { + const fs = require('node:fs'); + const os = require('node:os'); + const path = require('node:path'); + const { PaycomStore } = require('../../../plugins/paycom/backend/src/store'); + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'workforce-empty-')); + const database = path.join(root, 'paycom.sqlite3'); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + new PaycomStore(database).close(); + const client = new WorkforceClient({ port: new LocalPaycomWorkforcePort({ database }) }); + assert.equal((await client.employees()).status, 'not_initialized'); + assert.equal((await client.day({ date: '2026-09-08' })).status, 'not_initialized'); +}); diff --git a/dsp/runtime/supervisor/OVERVIEW.md b/dsp/runtime/supervisor/OVERVIEW.md new file mode 100644 index 0000000..ae7ae1f --- /dev/null +++ b/dsp/runtime/supervisor/OVERVIEW.md @@ -0,0 +1,7 @@ +# DSP process supervision + +This directory supervises the fixed built-in processes for one DSP: Auth Broker, Collection Manager, Runtime Gateway and Runtime Agent. A child failure stops the service, and systemd restarts it within bounded limits. + +The current backend is `native_service_v1`: separate Linux accounts and private data, shared immutable code, private Chrome debugging pipes, and per-service resource limits. `native-host-fixture.js` exercises two real accounts and file recovery. OCI definitions and examples remain for compatibility with previously installed releases. + +See [native DSP architecture and operations](../../core/installations/NATIVE-DSPS.md). diff --git a/dsp/runtime/supervisor/bin/dispatch-runtime-health b/dsp/runtime/supervisor/bin/dispatch-runtime-health new file mode 100755 index 0000000..a50d535 --- /dev/null +++ b/dsp/runtime/supervisor/bin/dispatch-runtime-health @@ -0,0 +1,4 @@ +#!/usr/bin/env node +'use strict'; +process.umask(0o077); +process.exitCode = require('../src/health').main(); diff --git a/dsp/runtime/supervisor/bin/dispatch-runtime-supervisor b/dsp/runtime/supervisor/bin/dispatch-runtime-supervisor new file mode 100755 index 0000000..aea769d --- /dev/null +++ b/dsp/runtime/supervisor/bin/dispatch-runtime-supervisor @@ -0,0 +1,4 @@ +#!/usr/bin/env node +'use strict'; +process.umask(0o077); +require('../src/supervisor').main().then(code => { process.exitCode = Number.isInteger(code) ? code : 1; }); diff --git a/dsp/runtime/supervisor/examples/build-native-fixture.js b/dsp/runtime/supervisor/examples/build-native-fixture.js new file mode 100644 index 0000000..abb734b --- /dev/null +++ b/dsp/runtime/supervisor/examples/build-native-fixture.js @@ -0,0 +1,19 @@ +'use strict'; +const fs = require('node:fs'), path = require('node:path'); +const { execFileSync } = require('node:child_process'); +const { buildNativeRuntime } = require('dispatch-core/core/installations/src/native-runtime-build.js'); +const { hashFileSync } = require('dispatch-core/core/installations/src/release-delivery-files.js'); +const { removeStage } = require('dispatch-core/core/installations/src/release-delivery-install.js'); +const root = path.resolve(__dirname, "../../.."), output = process.argv[2]; +if (!output || !path.isAbsolute(output) || fs.existsSync(output)) throw Error('fixture_output_invalid'); +fs.mkdirSync(output, { mode: 0o700 }); +const sourceCommit = execFileSync('/usr/bin/git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim(); +const result = buildNativeRuntime({ projectRoot: root, archive: path.join(output, 'runtime.tar.gz'), sourceCommit }); +const bridge = path.join(output, 'bridge'); +try { + require('dispatch-core/core/installations/src/create-bridge-artifact.js').main([bridge]); + fs.writeFileSync(path.join(output, 'descriptor.json'), JSON.stringify({ version: 1, backend: 'native_service_v1', + releaseId: 'dispatch_native_fixture', channel: 'fixture', sourceCommit, platform: 'linux/amd64', + runtimeAgentProtocol: 1, runtimeGatewayProtocol: 1, ...result, + bridgeManifestSha256: hashFileSync(path.join(bridge, 'manifest.json')) }) + '\n', { mode: 0o600 }); +} finally { removeStage(bridge); } diff --git a/dsp/runtime/supervisor/examples/native-host-fixture.js b/dsp/runtime/supervisor/examples/native-host-fixture.js new file mode 100644 index 0000000..bbb2300 --- /dev/null +++ b/dsp/runtime/supervisor/examples/native-host-fixture.js @@ -0,0 +1,176 @@ +'use strict'; +// Disposable two-account integration fixture. Never selects a dashboard DSP. +const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto'); +const { execFile } = require('node:child_process'); +const execute = require('node:util').promisify(execFile); +const { createOciFixtureDeploymentPlan, renderOciSystemUnit, renderOciBridgeSystemUnit, hostAccountName } = require('dispatch-core/core/installations/src/oci-deployment.js'); +const { MANAGED_INSTALLATION_DIRECTORY_FIELDS } = require('dispatch-protocol/paths/runtime-paths'); +const { CoreRuntimeAgentHub, createRuntimeAgentDispatchClient } = require('dispatch-core/core/agents/src/index.js'); +const ROOT = path.resolve(__dirname, "../../.."); +const RELEASE = '/opt/dispatch-runtime/releases/dispatch_native_fixture'; +async function run(command, args, options = {}) { + return execute(command, args, { encoding: 'utf8', timeout: 120000, maxBuffer: 16384, ...options }); +} +const sudo = (command, args, options) => run('/usr/bin/sudo', ['-n', command, ...args], options); +async function main(packageDirectory) { + if (!path.isAbsolute(packageDirectory) || fs.realpathSync(packageDirectory) !== packageDirectory + || fs.existsSync(RELEASE) || process.geteuid() === 0) throw Error('fixture_precondition'); + const release = JSON.parse(fs.readFileSync(path.join(packageDirectory, 'descriptor.json'))); + if (release.releaseId !== 'dispatch_native_fixture' || release.channel !== 'fixture') throw Error('fixture_package_invalid'); + const temporary = fs.mkdtempSync('/tmp/dispatch-native-host-'); fs.chmodSync(temporary, 0o700); + const centralRoot = `/run/user/${process.geteuid()}/dispatch-native-fixture-${process.pid}`; + fs.mkdirSync(centralRoot, { mode: 0o700 }); + const centralSocket = path.join(centralRoot, 'runtime-agent-hub.sock'); + const plans = [], accounts = [], units = [], authorities = {}; + const profile = '/etc/apparmor.d/dispatch-native-chrome', profileExisted = fs.existsSync(profile); + let hub; + try { + const bridge = path.join(temporary, 'bridge-artifact'); + require('dispatch-core/core/installations/src/create-bridge-artifact.js').main([bridge]); + if (require('dispatch-core/core/installations/src/release-delivery-files.js').hashFileSync(path.join(bridge, 'manifest.json')) !== release.bridgeManifestSha256) throw Error('fixture_bridge_mismatch'); + await sudo('/usr/bin/install', ['-d', '-m', '0755', RELEASE]); + await sudo('/usr/bin/node', ['--no-warnings', '-e', `const x=require(${JSON.stringify(path.join(ROOT, 'core/installations/src/native-runtime-artifact'))});x.unpackNativeRuntime(process.argv[1],process.argv[2],JSON.parse(require('node:fs').readFileSync(process.argv[3])));`, + path.join(packageDirectory, 'runtime.tar.gz'), path.join(RELEASE, 'runtime-artifact'), path.join(packageDirectory, 'descriptor.json')]); + await sudo('/usr/bin/cp', ['-R', bridge, path.join(RELEASE, 'bridge-artifact')]); + const diagnostic = path.join(temporary, 'diagnostic.js'); + fs.writeFileSync(diagnostic, `const fs=require('node:fs');const s=require('/opt/dispatch/runtime/supervisor/src/supervisor');try { const c=s.configuration();s.assertMountBoundary(c);console.log('native_boundary_verified');const {CollectionStore}=require('/opt/dispatch/runtime/collection-manager/src/store');const paths=require('/opt/dispatch/runtime/collection-manager/src/paths').defaultPaths();if(fs.existsSync(paths.database)){try{const store=new CollectionStore(paths);console.log('collection_storage_verified');store.close();}catch(e){console.error('fixture_collection_storage:'+e.message);throw e;}} } catch(e) { console.error(e.code);console.error(fs.readFileSync('/proc/self/mountinfo','utf8').split('\\n').filter(l=>/ \\/( |tmp |opt\\/dispatch |run\\/dispatch-agent |var\\/lib\\/dispatch\\/runtime_)/.test(l)).join('\\n'));process.exitCode=1; }`); + await sudo('/usr/bin/install', ['-m', '0444', diagnostic, path.join(RELEASE, 'diagnostic.js')]); + await sudo('/usr/bin/chown', ['-R', 'root:root', RELEASE]); + await sudo('/usr/bin/chmod', ['0555', RELEASE]); + await sudo('/usr/bin/node', ['--no-warnings', '-e', `require(${JSON.stringify(path.join(ROOT, 'core/installations/src/release-delivery-install'))}).installBrowserSandboxProfile()`]); + for (let i = 0; i < 2; i++) { + const key = `runtime_native_fixture_${i ? 'beta' : 'alpha'}`, uid = 20501 + i, name = hostAccountName(key); + for (const [database, identity] of [['passwd', name], ['passwd', String(uid)], ['group', name], ['group', String(uid)]]) { + try { await run('/usr/bin/getent', [database, identity]); throw Error('fixture_account_exists'); } + catch (error) { if (error.code !== 2) throw error; } + } + const manifest = { manifestVersion: 1, revision: 1, organization: { id: `org_native_fixture_${i}`, stationCode: 'DXX1', timezone: 'America/Chicago' }, + runtime: { key, templateId: 'isolated_dsp_v1', releaseId: release.releaseId } }; + const authority = { revision: 1, organization: manifest.organization, runtime: manifest.runtime }; + const plan = createOciFixtureDeploymentPlan(manifest, authority, release, + { name, uid, gid: uid, subuidStart: 300000 + i * 65536, subgidStart: 300000 + i * 65536, subidCount: 65536 }, + { version: 1, backend: release.backend, channel: 'fixture', organizationId: manifest.organization.id, + runtimeKey: key, manifestRevision: 1, releaseId: release.releaseId }); + if (fs.existsSync(plan.host.tenantRoot) || fs.existsSync(plan.host.bridgeRoot) + || fs.existsSync(plan.host.unitPath) || fs.existsSync(plan.host.bridgeUnitPath)) throw Error('fixture_path_exists'); + await sudo('/usr/sbin/groupadd', ['--system', '--gid', String(uid), name]); + accounts.push(name); plans.push(plan); + await sudo('/usr/sbin/useradd', ['--system', '--uid', String(uid), '--gid', String(uid), '--home-dir', plan.host.accountHome, + '--no-create-home', '--shell', '/usr/sbin/nologin', name]); + for (const directory of [plan.host.tenantRoot, plan.host.accountHome, path.dirname(plan.host.installationRoot), plan.host.installationRoot, + ...Object.values(MANAGED_INSTALLATION_DIRECTORY_FIELDS).map(p => path.join(plan.host.installationRoot, p))]) { + await sudo('/usr/bin/install', ['-d', '-o', name, '-g', name, '-m', '0700', directory]); + } + await sudo('/usr/bin/install', ['-d', '-o', 'root', '-g', 'root', '-m', '0711', plan.host.bridgeRoot]); + const token = crypto.randomBytes(32).toString('base64url'), tokenFile = path.join(temporary, `token-${i}`); + authorities[key] = crypto.createHash('sha256').update(token).digest('hex'); + fs.writeFileSync(tokenFile, token + '\n', { mode: 0o600 }); + await sudo('/usr/bin/install', ['-o', name, '-g', name, '-m', '0600', tokenFile, path.join(plan.host.installationRoot, 'secrets/runtime-agent/registration-token')]); + } + hub = new CoreRuntimeAgentHub({ socketPath: centralSocket, authorities, collectionCapacity: { workers: 1, recoveryMs: 0 } }); + await hub.start(); + for (const plan of plans) { + const bridge = renderOciBridgeSystemUnit(plan, { bridgeExecutable: path.join(RELEASE, 'bridge-artifact/core/agent-bridge/src/service-cli.js'), + centralSocket, centralUid: process.geteuid(), controllerUid: 0 }); + const runtimeUnit = renderOciSystemUnit(plan).replace('ExecStart=', `ExecStartPre=/opt/dispatch/dependencies/node/bin/node ${RELEASE}/diagnostic.js\nExecStart=`); + for (const [name, content] of [[plan.identity.bridgeUnitName, bridge], [plan.identity.unitName, runtimeUnit]]) { + const file = path.join(temporary, name); fs.writeFileSync(file, content); + await sudo('/usr/bin/install', ['-m', '0644', file, `/run/systemd/system/${name}`]); units.push(name); + } + } + await sudo('/usr/bin/systemctl', ['daemon-reload']); + await sudo('/usr/bin/systemctl', ['start', ...units]); + for (const plan of plans) { + const client = createRuntimeAgentDispatchClient({ runtimeKey: plan.runtimeKey, hub }); + let ready = false; + for (let attempt = 0; attempt < 30; attempt++) { + try { const response = await client.health(); if (response.ok) { ready = true; break; } } catch {} + await new Promise(resolve => setTimeout(resolve, 500)); + } + if (!ready) throw Error(`fixture_runtime_unhealthy:${plan.identity.suffix}`); + } + const first = plans[0], second = plans[1]; + const pid = (await sudo('/usr/bin/systemctl', ['show', first.identity.unitName, '--property=MainPID', '--value'])).stdout.trim(); + if (!/^[1-9][0-9]*$/.test(pid)) throw Error('fixture_pid_invalid'); + const probe = `const fs=require('node:fs');const {ChromeBrowserRuntime}=require('/opt/dispatch/runtime/auth-broker/src/browser-runtime');const {createTarget,CdpConnection}=require('/opt/dispatch/runtime/auth-broker/src/cdp');(async()=>{const browser=await new ChromeBrowserRuntime({stateRoot:process.env.DISPATCH_AUTH_STATE_ROOT,socketRoot:process.env.DISPATCH_RUNTIME_ROOT,executable:'/opt/dispatch/dependencies/browser/chrome',transport:'pipe'}).launch();try{const target=await createTarget(browser.endpoint,'about:blank');const connection=await CdpConnection.connect(target.webSocketDebuggerUrl);let value;try{value=await connection.evaluate('6*7');}finally{connection.close();}if(value!==42)throw Error('browser_evaluation_failed');if(fs.existsSync(${JSON.stringify(second.host.installationRoot)}))throw Error('cross_dsp_access');console.log(JSON.stringify({privateBrowser:true,result:value,separateData:true}));}finally{await browser.close();}})().catch(e=>{console.error(e.message);process.exitCode=1});`; + const output = await sudo('/usr/bin/nsenter', ['--target', pid, '--mount', `--setuid=${first.account.uid}`, `--setgid=${first.account.gid}`, '--', + '/usr/bin/env', '-i', 'PATH=/opt/dispatch/dependencies/node/bin:/usr/bin:/bin', 'HOME=/tmp', + ...Object.entries(first.guest.environment).map(([k, v]) => `${k}=${v}`), '/opt/dispatch/dependencies/node/bin/node', '--no-warnings', '-e', probe]); + process.stdout.write(output.stdout); + const capacity = async (plan, operation) => { + const processId = (await sudo('/usr/bin/systemctl', ['show', plan.identity.unitName, '--property=MainPID', '--value'])).stdout.trim(); + const script = `require('/opt/dispatch/runtime/collection-manager/src/capacity-runner').queryCapacity(process.env.DISPATCH_RUNTIME_AGENT_STATUS_SOCKET,{operation:${JSON.stringify(operation)},jobId:'a'.repeat(32),workers:6}).then(value=>console.log(JSON.stringify(value))).catch(()=>{process.exitCode=1});`; + const result = await sudo('/usr/bin/nsenter', ['--target', processId, '--mount', `--setuid=${plan.account.uid}`, `--setgid=${plan.account.gid}`, '--', + '/usr/bin/env', '-i', 'PATH=/opt/dispatch/dependencies/node/bin:/usr/bin:/bin', 'HOME=/tmp', + ...Object.entries(plan.guest.environment).map(([key, value]) => `${key}=${value}`), '/opt/dispatch/dependencies/node/bin/node', '--no-warnings', '-e', script]); + return JSON.parse(result.stdout); + }; + if ((await capacity(first, 'acquire')).workers !== 1 || (await capacity(second, 'acquire')).status !== 'waiting') throw Error('fixture_capacity_isolation_failed'); + await capacity(first, 'release'); + if ((await capacity(second, 'acquire')).workers !== 1) throw Error('fixture_capacity_handoff_failed'); + await capacity(second, 'release'); + process.stdout.write(JSON.stringify({ sharedCapacity: true, privateAgentBridges: true }) + '\n'); + await sudo('/usr/bin/systemctl', ['stop', first.identity.unitName]); + const other = await createRuntimeAgentDispatchClient({ runtimeKey: second.runtimeKey, hub }).health(); + if (!other.ok) throw Error('fixture_other_dsp_interrupted'); + await sudo('/usr/bin/systemctl', ['stop', ...units]); + const recovery = path.join(temporary, 'recovery'), capsuleModule = path.join(ROOT, 'core/installations/src/recovery-capsule'); + const roots = [{ source: RELEASE, target: RELEASE }, ...plans.map(plan => ({ source: plan.host.tenantRoot, target: plan.host.tenantRoot, + excludeContents: [`runtime/${plan.runtimeKey}/run`] })), + ...units.map(name => ({ source: `/run/systemd/system/${name}`, target: `/run/systemd/system/${name}` }))]; + const captured = await sudo('/usr/bin/node', ['--no-warnings', '-e', + `const c=require(${JSON.stringify(capsuleModule)});const roots=JSON.parse(process.argv[2]);const proof=c.capture(process.argv[1],roots,{kind:'native-fixture'},new Set([0,20501,20502]));c.verify(process.argv[1],proof.sha256,new Set(roots.map(r=>r.target)));console.log(JSON.stringify(proof));`, recovery, JSON.stringify(roots)]); + const proof = JSON.parse(captured.stdout); + for (const name of accounts) { + await sudo('/usr/sbin/userdel', [name]); + await sudo('/usr/sbin/groupdel', [name]).catch(error => { if (error.code !== 6) throw error; }); + } + for (const selected of roots) await sudo('/usr/bin/rm', ['-rf', '--', selected.target]); + for (const plan of plans) { + await sudo('/usr/sbin/groupadd', ['--system', '--gid', String(plan.account.gid), plan.account.name]); + await sudo('/usr/sbin/useradd', ['--system', '--uid', String(plan.account.uid), '--gid', String(plan.account.gid), + '--home-dir', plan.host.accountHome, '--no-create-home', '--shell', '/usr/sbin/nologin', plan.account.name]); + } + await sudo('/usr/bin/node', ['--no-warnings', '-e', + `require(${JSON.stringify(capsuleModule)}).installFresh(process.argv[1],process.argv[2],new Set(JSON.parse(process.argv[3])));`, + recovery, proof.sha256, JSON.stringify(roots.map(r => r.target))]); + await sudo('/usr/bin/systemctl', ['daemon-reload']); + await sudo('/usr/bin/systemctl', ['start', ...units]); + for (const plan of plans) { + let healthy = false; + for (let attempt = 0; attempt < 30; attempt++) { + try { if ((await createRuntimeAgentDispatchClient({ runtimeKey: plan.runtimeKey, hub }).health()).ok) { healthy = true; break; } } catch {} + await new Promise(resolve => setTimeout(resolve, 500)); + } + if (!healthy) throw Error('fixture_restored_runtime_unhealthy'); + } + await sudo('/usr/bin/rm', ['-rf', '--', recovery]); + process.stdout.write(JSON.stringify({ status: 'native_host_verified', dsps: plans.length, independentStop: true, restoredCodeDataAccountsAndServices: true }) + '\n'); + } catch (error) { + for (const name of units) { + const log = await sudo('/usr/bin/journalctl', ['-u', name, '-n', '8', '--no-pager']).catch(() => null); + if (log) process.stderr.write(log.stdout); + } + throw error; + } finally { + await sudo('/usr/bin/rm', ['-rf', '--', path.join(temporary, 'recovery')]).catch(() => {}); + for (const name of [...units].reverse()) await sudo('/usr/bin/systemctl', ['stop', name]).catch(() => {}); + await hub?.close(); + for (const name of units) await sudo('/usr/bin/rm', ['-f', `/run/systemd/system/${name}`]); + await sudo('/usr/bin/systemctl', ['daemon-reload']); + for (const name of accounts) { + await sudo('/usr/sbin/userdel', [name]).catch(() => {}); + await sudo('/usr/sbin/groupdel', [name]).catch(() => {}); + } + for (const plan of plans) await sudo('/usr/bin/rm', ['-rf', '--one-file-system', plan.host.tenantRoot, plan.host.bridgeRoot]); + await sudo('/usr/bin/rm', ['-rf', '--one-file-system', RELEASE]); + if (!profileExisted && fs.existsSync(profile)) { + await sudo('/usr/sbin/apparmor_parser', ['-R', profile]).catch(() => {}); + await sudo('/usr/bin/rm', ['-f', profile]); + } + require('dispatch-core/core/installations/src/release-delivery-install.js').removeStage(temporary); + fs.rmSync(centralRoot, { recursive: true, force: true }); + } +} +if (require.main === module) main(process.argv[2]).catch(error => { console.error(error.message); process.exitCode = 1; }); +module.exports = { main }; diff --git a/dsp/runtime/supervisor/examples/rootless-host-fixture.js b/dsp/runtime/supervisor/examples/rootless-host-fixture.js new file mode 100644 index 0000000..cbcf9ed --- /dev/null +++ b/dsp/runtime/supervisor/examples/rootless-host-fixture.js @@ -0,0 +1,796 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { + INSTALLATION_MANIFEST_VERSION, +} = require('dispatch-protocol/contracts/src'); +const { + MANAGED_INSTALLATION_DIRECTORY_FIELDS, +} = require('dispatch-protocol/paths/runtime-paths'); +const { + HOST_BRIDGE_ROOT, + HOST_TENANT_ROOT, + opaqueRuntimeSuffix, + hostAccountName, +} = require('dispatch-core/core/runtime-host-identity.js'); +const { + OCI_BACKEND, + SUBID_COUNT, + createOciFixtureDeploymentPlan, + renderOciSystemUnit, +} = require('dispatch-core/core/installations/src/oci-deployment.js'); +const { + CoreRuntimeAgentHub, + createRuntimeAgentDispatchClient, +} = require('dispatch-core/core/agents/src/index.js'); + +const IMAGE = 'localhost/dispatch-runtime:dev'; +const RUNTIMES = Object.freeze(['runtime_fixture_oci_alpha', 'runtime_fixture_oci_beta']); +const MAX_OUTPUT = 128 * 1024; +const SYSTEM_UNIT_RUNTIME_ROOT = '/run/systemd/system'; +const HOST_DISPATCH_ROOT = path.dirname(HOST_TENANT_ROOT); +const HOST_FIXTURE_LOCK = '/run/dispatch-rootless-host-fixture.lock'; +const REPOSITORY_ROOT = path.resolve(__dirname, "../../.."); +const { BRIDGE_ARTIFACT_FILES } = require('dispatch-core/core/installations/src/create-bridge-artifact.js'); +const REFERENCE_UNITS = Object.freeze(['dispatch-auth-broker.service', 'dispatch-collection-manager.service']); +let currentPhase = 'preflight'; + +function fail(code = 'rootless_host_fixture_failed', diagnostic = '') { + throw Object.assign(new Error(code), { code, diagnostic, phase: currentPhase }); +} + +function lexists(target) { + try { fs.lstatSync(target); return true; } catch (error) { + if (error?.code === 'ENOENT') return false; + throw error; + } +} + +function run(executable, args, { input, timeout = 120_000, allowFailure = false, environment = process.env } = {}) { + const result = spawnSync(executable, args, { + input, encoding: 'utf8', shell: false, timeout, maxBuffer: MAX_OUTPUT, env: environment, + }); + if (!allowFailure && (result.error || result.signal || result.status !== 0)) { + fail('rootless_host_fixture_failed', JSON.stringify({ + executable: path.basename(executable), + status: result.status, + signal: result.signal, + error: String(result.stderr || '').slice(-1024), + })); + } + return result; +} + +function sudo(args, options = {}) { + return run('/usr/bin/sudo', ['-n', ...args], options); +} + +function accountEnvironment(account) { + return Object.freeze({ + HOME: account.home, + XDG_DATA_HOME: account.engineDataRoot, + XDG_CONFIG_HOME: account.engineConfigRoot, + XDG_RUNTIME_DIR: `/run/user/${account.uid}`, + PATH: '/usr/bin:/bin', + LANG: 'C.UTF-8', + }); +} + +function accountCommand(account, executable, args, options = {}) { + const assignments = Object.entries(accountEnvironment(account)).map(([key, value]) => `${key}=${value}`); + return sudo(['-u', account.name, '/usr/bin/env', '-i', ...assignments, executable, ...args], options); +} + +function accountPodman(account, args, options = {}) { + return accountCommand(account, '/usr/bin/podman', args, options); +} + +function accountExists(name) { + return run('/usr/bin/getent', ['passwd', name], { allowFailure: true }).status === 0; +} + +function systemUnitExists(name) { + return run('/usr/bin/systemctl', ['show', name, '--property=LoadState', '--value'], { allowFailure: true }).stdout.trim() !== 'not-found'; +} + +function verifyUnitFragment(name, expectedPath) { + const output = run('/usr/bin/systemctl', [ + 'show', name, '--property=LoadState', '--property=FragmentPath', + ]).stdout; + if (!output.includes('LoadState=loaded\n') || !output.includes(`FragmentPath=${expectedPath}\n`)) fail(); +} + +function parseSubids(file) { + const ranges = []; + for (const line of fs.readFileSync(file, 'utf8').split('\n')) { + if (!line) continue; + const parts = line.split(':'); + if (parts.length !== 3) fail(); + const start = Number(parts[1]); + const count = Number(parts[2]); + if (!Number.isSafeInteger(start) || !Number.isSafeInteger(count) || count < 1) fail(); + ranges.push({ start, end: start + count - 1 }); + } + return ranges; +} + +function allocateSubids(count) { + const ranges = [...parseSubids('/etc/subuid'), ...parseSubids('/etc/subgid')]; + const maximum = ranges.reduce((value, range) => Math.max(value, range.end + 1), 1_000_000); + const aligned = Math.max(1_000_000, Math.ceil(maximum / SUBID_COUNT) * SUBID_COUNT); + return Array.from({ length: count }, (_, index) => aligned + index * SUBID_COUNT); +} + +function rangePresent(file, name, start) { + return fs.readFileSync(file, 'utf8').split('\n').includes(`${name}:${start}:${SUBID_COUNT}`); +} + +function referenceSnapshot() { + return Object.fromEntries(REFERENCE_UNITS.map(name => { + const active = run('/usr/bin/systemctl', ['--user', 'is-active', name], { allowFailure: true }).stdout.trim(); + const pid = run('/usr/bin/systemctl', ['--user', 'show', name, '--property=MainPID', '--value'], { allowFailure: true }).stdout.trim(); + return [name, { active, pid }]; + })); +} + +function imageIdentity() { + const value = JSON.parse(run('/usr/bin/podman', ['image', 'inspect', IMAGE]).stdout)[0]; + if (!/^sha256:[a-f0-9]{64}$/.test(value?.Digest || '') || !/^[a-f0-9]{64}$/.test(value?.Id || '')) fail(); + return Object.freeze({ digest: value.Digest, id: value.Id }); +} + +function sourceCommit() { + const value = run('/usr/bin/git', ['rev-parse', 'HEAD']).stdout.trim(); + if (!/^[a-f0-9]{40}$/.test(value)) fail(); + return value; +} + +function ensureHostRoot(target, mode) { + if (lexists(target)) { + const stat = fs.lstatSync(target); + if (!stat.isDirectory() || stat.uid !== 0 || stat.gid !== 0 || (stat.mode & 0o7777) !== mode + || fs.realpathSync(target) !== target) fail('unsafe_fixture_host_root'); + return false; + } + sudo(['/usr/bin/install', '-d', '-o', 'root', '-g', 'root', '-m', mode.toString(8).padStart(4, '0'), target]); + const stat = fs.lstatSync(target); + if (!stat.isDirectory() || stat.uid !== 0 || stat.gid !== 0 || (stat.mode & 0o7777) !== mode + || fs.realpathSync(target) !== target) fail('unsafe_fixture_host_root'); + return true; +} + +function createAccount(runtimeKey, subidStart) { + const suffix = opaqueRuntimeSuffix(runtimeKey); + const name = hostAccountName(runtimeKey); + const tenantRoot = path.join(HOST_TENANT_ROOT, suffix); + const home = path.join(tenantRoot, 'home'); + if (accountExists(name) || lexists(tenantRoot) || lexists(path.join(HOST_BRIDGE_ROOT, suffix))) fail(); + sudo(['/usr/bin/install', '-d', '-o', 'root', '-g', 'root', '-m', '0755', tenantRoot]); + sudo(['/usr/sbin/useradd', '--system', '--user-group', '--home-dir', home, '--create-home', '--shell', '/usr/sbin/nologin', name]); + const passwd = run('/usr/bin/getent', ['passwd', name]).stdout.trim().split(':'); + const uid = Number(passwd[2]); + const gid = Number(passwd[3]); + if (!Number.isSafeInteger(uid) || !Number.isSafeInteger(gid) || passwd[5] !== home || passwd[6] !== '/usr/sbin/nologin') fail(); + const passwordStatus = sudo(['/usr/bin/passwd', '--status', name]).stdout.trim().split(/\s+/); + if (passwordStatus[0] !== name || passwordStatus[1] !== 'L') fail('fixture_account_not_locked'); + sudo(['/usr/sbin/usermod', '--add-subuids', `${subidStart}-${subidStart + SUBID_COUNT - 1}`, '--add-subgids', `${subidStart}-${subidStart + SUBID_COUNT - 1}`, name]); + sudo(['/usr/bin/chown', `${name}:${name}`, tenantRoot]); + sudo(['/usr/bin/chmod', '0700', tenantRoot]); + const engineDataRoot = path.join(tenantRoot, 'engine-data'); + const engineConfigRoot = path.join(tenantRoot, 'engine-config'); + const installationRoot = path.join(tenantRoot, 'runtime', runtimeKey); + const bridgeRoot = path.join(HOST_BRIDGE_ROOT, suffix); + const ownedDirectories = [home, engineDataRoot, engineConfigRoot, path.dirname(installationRoot), installationRoot]; + const layoutDirectories = Object.values(MANAGED_INSTALLATION_DIRECTORY_FIELDS) + .map(relative => path.join(installationRoot, relative)); + for (const directory of [...ownedDirectories, ...layoutDirectories] + .sort((left, right) => left.split(path.sep).length - right.split(path.sep).length || left.localeCompare(right))) { + sudo(['/usr/bin/install', '-d', '-o', name, '-g', name, '-m', '0700', directory]); + } + sudo(['/usr/bin/install', '-d', '-o', 'root', '-g', 'root', '-m', '0711', bridgeRoot]); + sudo(['/usr/bin/loginctl', 'enable-linger', name]); + sudo(['/usr/bin/systemctl', 'start', `user@${uid}.service`]); + const groups = run('/usr/bin/id', ['-Gn', name]).stdout.trim().split(/\s+/); + if (groups.length !== 1 || groups[0] !== name || !rangePresent('/etc/subuid', name, subidStart) + || !rangePresent('/etc/subgid', name, subidStart)) fail(); + const runtimeDirectory = fs.lstatSync(`/run/user/${uid}`); + if (!runtimeDirectory.isDirectory() || runtimeDirectory.uid !== uid || (runtimeDirectory.mode & 0o7777) !== 0o700) fail(); + return { + runtimeKey, suffix, name, uid, gid, subuidStart: subidStart, subgidStart: subidStart, + subidCount: SUBID_COUNT, tenantRoot, home, engineDataRoot, engineConfigRoot, + installationRoot, bridgeRoot, + }; +} + +function writeToken(account, token, fixtureRoot) { + const temporary = path.join(fixtureRoot, `${account.suffix}.token`); + fs.writeFileSync(temporary, `${token}\n`, { mode: 0o600, flag: 'wx' }); + const target = path.join(account.installationRoot, 'secrets', 'runtime-agent', 'registration-token'); + sudo(['/usr/bin/install', '-o', account.name, '-g', account.name, '-m', '0600', temporary, target]); + fs.unlinkSync(temporary); +} + +function installRuntimeUnit(plan, fixtureRoot) { + const temporary = path.join(fixtureRoot, plan.identity.unitName); + const content = renderOciSystemUnit(plan); + fs.writeFileSync(temporary, content, { mode: 0o600, flag: 'wx' }); + run('/usr/bin/systemd-analyze', ['verify', temporary]); + const installed = path.join(SYSTEM_UNIT_RUNTIME_ROOT, plan.identity.unitName); + try { + sudo(['/usr/bin/install', '-o', 'root', '-g', 'root', '-m', '0644', temporary, installed]); + const stat = fs.lstatSync(installed); + if (!stat.isFile() || stat.uid !== 0 || stat.gid !== 0 || stat.nlink !== 1 + || (stat.mode & 0o7777) !== 0o644 || fs.readFileSync(installed, 'utf8') !== content) fail(); + return installed; + } catch (error) { + sudo(['/usr/bin/rm', '--force', installed], { allowFailure: true }); + throw error; + } +} + +function stageBridgeArtifact(artifactRoot) { + for (const relative of BRIDGE_ARTIFACT_FILES) { + const source = path.join(REPOSITORY_ROOT, relative); + const sourceInfo = fs.lstatSync(source); + if (!sourceInfo.isFile() || sourceInfo.isSymbolicLink() || sourceInfo.nlink !== 1 + || sourceInfo.uid !== process.geteuid() || (sourceInfo.mode & 0o022) !== 0 + || fs.realpathSync(source) !== source) fail('unsafe_fixture_source'); + const destination = path.join(artifactRoot, relative); + const segments = path.dirname(relative).split(path.sep); + for (let index = 1; index <= segments.length; index += 1) { + const directory = path.join(artifactRoot, ...segments.slice(0, index)); + sudo(['/usr/bin/install', '-d', '-o', 'root', '-g', 'root', '-m', '0555', directory]); + const info = fs.lstatSync(directory); + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== 0 || info.gid !== 0 + || (info.mode & 0o7777) !== 0o555 || fs.realpathSync(directory) !== directory) fail('unsafe_fixture_bridge_artifact'); + } + sudo(['/usr/bin/install', '-o', 'root', '-g', 'root', '-m', '0444', source, destination]); + const destinationInfo = fs.lstatSync(destination); + if (!destinationInfo.isFile() || destinationInfo.isSymbolicLink() || destinationInfo.nlink !== 1 + || destinationInfo.uid !== 0 || destinationInfo.gid !== 0 || (destinationInfo.mode & 0o7777) !== 0o444 + || fs.realpathSync(destination) !== destination + || !fs.readFileSync(source).equals(fs.readFileSync(destination))) fail('unsafe_fixture_bridge_artifact'); + } + return path.join(artifactRoot, 'core/agent-bridge/src/service-cli.js'); +} + +function bridgeUnit(account, centralSocket, centralUid, fixtureRoot, bridgeArtifactRoot, bridgeService) { + const name = `dispatch-fixture-bridge-${account.suffix}.service`; + if (bridgeService !== path.join(bridgeArtifactRoot, 'core/agent-bridge/src/service-cli.js')) fail(); + const content = [ + '[Unit]', + `Description=Dispatch fixture Runtime Agent bridge (${account.suffix})`, + '', + '[Service]', + 'Type=simple', + 'User=root', + 'Group=root', + 'Environment=PATH=/usr/bin:/bin', + `Environment=DISPATCH_RUNTIME_BRIDGE_KEY=${account.runtimeKey}`, + `Environment=DISPATCH_RUNTIME_BRIDGE_UPSTREAM_SOCKET=${centralSocket}`, + `Environment=DISPATCH_RUNTIME_BRIDGE_TENANT_UID=${account.uid}`, + `Environment=DISPATCH_RUNTIME_BRIDGE_TENANT_GID=${account.gid}`, + `Environment=DISPATCH_RUNTIME_BRIDGE_CENTRAL_UID=${centralUid}`, + `ExecStart=/usr/bin/node --no-warnings ${bridgeService}`, + 'Restart=on-failure', + 'RestartSec=1', + 'KillMode=control-group', + 'TimeoutStopSec=15', + 'UMask=0077', + 'NoNewPrivileges=true', + 'PrivateTmp=true', + 'ProtectSystem=strict', + 'ProtectHome=read-only', + 'ProtectKernelTunables=true', + 'ProtectKernelModules=true', + 'ProtectKernelLogs=true', + 'ProtectControlGroups=true', + 'ProtectClock=true', + 'ProtectHostname=true', + 'PrivateDevices=true', + 'RestrictNamespaces=true', + 'RestrictSUIDSGID=true', + 'SystemCallArchitectures=native', + 'CapabilityBoundingSet=CAP_CHOWN CAP_DAC_OVERRIDE CAP_FOWNER', + `ReadOnlyPaths=${bridgeArtifactRoot}`, + `ReadWritePaths=${account.bridgeRoot}`, + 'RestrictAddressFamilies=AF_UNIX', + '', + ].join('\n'); + const temporary = path.join(fixtureRoot, name); + const unitContent = `${content}\n`; + fs.writeFileSync(temporary, unitContent, { mode: 0o600, flag: 'wx' }); + run('/usr/bin/systemd-analyze', ['verify', temporary]); + const installed = path.join(SYSTEM_UNIT_RUNTIME_ROOT, name); + try { + sudo(['/usr/bin/install', '-o', 'root', '-g', 'root', '-m', '0644', temporary, installed]); + const stat = fs.lstatSync(installed); + if (!stat.isFile() || stat.uid !== 0 || stat.gid !== 0 || stat.nlink !== 1 + || (stat.mode & 0o7777) !== 0o644 || fs.readFileSync(installed, 'utf8') !== unitContent) fail(); + return { name, installed }; + } catch (error) { + sudo(['/usr/bin/rm', '--force', installed], { allowFailure: true }); + throw error; + } +} + +function inspectAccountImage(account, expectedId) { + const image = JSON.parse(accountPodman(account, ['image', 'inspect', IMAGE]).stdout)[0]; + const reference = (image?.RepoDigests || []).find(value => value.startsWith('localhost/dispatch-runtime@sha256:')); + if (!/^sha256:[a-f0-9]{64}$/.test(image?.Digest || '') + || reference !== `localhost/dispatch-runtime@${image.Digest}` + || image?.Id !== expectedId || image?.Config?.User !== '10001:10001') fail(); + return Object.freeze({ digest: image.Digest, id: image.Id, reference }); +} + +function verifyAccountImageManifest(account, reference, source) { + const code = 'const m=require("/opt/dispatch/runtime-release-manifest.json");process.stdout.write(JSON.stringify(m));'; + const value = JSON.parse(accountPodman(account, [ + 'run', '--rm', '--pull=never', '--network=none', '--read-only', '--entrypoint', '/usr/local/bin/node', + reference, '--no-warnings', '-e', code, + ], { timeout: 30_000 }).stdout); + if (value?.schemaVersion !== 1 || value.sourceCommit !== source || !['clean', 'dirty'].includes(value.sourceState) + || value.protocols?.runtimeAgent !== 1 || value.protocols?.runtimeGateway !== 1 + || typeof value.codeTreeDigest !== 'string' || !/^sha256:[a-f0-9]{64}$/.test(value.codeTreeDigest)) fail(); +} + +function inspectContainer(account, plan) { + const value = JSON.parse(accountPodman(account, ['inspect', plan.identity.containerName]).stdout)[0]; + const mounts = (value?.Mounts || []).map(item => ({ source: item.Source, destination: item.Destination })); + const runtimeMount = mounts.find(item => item.destination === plan.guest.installationRoot); + const bridgeMount = mounts.find(item => item.destination === plan.guest.bridgeRoot); + const processStatus = fs.readFileSync(`/proc/${value?.State?.Pid}/status`, 'utf8'); + if (value?.Config?.User !== '10001:10001' || value?.HostConfig?.ReadonlyRootfs !== true + || value?.HostConfig?.NetworkMode !== 'pasta' + || value?.HostConfig?.PidMode !== 'private' || value?.HostConfig?.IpcMode !== 'private' + || value?.HostConfig?.UTSMode !== 'private' + || value?.HostConfig?.PidsLimit !== 512 || value?.HostConfig?.Memory !== 4 * 1024 * 1024 * 1024 + || value?.HostConfig?.NanoCpus !== 2_000_000_000 || value?.HostConfig?.ShmSize !== 512 * 1024 * 1024 + || JSON.stringify(value?.HostConfig?.SecurityOpt) !== JSON.stringify(['no-new-privileges']) + || !value?.HostConfig?.Tmpfs?.['/tmp']?.includes('noexec') + || JSON.stringify(value?.EffectiveCaps || []) !== JSON.stringify(['CAP_SYS_CHROOT']) + || value?.Path !== '/usr/bin/tini' || (value?.NetworkSettings?.Ports && Object.keys(value.NetworkSettings.Ports).length !== 0) + || mounts.length !== 2 || runtimeMount?.source !== plan.host.installationRoot + || bridgeMount?.source !== plan.host.bridgeRoot + || !new RegExp(`^Uid:\\s+${account.uid}\\s`, 'm').test(processStatus)) fail(); + return value; +} + +async function waitFor(check, timeoutMs = 60_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { if (await check()) return; } catch {} + await new Promise(resolve => setTimeout(resolve, 200)); + } + fail(); +} + +function partialAccount(runtimeKey) { + const suffix = opaqueRuntimeSuffix(runtimeKey); + const name = hostAccountName(runtimeKey); + if (!accountExists(name)) return null; + const passwd = run('/usr/bin/getent', ['passwd', name]).stdout.trim().split(':'); + const uid = Number(passwd[2]); + const gid = Number(passwd[3]); + const tenantRoot = path.join(HOST_TENANT_ROOT, suffix); + return { + runtimeKey, + suffix, + name, + uid, + gid, + tenantRoot, + home: path.join(tenantRoot, 'home'), + engineDataRoot: path.join(tenantRoot, 'engine-data'), + engineConfigRoot: path.join(tenantRoot, 'engine-config'), + bridgeRoot: path.join(HOST_BRIDGE_ROOT, suffix), + }; +} + +function removeAccount(account) { + accountPodman(account, ['system', 'reset', '--force'], { allowFailure: true, timeout: 120_000 }); + sudo(['/usr/bin/loginctl', 'disable-linger', account.name], { allowFailure: true }); + sudo(['/usr/bin/systemctl', 'stop', `user@${account.uid}.service`], { allowFailure: true }); + sudo(['/usr/bin/systemctl', 'stop', `user-runtime-dir@${account.uid}.service`], { allowFailure: true }); + sudo(['/usr/sbin/userdel', '--remove', account.name], { allowFailure: true }); + sudo(['/usr/bin/rm', '--recursive', '--force', '--one-file-system', account.tenantRoot], { allowFailure: true }); + sudo(['/usr/bin/rm', '--recursive', '--force', '--one-file-system', account.bridgeRoot], { allowFailure: true }); +} + +async function runFixture() { + const before = referenceSnapshot(); + const bridgeArtifactRoot = path.join('/run', `dispatch-rootless-bridge-artifact-${process.pid}`); + if (lexists(bridgeArtifactRoot)) fail('fixture_preexisting'); + for (const runtimeKey of RUNTIMES) { + const suffix = opaqueRuntimeSuffix(runtimeKey); + const accountName = hostAccountName(runtimeKey); + const runtimeUnit = `dispatch-dsp-${suffix}.service`; + const bridgeName = `dispatch-fixture-bridge-${suffix}.service`; + if (accountExists(accountName) || run('/usr/bin/getent', ['group', accountName], { allowFailure: true }).status === 0 + || systemUnitExists(runtimeUnit) || systemUnitExists(bridgeName) + || lexists(path.join(SYSTEM_UNIT_RUNTIME_ROOT, runtimeUnit)) + || lexists(path.join(SYSTEM_UNIT_RUNTIME_ROOT, bridgeName)) + || lexists(path.join(HOST_TENANT_ROOT, suffix)) + || lexists(path.join(HOST_BRIDGE_ROOT, suffix)) + || sudo(['/usr/bin/test', '!', '-e', `/var/lib/systemd/linger/${accountName}`], { allowFailure: true }).status !== 0) { + fail('fixture_preexisting'); + } + } + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-rootless-host-')); + const accounts = []; + const runtimeUnits = []; + const bridgeUnits = []; + const createdHostRoots = []; + let hub; + let centralRoot; + let archiveRoot; + let archive; + let archiveRootIdentity; + let archiveIdentity; + let bridgeArtifactIdentity; + let bridgeArtifactCreated = false; + try { + if (ensureHostRoot(HOST_DISPATCH_ROOT, 0o755)) createdHostRoots.push(HOST_DISPATCH_ROOT); + if (ensureHostRoot(HOST_TENANT_ROOT, 0o755)) createdHostRoots.push(HOST_TENANT_ROOT); + if (ensureHostRoot(HOST_BRIDGE_ROOT, 0o711)) createdHostRoots.push(HOST_BRIDGE_ROOT); + currentPhase = 'image_archive'; + archiveRoot = fs.mkdtempSync(path.join('/var/tmp', 'dispatch-runtime-fixture-')); + fs.chmodSync(archiveRoot, 0o711); + const archiveRootInfo = fs.lstatSync(archiveRoot); + if (!archiveRootInfo.isDirectory() || archiveRootInfo.uid !== process.geteuid() + || archiveRootInfo.gid !== process.getegid() || (archiveRootInfo.mode & 0o7777) !== 0o711 + || fs.realpathSync(archiveRoot) !== archiveRoot) fail('unsafe_fixture_archive'); + archiveRootIdentity = Object.freeze({ dev: archiveRootInfo.dev, ino: archiveRootInfo.ino }); + archive = path.join(archiveRoot, 'runtime-image.tar'); + sudo(['/usr/bin/install', '-d', '-o', 'root', '-g', 'root', '-m', '0555', bridgeArtifactRoot]); + bridgeArtifactCreated = true; + const artifactInfo = fs.lstatSync(bridgeArtifactRoot); + if (!artifactInfo.isDirectory() || artifactInfo.uid !== 0 || artifactInfo.gid !== 0 + || (artifactInfo.mode & 0o7777) !== 0o555 || fs.realpathSync(bridgeArtifactRoot) !== bridgeArtifactRoot) { + fail('unsafe_fixture_bridge_artifact'); + } + bridgeArtifactIdentity = Object.freeze({ dev: artifactInfo.dev, ino: artifactInfo.ino }); + const bridgeService = stageBridgeArtifact(bridgeArtifactRoot); + const sourceImage = imageIdentity(); + const source = sourceCommit(); + run('/usr/bin/podman', ['save', '--format', 'docker-archive', '--output', archive, IMAGE], { timeout: 300_000 }); + fs.chmodSync(archive, 0o644); + const archiveInfo = fs.lstatSync(archive); + if (!archiveInfo.isFile() || archiveInfo.isSymbolicLink() || archiveInfo.nlink !== 1 + || archiveInfo.uid !== process.geteuid() || archiveInfo.gid !== process.getegid() + || (archiveInfo.mode & 0o7777) !== 0o644 || fs.realpathSync(archive) !== archive) fail('unsafe_fixture_archive'); + archiveIdentity = Object.freeze({ dev: archiveInfo.dev, ino: archiveInfo.ino }); + currentPhase = 'accounts'; + const subids = allocateSubids(RUNTIMES.length); + for (let index = 0; index < RUNTIMES.length; index += 1) accounts.push(createAccount(RUNTIMES[index], subids[index])); + if (accounts[0].uid === accounts[1].uid || accounts[0].gid === accounts[1].gid + || accounts[0].subuidStart + SUBID_COUNT > accounts[1].subuidStart) fail(); + + currentPhase = 'account_images'; + const authorities = {}; + let digest = null; + for (const account of accounts) { + const token = crypto.randomBytes(32).toString('base64url'); + currentPhase = `account_token_${account.suffix}`; + writeToken(account, token, fixtureRoot); + authorities[account.runtimeKey] = crypto.createHash('sha256').update(token, 'utf8').digest('hex'); + currentPhase = `account_image_load_${account.suffix}`; + const loaded = accountPodman(account, ['load', '--input', archive], { timeout: 300_000 }); + if (loaded.status !== 0) fail(); + currentPhase = `account_image_inspect_${account.suffix}`; + const loadedImage = inspectAccountImage(account, sourceImage.id); + verifyAccountImageManifest(account, loadedImage.reference, source); + if (digest !== null && loadedImage.digest !== digest) fail(); + digest = loadedImage.digest; + if (accountPodman(account, ['image', 'exists', loadedImage.reference], { allowFailure: true }).status !== 0) { + fail('rootless_host_fixture_failed', 'loaded_image_digest_not_addressable'); + } + currentPhase = `account_engine_inspect_${account.suffix}`; + const graphRoot = JSON.parse(accountPodman(account, ['info', '--format', 'json']).stdout)?.store?.graphRoot; + if (graphRoot !== path.join(account.engineDataRoot, 'containers', 'storage')) fail(); + } + + currentPhase = 'central_hub'; + const currentRuntimeRoot = process.env.XDG_RUNTIME_DIR || `/run/user/${process.geteuid()}`; + centralRoot = fs.mkdtempSync(path.join(currentRuntimeRoot, 'dispatch-rootless-hub-')); + fs.chmodSync(centralRoot, 0o700); + const centralSocket = path.join(centralRoot, 'runtime-agent-hub.sock'); + hub = new CoreRuntimeAgentHub({ + socketPath: centralSocket, + authorities, + heartbeatIntervalMs: 500, + heartbeatTimeoutMs: 3_000, + }); + await hub.start(); + + currentPhase = 'bridges'; + for (const account of accounts) { + bridgeUnits.push(bridgeUnit(account, centralSocket, process.geteuid(), fixtureRoot, bridgeArtifactRoot, bridgeService)); + } + sudo(['/usr/bin/systemctl', 'daemon-reload']); + for (const unit of bridgeUnits) verifyUnitFragment(unit.name, unit.installed); + for (const unit of bridgeUnits) sudo(['/usr/bin/systemctl', 'reset-failed', unit.name], { allowFailure: true }); + for (const unit of bridgeUnits) sudo(['/usr/bin/systemctl', 'start', unit.name]); + await waitFor(() => accounts.every(account => { + const socket = path.join(account.bridgeRoot, 'runtime-agent-hub.sock'); + const info = fs.lstatSync(socket); + const parent = fs.lstatSync(account.bridgeRoot); + const root = fs.lstatSync(HOST_BRIDGE_ROOT); + return info.isSocket() && info.uid === account.uid && info.gid === account.gid && (info.mode & 0o7777) === 0o600 + && parent.isDirectory() && parent.uid === 0 && parent.gid === 0 && (parent.mode & 0o7777) === 0o711 + && root.isDirectory() && root.uid === 0 && root.gid === 0 && (root.mode & 0o7777) === 0o711; + })); + const centralRootInfo = fs.lstatSync(centralRoot); + const centralSocketInfo = fs.lstatSync(centralSocket); + if (!centralRootInfo.isDirectory() || centralRootInfo.uid !== process.geteuid() + || (centralRootInfo.mode & 0o7777) !== 0o700 || !centralSocketInfo.isSocket() + || centralSocketInfo.uid !== process.geteuid() || (centralSocketInfo.mode & 0o7777) !== 0o600) fail(); + for (const unit of bridgeUnits) { + const properties = run('/usr/bin/systemctl', [ + 'show', unit.name, '--property=User', '--property=Group', '--property=ActiveState', + '--property=MainPID', '--property=ExecStart', '--property=CapabilityBoundingSet', + ]).stdout; + if (!properties.includes('User=root\n') || !properties.includes('Group=root\n') + || !properties.includes('ActiveState=active\n') || !properties.includes(bridgeService) + || !properties.includes('cap_chown') || !properties.includes('cap_dac_override') + || !properties.includes('cap_fowner')) fail(); + } + for (const account of accounts) { + const socket = path.join(account.bridgeRoot, 'runtime-agent-hub.sock'); + const before = fs.lstatSync(socket); + if (accountCommand(account, '/usr/bin/test', ['-w', account.bridgeRoot], { allowFailure: true }).status === 0 + || accountCommand(account, '/usr/bin/rm', ['--force', socket], { allowFailure: true }).status === 0) fail(); + const after = fs.lstatSync(socket); + if (before.dev !== after.dev || before.ino !== after.ino) fail(); + } + + currentPhase = 'runtime_units'; + const plans = accounts.map(account => { + const manifest = { + manifestVersion: INSTALLATION_MANIFEST_VERSION, + revision: 1, + organization: { id: `organization_${account.suffix}`, stationCode: 'DXX1', timezone: 'America/Chicago' }, + runtime: { key: account.runtimeKey, templateId: 'isolated_dsp_v1', releaseId: 'dispatch-runtime-fixture' }, + }; + const authority = { revision: 1, organization: { ...manifest.organization }, runtime: { ...manifest.runtime } }; + const release = { + version: 2, backend: OCI_BACKEND, releaseId: manifest.runtime.releaseId, + channel: 'fixture', image: `localhost/dispatch-runtime@${digest}`, + imageDigest: digest, sourceCommit: source, platform: 'linux/amd64', + imageId: sourceImage.id, + runtimeAgentProtocol: 1, runtimeGatewayProtocol: 1, + embeddedManifestSha256: 'c'.repeat(64), imageArchiveSha256: 'd'.repeat(64), + bridgeManifestSha256: 'e'.repeat(64), + }; + const deployment = { + version: 1, + backend: OCI_BACKEND, + channel: 'fixture', + organizationId: manifest.organization.id, + runtimeKey: manifest.runtime.key, + manifestRevision: manifest.revision, + releaseId: manifest.runtime.releaseId, + }; + return createOciFixtureDeploymentPlan(manifest, authority, release, { + name: account.name, uid: account.uid, gid: account.gid, + subuidStart: account.subuidStart, subgidStart: account.subgidStart, subidCount: SUBID_COUNT, + }, deployment); + }); + for (const plan of plans) runtimeUnits.push({ name: plan.identity.unitName, installed: installRuntimeUnit(plan, fixtureRoot) }); + sudo(['/usr/bin/systemctl', 'daemon-reload']); + for (const unit of runtimeUnits) verifyUnitFragment(unit.name, unit.installed); + for (const unit of runtimeUnits) sudo(['/usr/bin/systemctl', 'reset-failed', unit.name], { allowFailure: true }); + for (const unit of runtimeUnits) sudo(['/usr/bin/systemctl', 'start', unit.name]); + await waitFor(() => accounts.every(account => hub.connected(account.runtimeKey)), 90_000); + + for (const account of accounts) { + const result = await createRuntimeAgentDispatchClient({ hub, runtimeKey: account.runtimeKey }).system.status(); + if (!result?.ok) fail(); + } + currentPhase = 'isolation_verification'; + const containers = plans.map((plan, index) => inspectContainer(accounts[index], plan)); + if (containers[0].State.Pid === containers[1].State.Pid) fail(); + if (accountPodman(accounts[0], ['inspect', plans[1].identity.containerName], { allowFailure: true }).status === 0) fail(); + const betaToken = path.join(accounts[1].installationRoot, 'secrets', 'runtime-agent', 'registration-token'); + if (accountCommand(accounts[0], '/usr/bin/test', ['-r', betaToken], { allowFailure: true }).status === 0) fail(); + if (accountCommand(accounts[0], '/usr/bin/kill', ['-0', String(containers[1].State.Pid)], { allowFailure: true }).status === 0) fail(); + const betaSocket = path.join(accounts[1].bridgeRoot, 'runtime-agent-hub.sock'); + const crossSocket = accountCommand(accounts[0], '/usr/bin/node', ['-e', + 'const net=require("node:net");const s=net.createConnection(process.argv[1]);s.on("connect",()=>process.exit(1));s.on("error",()=>process.exit(0));setTimeout(()=>process.exit(0),1000);', betaSocket], { allowFailure: true }); + if (crossSocket.status !== 0) fail(); + + currentPhase = 'bridge_restart_recovery'; + const alphaBridgePid = run('/usr/bin/systemctl', ['show', bridgeUnits[0].name, '--property=MainPID', '--value']).stdout.trim(); + const betaBridgePid = run('/usr/bin/systemctl', ['show', bridgeUnits[1].name, '--property=MainPID', '--value']).stdout.trim(); + sudo(['/usr/bin/systemctl', 'kill', '--kill-whom=main', '--signal=KILL', bridgeUnits[0].name]); + let bridgeDisconnected = false; + await waitFor(() => { + if (!hub.connected(accounts[0].runtimeKey)) bridgeDisconnected = true; + const current = run('/usr/bin/systemctl', ['show', bridgeUnits[0].name, '--property=MainPID', '--value'], { allowFailure: true }).stdout.trim(); + return bridgeDisconnected && current && current !== '0' && current !== alphaBridgePid && hub.connected(accounts[0].runtimeKey); + }, 60_000); + if (run('/usr/bin/systemctl', ['show', bridgeUnits[1].name, '--property=MainPID', '--value']).stdout.trim() !== betaBridgePid + || !hub.connected(accounts[1].runtimeKey)) fail(); + + const browser = accountPodman(accounts[0], [ + 'exec', '--env', 'HOME=/tmp/browser-profile', plans[0].identity.containerName, + '/usr/bin/chromium', '--headless=new', '--disable-gpu', '--user-data-dir=/tmp/browser-profile', '--dump-dom', 'about:blank', + ], { timeout: 45_000 }); + if (!/ { + const result = accountPodman(accounts[0], [ + 'exec', plans[0].identity.containerName, '/usr/local/bin/node', '--no-warnings', + '/opt/dispatch/runtime/supervisor/src/chromium-sandbox-status.js', + ], { allowFailure: true }); + if (result.status !== 0) return false; + const value = JSON.parse(result.stdout.trim()); + return value.ok === true && value.status === 'sandboxed'; + }, 20_000); + accountPodman(accounts[0], [ + 'exec', plans[0].identity.containerName, '/usr/bin/pkill', '-TERM', '-f', + 'user-data-dir=/tmp/dispatch-sandbox-proof', + ]); + + currentPhase = 'restart_recovery'; + const betaMainPid = run('/usr/bin/systemctl', ['show', plans[1].identity.unitName, '--property=MainPID', '--value']).stdout.trim(); + const alphaMainPid = run('/usr/bin/systemctl', ['show', plans[0].identity.unitName, '--property=MainPID', '--value']).stdout.trim(); + accountPodman(accounts[0], ['exec', plans[0].identity.containerName, '/usr/bin/pkill', '-TERM', '-f', 'auth-broker/bin/dispatch-auth-broker']); + await waitFor(() => { + const current = run('/usr/bin/systemctl', ['show', plans[0].identity.unitName, '--property=MainPID', '--value'], { allowFailure: true }).stdout.trim(); + return current && current !== '0' && current !== alphaMainPid && hub.connected(accounts[0].runtimeKey); + }, 60_000); + if (run('/usr/bin/systemctl', ['show', plans[1].identity.unitName, '--property=MainPID', '--value']).stdout.trim() !== betaMainPid + || !hub.connected(accounts[1].runtimeKey)) fail(); + + const status = plans.map(plan => run('/usr/bin/systemctl', [ + 'show', plan.identity.unitName, '--property=User', '--property=Slice', '--property=MemoryMax', '--property=TasksMax', + '--property=CPUQuotaPerSecUSec', '--property=ActiveState', + ]).stdout); + for (let index = 0; index < plans.length; index += 1) { + if (!status[index].includes(`User=${accounts[index].name}\n`) || !status[index].includes('Slice=dispatch-dsp.slice\n') + || !status[index].includes('MemoryMax=4294967296\n') || !status[index].includes('TasksMax=512\n') + || !status[index].includes('CPUQuotaPerSecUSec=2s\n') + || !status[index].includes('ActiveState=active\n')) fail(); + if (accountPodman(accounts[index], ['healthcheck', 'run', plans[index].identity.containerName], { allowFailure: true }).status !== 0) fail(); + } + + } catch (error) { + if (!error.phase) error.phase = currentPhase; + throw error; + } finally { + currentPhase = 'cleanup'; + for (const unit of runtimeUnits) sudo(['/usr/bin/systemctl', 'stop', unit.name], { allowFailure: true }); + for (const unit of bridgeUnits) sudo(['/usr/bin/systemctl', 'stop', unit.name], { allowFailure: true }); + try { await hub?.close(); } catch {} + for (const unit of [...runtimeUnits, ...bridgeUnits]) sudo(['/usr/bin/rm', '--force', unit.installed], { allowFailure: true }); + sudo(['/usr/bin/systemctl', 'daemon-reload'], { allowFailure: true }); + for (const unit of [...runtimeUnits, ...bridgeUnits]) sudo(['/usr/bin/systemctl', 'reset-failed', unit.name], { allowFailure: true }); + for (const runtimeKey of [...RUNTIMES].reverse()) { + const account = accounts.find(value => value.runtimeKey === runtimeKey) || partialAccount(runtimeKey); + const suffix = opaqueRuntimeSuffix(runtimeKey); + if (account) removeAccount(account); + else { + sudo(['/usr/bin/rm', '--recursive', '--force', '--one-file-system', path.join(HOST_TENANT_ROOT, suffix)], { allowFailure: true }); + sudo(['/usr/bin/rm', '--recursive', '--force', '--one-file-system', path.join(HOST_BRIDGE_ROOT, suffix)], { allowFailure: true }); + } + } + try { + await waitFor(() => accounts.every(account => !lexists(`/run/user/${account.uid}`)), 10_000); + } catch { + fail('fixture_cleanup_failed', 'runtime_directory_cleanup_timeout'); + } + if (archiveRoot) { + const rootInfo = fs.lstatSync(archiveRoot); + if (!rootInfo.isDirectory() || rootInfo.isSymbolicLink() || rootInfo.uid !== process.geteuid() + || rootInfo.gid !== process.getegid() || (rootInfo.mode & 0o7777) !== 0o711 + || fs.realpathSync(archiveRoot) !== archiveRoot + || archiveRootIdentity && (rootInfo.dev !== archiveRootIdentity.dev || rootInfo.ino !== archiveRootIdentity.ino)) { + fail('fixture_cleanup_failed', 'unsafe_archive_root'); + } + if (archive && lexists(archive)) { + const fileInfo = fs.lstatSync(archive); + if (!fileInfo.isFile() || fileInfo.isSymbolicLink() || fileInfo.nlink !== 1 + || fileInfo.uid !== process.geteuid() || fileInfo.gid !== process.getegid() + || fs.realpathSync(archive) !== archive + || archiveIdentity && (fileInfo.dev !== archiveIdentity.dev || fileInfo.ino !== archiveIdentity.ino)) { + fail('fixture_cleanup_failed', 'unsafe_archive_file'); + } + fs.unlinkSync(archive); + } + fs.rmdirSync(archiveRoot); + } + if (bridgeArtifactCreated) { + const artifactInfo = fs.lstatSync(bridgeArtifactRoot); + if (!artifactInfo.isDirectory() || artifactInfo.isSymbolicLink() || artifactInfo.uid !== 0 || artifactInfo.gid !== 0 + || (artifactInfo.mode & 0o7777) !== 0o555 || fs.realpathSync(bridgeArtifactRoot) !== bridgeArtifactRoot + || bridgeArtifactIdentity && (artifactInfo.dev !== bridgeArtifactIdentity.dev || artifactInfo.ino !== bridgeArtifactIdentity.ino)) { + fail('fixture_cleanup_failed', 'unsafe_bridge_artifact'); + } + sudo(['/usr/bin/rm', '--recursive', '--force', '--one-file-system', bridgeArtifactRoot]); + if (lexists(bridgeArtifactRoot)) fail('fixture_cleanup_failed', 'bridge_artifact_remaining'); + } + if (centralRoot) fs.rmSync(centralRoot, { recursive: true, force: true }); + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + for (const root of [...createdHostRoots].reverse()) { + if (sudo(['/usr/bin/rmdir', root], { allowFailure: true }).status !== 0) fail('fixture_cleanup_failed'); + } + const after = referenceSnapshot(); + if (JSON.stringify(before) !== JSON.stringify(after)) fail('reference_service_changed'); + for (const runtimeKey of RUNTIMES) { + const suffix = opaqueRuntimeSuffix(runtimeKey); + const accountName = hostAccountName(runtimeKey); + const priorAccount = accounts.find(value => value.runtimeKey === runtimeKey); + if (accountExists(accountName) || run('/usr/bin/getent', ['group', accountName], { allowFailure: true }).status === 0 + || lexists(path.join(HOST_TENANT_ROOT, suffix)) + || lexists(path.join(HOST_BRIDGE_ROOT, suffix)) + || (priorAccount && lexists(`/run/user/${priorAccount.uid}`)) + || fs.readFileSync('/etc/subuid', 'utf8').split('\n').some(line => line.startsWith(`${accountName}:`)) + || fs.readFileSync('/etc/subgid', 'utf8').split('\n').some(line => line.startsWith(`${accountName}:`)) + || sudo(['/usr/bin/test', '!', '-e', `/var/lib/systemd/linger/${accountName}`], { allowFailure: true }).status !== 0 + || systemUnitExists(`dispatch-dsp-${suffix}.service`) + || systemUnitExists(`dispatch-fixture-bridge-${suffix}.service`)) fail('fixture_cleanup_failed'); + } + } + return Object.freeze({ + ok: true, + status: 'rootless_host_fixture_verified', + accounts: 2, + containers: 2, + uniqueSubidRanges: true, + lockedNonLoginAccounts: true, + imageManifestVerified: true, + resourceLimitsVerified: true, + systemdSupervised: true, + outboundAgentBridges: 2, + bridgeRestartRecovery: true, + crossedFilesystemDenied: true, + crossedEngineDenied: true, + crossedProcessDenied: true, + crossedSocketDenied: true, + browserSandbox: true, + restartRecovery: true, + referenceServicesUnchanged: true, + artifactsRemaining: 0, + }); +} + +async function main() { + process.umask(0o077); + if (process.geteuid() === 0 || run('/usr/bin/sudo', ['-n', 'true'], { allowFailure: true }).status !== 0) fail(); + if (sudo(['/usr/bin/mkdir', '--mode=0700', HOST_FIXTURE_LOCK], { allowFailure: true }).status !== 0) { + fail('rootless_host_fixture_busy'); + } + let receipt; + try { + const lock = fs.lstatSync(HOST_FIXTURE_LOCK); + if (!lock.isDirectory() || lock.uid !== 0 || lock.gid !== 0 || (lock.mode & 0o7777) !== 0o700 + || fs.realpathSync(HOST_FIXTURE_LOCK) !== HOST_FIXTURE_LOCK) fail('unsafe_fixture_host_root'); + receipt = await runFixture(); + } finally { + if (sudo(['/usr/bin/rmdir', HOST_FIXTURE_LOCK], { allowFailure: true }).status !== 0) { + fail('fixture_cleanup_failed', 'fixture_lock_cleanup_failed'); + } + } + process.stdout.write(`${JSON.stringify(receipt)}\n`); +} + +if (require.main === module) main().catch(error => { + process.stderr.write(`${JSON.stringify({ + ok: false, + status: error?.code || 'rootless_host_fixture_failed', + phase: error?.phase || currentPhase, + diagnostic: String(error?.diagnostic || '').replace(/[A-Za-z0-9_-]{43}/g, '[redacted]'), + })}\n`); + process.exitCode = 1; +}); + +module.exports = { main }; diff --git a/dsp/runtime/supervisor/examples/runtime-container.js b/dsp/runtime/supervisor/examples/runtime-container.js new file mode 100644 index 0000000..a9ebf29 --- /dev/null +++ b/dsp/runtime/supervisor/examples/runtime-container.js @@ -0,0 +1,305 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawn, spawnSync } = require('node:child_process'); +const { + MANAGED_INSTALLATION_LAYOUT_VERSION, + MANAGED_INSTALLATION_LAYOUT_TEMPLATE, + MANAGED_INSTALLATION_DIRECTORY_FIELDS, + managedInstallationRuntimeEnvironment, +} = require('dispatch-protocol/paths/runtime-paths'); +const { CoreRuntimeAgentHub, createRuntimeAgentDispatchClient } = require('dispatch-core/core/agents/src/index.js'); + +const IMAGE = 'localhost/dispatch-runtime:dev'; +const RUNTIME_KEY = 'runtime_container_fixture'; +const CONTAINER = `dispatch-runtime-container-fixture-${process.pid}`; +const GUEST_ROOT = `/var/lib/dispatch/${RUNTIME_KEY}`; +const MAX_OUTPUT = 64 * 1024; +let currentPhase = 'preflight'; +let lastDiagnostic = ''; + +function fail(code = 'runtime_container_fixture_failed', diagnostic = '') { + throw Object.assign(new Error(code), { code, diagnostic }); +} + +function command(args, { timeout = 30_000, allowFailure = false } = {}) { + const result = spawnSync('/usr/bin/podman', args, { + encoding: 'utf8', timeout, shell: false, + env: { HOME: os.homedir(), PATH: '/usr/bin:/bin', XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR || `/run/user/${process.geteuid()}` }, + maxBuffer: MAX_OUTPUT, + }); + if (!allowFailure && (result.error || result.signal || result.status !== 0)) fail(); + return result; +} + +function runtimeLayout(installationRoot, projectRoot) { + const directories = Object.fromEntries(Object.entries(MANAGED_INSTALLATION_DIRECTORY_FIELDS) + .map(([field, relative]) => [field, path.join(installationRoot, relative)])); + return Object.freeze({ + layoutVersion: MANAGED_INSTALLATION_LAYOUT_VERSION, + templateId: MANAGED_INSTALLATION_LAYOUT_TEMPLATE, + runtimeKey: RUNTIME_KEY, + projectRoot, + installationRoot, + directories: Object.freeze(directories), + }); +} + +function materialize(layout) { + fs.mkdirSync(layout.installationRoot, { mode: 0o700 }); + const selected = [...new Set(Object.values(layout.directories))] + .sort((left, right) => left.split(path.sep).length - right.split(path.sep).length || left.localeCompare(right)); + for (const directory of selected) fs.mkdirSync(directory, { mode: 0o700 }); + return layout.directories.runtimeAgentSecretsRoot; +} + +async function waitFor(check, timeoutMs = 30_000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { if (await check()) return; } catch {} + await new Promise(resolve => setTimeout(resolve, 100)); + } + fail(); +} + +function childExit(child) { + return new Promise(resolve => child.once('close', (code, signal) => resolve({ code, signal }))); +} + +async function withTimeout(promise, timeoutMs) { + let timer; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(Object.assign(new Error('runtime_container_fixture_failed'), { + code: 'runtime_container_fixture_failed', + })), timeoutMs); + }), + ]); + } finally { + clearTimeout(timer); + } +} + +async function main() { + process.umask(0o077); + if (process.geteuid() === 0) fail(); + if (command(['container', 'exists', CONTAINER], { allowFailure: true }).status === 0) fail('fixture_preexisting'); + const fixtureRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-container-')); + const hostRoot = path.join(fixtureRoot, RUNTIME_KEY); + const bridgeRoot = path.join(fixtureRoot, 'bridge'); + const hostLayout = runtimeLayout(hostRoot, path.resolve(__dirname, "../../..")); + const guestLayout = runtimeLayout(GUEST_ROOT, '/opt/dispatch'); + const token = crypto.randomBytes(32).toString('base64url'); + const tokenDigest = crypto.createHash('sha256').update(token, 'utf8').digest('hex'); + let hub; + let container; + let containerOutput = ''; + try { + currentPhase = 'materialize'; + const tokenRoot = materialize(hostLayout); + fs.writeFileSync(path.join(tokenRoot, 'registration-token'), `${token}\n`, { mode: 0o600, flag: 'wx' }); + fs.mkdirSync(bridgeRoot, { mode: 0o700 }); + const hubSocket = path.join(bridgeRoot, 'runtime-agent-hub.sock'); + currentPhase = 'hub_start'; + hub = new CoreRuntimeAgentHub({ + socketPath: hubSocket, + authorities: { [RUNTIME_KEY]: tokenDigest }, + heartbeatIntervalMs: 250, + heartbeatTimeoutMs: 2_000, + }); + await hub.start(); + + const environment = { + ...managedInstallationRuntimeEnvironment(guestLayout), + DISPATCH_RUNTIME_KEY: RUNTIME_KEY, + DISPATCH_RUNTIME_GATEWAY_SOCKET: path.join(GUEST_ROOT, 'run', 'runtime-gateway.sock'), + DISPATCH_RUNTIME_AGENT_TOKEN_FILE: path.join(GUEST_ROOT, 'secrets', 'runtime-agent', 'registration-token'), + DISPATCH_RUNTIME_AGENT_STATUS_SOCKET: path.join(GUEST_ROOT, 'run', 'runtime-agent-status.sock'), + }; + const args = [ + 'run', '--rm', '--name', CONTAINER, + '--label', 'io.dispatch.fixture=runtime-container-v1', + '--pull=never', '--http-proxy=false', '--network=none', '--pid=private', '--ipc=private', '--uts=private', '--read-only', + '--user', '10001:10001', '--userns', 'keep-id:uid=10001,gid=10001', + '--cap-drop', 'ALL', '--cap-add', 'SYS_CHROOT', '--security-opt', 'no-new-privileges', + '--memory', '4g', '--cpus', '2', '--pids-limit', '512', '--shm-size', '512m', + '--tmpfs', '/tmp:rw,noexec,nosuid,nodev,size=512m,mode=1777', + '--volume', `${hostRoot}:${GUEST_ROOT}:rw,rprivate,nosuid,nodev`, + '--volume', `${bridgeRoot}:/run/dispatch-agent:ro,rprivate,nosuid,nodev,noexec`, + ]; + for (const [key, value] of Object.entries(environment).sort(([a], [b]) => a.localeCompare(b))) { + args.push('--env', `${key}=${value}`); + } + args.push(IMAGE); + currentPhase = 'container_start'; + container = spawn('/usr/bin/podman', args, { + stdio: ['ignore', 'pipe', 'pipe'], shell: false, + env: { HOME: os.homedir(), PATH: '/usr/bin:/bin', XDG_RUNTIME_DIR: process.env.XDG_RUNTIME_DIR || `/run/user/${process.geteuid()}` }, + }); + const exited = childExit(container); + for (const stream of [container.stdout, container.stderr]) stream.on('data', chunk => { + containerOutput += chunk.toString('utf8'); + lastDiagnostic = containerOutput.slice(-2048); + if (Buffer.byteLength(containerOutput, 'utf8') > MAX_OUTPUT) container.kill('SIGKILL'); + }); + + currentPhase = 'agent_connect'; + await waitFor(() => hub.connected(RUNTIME_KEY)); + currentPhase = 'collector_definition'; + const applied = command([ + 'exec', CONTAINER, '/usr/local/bin/node', '--no-warnings', + '/opt/dispatch/runtime/collection-manager/bin/dispatch-collectionctl', + 'apply', '/opt/dispatch/plugins/paycom/backend/config/collection-manager.json', + ], { allowFailure: true }); + if (applied.status !== 0 || JSON.parse(applied.stdout.trim())?.ok !== true) { + const probe = command([ + 'exec', CONTAINER, '/usr/local/bin/node', '--no-warnings', '-e', + 'const fs=require("node:fs");const p="/opt/dispatch/plugins/paycom/backend/bin/dispatch-paycom-collector";const s=fs.lstatSync(p);console.log(JSON.stringify({uid:process.geteuid(),owner:s.uid,mode:s.mode&4095,link:s.nlink,real:fs.realpathSync(p)===p}));', + ], { allowFailure: true }); + fail('runtime_container_fixture_failed', JSON.stringify({ + status: applied.status, output: applied.stdout.trim(), error: applied.stderr.trim(), probe: probe.stdout.trim(), + })); + } + currentPhase = 'system_status'; + const systemStatus = await createRuntimeAgentDispatchClient({ hub, runtimeKey: RUNTIME_KEY }).system.status(); + if (!systemStatus?.ok) fail(); + currentPhase = 'container_health'; + const health = command(['healthcheck', 'run', CONTAINER], { allowFailure: true }); + if (health.status !== 0) { + const checks = [ + ['runtime/auth-broker/bin/dispatch-auth-brokerctl', 'health'], + ['runtime/collection-manager/bin/dispatch-collectionctl', 'status'], + ['runtime/gateway/bin/dispatch-runtime-gatewayctl', 'health'], + ['runtime/agent/bin/dispatch-runtime-agentctl', 'health'], + ].map(([relative, action]) => { + const result = command([ + 'exec', CONTAINER, '/usr/local/bin/node', '--no-warnings', `/opt/dispatch/${relative}`, action, + ], { allowFailure: true }); + return { component: path.basename(relative), status: result.status, output: result.stdout.trim() }; + }); + fail('runtime_container_fixture_failed', JSON.stringify(checks)); + } + const healthOutput = command([ + 'exec', CONTAINER, '/usr/local/bin/node', '--no-warnings', + '/opt/dispatch/runtime/supervisor/src/health.js', + ]); + const healthValue = JSON.parse(healthOutput.stdout.trim()); + if (!healthValue.ok || healthValue.status !== 'healthy' || healthValue.components !== 4) fail(); + + currentPhase = 'paycom_enrollment'; + const enrollment = { + command: 'enroll', requestId: `setup_${'a'.repeat(32)}`, expiresAt: Date.now() + 30_000, intent: 'create', + credentials: { clientCode: 'synthetic-client', username: 'synthetic-user', password: 'synthetic-enrollment-secret', + pin1: 'fixture-one', pin2: 'fixture-two', pin3: 'fixture-three', pin4: 'fixture-four', pin5: 'fixture-five' }, + }; + const enrolled = await hub.invoke(RUNTIME_KEY, 'paycom.setup', enrollment); + if (!enrolled.ok || enrolled.status !== 'succeeded' || enrolled.data?.configured !== true) { + fail('runtime_container_fixture_failed', JSON.stringify({ phase: currentPhase, status: enrolled.status })); + } + const replay = await hub.invoke(RUNTIME_KEY, 'paycom.setup', enrollment); + if (!replay.ok || replay.status !== 'succeeded') fail(); + const vaultCheck = command(['exec', CONTAINER, '/usr/local/bin/node', '--no-warnings', '-e', + 'const fs=require("node:fs"),path=require("node:path");const c=require("/opt/dispatch/runtime/gateway/src/managed-runtime").managedRuntimeConfiguration();const files=fs.readdirSync(c.paths.auth.databaseRoot);if(files.some(n=>fs.readFileSync(path.join(c.paths.auth.databaseRoot,n)).includes(Buffer.from("synthetic-enrollment-secret"))))process.exit(1);process.stdout.write("encrypted\\n");', + ]); + if (vaultCheck.status !== 0) fail(); + + currentPhase = 'paycom_configuration'; + const setupManifest = { manifestVersion: 1, revision: 1, + organization: { id: 'org_container_setup', stationCode: 'TST1', timezone: 'America/Chicago' }, + runtime: { key: RUNTIME_KEY, templateId: 'isolated_dsp_v1', releaseId: 'dispatch_current_1' } }; + const configureRequest = { command: 'start', requestId: `setup_${'a'.repeat(32)}`, step: 'configure', + manifest: setupManifest, manifestAuthority: { revision: 1, organization: setupManifest.organization, runtime: setupManifest.runtime }, parameters: {} }; + let configured = await hub.invoke(RUNTIME_KEY, 'paycom.setup', configureRequest); + const configureDeadline = Date.now() + 30_000; + while (configured.ok && configured.status === 'running' && Date.now() < configureDeadline) { + await new Promise(resolve => setTimeout(resolve, 100)); + configured = await hub.invoke(RUNTIME_KEY, 'paycom.setup', { ...configureRequest, command: 'status' }); + } + if (!configured.ok || configured.status !== 'succeeded' || configured.data?.plans !== 15) { + fail('runtime_container_fixture_failed', JSON.stringify({ phase: currentPhase, status: configured.status })); + } + + currentPhase = 'container_inspect'; + const inspect = JSON.parse(command(['inspect', CONTAINER]).stdout)[0]; + if (inspect?.Config?.User !== '10001:10001' || inspect?.HostConfig?.ReadonlyRootfs !== true + || inspect?.HostConfig?.NetworkMode !== 'none' || inspect?.HostConfig?.PidsLimit !== 512 + || inspect?.HostConfig?.PidMode !== 'private' || inspect?.HostConfig?.IpcMode !== 'private' + || inspect?.HostConfig?.UTSMode !== 'private' + || inspect?.HostConfig?.Memory !== 4 * 1024 * 1024 * 1024 || inspect?.HostConfig?.NanoCpus !== 2_000_000_000 + || inspect?.HostConfig?.ShmSize !== 512 * 1024 * 1024 + || JSON.stringify(inspect?.HostConfig?.SecurityOpt) !== JSON.stringify(['no-new-privileges']) + || !inspect?.HostConfig?.Tmpfs?.['/tmp']?.includes('noexec') + || JSON.stringify(inspect?.EffectiveCaps || []) !== JSON.stringify(['CAP_SYS_CHROOT'])) fail(); + const destinations = (inspect?.Mounts || []).map(item => item.Destination).sort(); + if (JSON.stringify(destinations) !== JSON.stringify([GUEST_ROOT, '/run/dispatch-agent'].sort())) fail(); + + currentPhase = 'browser'; + const browser = command([ + 'exec', '--env', 'HOME=/tmp/browser-profile', CONTAINER, + '/usr/bin/chromium', '--headless=new', '--disable-gpu', '--user-data-dir=/tmp/browser-profile', '--dump-dom', 'about:blank', + ], { timeout: 30_000 }); + if (!/ { + const result = command([ + 'exec', CONTAINER, '/usr/local/bin/node', '--no-warnings', + '/opt/dispatch/runtime/supervisor/src/chromium-sandbox-status.js', + ], { allowFailure: true }); + if (result.status !== 0) return false; + const value = JSON.parse(result.stdout.trim()); + return value.ok === true && value.status === 'sandboxed'; + }, 20_000); + command(['exec', CONTAINER, '/usr/bin/pkill', '-TERM', '-f', 'user-data-dir=/tmp/dispatch-sandbox-proof']); + + currentPhase = 'fail_fast'; + command(['exec', CONTAINER, '/usr/bin/pkill', '-TERM', '-f', 'auth-broker/bin/dispatch-auth-broker']); + const terminal = await withTimeout(exited, 20_000); + container = null; + if (terminal.code === 0 || terminal.signal) fail(); + const absent = command(['container', 'exists', CONTAINER], { allowFailure: true }); + if (absent.status === 0) fail(); + + } finally { + if (container) { + command(['stop', '--time', '5', CONTAINER], { allowFailure: true }); + command(['rm', '--force', '--ignore', CONTAINER], { allowFailure: true }); + } + try { await hub?.close(); } catch {} + fs.rmSync(fixtureRoot, { recursive: true, force: true }); + } + if (command(['container', 'exists', CONTAINER], { allowFailure: true }).status === 0 || fs.existsSync(fixtureRoot)) { + fail('runtime_container_fixture_cleanup_failed'); + } + process.stdout.write(`${JSON.stringify({ + ok: true, + status: 'runtime_container_fixture_verified', + services: 4, + outboundAgent: true, + readOnlyRoot: true, + paycomEnrollment: true, + browserSandbox: true, + failFast: true, + artifactsRemaining: 0, + })}\n`); +} + +if (require.main === module) main().catch(error => { + process.stderr.write(`${JSON.stringify({ + ok: false, + status: error?.code || 'runtime_container_fixture_failed', + phase: currentPhase, + diagnostic: String(error?.diagnostic || lastDiagnostic).replace(/[A-Za-z0-9_-]{43}/g, '[redacted]'), + })}\n`); + process.exitCode = 1; +}); + +module.exports = { main }; diff --git a/dsp/runtime/supervisor/package.json b/dsp/runtime/supervisor/package.json new file mode 100644 index 0000000..88d5b20 --- /dev/null +++ b/dsp/runtime/supervisor/package.json @@ -0,0 +1,15 @@ +{ + "name": "dispatch-runtime-container", + "version": "0.1.0", + "private": true, + "description": "Fixed non-root supervisor and health boundary for one isolated DSP runtime container", + "type": "commonjs", + "engines": { + "node": ">=22" + }, + "scripts": { + "build": "./scripts/build", + "test": "./scripts/test", + "verify": "./scripts/verify" + } +} diff --git a/dsp/runtime/supervisor/scripts/build b/dsp/runtime/supervisor/scripts/build new file mode 100755 index 0000000..ef4721c --- /dev/null +++ b/dsp/runtime/supervisor/scripts/build @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +for file in "$ROOT"/src/*.js "$ROOT"/tests/*.js; do + node --no-warnings --check "$file" +done +printf '%s\n' '{"ok":true,"status":"built"}' diff --git a/dsp/runtime/supervisor/scripts/measure-collection-capacity b/dsp/runtime/supervisor/scripts/measure-collection-capacity new file mode 100755 index 0000000..59ef349 --- /dev/null +++ b/dsp/runtime/supervisor/scripts/measure-collection-capacity @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +exec node --no-warnings "$ROOT/src/measure-collection-capacity.js" "$@" diff --git a/dsp/runtime/supervisor/scripts/test b/dsp/runtime/supervisor/scripts/test new file mode 100755 index 0000000..f6e53ac --- /dev/null +++ b/dsp/runtime/supervisor/scripts/test @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +cd "$ROOT" +node --no-warnings --test tests/*.test.js diff --git a/dsp/runtime/supervisor/scripts/verify b/dsp/runtime/supervisor/scripts/verify new file mode 100755 index 0000000..2a3e236 --- /dev/null +++ b/dsp/runtime/supervisor/scripts/verify @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +"$ROOT/tooling/build" +"$ROOT/tooling/test" diff --git a/dsp/runtime/supervisor/scripts/verify-rootless-host-fixture b/dsp/runtime/supervisor/scripts/verify-rootless-host-fixture new file mode 100755 index 0000000..66c82ab --- /dev/null +++ b/dsp/runtime/supervisor/scripts/verify-rootless-host-fixture @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)" +cd "$ROOT/../.." +node --no-warnings runtime/supervisor/examples/rootless-host-fixture.js diff --git a/dsp/runtime/supervisor/src/chromium-sandbox-status.js b/dsp/runtime/supervisor/src/chromium-sandbox-status.js new file mode 100644 index 0000000..6fe1f25 --- /dev/null +++ b/dsp/runtime/supervisor/src/chromium-sandbox-status.js @@ -0,0 +1,65 @@ +'use strict'; + +const fs = require('node:fs'); + +const PROFILE = '/tmp/dispatch-sandbox-proof'; + +function statusFields(pid) { + const fields = new Map(); + for (const line of fs.readFileSync(`/proc/${pid}/status`, 'utf8').split('\n')) { + const index = line.indexOf(':'); + if (index > 0) fields.set(line.slice(0, index), line.slice(index + 1).trim()); + } + return fields; +} + +function commandLine(pid) { + return fs.readFileSync(`/proc/${pid}/cmdline`).toString('utf8').split('\0').filter(Boolean); +} + +function chromiumProcesses() { + const rows = []; + for (const name of fs.readdirSync('/proc')) { + if (!/^[0-9]+$/.test(name)) continue; + try { + const argv = commandLine(name); + const text = argv.join(' '); + if (argv[0]?.startsWith('/usr/lib/chromium/chromium') && text.includes(`--user-data-dir=${PROFILE}`)) { + rows.push(Object.freeze({ pid: name, argv: Object.freeze(argv), text })); + } + } catch {} + } + return rows; +} + +function sandboxStatus() { + const rows = chromiumProcesses(); + const browser = rows.find(row => !row.text.includes('--type=')); + const renderer = rows.find(row => row.text.includes('--type=renderer')); + if (!browser || !renderer) return Object.freeze({ ok: false, status: 'not_ready' }); + if (rows.some(row => row.text.includes('--no-sandbox'))) return Object.freeze({ ok: false, status: 'sandbox_disabled' }); + + const fields = statusFields(renderer.pid); + const browserPidNamespace = fs.readlinkSync(`/proc/${browser.pid}/ns/pid`); + const rendererPidNamespace = fs.readlinkSync(`/proc/${renderer.pid}/ns/pid`); + const ok = fields.get('NoNewPrivs') === '1' + && fields.get('Seccomp') === '2' + && Number.parseInt(fields.get('Seccomp_filters') || '0', 10) >= 1 + && browserPidNamespace !== rendererPidNamespace; + return Object.freeze({ ok, status: ok ? 'sandboxed' : 'sandbox_inactive' }); +} + +function main() { + try { + const result = sandboxStatus(); + process.stdout.write(`${JSON.stringify(result)}\n`); + return result.ok ? 0 : 1; + } catch { + process.stdout.write('{"ok":false,"status":"unavailable"}\n'); + return 1; + } +} + +if (require.main === module) process.exitCode = main(); + +module.exports = { PROFILE, sandboxStatus, main }; diff --git a/dsp/runtime/supervisor/src/connections.js b/dsp/runtime/supervisor/src/connections.js new file mode 100644 index 0000000..c304552 --- /dev/null +++ b/dsp/runtime/supervisor/src/connections.js @@ -0,0 +1,2 @@ +'use strict'; +module.exports = require('dispatch-runtime-kit/supervisor/src/connections'); diff --git a/dsp/runtime/supervisor/src/create-release-manifest.js b/dsp/runtime/supervisor/src/create-release-manifest.js new file mode 100644 index 0000000..54be918 --- /dev/null +++ b/dsp/runtime/supervisor/src/create-release-manifest.js @@ -0,0 +1,130 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); +const { RUNTIME_GATEWAY_PROTOCOL_VERSION } = require('dispatch-protocol/gateway/protocol'); +const { RUNTIME_AGENT_PROTOCOL_VERSION } = require('dispatch-protocol/agent/protocol'); + +const ROOT = '/opt/dispatch'; +const MANIFEST_FILE = path.join(ROOT, 'runtime-release-manifest.json'); +const SOURCE_ROOTS = ['runtime', 'shared', 'plugins', 'compatibility/cdf', 'compatibility/paycom']; + +function fail(code) { + const error = new Error(code); + error.code = code; + throw error; +} + +function sourceFiles() { + const values = []; + const visit = relative => { + const absolute = path.join(ROOT, relative); + const stat = fs.lstatSync(absolute); + if (stat.isSymbolicLink()) fail('unsafe_runtime_source'); + if (stat.isDirectory()) { + for (const child of fs.readdirSync(absolute).sort()) visit(path.posix.join(relative, child)); + return; + } + if (!stat.isFile() || stat.nlink !== 1) fail('unsafe_runtime_source'); + values.push({ absolute, relative, mode: stat.mode & 0o777, uid: stat.uid, gid: stat.gid }); + }; + for (const relative of SOURCE_ROOTS) visit(relative); + return values; +} + +function treeDigest(files) { + const hash = crypto.createHash('sha256'); + for (const file of files) { + const body = fs.readFileSync(file.absolute); + hash.update(file.relative, 'utf8'); + hash.update('\0'); + hash.update(file.mode.toString(8), 'ascii'); + hash.update('\0'); + hash.update(String(file.uid), 'ascii'); + hash.update('\0'); + hash.update(String(file.gid), 'ascii'); + hash.update('\0'); + hash.update(String(body.length), 'ascii'); + hash.update('\0'); + hash.update(body); + hash.update('\0'); + } + return `sha256:${hash.digest('hex')}`; +} + +function packageVersion(relative) { + const value = JSON.parse(fs.readFileSync(path.join(ROOT, relative), 'utf8')); + if (typeof value.version !== 'string' || !/^\d+\.\d+\.\d+$/.test(value.version)) fail('invalid_component_version'); + return value.version; +} + +function installedPackageVersion(name) { + const value = execFileSync('/usr/bin/dpkg-query', ['-W', '-f=${Version}', name], { + encoding: 'utf8', + maxBuffer: 4096, + timeout: 5_000, + }).trim(); + if (!/^\d+[A-Za-z0-9.+:~_-]*$/.test(value)) fail('invalid_os_package_version'); + return value; +} + +function main(env = process.env) { + if (process.platform !== 'linux' || process.arch !== 'x64') fail('invalid_runtime_platform'); + const sourceCommit = env.DISPATCH_SOURCE_COMMIT; + const sourceState = env.DISPATCH_SOURCE_STATE; + const chromiumVersion = env.DISPATCH_CHROMIUM_VERSION; + if (!/^[a-f0-9]{40}$/.test(sourceCommit || '')) fail('invalid_source_commit'); + if (!['clean', 'dirty'].includes(sourceState)) fail('invalid_source_state'); + if (!/^\d+[A-Za-z0-9.+:~_-]*$/.test(chromiumVersion || '')) fail('invalid_chromium_version'); + const osPackages = { + caCertificates: installedPackageVersion('ca-certificates'), + chromium: installedPackageVersion('chromium'), + chromiumSandbox: installedPackageVersion('chromium-sandbox'), + fontsLiberation: installedPackageVersion('fonts-liberation'), + tini: installedPackageVersion('tini'), + utilLinux: installedPackageVersion('util-linux'), + }; + if (osPackages.chromium !== chromiumVersion || osPackages.chromiumSandbox !== chromiumVersion) { + fail('chromium_version_mismatch'); + } + const files = sourceFiles(); + const manifest = { + schemaVersion: 1, + sourceCommit, + sourceState, + baseImage: 'docker.io/library/node:22-bookworm-slim@sha256:4d676821dff059fd00d277ee4261ef34ea712317fed0737c03941481b5760c96', + platform: 'linux/amd64', + nodeVersion: process.version, + chromiumVersion, + osPackages, + protocols: { + runtimeAgent: RUNTIME_AGENT_PROTOCOL_VERSION, + runtimeGateway: RUNTIME_GATEWAY_PROTOCOL_VERSION, + }, + components: { + authBroker: packageVersion('runtime/auth-broker/package.json'), + collectionManager: packageVersion('runtime/collection-manager/package.json'), + runtimeAgent: packageVersion('runtime/agent/package.json'), + runtimeContainer: packageVersion('runtime/supervisor/package.json'), + runtimeGateway: packageVersion('runtime/gateway/package.json'), + sdk: packageVersion('runtime/sdk/package.json'), + }, + codeTreeDigest: treeDigest(files), + sourceFileCount: files.length, + }; + fs.writeFileSync(MANIFEST_FILE, `${JSON.stringify(manifest)}\n`, { mode: 0o444, flag: 'wx' }); + return manifest; +} + +if (require.main === module) { + try { + main(); + } catch (error) { + process.stderr.write(`${error?.code || 'runtime_manifest_failed'}\n`); + process.exitCode = 1; + } +} + +module.exports = { main, sourceFiles, treeDigest }; diff --git a/dsp/runtime/supervisor/src/diagnostics-data.js b/dsp/runtime/supervisor/src/diagnostics-data.js new file mode 100644 index 0000000..b128504 --- /dev/null +++ b/dsp/runtime/supervisor/src/diagnostics-data.js @@ -0,0 +1,54 @@ +'use strict'; + +const { periodFromEnd, buildTimecardUrl } = require('../../../plugins/paycom/backend/src/timecard-period'); + +const LABELS = ['SUN', 'MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT']; + +function timecardRecord(code = 'A001', end = '2026-09-05', variant = 1) { + const period = periodFromEnd(end); + return { + version: 2, + sourceFormat: 'paycom-timecard-dom.v2', + employeeCode: code, + periodStart: period.start, + periodEnd: period.end, + periodKey: period.key, + sourceUrl: buildTimecardUrl(code, period, variant), + pageTitle: 'Timecard Editor', + headers: ['date', 'paycode', 'i1', 'allocation1', 'o1', 'i2', 'allocation2', 'o2', 'hours', 'total_hours', 'amount', 'exception-points', 'waiver', 'comment', 'missing-punch', 'delete'], + additionalRows: [], + periodTotalHours: 8, + weeklyTotals: [8, 0], + approvals: [], + attestations: [], + mealWaivers: [], + days: period.dates.map((date, index) => ({ + date, + label: LABELS[index % 7], + payCode: '', + allocation1: '', + allocation2: '', + hours: index === 0 ? 8 : null, + totalHours: index === 0 ? 8 : null, + dollars: null, + exceptionText: '', + waiverChecked: null, + comments: [], + missingPunch: false, + unresolvedSlots: [], + punches: index === 0 ? [{ + ordinal: 1, rowIndex: 0, slot: 'i1', kind: 'IN DAY', displayTime: '09:00 AM', + actualTime: '09:00 AM', roundedTime: '09:00 AM', clockName: 'Clock', clockCode: 'WEB00', + comment: '', provenanceAvailable: true, changeRequestStatus: null, approved: false, + changeOperation: null, currentKind: null, currentTime: null, requestedKind: null, + requestedTime: null, changeNote: null, changeDetailState: 'not_applicable', + }] : [], + })), + }; +} + +function rosterRow(code = 'A001', name = 'Employee One') { + return { employeeCode: code, employeeName: name, isActive: true, isActiveDriver: true }; +} + +module.exports = { timecardRecord, rosterRow }; diff --git a/dsp/runtime/supervisor/src/diagnostics-seed.js b/dsp/runtime/supervisor/src/diagnostics-seed.js new file mode 100644 index 0000000..b191979 --- /dev/null +++ b/dsp/runtime/supervisor/src/diagnostics-seed.js @@ -0,0 +1,186 @@ +"use strict"; +// A fixed private Runtime Agent command: no scripts, paths, or employee data from HTTP. +const fs = require("node:fs"); +const crypto = require("node:crypto"); +const path = require("node:path"); +const { success, failure } = require("dispatch-protocol/contracts/src/result"); +const { ensurePrivateDirectory } = require("../../auth-broker/src/vault"); +const { + PaycomStore, + stageCandidate, + cleanupStage, +} = require("../../../plugins/paycom/backend/src/store"); +const { periodFromEnd } = require("../../../plugins/paycom/backend/src/timecard-period"); +const { + TIMECARD_SUMMARY, + ROUTE_VERSION, + linkRows, +} = require("../../../plugins/paycom/backend/src/resource-links"); +const { timecardRecord, rosterRow } = require("./diagnostics-data"); +function createDiagnosticsSeed(config) { + return ({ requestId }) => { + if ( + !/^org_[a-f0-9]{32}$/.test(requestId) || + ![`runtime_${requestId.slice(4)}`, `dsp_${requestId.slice(4)}`].includes(config.runtimeKey) + ) + return failure("runtime_identity_mismatch"); + const root = path.join(config.layout.directories.stateRoot, "diagnostics"); + ensurePrivateDirectory(root); + const receipt = path.join(root, "seed.json"); + let previous; + if (fs.existsSync(receipt)) { + const info = fs.lstatSync(receipt); + if ( + !info.isFile() || + info.isSymbolicLink() || + info.nlink !== 1 || + info.uid !== process.geteuid() || + (info.mode & 0o777) !== 0o600 + ) + return failure("runtime_boundary_violation"); + previous = JSON.parse(fs.readFileSync(receipt, "utf8")); + if (previous.requestId !== requestId) + return failure("runtime_identity_mismatch"); + if (previous.data) return success("succeeded", previous.data); + } + const store = new PaycomStore(config.paths.paycom.database); + try { + // A regular DSP with existing publications can never be converted into a fixture. + if ( + !previous && + store.db.prepare("SELECT 1 FROM publications LIMIT 1").get() + ) + return failure("installation_operation_not_allowed"); + const end = new Date(); + end.setUTCDate(end.getUTCDate() + (6 - end.getUTCDay())); + const target = previous?.target || end.toISOString().slice(0, 10); + const write = (value) => { + const temporary = + receipt + "." + crypto.randomBytes(8).toString("hex") + ".tmp"; + const fd = fs.openSync( + temporary, + fs.constants.O_WRONLY | + fs.constants.O_CREAT | + fs.constants.O_EXCL | + fs.constants.O_NOFOLLOW, + 0o600, + ); + try { + fs.writeFileSync(fd, JSON.stringify(value)); + fs.fsyncSync(fd); + } finally { + fs.closeSync(fd); + } + fs.renameSync(temporary, receipt); + }; + const collectedAt = previous?.collectedAt || new Date().toISOString(); + write({ requestId, target, collectedAt }); + const period = periodFromEnd(target); + const STAGING_ROOT = config.paths.paycom.stagingRoot; + function publish(candidate) { + const stage = stageCandidate(STAGING_ROOT, { + attempt: 1, + collectedAt, + ...candidate, + runId: candidate.runId, + }); + try { + return store.publish(stage); + } finally { + cleanupStage(stage, STAGING_ROOT); + } + } + const payPeriods = publish({ + kind: "pay_periods", + target: period.end, + runId: "run_periods_fixture", + metadata: {}, + rows: [ + { + start: period.start, + end: period.end, + key: period.key, + relation: "current", + }, + ], + }); + const rows = [ + rosterRow("Z999", "Synthetic Test Driver"), + rosterRow("Z998", "Synthetic Test Dispatcher"), + ]; + const roster = publish({ + kind: "roster", + target: period.end, + runId: "run_fixture_roster", + metadata: {}, + rows, + }); + const timecards = publish({ + kind: "timecards", + target: period.end, + periodKey: period.key, + runId: "run_fixture_timecards", + metadata: { + periodStart: period.start, + periodEnd: period.end, + mode: "full", + rosterPublicationId: roster.publicationId, + rosterContentSha256: roster.contentSha256, + }, + rows: rows.map((row) => ({ + employeeCode: row.employeeCode, + employeeName: row.employeeName, + record: timecardRecord(row.employeeCode, period.end), + sourceSha256: "b".repeat(64), + })), + }); + const links = publish({ + kind: "resource_links", + target: period.end, + periodKey: period.key, + runId: "run_fixture_links", + metadata: { + resourceType: TIMECARD_SUMMARY, + periodStart: period.start, + periodEnd: period.end, + rosterPublicationId: roster.publicationId, + rosterContentSha256: roster.contentSha256, + routeVersion: ROUTE_VERSION, + }, + rows: linkRows(TIMECARD_SUMMARY, rows, period), + }); + + { + const { + CollectionStore, + } = require("dispatch-runtime-kit/collection-manager/src/store"); + const { + materializeSpec, + } = require("../../collection-manager/src/control-cli"); + const spec = JSON.parse( + fs.readFileSync( + path.resolve( + __dirname, + "../../../plugins/paycom/backend/config/collection-manager.json", + ), + ), + ); + for (const sync of spec.syncs || []) sync.desiredState = "stopped"; + for (const plan of spec.plans) plan.schedule = { type: "manual" }; + const collection = new CollectionStore(config.paths.collection); + try { + collection.applySpec(materializeSpec(spec, config.paths.projectRoot)); + } finally { + collection.close(); + } + } + + const data = { target, payPeriods, roster, timecards, links }; + write({ requestId, target, collectedAt, data }); + return success("succeeded", data); + } finally { + store.close(); + } + }; +} +module.exports = { createDiagnosticsSeed }; diff --git a/dsp/runtime/supervisor/src/egress-relay.js b/dsp/runtime/supervisor/src/egress-relay.js new file mode 100644 index 0000000..7609dcd --- /dev/null +++ b/dsp/runtime/supervisor/src/egress-relay.js @@ -0,0 +1,47 @@ +'use strict'; + +const net = require('node:net'); +const PROXY_PORT = 17891; +const EGRESS_SOCKET = '/run/dispatch-agent/egress.sock'; + +// This loopback listener lives inside one private network namespace. Only the +// host's constrained CONNECT server can turn a request into an external socket. +function createEgressRelay({ socketPath = EGRESS_SOCKET, port = PROXY_PORT } = {}) { + const sockets = new Set(); + const track = socket => { + sockets.add(socket); socket.on('error', () => socket.destroy()); + socket.once('close', () => sockets.delete(socket)); + socket.setTimeout(120000, () => socket.destroy()); + return socket; + }; + const server = net.createServer(client => { + track(client); client.pause(); + const upstream = track(net.createConnection(socketPath)); + client.once('close', () => upstream.destroy()); + upstream.once('close', () => client.destroy()); + upstream.once('connect', () => { client.pipe(upstream); upstream.pipe(client); client.resume(); }); + }); + server.maxConnections = 64; + return { + server, + start: () => new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(port, '127.0.0.1', () => { server.off('error', reject); resolve(); }); + }), + close: async () => { + for (const socket of sockets) socket.destroy(); + await new Promise(resolve => server.close(resolve)); + }, + }; +} + +async function main() { + if (process.env.DISPATCH_RUNTIME_BACKEND !== 'directory_service_v1') throw new Error('runtime_boundary_violation'); + const relay = createEgressRelay(); + await relay.start(); + relay.server.on('error', () => { relay.close().finally(() => { process.exitCode = 1; }); }); + const close = () => { relay.close().catch(() => { process.exitCode = 1; }); }; + process.once('SIGTERM', close); process.once('SIGINT', close); +} +if (require.main === module) main().catch(() => { process.exitCode = 1; }); +module.exports = { createEgressRelay, PROXY_PORT, EGRESS_SOCKET }; diff --git a/dsp/runtime/supervisor/src/execution.js b/dsp/runtime/supervisor/src/execution.js new file mode 100644 index 0000000..2e4d3ae --- /dev/null +++ b/dsp/runtime/supervisor/src/execution.js @@ -0,0 +1,98 @@ +'use strict'; + +const path = require('node:path'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const control = require('../../collection-manager/src/execution-control'); +const { nextWake } = require('../../collection-manager/src/next-wake'); +const { request } = require('dispatch-runtime-kit/auth-broker/src/client'); +const { catalog, pluginEntry, ROOT } = require('dispatch-protocol/plugin-sdk/catalog'); +const { saveStatus } = require('dispatch-protocol/published/status'); +const { success, failure } = require('dispatch-protocol/contracts/src/result'); +const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); + +function nextObservation(values, now) { + const dates = [...(values.connections?.data?.items || []).map(item => item.retryAt), values['paycom-readiness']?.data?.retryAt] + .filter(Boolean).map(Date.parse).filter(value => Number.isSafeInteger(value) && value > now); + return dates.length ? Math.min(...dates) : null; +} + +function createExecution({ configuration, client, plugins, activeRequests = () => 0, authRequest = request }) { + let pending = null, lastSystem = null, systemAt = 0; + const publications = new Map(); + const directory = path.join(configuration.paths.dataRoot, 'published'); + async function execute(input) { + const store = new CollectionStore(configuration.paths.collection); + try { + const now = Date.now(); + if (input.command !== 'snapshot') control.command(store, input.command, input.scheduledAt); + if (input.command === 'restore') return success('restored', {}); + // The manager must observe drain after its last asynchronous scheduler tick. + if (input.command === 'drain') { + const deadline = now + 3000; + while (Date.now() < deadline) { + const state = control.read(store.db); + if (state?.draining && state.acknowledged === state.generation) break; + await pause(25); + } + } + const activity = await authRequest(configuration.paths.auth.socket, { action: 'activity' }); + if (!activity?.ok || typeof activity.busy !== 'boolean') return failure('execution_state_unavailable', { recoverable: true }); + const connections = await client.connectionsManage({ command: 'list' }); + if (!connections.ok) return failure('execution_state_unavailable', { recoverable: true }); + if (!lastSystem || now - systemAt >= 60000 || input.command === 'adopt' || input.command === 'drain') { + lastSystem = await client.system.status(); systemAt = now; + } + const values = { system: lastSystem, connections, plugins: await plugins.manage({ command: 'status' }) }; + values.collections = await client.collections.health(); + const enabled = catalog().filter(definition => plugins.enabled(definition.id)); + for (const definition of enabled) { + for (const id of definition.syncs) values[`sync:${id}`] = await client.sync.status(id); + const publisher = process.env.DISPATCH_PLUGIN_BACKEND === 'core_v1' + ? () => require('dispatch-sdk/runtime').createFrameworkClient().request('plugin.publish', { pluginId: definition.id, request: {} }) + : definition.runtime && require(pluginEntry(ROOT, definition, 'runtime')).publish; + if (publisher) { + const source = store.sources().find(item => definition.collectors.includes(item.collector)); + if (source?.config?.timezone && !publications.has(definition.id)) { + const task = { pending: true, error: null }; + publications.set(definition.id, task); + task.promise = Promise.resolve().then(() => publisher({ paths: configuration.paths, directory, timezone: source.config.timezone })) + .catch(error => { task.error = error; }).finally(() => { task.pending = false; }); + // Unchanged pointers finish immediately. Full publication runs in a + // temporary worker; its pending state prevents idle shutdown. + await Promise.race([task.promise, pause(25)]); + } + const task = publications.get(definition.id); + if (task && !task.pending) { + publications.delete(definition.id); + if (task.error) throw task.error; + } + } + } + if (enabled.some(definition => definition.id === 'paycom')) { + const readiness = await authRequest(configuration.paths.auth.socket, { action: 'profile-readiness', profile: 'paycom-main' }); + if (readiness?.ok && readiness.readiness) values['paycom-readiness'] = success('succeeded', readiness.readiness); + } + const state = control.read(store.db); + const running = store.db.prepare("SELECT count(*) n FROM runs WHERE status='running'").get().n; + const queuedNow = store.db.prepare("SELECT count(*) n FROM runs WHERE status='queued' AND run_after<=?").get(Date.now()).n; + const deadlines = [await nextWake(store, Date.now()), nextObservation(values, now)].filter(Number.isSafeInteger); + const next = deadlines.length ? Math.min(...deadlines) : null; + const unsettledClock = state && state.requestedAt !== null && (state.completedAt === null || state.requestedAt > state.completedAt); + // Disabled plugins can still have a publication that must finish safely. + for (const [id, task] of publications) if (!task.pending && !enabled.some(item => item.id === id)) publications.delete(id); + const busy = Boolean(publications.size || activity.busy || plugins.busy() || activeRequests() || running || queuedNow || unsettledClock + || enabled.some(definition => !definition.published)); + const execution = { version: 1, busy, nextWakeAt: next, + drained: Boolean(state?.draining && state.acknowledged === state.generation && !busy), observedAt: Date.now() }; + values.execution = execution; + saveStatus(directory, values, execution.observedAt); + return success('found', execution); + } finally { store.close(); } + } + return input => { + if (pending) return Promise.resolve(failure('execution_busy', { recoverable: true })); + pending = execute(input).finally(() => { pending = null; }); + return pending; + }; +} +module.exports = { createExecution, nextObservation }; diff --git a/dsp/runtime/supervisor/src/health.js b/dsp/runtime/supervisor/src/health.js new file mode 100644 index 0000000..0be9163 --- /dev/null +++ b/dsp/runtime/supervisor/src/health.js @@ -0,0 +1,57 @@ +'use strict'; + +const { spawnSync } = require('node:child_process'); +const path = require('node:path'); +const { configuration, PROJECT_ROOT } = require('./supervisor'); + +const CHECKS = Object.freeze([ + Object.freeze({ id: 'auth_broker', relative: 'runtime/auth-broker/bin/dispatch-auth-brokerctl', args: ['health'] }), + Object.freeze({ id: 'collection_manager', relative: 'runtime/collection-manager/bin/dispatch-collectionctl', args: ['status'] }), + Object.freeze({ id: 'runtime_gateway', relative: 'runtime/gateway/bin/dispatch-runtime-gatewayctl', args: ['health'] }), + Object.freeze({ id: 'runtime_agent', relative: 'runtime/agent/bin/dispatch-runtime-agentctl', args: ['health'] }), +]); +const MAX_OUTPUT_BYTES = 16_384; + +function healthyResponse(text) { + if (typeof text !== 'string' || Buffer.byteLength(text) > MAX_OUTPUT_BYTES + || !text.endsWith('\n') || text.includes('\r') || text.slice(0, -1).includes('\n')) return false; + let value; + try { value = JSON.parse(text.slice(0, -1)); } catch { return false; } + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) && value.ok === true; +} + +function checkRuntime({ environment = process.env, spawnImpl = spawnSync } = {}) { + let config; + try { config = configuration(environment); } catch { return false; } + for (const check of CHECKS) { + const script = path.join(PROJECT_ROOT, check.relative); + let result; + try { + result = spawnImpl(process.execPath, ['--no-warnings', script, ...check.args], { + cwd: path.dirname(script), + env: config.childEnvironment, + encoding: 'utf8', + timeout: 5_000, + maxBuffer: MAX_OUTPUT_BYTES, + shell: false, + stdio: ['ignore', 'pipe', 'ignore'], + }); + } catch { return false; } + if (result.error || result.signal || result.status !== 0 || !healthyResponse(result.stdout)) return false; + } + return true; +} + +function main() { + const healthy = checkRuntime(); + process.stdout.write(`${JSON.stringify({ + ok: healthy, + status: healthy ? 'healthy' : 'runtime_unavailable', + components: CHECKS.length, + })}\n`); + return healthy ? 0 : 1; +} + +if (require.main === module) process.exitCode = main(); + +module.exports = { CHECKS, MAX_OUTPUT_BYTES, healthyResponse, checkRuntime, main }; diff --git a/dsp/runtime/supervisor/src/install-legacy-entrypoints.js b/dsp/runtime/supervisor/src/install-legacy-entrypoints.js new file mode 100644 index 0000000..3d74c76 --- /dev/null +++ b/dsp/runtime/supervisor/src/install-legacy-entrypoints.js @@ -0,0 +1,24 @@ +'use strict'; + +// Collector commands are persisted in existing DSP databases and queued runs. +// Keep those executable addresses working without changing tenant data on upgrade. +const fs = require('node:fs'); +const path = require('node:path'); +const ROOT = '/opt/dispatch'; +const LEGACY_ENTRYPOINTS = require('dispatch-protocol/legacy-entrypoints'); +function install() { + for (const [provider, executables] of Object.entries(LEGACY_ENTRYPOINTS)) { + const directory = path.join(ROOT, 'plugins', provider, 'bin'); + fs.mkdirSync(directory, { recursive: true, mode: 0o755 }); + for (const executable of executables) { + const base = require('dispatch-protocol/plugin-sdk/catalog').plugin(provider) ? ['plugins', provider, 'backend'] : ['compatibility', provider]; + const target = path.join(ROOT, ...base, 'bin', executable); + if (!fs.lstatSync(target).isFile()) throw new Error('missing_collector_entrypoint'); + const file = path.join(directory, executable); + fs.writeFileSync(file, `#!/usr/bin/env -S node --no-warnings\n'use strict';\nrequire(${JSON.stringify(target)});\n`, { flag: 'wx', mode: 0o755 }); + fs.chownSync(file, 10001, 10001); + } + } +} +if (require.main === module) install(); +module.exports = { install, LEGACY_ENTRYPOINTS }; diff --git a/dsp/runtime/supervisor/src/measure-collection-capacity.js b/dsp/runtime/supervisor/src/measure-collection-capacity.js new file mode 100644 index 0000000..59e240f --- /dev/null +++ b/dsp/runtime/supervisor/src/measure-collection-capacity.js @@ -0,0 +1,107 @@ +'use strict'; +// Synthetic, local-only browser work. Never enrolls credentials or contacts a +// provider. Measurements are a sizing aid, not a Paycom throughput guarantee. +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { performance } = require('node:perf_hooks'); +const { ChromeBrowserRuntime } = require('../../auth-broker/src/browser-runtime'); +const { CdpConnection, createTarget } = require('../../auth-broker/src/cdp'); +function parse(argv) { + const options = { dsps: [1, 2, 4], workers: 2, seconds: 5 }; + for (let i = 0; i < argv.length; i += 2) { + const name = argv[i]; + if (!['--dsps', '--workers', '--seconds'].includes(name) || argv[i + 1] === undefined) throw Error('invalid_probe_options'); + options[name.slice(2)] = name === '--dsps' ? argv[i + 1].split(',').map(Number) : Number(argv[i + 1]); + } + if (options.dsps.length < 1 || options.dsps.length > 8 || options.dsps.some(n => !Number.isInteger(n) || n < 1 || n > 8) + || !Number.isInteger(options.workers) || options.workers < 1 || options.workers > 6 + || !Number.isInteger(options.seconds) || options.seconds < 1 || options.seconds > 60) throw Error('invalid_probe_options'); + return options; +} +function cpu() { + return os.cpus().reduce((sum, item) => ({ idle: sum.idle + item.times.idle, + total: sum.total + Object.values(item.times).reduce((a, b) => a + b, 0) }), { idle: 0, total: 0 }); +} +function descendantsMemory() { + const processes = []; + for (const pid of fs.readdirSync('/proc').filter(name => /^\d+$/.test(name))) { + try { + const value = fs.readFileSync(`/proc/${pid}/status`, 'utf8'); + processes.push({ pid: Number(pid), parent: Number(/^PPid:\s+(\d+)/m.exec(value)?.[1]), rss: Number(/^VmRSS:\s+(\d+)/m.exec(value)?.[1] || 0) * 1024 }); + } catch {} + } + const selected = new Set([process.pid]); + let changed = true; + while (changed) { changed = false; for (const item of processes) if (!selected.has(item.pid) && selected.has(item.parent)) { selected.add(item.pid); changed = true; } } + return processes.filter(item => selected.has(item.pid) && item.pid !== process.pid).reduce((sum, item) => sum + item.rss, 0); +} +const expression = `(() => { + document.body.innerHTML = '' + Array.from({length: 500}, (_, i) => '').join('') + '
Fixture ' + i + '8.00
'; + return Array.from(document.querySelectorAll('tr')).reduce((sum, row) => sum + Number(row.cells[1].textContent), 0); +})()`; +async function measure({ dsps, workers, seconds }, { browser = process.env.DISPATCH_CHROME_EXECUTABLE || '/opt/google/chrome/chrome' } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dprobe-')); + const browsers = [], connections = [], samples = []; + let timer; + const begun = performance.now(); + const cpuBefore = cpu(); + let peakRss = 0; + let minFreeMemory = os.freemem(); + const sample = () => { peakRss = Math.max(peakRss, descendantsMemory()); minFreeMemory = Math.min(minFreeMemory, os.freemem()); }; + try { + timer = setInterval(sample, 250); + // Launch failures are settled before cleanup so a late launch cannot leak. + const launches = await Promise.allSettled(Array.from({ length: dsps }, async (_, i) => { + const directory = path.join(root, String(i)); fs.mkdirSync(directory, { mode: 0o700 }); + const runtime = new ChromeBrowserRuntime({ stateRoot: path.join(directory, 'sessions'), socketRoot: directory, + executable: browser, headless: true, transport: 'pipe' }); + const instance = await runtime.launch(); browsers.push(instance); + const targets = await Promise.allSettled(Array.from({ length: workers }, async () => { + const target = await createTarget(instance.endpoint, 'about:blank'); + const connection = await CdpConnection.connect(target.webSocketDebuggerUrl); connections.push(connection); + })); + if (targets.some(result => result.status === 'rejected')) throw Error('probe_target_failed'); + })); + if (launches.some(result => result.status === 'rejected')) throw Error('probe_browser_failed'); + const startupMs = performance.now() - begun; + const deadline = performance.now() + seconds * 1000; + let failures = 0; + await Promise.all(connections.map(async connection => { + while (performance.now() < deadline) { + const start = performance.now(); + try { + const value = await connection.command('Runtime.evaluate', { expression, returnByValue: true }); + if (value.result?.value !== 4000) throw Error('probe_result_invalid'); + samples.push(performance.now() - start); + } catch { failures += 1; break; } + } + })); + sample(); + const cpuAfter = cpu(); + samples.sort((a, b) => a - b); + const percentile = fraction => samples.length ? Math.round(samples[Math.min(samples.length - 1, Math.floor(samples.length * fraction))]) : null; + return { dsps, workersPerDsp: workers, totalWorkers: connections.length, startupMs: Math.round(startupMs), seconds, + successfulOperations: samples.length, failures, operationP50Ms: percentile(0.5), operationP95Ms: percentile(0.95), + peakBrowserRssBytes: peakRss, minHostFreeMemoryBytes: minFreeMemory, + hostCpuBusyPercent: Math.round(100 * (1 - (cpuAfter.idle - cpuBefore.idle) / Math.max(1, cpuAfter.total - cpuBefore.total))) }; + } finally { + clearInterval(timer); + for (const connection of connections) connection.close(); + const results = await Promise.allSettled(browsers.map(browser => browser.close())); + fs.rmSync(root, { recursive: true, force: true }); + if (results.some(result => result.status === 'rejected')) throw Error('probe_cleanup_failed'); + } +} +async function main(argv) { + const options = parse(argv); + const samples = []; + for (const dsps of options.dsps) samples.push(await measure({ ...options, dsps })); + return { workload: 'synthetic_local_browser', livePaycomVerified: false, hostCpus: os.availableParallelism(), + hostMemoryBytes: os.totalmem(), samples }; +} +if (require.main === module) main(process.argv.slice(2)).then(result => { + process.stdout.write(`${JSON.stringify(result, null, 2)}\n`); + if (result.samples.some(sample => sample.failures)) process.exitCode = 1; +}).catch(() => { process.stderr.write('Collection capacity probe failed.\n'); process.exitCode = 1; }); +module.exports = { parse, measure }; diff --git a/dsp/runtime/supervisor/src/paycom-activation-runtime.js b/dsp/runtime/supervisor/src/paycom-activation-runtime.js new file mode 100644 index 0000000..b3f35f9 --- /dev/null +++ b/dsp/runtime/supervisor/src/paycom-activation-runtime.js @@ -0,0 +1,3 @@ +'use strict'; +// Compatibility entrypoint; the implementation belongs to the Paycom plugin. +module.exports = require('../../../plugins/paycom/backend/runtime/activation.js'); diff --git a/dsp/runtime/supervisor/src/paycom-definition.js b/dsp/runtime/supervisor/src/paycom-definition.js new file mode 100644 index 0000000..320ec06 --- /dev/null +++ b/dsp/runtime/supervisor/src/paycom-definition.js @@ -0,0 +1,3 @@ +'use strict'; +// Compatibility entrypoint; the implementation belongs to the Paycom plugin. +module.exports = require('../../../plugins/paycom/backend/runtime/definition.js'); diff --git a/dsp/runtime/supervisor/src/paycom-setup.js b/dsp/runtime/supervisor/src/paycom-setup.js new file mode 100644 index 0000000..d893d32 --- /dev/null +++ b/dsp/runtime/supervisor/src/paycom-setup.js @@ -0,0 +1,3 @@ +'use strict'; +// Compatibility entrypoint; the implementation belongs to the Paycom plugin. +module.exports = require('../../../plugins/paycom/backend/runtime/setup.js'); diff --git a/dsp/runtime/supervisor/src/supervisor.js b/dsp/runtime/supervisor/src/supervisor.js new file mode 100644 index 0000000..2f818c3 --- /dev/null +++ b/dsp/runtime/supervisor/src/supervisor.js @@ -0,0 +1,228 @@ +'use strict'; + +const { spawn } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); +const { INSTALLATION_IDENTIFIER_RE } = require('dispatch-protocol/contracts/src'); +const { managedRuntimeEnvironmentFromProcess } = require('dispatch-protocol/paths/runtime-paths'); + +const PROJECT_ROOT = '/opt/dispatch'; +const CONTAINER_STORAGE_ROOT = '/var/lib/dispatch'; +const AGENT_BRIDGE_ROOT = '/run/dispatch-agent'; +const STOP_GRACE_MS = 20_000; +const serviceBackend = value => ['native_service_v1', 'directory_service_v1'].includes(value); +const FORBIDDEN_ENVIRONMENT = Object.freeze([ + 'DISPATCH_LOCAL_ROOT', + 'DISPATCH_ACCESS_CONTROL_DATABASE_ROOT', + 'NODE_OPTIONS', + 'LD_PRELOAD', + 'LD_LIBRARY_PATH', +]); +const COMPONENTS = Object.freeze([ + Object.freeze({ id: 'auth_broker', relative: 'runtime/auth-broker/bin/dispatch-auth-broker' }), + Object.freeze({ id: 'collection_manager', relative: 'runtime/collection-manager/bin/dispatch-collection-manager' }), + Object.freeze({ id: 'runtime_gateway', relative: 'runtime/gateway/bin/dispatch-runtime-gateway' }), + Object.freeze({ id: 'runtime_agent', relative: 'runtime/agent/bin/dispatch-runtime-agent' }), +]); + +function fail(code = 'runtime_boundary_violation') { + throw Object.assign(new Error(code), { code }); +} + +function absolute(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value + || /[\0\r\n]/.test(value)) fail(); + return value; +} + +function configuration(environment = process.env) { + if (!environment || typeof environment !== 'object') fail(); + if (FORBIDDEN_ENVIRONMENT.some(key => Object.hasOwn(environment, key))) fail(); + const runtimeKey = environment.DISPATCH_RUNTIME_KEY; + if (typeof runtimeKey !== 'string' || runtimeKey === 'local' + || !INSTALLATION_IDENTIFIER_RE.test(runtimeKey)) fail('runtime_identity_mismatch'); + const managed = managedRuntimeEnvironmentFromProcess(environment); + const installationRoot = path.dirname(managed.DISPATCH_DATA_ROOT); + if (managed.DISPATCH_PROJECT_ROOT !== PROJECT_ROOT + || path.dirname(installationRoot) !== CONTAINER_STORAGE_ROOT + || path.basename(installationRoot) !== runtimeKey) fail('runtime_identity_mismatch'); + const gatewaySocket = absolute(environment.DISPATCH_RUNTIME_GATEWAY_SOCKET); + const hubSocket = absolute(environment.DISPATCH_RUNTIME_AGENT_HUB_SOCKET); + const tokenFile = absolute(environment.DISPATCH_RUNTIME_AGENT_TOKEN_FILE); + const statusSocket = absolute(environment.DISPATCH_RUNTIME_AGENT_STATUS_SOCKET); + if (gatewaySocket !== path.join(managed.DISPATCH_RUNTIME_ROOT, 'runtime-gateway.sock') + || tokenFile !== path.join(managed.DISPATCH_SECRETS_ROOT, 'runtime-agent', 'registration-token') + || statusSocket !== path.join(managed.DISPATCH_RUNTIME_ROOT, 'runtime-agent-status.sock') + || hubSocket !== path.join(AGENT_BRIDGE_ROOT, 'runtime-agent-hub.sock') + || environment.DISPATCH_CHROME_EXECUTABLE !== (serviceBackend(environment.DISPATCH_RUNTIME_BACKEND) + ? '/opt/dispatch/dependencies/browser/chrome' : '/usr/bin/chromium')) fail(); + const childEnvironment = Object.freeze({ + HOME: serviceBackend(environment.DISPATCH_RUNTIME_BACKEND) ? '/tmp/dispatch-home' : '/home/dispatch', + PATH: environment.DISPATCH_RUNTIME_BACKEND === 'directory_service_v1' ? '/opt/dispatch-tools:/usr/bin:/bin' + : environment.DISPATCH_RUNTIME_BACKEND === 'native_service_v1' ? '/opt/dispatch/dependencies/node/bin:/usr/bin:/bin' : '/usr/local/bin:/usr/bin:/bin', + LANG: 'C.UTF-8', + LC_ALL: 'C.UTF-8', + TZ: 'UTC', + NODE_NO_WARNINGS: '1', + DISPATCH_MANAGED_RUNTIME: '1', + ...managed, + DISPATCH_RUNTIME_KEY: runtimeKey, + DISPATCH_RUNTIME_GATEWAY_SOCKET: gatewaySocket, + DISPATCH_RUNTIME_AGENT_HUB_SOCKET: hubSocket, + DISPATCH_RUNTIME_AGENT_TOKEN_FILE: tokenFile, + DISPATCH_RUNTIME_AGENT_STATUS_SOCKET: statusSocket, + DISPATCH_CHROME_EXECUTABLE: environment.DISPATCH_CHROME_EXECUTABLE, + ...(serviceBackend(environment.DISPATCH_RUNTIME_BACKEND) ? { DISPATCH_RUNTIME_BACKEND: environment.DISPATCH_RUNTIME_BACKEND } : {}), + ...(environment.DISPATCH_PLUGIN_BACKEND === 'core_v1' ? { DISPATCH_PLUGIN_BACKEND: 'core_v1' } : {}), + }); + return Object.freeze({ runtimeKey, childEnvironment }); +} + +function definitions(config) { + if (!config || config.childEnvironment?.DISPATCH_PROJECT_ROOT !== PROJECT_ROOT) fail(); + let components = config.childEnvironment.DISPATCH_RUNTIME_BACKEND === 'directory_service_v1' + ? [{ id: 'egress_relay', relative: 'runtime/supervisor/src/egress-relay.js' }, ...COMPONENTS] : COMPONENTS; + if (config.childEnvironment.DISPATCH_PLUGIN_BACKEND === 'core_v1') components = components.filter(item => item.id !== 'auth_broker'); + return Object.freeze(components.map(component => Object.freeze({ + id: component.id, + executable: process.execPath, + arguments: Object.freeze(['--no-warnings', path.join(PROJECT_ROOT, component.relative)]), + workingDirectory: path.dirname(path.join(PROJECT_ROOT, component.relative)), + environment: config.childEnvironment, + }))); +} + +function decodeMountPath(value) { + return value.replace(/\\([0-7]{3})/g, (_, octal) => String.fromCharCode(Number.parseInt(octal, 8))); +} + +function assertMountBoundary(config, mountInfo = fs.readFileSync('/proc/self/mountinfo', 'utf8'), exists = fs.existsSync) { + if (!config?.childEnvironment || typeof mountInfo !== 'string' + || Buffer.byteLength(mountInfo, 'utf8') > 4 * 1024 * 1024 || typeof exists !== 'function') fail(); + const mounts = mountInfo.trim().split('\n').filter(Boolean).map(line => { + const fields = line.split(' '); + if (fields.indexOf('-') < 6) fail(); + return { point: decodeMountPath(fields[4]), options: new Set(fields[5].split(',')) }; + }); + const exact = point => mounts.find(mount => mount.point === point); + const runtimeRoot = path.dirname(config.childEnvironment.DISPATCH_DATA_ROOT); + const root = exact('/'); + const runtime = exact(runtimeRoot); + const bridge = exact(AGENT_BRIDGE_ROOT); + const temporary = exact('/tmp'); + if (!root?.options.has('ro') || !runtime?.options.has('rw') || !bridge?.options.has('ro') + || !temporary?.options.has('rw') || !temporary.options.has('noexec') + || !temporary.options.has('nosuid') || !temporary.options.has('nodev')) fail(); + if (serviceBackend(config.childEnvironment.DISPATCH_RUNTIME_BACKEND)) { + if (!exact(PROJECT_ROOT)?.options.has('ro') + || mounts.some(mount => mount.point.startsWith(`${PROJECT_ROOT}/`) && !mount.options.has('ro'))) fail(); + if (config.childEnvironment.DISPATCH_RUNTIME_BACKEND === 'directory_service_v1' + && mounts.some(mount => (mount.point === runtimeRoot || mount.point.startsWith(`${runtimeRoot}/`)) + && !mount.options.has('noexec'))) fail(); + } else if (mounts.some(mount => mount.point === PROJECT_ROOT || mount.point.startsWith(`${PROJECT_ROOT}/`))) fail(); + for (const socket of ['/run/podman/podman.sock', '/var/run/docker.sock']) { + if (!exists(socket)) continue; + if (!serviceBackend(config.childEnvironment.DISPATCH_RUNTIME_BACKEND)) fail(); + let accessible = false; + try { fs.accessSync(socket, fs.constants.R_OK | fs.constants.W_OK); accessible = true; } catch {} + if (accessible) fail(); + } + return true; +} + +function createSupervisor({ environment = process.env, spawnImpl = spawn, stopGraceMs = STOP_GRACE_MS } = {}) { + if (typeof spawnImpl !== 'function' || !Number.isSafeInteger(stopGraceMs) + || stopGraceMs < 1 || stopGraceMs > STOP_GRACE_MS) fail(); + const config = configuration(environment); + const selected = definitions(config); + const children = new Map(); + let stopping = false; + let resolveStopped; + const stopped = new Promise(resolve => { resolveStopped = resolve; }); + + function signalGroup(child, signal) { + if (!child?.pid) return; + try { process.kill(-child.pid, signal); } + catch { try { child.kill(signal); } catch {} } + } + + async function stop(exitCode) { + if (stopping) return stopped; + stopping = true; + for (const child of children.values()) signalGroup(child, 'SIGTERM'); + const deadline = Date.now() + stopGraceMs; + while (children.size > 0 && Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 25)); + } + for (const child of children.values()) signalGroup(child, 'SIGKILL'); + while (children.size > 0) await new Promise(resolve => setTimeout(resolve, 10)); + resolveStopped(exitCode); + return stopped; + } + + function start() { + for (const definition of selected) { + let child; + try { + child = spawnImpl(definition.executable, [...definition.arguments], { + cwd: definition.workingDirectory, + env: definition.environment, + detached: true, + shell: false, + stdio: ['ignore', 'inherit', 'inherit'], + }); + } catch { + stop(1).catch(() => {}); + break; + } + children.set(definition.id, child); + child.once('error', () => { stop(1).catch(() => {}); }); + child.once('close', code => { + children.delete(definition.id); + if (!stopping) stop(code === 0 ? 1 : code || 1).catch(() => {}); + }); + } + if (!stopping && children.size !== selected.length) stop(1).catch(() => {}); + return stopped; + } + + return Object.freeze({ + config, + definitions: selected, + start, + stop, + childCount: () => children.size, + }); +} + +async function main() { + process.umask(0o077); + if (process.geteuid() === 0) return 1; + let supervisor; + try { + supervisor = createSupervisor(); + assertMountBoundary(supervisor.config); + if (serviceBackend(supervisor.config.childEnvironment.DISPATCH_RUNTIME_BACKEND)) { + fs.mkdirSync(supervisor.config.childEnvironment.HOME, { recursive: true, mode: 0o700 }); + } + } + catch { return 1; } + process.once('SIGTERM', () => { supervisor.stop(0).catch(() => {}); }); + process.once('SIGINT', () => { supervisor.stop(0).catch(() => {}); }); + return supervisor.start(); +} + +if (require.main === module) main().then(code => { process.exitCode = Number.isInteger(code) ? code : 1; }); + +module.exports = { + PROJECT_ROOT, + CONTAINER_STORAGE_ROOT, + AGENT_BRIDGE_ROOT, + COMPONENTS, + STOP_GRACE_MS, + configuration, + definitions, + assertMountBoundary, + createSupervisor, + main, +}; diff --git a/dsp/runtime/supervisor/tests/execution.test.js b/dsp/runtime/supervisor/tests/execution.test.js new file mode 100644 index 0000000..9a02dc0 --- /dev/null +++ b/dsp/runtime/supervisor/tests/execution.test.js @@ -0,0 +1,71 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { resolveLocalRuntimePaths } = require('dispatch-protocol/paths/runtime-paths'); +const { CredentialVault } = require('../../auth-broker/src/vault'); +const { AuthBrokerServer } = require('../../auth-broker/src/server'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { CollectionManager } = require('../../collection-manager/src/manager'); +const { createManagedRuntimeDispatchClient } = require('../../gateway/src/managed-runtime'); +const { createRuntimePlugins } = require('../../plugin-host'); +const { createRuntimeConnections } = require('dispatch-runtime-kit/supervisor/src/connections'); +const { createExecution } = require('../src/execution'); +const { readStatus } = require('dispatch-protocol/published/status'); + +test('saved connection cooldowns schedule an observation without performing a login', () => { + const { nextObservation } = require('../src/execution'); + const now = Date.parse('2026-09-11T12:00:00Z'); + assert.equal(nextObservation({ connections: { data: { items: [{ retryAt: '2026-09-11T12:01:00Z' }] } } }, now), now + 60000); + assert.equal(nextObservation({ 'paycom-readiness': { data: { retryAt: '2026-09-11T12:02:00Z' } } }, now), now + 120000); + assert.equal(nextObservation({ connections: { data: { items: [{ retryAt: '2026-09-11T11:59:00Z' }] } } }, now), null); +}); + +test('real broker and collection manager acknowledge quiescence before a published checkpoint permits sleep', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'runtime-quiescence-')); + const paths = resolveLocalRuntimePaths({ localRoot: root }); + for (const name of ['data', 'state', 'secrets', 'run', 'staging']) fs.mkdirSync(path.join(root, name), { mode: 0o700 }); + const vault = new CredentialVault(paths.auth); vault.close(); + const auth = new AuthBrokerServer(paths.auth); + const store = new CollectionStore(paths.collection), manager = new CollectionManager(store, { tickMs: 20 }); + const configuration = { paths, layout: { directories: { stateRoot: paths.stateRoot } } }; + const client = Object.assign(Object.create(createManagedRuntimeDispatchClient(configuration)), { connectionsManage: createRuntimeConnections(configuration) }); + const plugins = createRuntimePlugins(configuration, client); + client.system.includePaycom = () => plugins.enabled('paycom'); + const execution = createExecution({ configuration, client, plugins }); + t.after(async () => { await manager.stop(); store.close(); await auth.close(); fs.rmSync(root, { recursive: true, force: true }); }); + await auth.start(); await manager.start(); + const adopted = await execution({ command: 'adopt', scheduledAt: Date.now() }); + assert.equal(adopted.ok, true, JSON.stringify(adopted)); + await new Promise(resolve => setTimeout(resolve, 40)); + const drained = await execution({ command: 'drain', scheduledAt: null }); + assert.equal(drained.data.drained, true, JSON.stringify(drained)); + assert.equal(readStatus(path.join(paths.dataRoot, 'published'), 'connections').value.data.items.length, 2); + auth.serviceConnections.operations.set('cortex', Promise.resolve()); + const busy = await execution({ command: 'drain', scheduledAt: null }); + assert.equal(busy.data.drained, false); assert.equal(busy.data.busy, true); + auth.serviceConnections.operations.clear(); + const spec = require('../../collection-manager/tests/helpers').spec(); + spec.collectors[0].id = 'paycom'; spec.sources[0].collector = 'paycom'; + spec.collectors[0].sourceSchema.properties.timezone = { type: 'string' }; + spec.sources[0].config.timezone = 'UTC'; store.applySpec(spec); + const plugin = require('../../../plugins/paycom/backend/plugin'), originalPublish = plugin.publish; + let releasePublication, enabled = true; + plugin.publish = () => new Promise(resolve => { releasePublication = resolve; }); + t.after(() => { plugin.publish = originalPublish; }); + const publishing = createExecution({ configuration, client, plugins: { + enabled: () => enabled, busy: () => false, manage: () => plugins.manage({ command: 'status' }), + } }); + const building = await publishing({ command: 'snapshot', scheduledAt: null }); + assert.equal(building.data.busy, true, 'a temporary publisher keeps the DSP alive'); + enabled = false; + assert.equal((await publishing({ command: 'drain', scheduledAt: null })).data.drained, false, + 'disabling a plugin cannot interrupt its active publication'); + releasePublication({ changed: 1 }); await new Promise(resolve => setImmediate(resolve)); + assert.equal((await publishing({ command: 'drain', scheduledAt: null })).data.drained, true); + plugin.publish = originalPublish; + const restored = await execution({ command: 'restore', scheduledAt: null }); + assert.equal(restored.status, 'restored'); +}); diff --git a/dsp/runtime/supervisor/tests/paycom-setup.test.js b/dsp/runtime/supervisor/tests/paycom-setup.test.js new file mode 100644 index 0000000..0263df3 --- /dev/null +++ b/dsp/runtime/supervisor/tests/paycom-setup.test.js @@ -0,0 +1,127 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const net = require('node:net'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { createContainerPaycomSetup } = require('../../../plugins/paycom/backend/runtime/setup'); +const { success, failure } = require('dispatch-protocol/contracts/src/result'); +const { + MANAGED_INSTALLATION_LAYOUT_VERSION, MANAGED_INSTALLATION_LAYOUT_TEMPLATE, + MANAGED_INSTALLATION_DIRECTORY_FIELDS, resolveManagedInstallationRuntimePaths, +} = require('dispatch-protocol/paths/runtime-paths'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'paycom-setup-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const runtimeKey = 'runtime_login'; + const installationRoot = path.join(root, runtimeKey); + const layout = { layoutVersion: MANAGED_INSTALLATION_LAYOUT_VERSION, + templateId: MANAGED_INSTALLATION_LAYOUT_TEMPLATE, runtimeKey, + projectRoot: '/opt/dispatch', installationRoot, + directories: Object.fromEntries(Object.entries(MANAGED_INSTALLATION_DIRECTORY_FIELDS) + .map(([key, relative]) => [key, path.join(installationRoot, relative)])), + }; + fs.mkdirSync(layout.directories.stateRoot, { recursive: true, mode: 0o700 }); + const config = { layout, runtimeKey, paths: resolveManagedInstallationRuntimePaths(layout) }; + const manifest = { manifestVersion: 1, revision: 1, + organization: { id: 'org_login', stationCode: 'DWA1', timezone: 'UTC' }, + runtime: { key: runtimeKey, templateId: 'isolated_dsp_v1', releaseId: 'dispatch_v1' } }; + const input = { command: 'start', requestId: `setup_${'a'.repeat(32)}`, step: 'test', manifest, + manifestAuthority: { revision: 1, organization: manifest.organization, runtime: manifest.runtime }, parameters: {} }; + return { root, config, input }; +} + +async function complete(setup, input) { + let result = await setup(input); + for (let i = 0; result.status === 'running' && i < 10; i++) { + await new Promise(resolve => setImmediate(resolve)); + result = await setup({ ...input, command: 'status' }); + } + return result; +} + +test('runtime login check touches authentication only and preserves failures across polling', async t => { + const { config, input } = fixture(t); + const calls = []; + let response = failure('manual_verification_required'); + const unused = new Proxy({}, { get() { assert.fail('Login must not call collection, sync or workforce methods'); } }); + const setup = createContainerPaycomSetup(config, { + auth: { + async profileStatus(profile) { calls.push(['status', profile]); return success('configured', { profile: { configured: true, provider: 'paycom' } }); }, + async testProfile(profile) { calls.push(['test', profile]); return response; }, + health() { assert.fail('Login does not depend on infrastructure health checks'); }, + }, + collections: unused, sync: unused, paycom: unused, + }); + assert.equal((await complete(setup, input)).status, 'manual_verification_required'); + assert.equal((await complete(setup, input)).status, 'manual_verification_required'); + assert.equal(calls.length, 2, 'Polling must not repeat login attempts'); + response = success('authenticated', { profile: 'paycom-main', provider: 'paycom', testedAt: new Date().toISOString() }); + const retry = { ...input, requestId: `setup_${'b'.repeat(32)}` }; + const result = await complete(setup, retry); + assert.equal(result.status, 'succeeded'); + assert.equal(result.data.status, 'authenticated'); + assert.equal(calls.length, 4); + assert.equal(fs.existsSync(config.paths.collection.database), false); +}); + +for (const status of ['profile_not_configured', 'profile_locked']) { + test(`credential resubmission handles ${status} without bypassing other failures`, async t => { + const { config } = fixture(t); + fs.mkdirSync(path.dirname(config.paths.auth.socket), { recursive: true, mode: 0o700 }); + const calls = []; + const server = net.createServer(socket => { + let text = ''; + socket.on('data', chunk => { + text += chunk; + if (!text.includes('\n')) return; + const request = JSON.parse(text); + calls.push(request); + const response = calls.length === 1 ? { ok: false, status } : { ok: true, status: 'configured' }; + socket.end(`${JSON.stringify(response)}\n`); + }); + }); + await new Promise(resolve => server.listen(config.paths.auth.socket, resolve)); + fs.chmodSync(config.paths.auth.socket, 0o600); + t.after(() => new Promise(resolve => server.close(resolve))); + const credentials = { clientCode: 'fixture-code', username: 'fixture-user', password: 'fixture-password-never-persist', + pin1: 'one', pin2: 'two', pin3: 'three', pin4: 'four', pin5: 'five' }; + const setup = createContainerPaycomSetup(config, {}); + const input = { command: 'enroll', requestId: `setup_${'c'.repeat(32)}`, expiresAt: Date.now() + 30_000, intent: 'replace', credentials }; + const result = await setup(input); + assert.equal(result.status, status === 'profile_not_configured' ? 'succeeded' : 'profile_locked'); + assert.deepEqual(calls.map(item => item.intent), status === 'profile_not_configured' ? ['replace', 'create'] : ['replace']); + assert.ok(calls.every(item => item.action === 'enroll-paycom')); + assert.deepEqual(calls[0].credentials, credentials); + await setup(input); + assert.equal(calls.length, status === 'profile_not_configured' ? 2 : 1, 'Request replay must not resubmit credentials'); + const stateRoot = path.join(config.layout.directories.stateRoot, 'paycom-setup'); + for (const name of fs.existsSync(stateRoot) ? fs.readdirSync(stateRoot) : []) assert.equal(fs.readFileSync(path.join(stateRoot, name), 'utf8').includes(credentials.password), false); + }); +} + +test('readiness reads the broker each time without cached receipts, browsers, or collection work', async t => { + const { config, input } = fixture(t); + fs.mkdirSync(path.dirname(config.paths.auth.socket), { recursive: true, mode: 0o700 }); + let calls = 0; + const server = net.createServer(socket => socket.once('data', bytes => { + assert.deepEqual(JSON.parse(bytes), { action: 'profile-readiness', profile: 'paycom-main' }); + calls++; + socket.end(JSON.stringify({ ok: true, status: 'found', readiness: { + state: calls === 1 ? 'manual' : 'ready', retryAllowed: calls !== 1, retryAt: null, + }, lastAuthentication: { privateDiagnostic: 'not-for-the-dashboard' } }) + '\n'); + })); + await new Promise(resolve => server.listen(config.paths.auth.socket, resolve)); + t.after(() => new Promise(resolve => server.close(resolve))); + const setup = createContainerPaycomSetup(config, {}); + const value = { ...input, command: 'status', step: 'readiness' }; + assert.equal((await setup(value)).data.retryAllowed, false); + const next = await setup(value); + assert.equal(next.data.retryAllowed, true); + assert.equal(JSON.stringify(next).includes('not-for-the-dashboard'), false); + const receiptRoot = path.join(config.layout.directories.stateRoot, 'paycom-setup'); + assert.deepEqual(fs.existsSync(receiptRoot) ? fs.readdirSync(receiptRoot) : [], []); +}); diff --git a/dsp/runtime/supervisor/tests/plugins.test.js b/dsp/runtime/supervisor/tests/plugins.test.js new file mode 100644 index 0000000..09a392a --- /dev/null +++ b/dsp/runtime/supervisor/tests/plugins.test.js @@ -0,0 +1,113 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const test = require('node:test'); +const { fixture } = require('../../collection-manager/tests/helpers'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { createRuntimePlugins } = require('../../plugin-host/index'); +const { success } = require('dispatch-protocol/contracts/src/result'); +function configuration(f) { + const path = require('node:path'); + return { paths: { projectRoot: path.resolve(__dirname, '../../..'), + dataRoot: path.join(f.root, 'data'), stateRoot: f.root, + stagingRoot: path.join(f.root, 'staging'), collection: f.paths }, + layout: { directories: { stateRoot: f.root } } }; +} +test('runtime host gates legacy and registered actions, retains explicit uninstall, and retries cleanup', async t => { + const f = fixture(); t.after(() => fs.rmSync(f.root, { recursive: true, force: true })); + let cleanupFails = false; let invokes = 0; let configured = false; let unlocks = 0; + const host = createRuntimePlugins({ paths: { collection: f.paths } }, { + auth: { profileStatus: async () => success('configured', { profile: { configured } }) }, + }, { createStore: () => new CollectionStore(f.paths), load: () => ({ + invoke: async () => { invokes++; return success('found', { value: 1 }); }, + setup: async () => success('succeeded', {}), + enable: async () => { unlocks++; }, + disable: async () => { if (cleanupFails) throw new Error('synthetic cleanup interruption'); }, + }) }); + assert.equal((await host.authorize('workforce.day', { query: {} })).status, 'plugin_disabled'); + assert.equal((await host.invoke({ pluginId: 'paycom', action: 'workforce.day', input: { query: {} } })).status, 'plugin_disabled'); + assert.equal(invokes, 0); + const apply = (state, revision) => host.manage({ command: 'apply', pluginId: 'paycom', version: require('../../../plugins/paycom/dispatch-plugin.json').version, state, revision }); + assert.equal((await apply('enabled', 1)).status, 'applied'); assert.equal(unlocks, 1); + assert.equal(await host.authorize('workforce.day', { query: {} }), null); + assert.equal((await host.invoke({ pluginId: 'paycom', action: 'workforce.day', input: { query: {} } })).ok, true); + cleanupFails = true; + assert.equal((await apply('disabled', 2)).ok, false); + assert.equal((await host.authorize('connections.manage', { command: 'save', service: 'paycom' })).status, 'plugin_disabled'); + assert.equal((await host.authorize('sync.run_now', { id: 'paycom-main-workforce' })).status, 'plugin_disabled'); + assert.equal(await host.authorize('connections.manage', { command: 'test', service: 'cortex' }), null); + cleanupFails = false; assert.equal((await apply('disabled', 2)).ok, true); + assert.equal((await apply('enabled', 1)).status, 'plugin_revision_conflict'); + assert.equal((await apply('uninstalled', 3)).ok, true); + configured = true; + assert.equal((await host.manage({ command: 'status' })).data.items[0].state, 'uninstalled'); + assert.equal(invokes, 1); +}); + +test('credential-only legacy enrollment is adopted without unlocking or changing the profile', async t => { + const f = fixture(); t.after(() => fs.rmSync(f.root, { recursive: true, force: true })); + let unlocks = 0; + const host = createRuntimePlugins({ paths: { collection: f.paths } }, { + auth: { profileStatus: async () => success('configured', { profile: { configured: true } }) }, + }, { createStore: () => new CollectionStore(f.paths), load: () => ({ enable: async () => { unlocks++; } }) }); + assert.deepEqual((await host.manage({ command: 'status' })).data.items[0], { id: 'paycom', version: require('../../../plugins/paycom/dispatch-plugin.json').version, state: 'enabled', revision: 0 }); + await host.manage({ command: 'apply', pluginId: 'paycom', version: require('../../../plugins/paycom/dispatch-plugin.json').version, state: 'enabled', revision: 1 }); + assert.equal(unlocks, 0); +}); + +test('real Paycom lifecycle retains an existing authentication guard through reinstall', async t => { + const f = fixture(); t.after(() => fs.rmSync(f.root, { recursive: true, force: true })); + const path = require('node:path'); + const { AttemptGuard } = require('../../auth-broker/src/attempt-guard'); + const guard = new AttemptGuard(path.join(f.root, 'auth', 'attempts.json')); + guard.lock('paycom-main'); + const before = fs.readFileSync(guard.file); + const profileMutations = []; + const host = createRuntimePlugins(configuration(f), { + auth: { + lockProfile: async profile => { profileMutations.push('lock'); guard.lock(profile); return success('locked', {}); }, + unlockProfile: async profile => { profileMutations.push('unlock'); guard.unlock(profile); return success('unlocked', {}); }, + }, + }); + let revision = 0; + for (const state of ['enabled', 'disabled', 'enabled', 'uninstalled', 'enabled']) { + assert.equal((await host.manage({ command: 'apply', pluginId: 'paycom', version: require('../../../plugins/paycom/dispatch-plugin.json').version, state, revision: ++revision })).status, 'applied'); + assert.deepEqual(fs.readFileSync(guard.file), before); + assert.equal(guard.status('paycom-main'), 'manual_verification_required'); + } + assert.deepEqual(profileMutations, []); +}); + +test('first Paycom installation creates private feature state in a fresh DSP without provider access', async t => { + const f = fixture(); t.after(() => fs.rmSync(f.root, { recursive: true, force: true })); + const path = require('node:path'); + const featureRoot = path.join(f.root, 'plugins', 'paycom'); + assert.equal(fs.existsSync(path.dirname(featureRoot)), false); + const host = createRuntimePlugins(configuration(f), {}); + const request = { command: 'apply', pluginId: 'paycom', version: require('../../../plugins/paycom/dispatch-plugin.json').version, state: 'enabled', revision: 1 }; + assert.equal((await host.manage(request)).status, 'applied'); + assert.equal(host.enabled('paycom'), true); + for (const directory of [path.dirname(featureRoot), featureRoot]) { + assert.equal(fs.statSync(directory).mode & 0o777, 0o700); + assert.equal(fs.statSync(directory).uid, process.geteuid()); + } + assert.deepEqual(fs.readdirSync(featureRoot), []); + assert.equal((await host.manage(request)).status, 'applied'); +}); + +test('Paycom installation rejects unsafe feature parents without enabling the plugin', async t => { + const path = require('node:path'); + for (const kind of ['symlink', 'writable']) { + const f = fixture(); t.after(() => fs.rmSync(f.root, { recursive: true, force: true })); + const parent = path.join(f.root, 'plugins'); + const other = path.join(f.root, 'other'); + fs.mkdirSync(other, { mode: 0o700 }); + if (kind === 'symlink') fs.symlinkSync(other, parent); + else { fs.mkdirSync(parent); fs.chmodSync(parent, 0o777); } + const host = createRuntimePlugins(configuration(f), {}); + assert.equal((await host.manage({ command: 'apply', pluginId: 'paycom', version: require('../../../plugins/paycom/dispatch-plugin.json').version, + state: 'enabled', revision: 1 })).status, 'plugin_unavailable'); + assert.equal(host.enabled('paycom'), false); + assert.deepEqual(fs.readdirSync(other), []); + } +}); diff --git a/dsp/runtime/supervisor/tests/runtime-container.test.js b/dsp/runtime/supervisor/tests/runtime-container.test.js new file mode 100644 index 0000000..83ba1ee --- /dev/null +++ b/dsp/runtime/supervisor/tests/runtime-container.test.js @@ -0,0 +1,135 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const path = require('node:path'); +const test = require('node:test'); +const { + MANAGED_INSTALLATION_LAYOUT_VERSION, + MANAGED_INSTALLATION_LAYOUT_TEMPLATE, + MANAGED_INSTALLATION_DIRECTORY_FIELDS, + managedInstallationRuntimeEnvironment, +} = require('dispatch-protocol/paths/runtime-paths'); +const { + COMPONENTS, + PROJECT_ROOT, + CONTAINER_STORAGE_ROOT, + AGENT_BRIDGE_ROOT, + configuration, + definitions, + assertMountBoundary, +} = require('../src/supervisor'); +const { healthyResponse, checkRuntime } = require('../src/health'); + +function environment() { + const runtimeKey = 'runtime_container_alpha'; + const runtimeRoot = path.join(CONTAINER_STORAGE_ROOT, runtimeKey); + const directories = Object.fromEntries(Object.entries(MANAGED_INSTALLATION_DIRECTORY_FIELDS) + .map(([field, relative]) => [field, path.join(runtimeRoot, relative)])); + return { + DISPATCH_MANAGED_RUNTIME: '1', + ...managedInstallationRuntimeEnvironment({ + layoutVersion: MANAGED_INSTALLATION_LAYOUT_VERSION, + templateId: MANAGED_INSTALLATION_LAYOUT_TEMPLATE, + runtimeKey, + projectRoot: PROJECT_ROOT, + installationRoot: runtimeRoot, + directories, + }), + DISPATCH_RUNTIME_KEY: runtimeKey, + DISPATCH_RUNTIME_GATEWAY_SOCKET: path.join(runtimeRoot, 'run', 'runtime-gateway.sock'), + DISPATCH_RUNTIME_AGENT_HUB_SOCKET: path.join(AGENT_BRIDGE_ROOT, 'runtime-agent-hub.sock'), + DISPATCH_RUNTIME_AGENT_TOKEN_FILE: path.join(runtimeRoot, 'secrets', 'runtime-agent', 'registration-token'), + DISPATCH_RUNTIME_AGENT_STATUS_SOCKET: path.join(runtimeRoot, 'run', 'runtime-agent-status.sock'), + DISPATCH_CHROME_EXECUTABLE: '/usr/bin/chromium', + }; +} + +test('container runtime configuration is fixed, closed, and runtime-bound', () => { + const selected = configuration({ ...environment(), IGNORED_HOST_VALUE: 'not-forwarded' }); + assert.equal(selected.runtimeKey, 'runtime_container_alpha'); + assert.equal(Object.hasOwn(selected.childEnvironment, 'IGNORED_HOST_VALUE'), false); + assert.deepEqual(definitions(selected).map(item => item.id), COMPONENTS.map(item => item.id)); + assert.equal(definitions(selected).every(item => item.executable === process.execPath), true); + assert.equal(definitions(selected).every(item => item.arguments.length === 2), true); + assert.throws(() => configuration({ ...environment(), DISPATCH_LOCAL_ROOT: '/tmp/crossed' }), + error => error.code === 'runtime_boundary_violation'); + assert.throws(() => configuration({ ...environment(), DISPATCH_RUNTIME_KEY: 'local' }), + error => error.code === 'runtime_identity_mismatch'); + assert.throws(() => configuration({ + ...environment(), DISPATCH_RUNTIME_AGENT_HUB_SOCKET: '/run/another/hub.sock', + }), error => error.code === 'runtime_boundary_violation'); +}); + +test('container health accepts only one successful JSON line from all fixed components', () => { + assert.equal(healthyResponse('{"ok":true,"status":"healthy"}\n'), true); + assert.equal(healthyResponse('{"ok":true}\n{"ok":true}\n'), false); + assert.equal(healthyResponse('{"ok":false,"status":"unavailable"}\n'), false); + const invocations = []; + const healthy = checkRuntime({ + environment: environment(), + spawnImpl(executable, args, options) { + invocations.push({ executable, args, options }); + return { status: 0, signal: null, error: null, stdout: '{"ok":true,"status":"healthy"}\n' }; + }, + }); + assert.equal(healthy, true); + assert.deepEqual(invocations.map(item => path.basename(item.args[1])), [ + 'dispatch-auth-brokerctl', 'dispatch-collectionctl', 'dispatch-runtime-gatewayctl', 'dispatch-runtime-agentctl', + ]); + assert.equal(invocations.every(item => item.options.shell === false), true); + assert.equal(checkRuntime({ + environment: environment(), + spawnImpl: () => ({ status: 1, signal: null, error: null, stdout: '{"ok":false}\n' }), + }), false); +}); + +test('container supervisor requires the read-only image and exact writable/private mounts', () => { + const config = configuration(environment()); + const mountInfo = [ + '1 0 0:1 / / ro,relatime - overlay overlay ro', + '2 1 0:2 / /var/lib/dispatch/runtime_container_alpha rw,nosuid,nodev - ext4 state rw', + '3 1 0:3 / /run/dispatch-agent ro,nosuid,nodev,noexec - ext4 bridge ro', + '4 1 0:4 / /tmp rw,nosuid,nodev,noexec - tmpfs tmpfs rw', + ].join('\n'); + const absent = () => false; + assert.equal(assertMountBoundary(config, mountInfo, absent), true); + assert.throws(() => assertMountBoundary(config, mountInfo.replace(' / ro,', ' / rw,'), absent), + error => error.code === 'runtime_boundary_violation'); + assert.throws(() => assertMountBoundary(config, `${mountInfo}\n5 1 0:5 / /opt/dispatch rw - ext4 source rw`, absent), + error => error.code === 'runtime_boundary_violation'); +}); + +test('directory service keeps its backend and trusted browser path in every child', () => { + const { environment: directoryEnvironment } = require('dispatch-core/host/services/service.js'); + const env = directoryEnvironment('dsp_' + 'a'.repeat(32)); + const config = configuration({ ...env, IGNORED_HOST_VALUE: 'not-forwarded' }); + for (const child of definitions(config)) { + assert.equal(child.environment.DISPATCH_RUNTIME_BACKEND, 'directory_service_v1'); + assert.equal(child.environment.DISPATCH_CHROME_EXECUTABLE, '/opt/dispatch/dependencies/browser/chrome'); + assert.equal(child.environment.PATH, '/opt/dispatch-tools:/usr/bin:/bin'); + assert.equal(child.environment.IGNORED_HOST_VALUE, undefined); + } + assert.throws(() => configuration({ ...env, DISPATCH_CHROME_EXECUTABLE: '/tmp/chrome' })); + assert.throws(() => directoryEnvironment('../escape')); +}); + +test('service mounts reject writable source children and executable DSP storage', () => { + const { environment: directoryEnvironment } = require('dispatch-core/host/services/service.js'); + const env = directoryEnvironment('dsp_' + 'b'.repeat(32)); + const config = configuration(env), root = path.dirname(env.DISPATCH_DATA_ROOT); + const mounts = [ + '1 0 0:1 / / ro - ext4 root ro', + `2 1 0:2 / ${root} rw,noexec - ext4 state rw`, + '3 1 0:3 / /run/dispatch-agent ro,noexec - ext4 bridge ro', + '4 1 0:4 / /tmp rw,nosuid,nodev,noexec - tmpfs tmpfs rw', + '5 1 0:5 / /opt/dispatch ro - ext4 code ro', + '6 5 0:6 / /opt/dispatch/protocol ro - ext4 source ro', + `7 2 0:7 / ${root}/data rw,noexec - ext4 data rw`, + ].join('\n'); + assert.equal(assertMountBoundary(config, mounts, () => false), true); + assert.throws(() => assertMountBoundary(config, mounts.replace('/protocol ro', '/protocol rw'), () => false)); + assert.throws(() => assertMountBoundary(config, mounts.replace('/data rw,noexec', '/data rw'), () => false)); + const native = configuration({ ...environment(), DISPATCH_RUNTIME_BACKEND: 'native_service_v1', + DISPATCH_CHROME_EXECUTABLE: '/opt/dispatch/dependencies/browser/chrome' }); + assert.equal(native.childEnvironment.DISPATCH_RUNTIME_BACKEND, 'native_service_v1'); +}); diff --git a/dsp/runtime/workers/README.md b/dsp/runtime/workers/README.md new file mode 100644 index 0000000..f1df3ca --- /dev/null +++ b/dsp/runtime/workers/README.md @@ -0,0 +1,14 @@ +# Temporary authentication workers + +`AuthenticationWorker` opens only the supplied DSP vault, uses the selected +reviewed adapter and preserves existing persistent profiles and authentication +guards. It returns browser access, never credentials. Renewal and shutdown reuse +the current session manager; vault/maintenance locks remain held until browser +cleanup succeeds. + +This is the worker implementation, not yet a process entrypoint or OS launcher. +Tests use disposable encrypted vaults and simulated browser processes. They do +not establish filesystem/process isolation. The host worker mount plan excludes +vaults from ordinary plugin workers and excludes plugin databases from auth +workers. The launcher and real isolation acceptance remain required before +provisioning can use these workers. diff --git a/dsp/runtime/workers/authentication-server.js b/dsp/runtime/workers/authentication-server.js new file mode 100644 index 0000000..b42469b --- /dev/null +++ b/dsp/runtime/workers/authentication-server.js @@ -0,0 +1,62 @@ +'use strict'; +const path = require('node:path'); +const { AuthBrokerServer } = require('../auth-broker/src/server'); +const { ChromeBrowserRuntime } = require('../auth-broker/src/browser-runtime'); +const { verifyPackage } = require('dispatch-protocol/plugin-sdk/package-files'); +const { validateDspId } = require('dispatch-protocol/paths/platform-paths'); +const { createEgressRelay } = require('../supervisor/src/egress-relay'); + +// Every Chrome launch, including owner checks and verification, belongs to this +// one admitted worker. Its namespace contains this DSP's auth storage only. +function singleBrowser(runtime) { + let occupied = null; + return { reconcile: () => runtime.reconcile(), async launch(options) { + if (occupied) throw Object.assign(new Error('session_busy'), { code: 'session_busy' }); + const owner = Symbol('browser'); occupied = owner; + const browser = await runtime.launch(options); + return { ...browser, async close() { await browser.close(); if (occupied === owner) occupied = null; } }; + } }; +} +async function main() { + process.umask(0o077); + if (process.geteuid() === 0) throw new Error('worker_boundary_invalid'); + const input = require('dispatch-protocol/transport/private-file').privateResult('/run/dispatch-plugin/request.json'); + if (Object.keys(input).sort().join(',') !== 'dspId,plugins,schemaVersion' || input.schemaVersion !== 1 || !Array.isArray(input.plugins) + || input.plugins.length > 16) throw new Error('worker_request_invalid'); + validateDspId(input.dspId); + const adapters = { 'amazon-logistics': require('../auth-broker/src/adapters/amazon-logistics').amazonLogisticsAdapter }; + for (const plugin of input.plugins) { + if (Object.keys(plugin).sort().join(',') !== 'digest,id' || !/^[a-z][a-z0-9-]{0,63}$/.test(plugin.id)) throw new Error('worker_package_invalid'); + const root = '/opt/dispatch-auth/plugins/' + plugin.id; + const manifest = verifyPackage(root, plugin.digest); + if (manifest.plugin.id !== plugin.id) throw new Error('worker_package_invalid'); + const declared = path.join(root, 'backend/authentication.js'); + if (manifest.files.some(file => file.path === 'backend/authentication.js')) { + const values = require(declared); + for (const adapter of Object.values(values)) if (adapter?.provider) { + if (adapters[adapter.provider]) throw new Error('authentication_adapter_conflict'); + adapters[adapter.provider] = adapter; + } + } + } + const root = `/var/lib/dispatch/${input.dspId}`; + const paths = { projectRoot: '/opt/dispatch', databaseRoot: root + '/data/auth-broker', secretRoot: root + '/secrets/auth-broker', + stateRoot: root + '/state/auth-broker', runtimeRoot: root + '/run', socket: root + '/run/auth.sock', + database: root + '/data/auth-broker/credentials.sqlite3', key: root + '/secrets/auth-broker/master.key', + browserSessions: root + '/state/auth-broker/browser-sessions', attempts: root + '/state/auth-broker/authentication-attempts.json' }; + const relay = createEgressRelay({ socketPath: paths.runtimeRoot + '/egress.sock' }); + await relay.start(); + const server = new AuthBrokerServer(paths, { adapters, + browserRuntime: singleBrowser(new ChromeBrowserRuntime({ stateRoot: paths.browserSessions, socketRoot: paths.runtimeRoot })), + assistancePermitted: id => input.plugins.some(plugin => plugin.id === id), + browserAssistance: options => require('../auth-broker/src/browser-assistance').assistBrowser({ ...options, + runtimeRoot: paths.runtimeRoot, socketPath: paths.runtimeRoot + '/browser-assist.sock' }), + }); + let closing; + const close = () => closing ||= server.close().finally(() => relay.close()); + try { await server.start(); } catch (error) { await close(); throw error; } + process.once('SIGTERM', () => close().catch(() => { process.exitCode = 1; })); + process.once('SIGINT', () => close().catch(() => { process.exitCode = 1; })); +} +if (require.main === module) main().catch(() => { process.exitCode = 1; }); +module.exports = { main, singleBrowser }; diff --git a/dsp/runtime/workers/authentication.js b/dsp/runtime/workers/authentication.js new file mode 100644 index 0000000..4feb167 --- /dev/null +++ b/dsp/runtime/workers/authentication.js @@ -0,0 +1,75 @@ +'use strict'; +const path = require('node:path'); +const { CredentialVault, ensurePrivateDirectory } = require('../auth-broker/src/vault'); +const { BrowserSessionManager } = require('../auth-broker/src/session-manager'); +const { ChromeBrowserRuntime } = require('../auth-broker/src/browser-runtime'); +const { AttemptGuard } = require('../auth-broker/src/attempt-guard'); +const { AuthenticationDiagnostics } = require('../auth-broker/src/authentication-diagnostics'); +const { acquireMaintenanceLock } = require('../auth-broker/src/maintenance-lock'); + +// This class runs inside the DSP authentication worker. Core passes connection +// references over the worker transport; it never imports or instantiates this +// class in its own process. The host supplies paths and the reviewed adapter. +class AuthenticationWorker { + constructor({ paths, adapter, profile, pluginId, jobId, ttlMs = 90000, browserRuntime = null, assistance = null }) { + if (typeof adapter?.authenticate !== 'function' || !/^[a-z][a-z0-9_-]{0,47}$/.test(profile) + || !/^[a-z][a-z0-9_.-]{0,63}$/.test(pluginId) || !/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(jobId) + || !Number.isInteger(ttlMs) || ttlMs < 30000 || ttlMs > 300000) throw new TypeError('authentication_worker_invalid'); + Object.assign(this, { paths, adapter, profile, pluginId, jobId, ttlMs, browserRuntime, assistance }); + this.vault = null; this.sessions = null; this.session = null; this.releaseLock = null; + this.controller = new AbortController(); this.pending = null; this.closing = null; this.closed = false; + } + async start({ signal } = {}) { + if (this.pending || this.sessions || this.closed) throw new Error('authentication_worker_busy'); + const cancel = () => this.controller.abort(); + signal?.addEventListener('abort', cancel, { once: true }); + if (signal?.aborted) cancel(); + this.pending = (async () => { + if (this.controller.signal.aborted) throw new Error('acquisition_cancelled'); + ensurePrivateDirectory(this.paths.stateRoot); + this.releaseLock = acquireMaintenanceLock(this.paths); + this.vault = new CredentialVault(this.paths, { readOnly: true }); + const metadata = this.vault.status(this.profile); + if (!metadata.configured || metadata.provider !== this.adapter.provider) throw new Error('profile_not_configured'); + const runtime = this.browserRuntime || new ChromeBrowserRuntime({ stateRoot: this.paths.browserSessions }); + if (runtime.reconcile) await runtime.reconcile(); + this.sessions = new BrowserSessionManager({ vault: this.vault, browserRuntime: runtime, + adapters: { [this.adapter.provider]: this.adapter }, attemptGuard: new AttemptGuard(this.paths.attempts), + diagnostics: new AuthenticationDiagnostics(path.join(this.paths.stateRoot, 'authentication-diagnostics.json')), + browserAssistance: this.assistance, assistancePermitted: id => id === this.pluginId }); + this.session = await this.sessions.acquire({ profile: this.profile, collector: this.pluginId, + runId: this.jobId, ttlSeconds: Math.ceil(this.ttlMs / 1000) }, { signal: this.controller.signal }); + if (this.controller.signal.aborted) throw new Error('acquisition_cancelled'); + return { ...this.session.browser }; + })(); + try { return await this.pending; } + catch (error) { + this.pending = null; + await this.close(); + throw error; + } finally { this.pending = null; signal?.removeEventListener('abort', cancel); } + } + renew() { + if (!this.session || this.closed || this.controller.signal.aborted) throw new Error('lease_lost'); + this.sessions.renew(this.session.lease, Math.ceil(this.ttlMs / 1000)); + return { renewed: true }; + } + async close() { + if (this.closing) return this.closing; + this.controller.abort(); this.closed = true; + this.closing = (async () => { + if (this.pending) await this.pending.catch(() => {}); + if (this.sessions) { + await this.sessions.close(); + for (const lease of this.sessions.sessions.keys()) await this.sessions.release(lease, 'revoked'); + if (this.sessions.sessions.size) throw new Error('browser_cleanup_failed'); + } + this.sessions = null; this.session = null; + this.vault?.close(); this.vault = null; + this.releaseLock?.(); this.releaseLock = null; + return true; + })().finally(() => { this.closing = null; }); + return this.closing; + } +} +module.exports = { AuthenticationWorker }; diff --git a/dsp/runtime/workers/plugin.js b/dsp/runtime/workers/plugin.js new file mode 100644 index 0000000..bdcb8d5 --- /dev/null +++ b/dsp/runtime/workers/plugin.js @@ -0,0 +1,41 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { boundedJson, MAX_INPUT_BYTES, MAX_RESULT_BYTES } = require('dispatch-sdk/protocol'); +const { createWorkerClient } = require('dispatch-sdk/node'); +const { verifyPackage } = require('dispatch-protocol/plugin-sdk/package-files'); + +async function main() { + process.umask(0o077); + if (process.geteuid() === 0) throw new Error('worker_boundary_invalid'); + const input = boundedJson(require('dispatch-protocol/transport/private-file').privateResult('/run/dispatch-plugin/request.json'), MAX_INPUT_BYTES); + if (input.schemaVersion !== 1 || !['initialize', 'invoke', 'collect', 'publish', 'read', 'inspect', 'evidence'].includes(input.kind) + || Object.keys(input).sort().join(',') !== 'action,digest,input,kind,pluginId,schemaVersion,timezone') throw new Error('worker_request_invalid'); + const manifest = verifyPackage('/opt/dispatch-plugin', input.digest); + if (manifest.plugin.id !== input.pluginId || !manifest.plugin.runtime) throw new Error('worker_package_invalid'); + const dispatch = createWorkerClient(); + const implementation = require(path.join('/opt/dispatch-plugin', manifest.plugin.runtime)); + const controller = new AbortController(); + const cancel = () => controller.abort(); + process.once('SIGTERM', cancel); process.once('SIGINT', cancel); + let value; + if (input.kind === 'invoke') { + const operation = manifest.plugin.actions.find(item => item.id === input.action); + if (!operation) throw new Error('plugin_action_denied'); + const { operationInput, operationOutput } = require('dispatch-sdk/operations'); + value = await implementation.createPlugin({ dispatch }).invoke(input.action, operationInput(operation, input.input), { signal: controller.signal }); + if (value?.ok === true) operationOutput(operation, value.data); + } else { + if (typeof implementation[input.kind] !== 'function') throw new Error('plugin_entrypoint_invalid'); + value = await implementation[input.kind]({ dispatch, request: input.input, timezone: input.timezone, signal: controller.signal }); + } + if (controller.signal.aborted) throw new Error('cancelled'); + const bytes = JSON.stringify({ ok: true, value: boundedJson(value, MAX_RESULT_BYTES - 128) }); + fs.writeFileSync('/run/dispatch-plugin/result.json', bytes, { mode: 0o600, flag: 'wx' }); +} +if (require.main === module) main().catch(error => { + const code = /^[a-z][a-z0-9_]{0,79}$/.test(error.code || '') ? error.code : 'plugin_worker_failed'; + try { fs.writeFileSync('/run/dispatch-plugin/result.json', JSON.stringify({ ok: false, code }), { mode: 0o600, flag: 'wx' }); } catch {} + process.exitCode = 1; +}); +module.exports = { main }; diff --git a/dsp/runtime/workers/tests/authentication.test.js b/dsp/runtime/workers/tests/authentication.test.js new file mode 100644 index 0000000..dfb5174 --- /dev/null +++ b/dsp/runtime/workers/tests/authentication.test.js @@ -0,0 +1,50 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { AuthenticationWorker } = require('../authentication'); +const { CredentialVault } = require('../../auth-broker/src/vault'); +const { defaultPaths } = require('../../auth-broker/src/paths'); +const { AttemptGuard } = require('../../auth-broker/src/attempt-guard'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-worker-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const paths = defaultPaths({ databaseRoot: path.join(root, 'db'), secretRoot: path.join(root, 'keys'), + stateRoot: path.join(root, 'state'), runtimeRoot: path.join(root, 'run') }); + const vault = new CredentialVault(paths); + vault.put('site-main', 'basic', { username: 'fixture-user', password: 'fixture-password' }); vault.close(); + let closed = 0; + const browserRuntime = { launch: async () => ({ endpoint: 'http://127.0.0.1:9500', close: async () => { closed++; } }) }; + return { paths, browserRuntime, closed: () => closed }; +} +test('temporary auth worker opens the DSP vault and returns only browser access', async t => { + const { paths, browserRuntime, closed } = fixture(t); + const before = fs.readFileSync(paths.database), key = fs.readFileSync(paths.key); + const worker = new AuthenticationWorker({ paths, browserRuntime, profile: 'site-main', pluginId: 'sample', jobId: 'job_1', + adapter: { provider: 'basic', authenticate: async (_, credentials) => { + assert.equal(credentials.password, 'fixture-password'); return { status: 'authenticated' }; + } } }); + try { + const lease = await worker.start(); + assert.deepEqual(Object.keys(lease).sort(), ['access', 'endpoint', 'protocol']); + assert.equal(JSON.stringify(lease).includes('fixture-password'), false); + assert.deepEqual(worker.renew(), { renewed: true }); + } finally { await worker.close(); } + assert.equal(closed(), 1); + assert.deepEqual(fs.readFileSync(paths.database), before); + assert.deepEqual(fs.readFileSync(paths.key), key); + assert.equal(worker.vault, null); +}); +test('existing verification guard blocks login without changing credentials or opening a browser', async t => { + const { paths, browserRuntime, closed } = fixture(t); + fs.mkdirSync(paths.stateRoot, { mode: 0o700 }); + new AttemptGuard(paths.attempts).lock('site-main'); + const worker = new AuthenticationWorker({ paths, browserRuntime, profile: 'site-main', pluginId: 'sample', jobId: 'job_1', + adapter: { provider: 'basic', authenticate: async () => assert.fail('guard was bypassed') } }); + await assert.rejects(worker.start(), { code: 'manual_verification_required' }); + assert.equal(closed(), 0); + assert.equal(new AttemptGuard(paths.attempts).status('site-main'), 'manual_verification_required'); +}); diff --git a/dsp/runtime/workers/tests/single-browser.test.js b/dsp/runtime/workers/tests/single-browser.test.js new file mode 100644 index 0000000..b5471b8 --- /dev/null +++ b/dsp/runtime/workers/tests/single-browser.test.js @@ -0,0 +1,14 @@ +'use strict'; +const test=require('node:test'),assert=require('node:assert/strict'); +const {singleBrowser}=require('../authentication-server'); +test('a worker never frees browser capacity after an uncertain launch or an old handle closes twice', async () => { + const runtime = singleBrowser({ reconcile() {}, async launch() { return { async close() {} }; } }); + const first = await runtime.launch({}); + await assert.rejects(runtime.launch({}), { code: 'session_busy' }); + await first.close(); const second = await runtime.launch({}); + await first.close(); await assert.rejects(runtime.launch({}), { code: 'session_busy' }); + await second.close(); await (await runtime.launch({})).close(); + const failed = singleBrowser({ reconcile() {}, async launch() { throw new Error('uncertain_cleanup'); } }); + await assert.rejects(failed.launch({}), /uncertain_cleanup/); + await assert.rejects(failed.launch({}), { code: 'session_busy' }); +}); diff --git a/dsp/tooling/check-public-tree.py b/dsp/tooling/check-public-tree.py new file mode 100644 index 0000000..03d6fbe --- /dev/null +++ b/dsp/tooling/check-public-tree.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Check every entry, including ignored/hidden files, without printing contents.""" +import argparse +import hashlib +import json +import os +import pathlib +import re +import stat + +def inspect(root, policy): + root = root.resolve() + findings = [] + inventory = [] + forbidden = {'.git', 'node_modules', '__pycache__', '.env', '.npmrc', '.ssh'} + if policy.get('profile') != 'repository': forbidden.add('.github') + private_extensions = {'.sqlite', '.sqlite3', '.db', '.log', '.har', '.pem', '.key', '.p12', '.pfx'} + patterns = [(label, re.compile(expression, re.I)) for label, expression in policy.get('patterns', {}).items()] + literal_patterns = [(f'private-term-{i}', re.compile(re.escape(value), re.I)) for i, value in enumerate(policy.get('privateTerms', []))] + patterns += literal_patterns + patterns += [ + ('private-key', re.compile(r'-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----')), + ('github-token', re.compile(r'\b(?:gh[pousr]_[A-Za-z0-9]{30,}|github_pat_[A-Za-z0-9_]{50,})\b')), + ] + allowed_binary = policy.get('allowedBinaries', {}) + for base, dirs, files in os.walk(root, followlinks=False): + for name in sorted(dirs + files): + file = pathlib.Path(base, name) + relative = file.relative_to(root).as_posix() + info = file.lstat() + def finding(reason): findings.append({'path': relative, 'rule': reason}) + if file.is_symlink(): + finding('symlink'); continue + if name in forbidden or file.suffix.lower() in private_extensions: + finding('private-or-generated-entry') + for label, expression in patterns: + if expression.search(relative): finding(label + '-filename') + if stat.S_ISDIR(info.st_mode): continue + if not stat.S_ISREG(info.st_mode): + finding('special-file'); continue + if info.st_nlink != 1: finding('hardlink') + content = file.read_bytes() + digest = hashlib.sha256(content).hexdigest() + inventory.append({'path': relative, 'bytes': len(content), 'sha256': digest}) + try: text = content.decode('utf-8') + except UnicodeDecodeError: + if allowed_binary.get(relative) != digest: finding('unreviewed-binary') + continue + for label, expression in patterns: + if expression.search(text): finding(label) + return {'ok': not findings, 'files': len(inventory), 'findings': findings, 'inventory': inventory, + 'scope': 'Full tree scan with private policy; complements manual review and secret scanning.'} + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('root', type=pathlib.Path) + parser.add_argument('--policy', required=True, type=pathlib.Path) + parser.add_argument('--report', required=True, type=pathlib.Path) + args = parser.parse_args() + root = args.root.resolve() + if args.policy.resolve().is_relative_to(root) or args.report.resolve().is_relative_to(root): + parser.error('Keep the private policy and report outside the public tree') + result = inspect(root, json.loads(args.policy.read_text())) + args.report.write_text(json.dumps(result, indent=2) + '\n') + print(json.dumps({key: result[key] for key in ['ok', 'files', 'findings']})) + raise SystemExit(0 if result['ok'] else 1) diff --git a/dsp/tooling/ci-browser-smoke.js b/dsp/tooling/ci-browser-smoke.js new file mode 100644 index 0000000..d35424c --- /dev/null +++ b/dsp/tooling/ci-browser-smoke.js @@ -0,0 +1,35 @@ +'use strict'; + +// A blank-page fixture only: never use this diagnostic with provider credentials. +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); +const { ChromeBrowserRuntime } = require('../runtime/auth-broker/src/browser-runtime'); + +async function main() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-ci-browser-')); + fs.chmodSync(root, 0o700); + let diagnostics = '', browser; + // This checks tool availability on a cold hosted runner, not startup latency. + // Production and the real browser tests retain their normal startup limits. + const runtime = new ChromeBrowserRuntime({ stateRoot: path.join(root, 'profiles'), directoryNetwork: false, startTimeoutMs: 60000, + spawnImpl(command, args, options) { + const stdio = [...options.stdio];stdio[2] = 'pipe'; + const child = spawn(command, args, { ...options, stdio }); + child.stderr.on('data', block => { if (diagnostics.length < 16000) diagnostics += block.toString().slice(0, 16000 - diagnostics.length); }); + return child; + } }); + try { + browser = await runtime.launch({ provider: 'paycom', profile: 'fixture', nativeInput: true }); + console.log('Blank native Chrome window started and will be cleaned up'); + } catch (error) { + console.error(diagnostics); + throw error; + } finally { + await browser?.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(error => { console.error(error.message); process.exitCode = 1; }); diff --git a/dsp/tooling/fetch-platform.py b/dsp/tooling/fetch-platform.py new file mode 100644 index 0000000..47fa105 --- /dev/null +++ b/dsp/tooling/fetch-platform.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Download only a configured, digest-pinned public dependency bundle.""" +import hashlib,json,os,pathlib,re,sys,tarfile,urllib.parse,urllib.request + +def main(): + config=json.loads(pathlib.Path(__file__).with_name('platform-dependencies.json').read_text()) + url=os.environ.get('DISPATCH_PLATFORM_PACKAGES_URL') or config['url'] + digest=os.environ.get('DISPATCH_PLATFORM_PACKAGES_SHA256') or config['sha256'] + if not url or not digest or not re.fullmatch('[a-f0-9]{64}',digest):raise ValueError('Configure the Core platform package URL and SHA-256 first') + if urllib.parse.urlparse(url).scheme!='https':raise ValueError('HTTPS required') + target=pathlib.Path(sys.argv[1]).resolve() + if target.exists():raise ValueError('Output must be a new directory') + target.mkdir(parents=True,mode=0o700) + archive=target.with_suffix('.tar.gz') + with urllib.request.urlopen(url,timeout=60) as response,archive.open('xb') as output: + total=0;actual=hashlib.sha256() + while block:=response.read(1024*1024): + total+=len(block) + if total>128*1024*1024:raise ValueError('Bundle too large') + output.write(block);actual.update(block) + if actual.hexdigest()!=digest:raise ValueError('Bundle digest mismatch') + with tarfile.open(archive) as handle: + members=handle.getmembers() + if len(members)>10000 or sum(item.size for item in members)>256*1024*1024:raise ValueError('Bundle too large') + for member in members: + name=pathlib.PurePosixPath(member.name) + if name.is_absolute() or '..' in name.parts or not(member.isfile() or member.isdir()):raise ValueError('Invalid bundle entry') + for member in members: + # Explicitly reject links and paths above before using the portable extractor. + handle.extract(member,target,filter='data') + if not(target/'install.cjs').is_file():raise ValueError('Bundle installer missing') + print(json.dumps({'ok':True,'sha256':digest})) +if __name__=='__main__':main() diff --git a/dsp/tooling/frontend/env.d.ts b/dsp/tooling/frontend/env.d.ts new file mode 100644 index 0000000..cbe652d --- /dev/null +++ b/dsp/tooling/frontend/env.d.ts @@ -0,0 +1 @@ +declare module "*.css"; diff --git a/dsp/tooling/frontend/package-lock.json b/dsp/tooling/frontend/package-lock.json new file mode 100644 index 0000000..dace080 --- /dev/null +++ b/dsp/tooling/frontend/package-lock.json @@ -0,0 +1,3689 @@ +{ + "name": "dispatch-dsp-build-tools", + "version": "0.4.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "dispatch-dsp-build-tools", + "version": "0.4.0", + "dependencies": { + "@fontsource-variable/inter": "^5.3.0", + "@tanstack/react-query": "^5.102.8", + "class-variance-authority": "^0.7.1", + "clsx": "^2.1.1", + "get-nonce": "^1.0.1", + "lucide-react": "^1.41.0", + "radix-ui": "^1.6.7", + "react": "^19.2.8", + "react-dom": "^19.2.8", + "tailwind-merge": "^3.6.0", + "tw-animate-css": "^1.4.0" + }, + "devDependencies": { + "@playwright/test": "^1.63.0", + "@tailwindcss/vite": "^4.3.3", + "@types/node": "^26.4.1", + "@types/react": "^19.2.18", + "@types/react-dom": "^19.2.7", + "@vitejs/plugin-react": "^6.1.1", + "prettier": "^3.9.6", + "tailwindcss": "^4.3.3", + "typescript": "^7.0.2", + "vite": "^8.2.2" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@floating-ui/core": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", + "integrity": "sha512-0CIZ5itps/8x7BG8dEIhs53BvCUH2PCoogtakwRTut+Arm58sJooJ0AuZhLw2HJYIR5cMLNPBSS728sPho2khQ==", + "license": "MIT", + "dependencies": { + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/dom": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.8.0.tgz", + "integrity": "sha512-yXSrzeHZBTZadLOlfyhCkJHNeLJnHRnRInwdZ40L7ZiaAtrBwoYlsDrX3v5zB1Utk7CLfzcOVnVVWoXEky7Ceg==", + "license": "MIT", + "dependencies": { + "@floating-ui/core": "^1.8.0", + "@floating-ui/utils": "^0.2.12" + } + }, + "node_modules/@floating-ui/react-dom": { + "version": "2.1.9", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.9.tgz", + "integrity": "sha512-JDjEFGCpImxDCA7JJKviA0M9+RtmJdj0m/NVU5IMgBK+AmZouAQQ7/+2GLH0GXXY0YMw9oXPB8hKdbPYg5QLYg==", + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.8.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@floating-ui/utils": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.12.tgz", + "integrity": "sha512-HpCo8tmWzLVad5s2d19EhAz5zqrrQ6s69qd6moPMQvkOuSwDT1YgRfWSVuc4ennqrgv3OHppiOGMQ7oC13yIww==", + "license": "MIT" + }, + "node_modules/@fontsource-variable/inter": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/@fontsource-variable/inter/-/inter-5.3.0.tgz", + "integrity": "sha512-OupL48va4JNofb97w6NYeF9S7W/kHNKM0Er8Dem5nqi4jeOLrVJDoE8tZEpnMJmtkvNbB1EIPPwHcdkF6b1oUA==", + "license": "OFL-1.1" + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.148.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.148.0.tgz", + "integrity": "sha512-Nm4s/jB+4FpFsPhWGEC4h7rzksesmtnMXomo6rCMcg/b8zLQuOziRgkCS1fxDCXOlJB/6Q8oABOZ/OP6RIPj9A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@playwright/test": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz", + "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@radix-ui/number": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/number/-/number-1.1.3.tgz", + "integrity": "sha512-Road2bidD0uu/1BGDOWNdPI06g0lIRy6IF9GZcIrDK2KGItfor8IQwQa+yM2ERgHM1MmHxaxpTzk0/Jp42lNfA==", + "license": "MIT" + }, + "node_modules/@radix-ui/primitive": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.7.tgz", + "integrity": "sha512-rqWnm76nYT8HoNNqEjpgJ7Pw/DrBj5iBTrmEPo6HTX5+VJyBNOqTdv4g89G63HuR5g0AaENoAcH7Is5fF2kZ8Q==", + "license": "MIT" + }, + "node_modules/@radix-ui/react-accessible-icon": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accessible-icon/-/react-accessible-icon-1.1.15.tgz", + "integrity": "sha512-WTQwcAvQf5sOcuUyi90lKPbhwcvQ+j55cjrSmeaN+L2vKU3DooOvlKw2MDeiJ5IkV5N905KW0/fGojKOBhD11A==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-accordion": { + "version": "1.2.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-accordion/-/react-accordion-1.2.20.tgz", + "integrity": "sha512-jDhG9FvAEnlhnjrsINbNXcUa4G+L1KqSkJSunkbKEzFRcAb52jvM0PjPxPRvhe1HNc5F5yc0yzzWeeqlH4yBIg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-alert-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-alert-dialog/-/react-alert-dialog-1.1.23.tgz", + "integrity": "sha512-VAYOiQRqj3GPpYJE0I9J+X8Ip05cyVlNdKOFeiGS2Ou1HHGfpl0BxOyZm6nmVDyU+W+NF3/XLzmjHmVGydhwgA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-arrow": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.15.tgz", + "integrity": "sha512-v4zggRcjadnI+ClKDuijlQEW4tw3NoaeHc/PwpKnLoLLKNUG4InLegkstooLcRIUWCs+8L22dGURCVuFfOKfnA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-aspect-ratio": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-aspect-ratio/-/react-aspect-ratio-1.1.15.tgz", + "integrity": "sha512-fy+dyVR+90nelK8rqIznFlxzx7uPcGbhxH8Nfr2bHb4UfSe+e3hklOC0luK0hDwVwnRX7xTRySpsrQVeW+/oNQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-avatar": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-avatar/-/react-avatar-1.2.6.tgz", + "integrity": "sha512-4ULOTJ/mqy2hT9GlWa/MFHxHSvH3nJzHnZM1waNsc5Bonv7i70aNenghXmD97S6OJ81ekXONGGt4nT1r0PfEdA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-checkbox": { + "version": "1.3.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-checkbox/-/react-checkbox-1.3.11.tgz", + "integrity": "sha512-Gnptr9pDDQxD3hgq2dtPbtrp/c2qH1mBwIzw3X/ivrMb2e1t0jMTi606fVEqFPaQR1ggXIVQWKj3P2WW9v7zGQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collapsible": { + "version": "1.1.20", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collapsible/-/react-collapsible-1.1.20.tgz", + "integrity": "sha512-mcGesGplBnzN2sbvJETzpCNfSMyPnb29q1GRLU+Ib7bJrpIG2ywmRoh2V5VbA2uNvKikKUlVbAPks7JDjz4A8Q==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collection": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.15.tgz", + "integrity": "sha512-9W+B9NPF0NaaPh/1NJd3+KqsnlLqU9H7T2rvww+fp+T/evVXdNAyYcnfRQZFOjkR1ajQp3yORlqnI8soawLvNA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-compose-refs": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.5.tgz", + "integrity": "sha512-+48PbAAbq3didjJxa+OaWY2ZwgAKsNiRGyeHKszblZMQ+kcpd9pAaT11cMkGEie0vsOi3QdeTE6d5Fe3Gn61kA==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.2.2.tgz", + "integrity": "sha512-RHCUGwKHDr0hDGg4X7ma4JG4/+12qxw8rkh5QKdDldlCvtja6nUx1Ef/8HVrJze81lEsgLQlqjzjGNHantgnQA==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context-menu": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context-menu/-/react-context-menu-2.3.7.tgz", + "integrity": "sha512-CtXP35dxaB5T3zXSd+E3uHe/QpXcpYnZmxp6OaIbfthtfW4wyb77M23BG+bwIJDtsMwEP/YssdsmNyZu7jhWew==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dialog": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.1.23.tgz", + "integrity": "sha512-Ksw4WeROkO4rC9k/onilX/Ao2Cr1ku1unMNH+XSCcP4jSXYu7HDsg9n4ojMjVb22XpYjAQ9qfrFlVbru1vXDUA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-direction": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.4.tgz", + "integrity": "sha512-5pzg4FGQNpExhnhT2zlrP1wZFaYCd1K0nYWoFAdcYoYK868IEigqMX3B3f8yIoRlAhAeDWciLI6ZdCKHF9P4Vg==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dismissable-layer": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.19.tgz", + "integrity": "sha512-8g4pfOL9HoKKLWGiypT+dphVqjFfmcXO5GBnhsG6zI+lxAx/8feQpr+1LSN8Re3hiZ+XkLNS4O9ztK11/LzQ6w==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-effect-event": "0.0.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dropdown-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dropdown-menu/-/react-dropdown-menu-2.1.24.tgz", + "integrity": "sha512-geq8l2rJkxvkXsT9RMgtUE3P8pITFpTsvYpbySi1IH4fZEABD/Gp85myayFgxk0ktljGMJnCbeFkyTusvSvv7g==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-guards": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.6.tgz", + "integrity": "sha512-RNOJjfZMTyBM6xYmV3IVGXkPjIhcBAuv48POevAXwrGJhkWZ9p1rFoIS1JFooPuT193AZmRsCPhpoVJxx6OPoQ==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-scope": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.16.tgz", + "integrity": "sha512-wmRZ2WWLvmt6KHy2rNPOdPUjwq5xOHY02+m+udwJTn0aNIox/rkskAvJTyTLGhPK6KgrUjlJUJpgmx/+wFiFIQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-form": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-form/-/react-form-0.1.16.tgz", + "integrity": "sha512-Q4TLEn2A7TAypxwmd6R9EwrlXDvkfYSDMrq9/887AXAGh+G1rH+kYJKSTv+Si9Y0JPKTwKYv6PviAJosysNimA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-hover-card": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-hover-card/-/react-hover-card-1.1.23.tgz", + "integrity": "sha512-H8qONfZd3ltrU3+jHCIgITbWo6e1iTKvP9DHdrvYbX48ooRM5FjEDTn16AMwdfuOGkWdZEhpl3PLL/Wk/AnHDQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-id": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.4.tgz", + "integrity": "sha512-TMQp2llA+RYn7JcjnrMnz7wN4pcVttPZnRZo52PLQsoLVKzNlVwUeHmfePgTgRluXFvlD3GD5g5MOVVTJCO0qA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-label": { + "version": "2.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-label/-/react-label-2.1.15.tgz", + "integrity": "sha512-o/rdYEwZTTo5tjknnPeyQFU45kUC4i/XyeDPP+HGyi6XqpOP6Zf5Ya5vh/Yfe9Id5JiuWnnAx2XqIeD3UYZt0g==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menu": { + "version": "2.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menu/-/react-menu-2.1.24.tgz", + "integrity": "sha512-uW7RVuU6Lp/ZtfeY4b3kL32zccgEWvPv1+cf17ubYzHa9cL8AHokmk36cG/XEiH/smbQvumnieXX9j/e9RqJWA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menubar": { + "version": "1.1.24", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menubar/-/react-menubar-1.1.24.tgz", + "integrity": "sha512-eeVs0vf7cuqXaM0qLQCPcufImiJNVBXdJDLu7ZGYl2732UH23Qat/foNGrr6vYV3/DdTsBqASoggUFgH14OcZA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-navigation-menu": { + "version": "1.2.22", + "resolved": "https://registry.npmjs.org/@radix-ui/react-navigation-menu/-/react-navigation-menu-1.2.22.tgz", + "integrity": "sha512-ou7iLEJ+yrhQndkkA4U21XIdS/CS45F4iXIkTZcb6/Ne9EMsOuDudVmCwmDnfFZZ+y1FZqXRNSIgBy+YMvZVZg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-one-time-password-field": { + "version": "0.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-one-time-password-field/-/react-one-time-password-field-0.1.16.tgz", + "integrity": "sha512-Tj9P6ntAJEw52oq/F0AGknXR4XncxEt7XU47O3xJQOiWfLzEy3d9gtgKfvjSzGxzHkfL+VzvxGu2KTFsloJqXw==", + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-password-toggle-field": { + "version": "0.1.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-password-toggle-field/-/react-password-toggle-field-0.1.11.tgz", + "integrity": "sha512-4gvFnmDXu3dgj21CqsufzIameRvlRd4SBqaWhcrlrNhRo0Y5i/49AmRJYe1fdAM3G2VNBbmin4b0D6cdQocwgw==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-is-hydrated": "0.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popover": { + "version": "1.1.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popover/-/react-popover-1.1.23.tgz", + "integrity": "sha512-mw58MrBlyHWFisTOYignD0vf/3gdcgAR+9of1s9G/38CbFiUwH1nCDkc0AUM9IrXFgN5Ue8n45j9WCgyM1sbiQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popper": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.3.7.tgz", + "integrity": "sha512-UsJrrd7w4wuKKTdvd/DNERVlwSlUcyXzjhyDwBk+3aPOsCjOY6ZSbxuw8E6lZTjjfP8Cpd0J8VVkrYUWyGYXyg==", + "license": "MIT", + "dependencies": { + "@floating-ui/react-dom": "^2.0.0", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-rect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-portal": { + "version": "1.1.17", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.17.tgz", + "integrity": "sha512-vKQLcWypUnwZVvfV7UkGahH2g6ySe8M8R+zYBwPrv5byZ9QAW6cQVvNKo7GgmD+p8aYb6D9JBuvy8/WhOno2wQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-presence": { + "version": "1.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.10.tgz", + "integrity": "sha512-3wyzCQ6+ubRA+D4uv9m95JYLXxmOHp05qjrkjeA7uKHHtjpPggQzc6DAb0URl7j67oR0K2foO4ip27TiX037Bw==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-primitive": { + "version": "2.1.10", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.10.tgz", + "integrity": "sha512-MucOnzh6hR5mid6VpkbglRAMYMjKLqRnGBbjXkzjK52fuQDd1qbkx78a5P40mkcnVXJdEVxm26E9OPAiUq7nBg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.3.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-progress": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-progress/-/react-progress-1.1.16.tgz", + "integrity": "sha512-5XnomAsoZZCY+KNTxbIghpGqPruZvKFNlvcAljVAOdDRDsH4/OZQxhtwo5wdtoDM5R6MhJBb2sPnDuRFep3lzg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-radio-group": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-radio-group/-/react-radio-group-1.4.7.tgz", + "integrity": "sha512-cgYFEkntCxppHZgtSZ+7vh0wbZQ+IC7PPMw8DSnRG27B6kDd32/Zw0OJt7dGDigCoprMuWHjg2PvUn3PYvPFoQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-roving-focus": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.19.tgz", + "integrity": "sha512-V9jI6hDjT7l3jsCQD9bLNvDLM3tH/gdbOTp7Tefp3hbbgCGQoK7tUvrWiRlcoBHIZ809ElXwNQwVo0B98LuTXQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-scroll-area": { + "version": "1.2.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-scroll-area/-/react-scroll-area-1.2.18.tgz", + "integrity": "sha512-Zn5Cd171wxsO3Dfg8HaW6RifTb9CYTKQJHs/G4+LN1GfmJpaQMZQyQxMprVPHpaz7QY4l9BxK2JwQuzHsXC8nA==", + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-select": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-select/-/react-select-2.3.7.tgz", + "integrity": "sha512-WFGImkmbzcfxeIwq/+4HvRN0pizBwbwQUED4I13ezQsDdfl38ZntN6TmR8XaSzPBqoCToe8rF75j6NPNDSzhbg==", + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-separator": { + "version": "1.1.15", + "resolved": "https://registry.npmjs.org/@radix-ui/react-separator/-/react-separator-1.1.15.tgz", + "integrity": "sha512-jOLO4lssEzWpoDu7G+Ze4VjwMRUBt291pnZD0gmalREZipnTX3wadQo7Fy48GCTfe14/YRN6rw/rOJqrE85Wxw==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slider": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slider/-/react-slider-1.4.7.tgz", + "integrity": "sha512-mTSLf1GC/C0moWjTbvCM6Qn/gBjvlFt1azuWF2v7MN5C3Zq2U2J2lN3ZEYkpujuOU5Ro7A28wkviSxaKnG0BYg==", + "license": "MIT", + "dependencies": { + "@radix-ui/number": "1.1.3", + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-previous": "1.1.4", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slot": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.3.3.tgz", + "integrity": "sha512-qx7oqnYbxnK9kYI9m317qmFmEgo6ywqWvbTogdj7cL9p3/yx4M48p7Rnw5z3H890cL/ow/EeWJsuTykeZVXP5Q==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.5" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-switch": { + "version": "1.3.7", + "resolved": "https://registry.npmjs.org/@radix-ui/react-switch/-/react-switch-1.3.7.tgz", + "integrity": "sha512-48tB/4dn2UVLBCYhTu9AuR63IHl73l/qLbLgxd86noTUor4/K4LFDAcYjK+isP5313qxaFpjPVogE7+Y0/V3Kw==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-size": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tabs": { + "version": "1.1.21", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.21.tgz", + "integrity": "sha512-UKxJlZid7FVtsk/WTxj4i4uSEgj2Au+KBbS7SQyTlzMhhn+86Cz3tISZdTa87bfEfcuvZezf2ZsxD4xuEKtkog==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toast": { + "version": "1.2.23", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toast/-/react-toast-1.2.23.tgz", + "integrity": "sha512-ofhyAsYaocRGOs/n0XWdUOSVzEAG6BfrMVM8z0c0kLEWY38w/0WuMFPTJP/HVaZPYkMvHZoKIIhNcjbTCBILPg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle": { + "version": "1.1.18", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle/-/react-toggle-1.1.18.tgz", + "integrity": "sha512-7lonPlKfSacd20GlOBx2ltuVKz9oqWYZz+oMQyOltw6t1y2nyftj2ZmwwUHYn49kqfDWcp8dNZm5NgV+5Z+mug==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toggle-group": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toggle-group/-/react-toggle-group-1.1.19.tgz", + "integrity": "sha512-OtnwuSVjd1Ofi+AdnvhsjQdyuhCDwYs1w9RyB5BN/OavXOVQo42SYqQjwUnbPnaiPFBpQ9aX70dWeee+v2oBLA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-use-controllable-state": "1.2.6" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-toolbar": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/@radix-ui/react-toolbar/-/react-toolbar-1.1.19.tgz", + "integrity": "sha512-Ph0IvtYw4VB12ZnZg+YtrGs8yJQsnizwo/zu0R4Y/nWugtJzA7Pg1eWeuDR9+LSqn+xjamss+UOSOJJJ4gx8jw==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-toggle-group": "1.1.19" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tooltip": { + "version": "1.2.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.2.16.tgz", + "integrity": "sha512-6EamKFRRnlpdadndbZ6LMwycfwkwPte1B42hs6QA0gYhjaOKqW4PZ4pjaW9UrlDX5eVt/OjncE7BFTPL5nmZhg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-id": "1.1.4", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-callback-ref": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.4.tgz", + "integrity": "sha512-R6OUY2e2fA6Yn6s+VSx5KBV6Nx8LQEhu+cz7LCej18rQ1HLyg9PSC9jP/ZNx0o6FAIK9c0F1kHylzSxKsdlkrQ==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-controllable-state": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.6.tgz", + "integrity": "sha512-uEQJGT97ZA/TgP/Hydw47lHu+/vQj6z/0jA+WeTbK1o9Rx45GImjpD0tc3W5ad3D6XTSR6e1yEO0FvGq6WQfVQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-effect-event": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.5.tgz", + "integrity": "sha512-7cshFL8HGS/7HEiHH+9kL9HBwp2sa9yX18Knwek6KYWmXwM7pegMgta2AXMQKI+rq3JnfSj9x8wYqFMTdG1Jgg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-escape-keydown": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.5.tgz", + "integrity": "sha512-ge3ipobwSXTj4JyVtswQ7qZj0ZHdtbGuOno/LrgAAeSxtsJ6Vs4Gz5IkPH2bmqpjcLUFoqGhA/mueuIf63UXlA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-callback-ref": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-is-hydrated": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-is-hydrated/-/react-use-is-hydrated-0.1.3.tgz", + "integrity": "sha512-umO/aJ+82CpOnhDZUTbILCQf7kU/g0iv+oGs/Q8jw7IkhWBzaEP4sA268PhFAJTFetbwp3ICc6ktpI4TqtxcIw==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-layout-effect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.4.tgz", + "integrity": "sha512-K20DkRkUwDnxEYMBPcg3Y6voLkEy5p5QQmszZgLngKKiC7dzBR/aEuK3w1qlx2JWDUNH6FluahYdgR3BP+QbYw==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-previous": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-previous/-/react-use-previous-1.1.4.tgz", + "integrity": "sha512-XoSLhbRbqxFtgJoi2fNHA3C6pDlY34x508vUpUGoFZfvePfHXHbE1lC4FYFMnJWgiCRroSTw6fOsXQoVS9RwZg==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-rect": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.4.tgz", + "integrity": "sha512-cSOCh6JlkmfjLyNcLiu2nB4v+nm+dkZ+Q5KHWk/soo4U7ZLiEQFKHK9/YmtBHjfCEaU43IBKQOc4/uJmCaiCTQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/rect": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-size": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.4.tgz", + "integrity": "sha512-D3anSY15EJoxrihpsXI6SMrmmonnQtR2ni7arO+Lfdg3O95b9hNXxONk8jA5C8ANdF/h5HMAxejgs8PWJ6rlhw==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.4" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-visually-hidden": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.2.11.tgz", + "integrity": "sha512-NFS86RYYZb4/exihaESBGOpMJFz8MGLAfu3mOBSGByVnVPC9JPASfYubxd/8KbkQK0sYAv8lVQDEQukDX/qXvQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.10" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/rect": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.3.tgz", + "integrity": "sha512-JtyZR+mqgBibTo8xea3B6ZRmzZiM/YeVBtUkas6zMuXjAlfIFIW2FgqeM9eLyvEaYX66vr6DJMK+4U6LV0KhNw==", + "license": "MIT" + }, + "node_modules/@rolldown/binding-android-arm-eabi": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.7.tgz", + "integrity": "sha512-EypzgnYCwyVY4NDHKzGmNJT5b+XaQEBniHxsMdeIQLB/tcCzZnhqrzHpZFbX9iaxx+5RiB8caATBtfvZP7zVxQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.7.tgz", + "integrity": "sha512-l17HE9EweWaqJZhuUuNBN/FzM62xw+DECVnJyvMsxn8vJFAGLy5QfLDoYAcronkAN8VxKZHezDpulHDPx95vFw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.7.tgz", + "integrity": "sha512-8ED8ELFvHXc6OCETIn4gXObPiaR6bckM/ipXtbzlPVDRMBfEGjCKgO90F9YtfdpDatVx/ZQw7aZ1vUMf/+T3Mw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.7.tgz", + "integrity": "sha512-/WPripjtiAIZ2tWY7ddijORT0Ujg87wxWW/qcoFVCKAWVDPhtY0xr7Dj0M3GyNGz60jGwTElhro/mkF9dT7dDQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.7.tgz", + "integrity": "sha512-14DI4NcqpvbICxSnGLx3PmtDaWqRP/KGSGb6C+JLLVPeZRl6dKdHba3pGsqT3vpdTqhEYIPG0MMQ8c0xYqoJxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.7.tgz", + "integrity": "sha512-bxrWIRvHWQvbJwi+VIie/kDJmQxcNE6xxWwZdqF/ExVAigtHkv54WTLQPb+QsZdnFy18fg7JPfWGL0RH6vwIlQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.7.tgz", + "integrity": "sha512-toOY2BChBZyuxU7OYX6Tn389di4IzAqPTycVcci0O7FSfBqzRB3RZn+K5Is6ANf4tmgRd/K1yZTsNTXbkXsnLg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.7.tgz", + "integrity": "sha512-lAIXTH/aiLRLxsTgQvfhjo4K1ydWIp00+V0voOr9beb/9ZmkUFrSIb03dXNFRgMNvkE6oGsF10ioQ6UsI+vS5Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.7.tgz", + "integrity": "sha512-kdnwS28Pkenp/mZMRwjXXXwxQ7pIsm+bF919LUK93BOyhcLsrVKdP2p9fxpiPNPAbNuch8ypQt0pm2P2LYCAGg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.7.tgz", + "integrity": "sha512-516OdsyLdr5E65paF3yBF55t8mfm9+gmtCsK3xI7XKXIT7EfRlHhxL8K/NR6Hu8BWSgF5+1w74lTL0+nxcc8Qw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.7.tgz", + "integrity": "sha512-r8/z8n7GFaYRln3xmP1Cxy0HH/HLM0uBUPkEuSVEfKGDA89M0FsZRZJRSwe/tJjRx+fpH/gjorfhB8tmEbSFLA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.7.tgz", + "integrity": "sha512-pAsE8iiDxUg1xBqdhrTfg45AVDVpirjz00sblEYClGNNcMnDb+e8beQgqIAw6LvauX/APvgxUnwrgun/YYGBhw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.7.tgz", + "integrity": "sha512-lTcIYmmnQQA8Or/2DatS6oSqcdLHvendjS+zLu+FwgToynWMRSmQdpM65fTANJgIS4mjbMOo5KT2lnT9SAb96w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.7.tgz", + "integrity": "sha512-e3Gu3WxbNk/UqQhxqU7YIYO+9ZBvWNz3U+h/qRFosscMFzdRPbXYSaSWgSnklv2fz1TgzBTcti2z35c/7irsHw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.7.tgz", + "integrity": "sha512-W/jg5qoRSqjsEv0+dZi4e687mcHqmVuU0P4fK6qS/xjetW2Gmc1W8j//z5nAeNcC8Ttm0hV46IjcYeuVwYhuiw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tailwindcss/node": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.3.3.tgz", + "integrity": "sha512-/T8IKEsf9VTU6tLjgC7+sv2mOPtQxzE2jMw7u4Tt40Tx+QSZxpzh95/H6cMKoja9XuW7iMdLJYBB0o9G1CaAgg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/remapping": "^2.3.5", + "enhanced-resolve": "^5.24.1", + "jiti": "^2.7.0", + "lightningcss": "1.32.0", + "magic-string": "^0.30.21", + "source-map-js": "^1.2.1", + "tailwindcss": "4.3.3" + } + }, + "node_modules/@tailwindcss/oxide": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide/-/oxide-4.3.3.tgz", + "integrity": "sha512-krXjAikiaFSPaK/FkAQT5UTx3VormQaiZ5hBFlJZ9UFQGB/rwg1MZIhHAG9smMQRTdyJxP6Qt5MwMtdyU5FWrA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 20" + }, + "optionalDependencies": { + "@tailwindcss/oxide-android-arm64": "4.3.3", + "@tailwindcss/oxide-darwin-arm64": "4.3.3", + "@tailwindcss/oxide-darwin-x64": "4.3.3", + "@tailwindcss/oxide-freebsd-x64": "4.3.3", + "@tailwindcss/oxide-linux-arm-gnueabihf": "4.3.3", + "@tailwindcss/oxide-linux-arm64-gnu": "4.3.3", + "@tailwindcss/oxide-linux-arm64-musl": "4.3.3", + "@tailwindcss/oxide-linux-x64-gnu": "4.3.3", + "@tailwindcss/oxide-linux-x64-musl": "4.3.3", + "@tailwindcss/oxide-wasm32-wasi": "4.3.3", + "@tailwindcss/oxide-win32-arm64-msvc": "4.3.3", + "@tailwindcss/oxide-win32-x64-msvc": "4.3.3" + } + }, + "node_modules/@tailwindcss/oxide-android-arm64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-android-arm64/-/oxide-android-arm64-4.3.3.tgz", + "integrity": "sha512-Y85A2gmPSkl5Ve5qR86GL4HT509cFqQh1aes9p3sSkyTPwt0Pppf3GkwGe4JPACcRYjgJIEhQgM6dBClnr0NYw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-arm64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-arm64/-/oxide-darwin-arm64-4.3.3.tgz", + "integrity": "sha512-BiaWatpBcERQFDlOjRDpIVXuFK5PJez5SA4JMg6VYZdBYU+qKfV/vqjcIs+IYmtitf1xYQZTwXvU/8y4lfZUGw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-darwin-x64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-darwin-x64/-/oxide-darwin-x64-4.3.3.tgz", + "integrity": "sha512-fAeUqfV5ndhxRwai8cXGzdLvul9utWOmeTkv69unv4ZXixjn61Z+p9lCWdwOwA3TYboG3BwdVuN/RDjhBRl0mw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-freebsd-x64": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-freebsd-x64/-/oxide-freebsd-x64-4.3.3.tgz", + "integrity": "sha512-iyf5bV6+wnAlflVeEy7R25dupxTNECZN5QMI0qNT6eT+EgaGdZcKhGkr5SdoaWiLJ3spLqIY9VCeSGrwmtg4kw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm-gnueabihf": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm-gnueabihf/-/oxide-linux-arm-gnueabihf-4.3.3.tgz", + "integrity": "sha512-aAYUprJAJQWWbRrPvtjdroZ56Md+JM8pMiopS6xGEwDfLhqj+2ver2p4nU4Mb3CRqcMmNBjo8KkUgcxhkzVQGQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-gnu": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-gnu/-/oxide-linux-arm64-gnu-4.3.3.tgz", + "integrity": "sha512-nDxldcEENOxZRzC2uu9jrutZdAAQtb+8WWDCSnWL1zvBk1+FN+x6MtDViPB5AJMfttVCUhehGWus3XBPgatM/w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-arm64-musl": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-arm64-musl/-/oxide-linux-arm64-musl-4.3.3.tgz", + "integrity": "sha512-Md44bD6veX/PC5iyF8cDVnw4HBIANZepRZZ7a8DQOvkfo5WUBwcp6iAuCUz23u+4SUkhJlD3eL7hNdW8ezd/kA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-gnu": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-gnu/-/oxide-linux-x64-gnu-4.3.3.tgz", + "integrity": "sha512-tx7us1muwOKAKWao2v/GaafFeQboE6aj88vC6ziN2NCGcRm8gWUhwjzg+YdVB1e4boAtdtma4L43onunI6NS4w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-linux-x64-musl": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-linux-x64-musl/-/oxide-linux-x64-musl-4.3.3.tgz", + "integrity": "sha512-SJxX60smvHgasZoBy11dX6YRjXJFovwWBoedhbQPOBzgFWBHGB+TVPWB9BxzR7TTxU8FQZAI2AyiNCMzFm8Img==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-wasm32-wasi": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-wasm32-wasi/-/oxide-wasm32-wasi-4.3.3.tgz", + "integrity": "sha512-jx1+rPhY/5Ympkktd656HBWEBLxP7dH06losBLjjf5vgCODXvi9KhtftWcMIwTFIDqBr7cRnQkdLnAG+IOlGvQ==", + "bundleDependencies": [ + "@napi-rs/wasm-runtime", + "@emnapi/core", + "@emnapi/runtime", + "@tybys/wasm-util", + "@emnapi/wasi-threads", + "tslib" + ], + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "^1.11.1", + "@emnapi/runtime": "^1.11.1", + "@emnapi/wasi-threads": "^1.2.2", + "@napi-rs/wasm-runtime": "^1.1.4", + "@tybys/wasm-util": "^0.10.2", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@tailwindcss/oxide-win32-arm64-msvc": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-arm64-msvc/-/oxide-win32-arm64-msvc-4.3.3.tgz", + "integrity": "sha512-3rc292Ca2ceK6Ulcc/bAVnTs/3nDtoPhyEKlgPv+yQJQi/JS/AMJlqzxvlDacL1nekbrcf6bTqp/jV4qgnPxNQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/oxide-win32-x64-msvc": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/oxide-win32-x64-msvc/-/oxide-win32-x64-msvc-4.3.3.tgz", + "integrity": "sha512-yJ0pwIVc/nYeGoV02WtsN8KYyLQv7kyI2wDnkezyJlGGjkd4QLwDGAwl47YpPJeuI0M0ObaXGSPjvWDPeTPggw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 20" + } + }, + "node_modules/@tailwindcss/vite": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/@tailwindcss/vite/-/vite-4.3.3.tgz", + "integrity": "sha512-yYU8cogLeSh/ms2jh8Fj7jaba/EWa7Ja6GoUqYZaraEuCI5YS6ms6ObZgjjedm+jm6XZjdNRWBpPP6Z86oOxcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@tailwindcss/node": "4.3.3", + "@tailwindcss/oxide": "4.3.3", + "tailwindcss": "4.3.3" + }, + "peerDependencies": { + "vite": "^5.2.0 || ^6 || ^7 || ^8" + } + }, + "node_modules/@tanstack/query-core": { + "version": "5.102.8", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.102.8.tgz", + "integrity": "sha512-ZNjkJ33CqvPNec/6lZBnHqLc3EVGPZ9ySLhYahU9TcuRFdmwXewuj0c4hwSWcGHqEUwcSrKeZ+oGcvPBqXcQcg==", + "license": "MIT" + }, + "node_modules/@tanstack/react-query": { + "version": "5.102.8", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.102.8.tgz", + "integrity": "sha512-TYBea4OuXWD7MhaSHq069TWbFe7rcwWN6kzT7JF0OKi1K6c1gTv2IzD6A6ExJsCMozdkqBWeuIUZmu4KQg0O5A==", + "license": "MIT", + "dependencies": { + "@tanstack/query-core": "5.102.8" + }, + "peerDependencies": { + "react": "^18 || ^19" + } + }, + "node_modules/@types/node": { + "version": "26.4.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.4.1.tgz", + "integrity": "sha512-k97ENvZWtvA6yqz5/FS6a7duDgOPEeOQOc2iKS/nY6mX6qJUKtLnWzQS+Xj6tXweyj6ZcTAK2Qecetnvi9nCLA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/@types/react": { + "version": "19.2.18", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.18.tgz", + "integrity": "sha512-AnzbBERsrLKtk2XSfTbYRLjQPdy116Sty4q+T+Bp3IC4l6jNBvreVPAHmpq9qhXQM7CXZPjLVmGMw9sy+hxQ3w==", + "devOptional": true, + "license": "MIT", + "dependencies": { + "csstype": "^3.2.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.2.7", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.7.tgz", + "integrity": "sha512-I8bPpDLcHBv1qiIiXDCy71Rt8eQDKJP0sMSWJphDdAcdqiJ1sGpZamavoEIRZmYzjia9LuEb2HlYdDpmoENpvQ==", + "devOptional": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.2.0" + } + }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-6.1.1.tgz", + "integrity": "sha512-yxLaQV9gkhS8ezJqCM6+ndU7mDY6gqAg75NQ+0IjwEI8IYOmQCgkRwHKVSfWXW076DsqMo0Dk+0FK1U+M5RgFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rolldown/pluginutils": "^1.0.1" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "peerDependencies": { + "@rolldown/plugin-babel": "^0.1.7 || ^0.2.0", + "babel-plugin-react-compiler": "^1.0.0", + "oxc-transform-react": "^0.145.0", + "vite": "^8.0.0" + }, + "peerDependenciesMeta": { + "@rolldown/plugin-babel": { + "optional": true + }, + "babel-plugin-react-compiler": { + "optional": true + }, + "oxc-transform-react": { + "optional": true + } + } + }, + "node_modules/aria-hidden": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz", + "integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/class-variance-authority": { + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/class-variance-authority/-/class-variance-authority-0.7.1.tgz", + "integrity": "sha512-Ka+9Trutv7G8M6WT6SeiRWz792K5qEqIGEGzXKhAE6xOWAY6pPH8U+9IY3oCMv6kqTmLsv7Xh/2w2RigkePMsg==", + "license": "Apache-2.0", + "dependencies": { + "clsx": "^2.1.1" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "devOptional": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/detect-node-es": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", + "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==", + "license": "MIT" + }, + "node_modules/enhanced-resolve": { + "version": "5.24.5", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.24.5.tgz", + "integrity": "sha512-L1l8TNvomm6UVW5B253AGxQagSQr+vGwhMlrrfRS2qmhx46AMpMVJKQYLvWYbysTMY8VoicOvzHzoHMbyzB+4A==", + "dev": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.3.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-nonce": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", + "integrity": "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/jiti": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-2.7.0.tgz", + "integrity": "sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==", + "dev": true, + "license": "MIT", + "bin": { + "jiti": "lib/jiti-cli.mjs" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/lucide-react": { + "version": "1.41.0", + "resolved": "https://registry.npmjs.org/lucide-react/-/lucide-react-1.41.0.tgz", + "integrity": "sha512-6lksP35l6KszDKUeRTi4LV7i6DEe0Yzl2ALJm9j4c5xEYN91GdW1xGsawGMOg2mgjF5GHBVX8pKX9kP+cWsP3Q==", + "license": "ISC", + "peerDependencies": { + "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/nanoid": { + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", + "dev": true, + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", + "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/postcss": { + "version": "8.5.28", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", + "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "dev": true, + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.18", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prettier": { + "version": "3.9.6", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.6.tgz", + "integrity": "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/radix-ui": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/radix-ui/-/radix-ui-1.6.7.tgz", + "integrity": "sha512-QBdhh1arIEUvPC0dQ5+nwWAxt7+N+oP/9jPwjJkGFoSk/sqxg32gJtSXGtFh8frAIcS6oC9cx2Q+7KYCQLOAeA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.7", + "@radix-ui/react-accessible-icon": "1.1.15", + "@radix-ui/react-accordion": "1.2.20", + "@radix-ui/react-alert-dialog": "1.1.23", + "@radix-ui/react-arrow": "1.1.15", + "@radix-ui/react-aspect-ratio": "1.1.15", + "@radix-ui/react-avatar": "1.2.6", + "@radix-ui/react-checkbox": "1.3.11", + "@radix-ui/react-collapsible": "1.1.20", + "@radix-ui/react-collection": "1.1.15", + "@radix-ui/react-compose-refs": "1.1.5", + "@radix-ui/react-context": "1.2.2", + "@radix-ui/react-context-menu": "2.3.7", + "@radix-ui/react-dialog": "1.1.23", + "@radix-ui/react-direction": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.19", + "@radix-ui/react-dropdown-menu": "2.1.24", + "@radix-ui/react-focus-guards": "1.1.6", + "@radix-ui/react-focus-scope": "1.1.16", + "@radix-ui/react-form": "0.1.16", + "@radix-ui/react-hover-card": "1.1.23", + "@radix-ui/react-label": "2.1.15", + "@radix-ui/react-menu": "2.1.24", + "@radix-ui/react-menubar": "1.1.24", + "@radix-ui/react-navigation-menu": "1.2.22", + "@radix-ui/react-one-time-password-field": "0.1.16", + "@radix-ui/react-password-toggle-field": "0.1.11", + "@radix-ui/react-popover": "1.1.23", + "@radix-ui/react-popper": "1.3.7", + "@radix-ui/react-portal": "1.1.17", + "@radix-ui/react-presence": "1.1.10", + "@radix-ui/react-primitive": "2.1.10", + "@radix-ui/react-progress": "1.1.16", + "@radix-ui/react-radio-group": "1.4.7", + "@radix-ui/react-roving-focus": "1.1.19", + "@radix-ui/react-scroll-area": "1.2.18", + "@radix-ui/react-select": "2.3.7", + "@radix-ui/react-separator": "1.1.15", + "@radix-ui/react-slider": "1.4.7", + "@radix-ui/react-slot": "1.3.3", + "@radix-ui/react-switch": "1.3.7", + "@radix-ui/react-tabs": "1.1.21", + "@radix-ui/react-toast": "1.2.23", + "@radix-ui/react-toggle": "1.1.18", + "@radix-ui/react-toggle-group": "1.1.19", + "@radix-ui/react-toolbar": "1.1.19", + "@radix-ui/react-tooltip": "1.2.16", + "@radix-ui/react-use-callback-ref": "1.1.4", + "@radix-ui/react-use-controllable-state": "1.2.6", + "@radix-ui/react-use-effect-event": "0.0.5", + "@radix-ui/react-use-escape-keydown": "1.1.5", + "@radix-ui/react-use-is-hydrated": "0.1.3", + "@radix-ui/react-use-layout-effect": "1.1.4", + "@radix-ui/react-use-size": "1.1.4", + "@radix-ui/react-visually-hidden": "1.2.11" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/react": { + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.8.tgz", + "integrity": "sha512-PWaYA1L/q9u2u7xYQi+Y3L3Yfnie7XyLeaJICV1MGD6LprsBxcAqGjYyr0eY3p+QdsA+x/Irkt4Qif8D63+Sbw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.2.8", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.2.8.tgz", + "integrity": "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.27.0" + }, + "peerDependencies": { + "react": "^19.2.8" + } + }, + "node_modules/react-remove-scroll": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz", + "integrity": "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==", + "license": "MIT", + "dependencies": { + "react-remove-scroll-bar": "^2.3.7", + "react-style-singleton": "^2.2.3", + "tslib": "^2.1.0", + "use-callback-ref": "^1.3.3", + "use-sidecar": "^1.1.3" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-remove-scroll-bar": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz", + "integrity": "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==", + "license": "MIT", + "dependencies": { + "react-style-singleton": "^2.2.2", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-style-singleton": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz", + "integrity": "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==", + "license": "MIT", + "dependencies": { + "get-nonce": "^1.0.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/rolldown": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.7.tgz", + "integrity": "sha512-g0EtLvBjTUB7jhyV0S/TCup3v/XSVl45vUIGbOGU4QPiyjTenCe4mKuFvW9fEgYmS2Fo42AUssRmNuMziXdrig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.148.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm-eabi": "1.2.7", + "@rolldown/binding-android-arm64": "1.2.7", + "@rolldown/binding-darwin-arm64": "1.2.7", + "@rolldown/binding-darwin-x64": "1.2.7", + "@rolldown/binding-freebsd-x64": "1.2.7", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.7", + "@rolldown/binding-linux-arm64-gnu": "1.2.7", + "@rolldown/binding-linux-arm64-musl": "1.2.7", + "@rolldown/binding-linux-ppc64-gnu": "1.2.7", + "@rolldown/binding-linux-s390x-gnu": "1.2.7", + "@rolldown/binding-linux-x64-gnu": "1.2.7", + "@rolldown/binding-linux-x64-musl": "1.2.7", + "@rolldown/binding-openharmony-arm64": "1.2.7", + "@rolldown/binding-win32-arm64-msvc": "1.2.7", + "@rolldown/binding-win32-x64-msvc": "1.2.7" + } + }, + "node_modules/scheduler": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", + "integrity": "sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==", + "license": "MIT" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tailwind-merge": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.6.0.tgz", + "integrity": "sha512-uxL7qAVQriqRQPAyK3pj66VqskWqoZ37PW94jwOTwNfq/z9oyu1V+eqrZqtR2+fCiXdYOZe/Modt8GtvqNzu+w==", + "license": "MIT" + }, + "node_modules/tailwindcss": { + "version": "4.3.3", + "resolved": "https://registry.npmjs.org/tailwindcss/-/tailwindcss-4.3.3.tgz", + "integrity": "sha512-gOhV3P7ufE62QDGg1zVaTgCR+EtPv92k2nIhVcVKcLmxT1sUBsQGhnZj175j+MqRt4zLF7ic+sCYjfhxMxj7YQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/tapable": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.3.3.tgz", + "integrity": "sha512-uxc/zpqFg6x7C8vOE7lh6Lbda8eEL9zmVm/PLeTPBRhh1xCgdWaQ+J1CUieGpIfm2HdtsUpRv+HshiasBMcc6A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/tw-animate-css": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/tw-animate-css/-/tw-animate-css-1.4.0.tgz", + "integrity": "sha512-7bziOlRqH0hJx80h/3mbicLW7o8qLsH5+RaLR2t+OHM3D0JlWGODQKQ4cxbK7WlvmUxpcj6Kgu6EKqjrGFe3QQ==", + "license": "MIT" + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/use-callback-ref": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz", + "integrity": "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sidecar": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.3.tgz", + "integrity": "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==", + "license": "MIT", + "dependencies": { + "detect-node-es": "^1.1.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/vite": { + "version": "8.2.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.2.tgz", + "integrity": "sha512-cFKLV/PRgAUlIRm5WjMjJ86jrftzpqcgH+Us+DS8mI3CDNiH30Whrz8uHL3+MOLPAgqbMBAqWdAHAphOAM+z/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.33.0", + "picomatch": "^4.0.5", + "postcss": "^8.5.26", + "rolldown": "~1.2.4", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.4.0 || ^0.5.0", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/lightningcss": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", + "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.33.0", + "lightningcss-darwin-arm64": "1.33.0", + "lightningcss-darwin-x64": "1.33.0", + "lightningcss-freebsd-x64": "1.33.0", + "lightningcss-linux-arm-gnueabihf": "1.33.0", + "lightningcss-linux-arm64-gnu": "1.33.0", + "lightningcss-linux-arm64-musl": "1.33.0", + "lightningcss-linux-x64-gnu": "1.33.0", + "lightningcss-linux-x64-musl": "1.33.0", + "lightningcss-win32-arm64-msvc": "1.33.0", + "lightningcss-win32-x64-msvc": "1.33.0" + } + }, + "node_modules/vite/node_modules/lightningcss-android-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", + "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-darwin-arm64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", + "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-darwin-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", + "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-freebsd-x64": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", + "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", + "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", + "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-arm64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", + "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-x64-gnu": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", + "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-linux-x64-musl": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", + "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", + "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/vite/node_modules/lightningcss-win32-x64-msvc": { + "version": "1.33.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", + "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + } + } + } +} diff --git a/dsp/tooling/frontend/package.json b/dsp/tooling/frontend/package.json new file mode 100644 index 0000000..68638e7 --- /dev/null +++ b/dsp/tooling/frontend/package.json @@ -0,0 +1,37 @@ +{ + "name": "dispatch-dsp-build-tools", + "version": "0.4.0", + "private": true, + "description": "Frontend compilation dependencies for installed DSP plugins", + "type": "commonjs", + "main": "server/main.js", + "engines": { + "node": ">=22" + }, + "scripts": {}, + "dependencies": { + "@fontsource-variable/inter": "^5.3.0", + "@tanstack/react-query": "^5.102.8", + "class-variance-authority": "^0.7.1", + "clsx": "^2.1.1", + "get-nonce": "^1.0.1", + "lucide-react": "^1.41.0", + "radix-ui": "^1.6.7", + "react": "^19.2.8", + "react-dom": "^19.2.8", + "tailwind-merge": "^3.6.0", + "tw-animate-css": "^1.4.0" + }, + "devDependencies": { + "@playwright/test": "^1.63.0", + "@tailwindcss/vite": "^4.3.3", + "@types/node": "^26.4.1", + "@types/react": "^19.2.18", + "@types/react-dom": "^19.2.7", + "@vitejs/plugin-react": "^6.1.1", + "prettier": "^3.9.6", + "tailwindcss": "^4.3.3", + "typescript": "^7.0.2", + "vite": "^8.2.2" + } +} diff --git a/dsp/tooling/platform-dependencies.json b/dsp/tooling/platform-dependencies.json new file mode 100644 index 0000000..2f38568 --- /dev/null +++ b/dsp/tooling/platform-dependencies.json @@ -0,0 +1,5 @@ +{ + "url": "https://github.com/dillonlille/dispatch-core/releases/download/v0.0.1/platform-packages.tar.gz", + "sha256": "b233ad949d4bfc931e19ef7e988c3e15e109241e9aa479cbdeb54404ac013990", + "description": "Use the verified platform package bundle published with Core 0.0.1 for DSP checks, builds, and releases. Upgrade the URL and SHA-256 together through a reviewed PR." +} diff --git a/dsp/tooling/platform-lock.py b/dsp/tooling/platform-lock.py new file mode 100644 index 0000000..3822183 --- /dev/null +++ b/dsp/tooling/platform-lock.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Resolve a reviewed dependency lock; source builds are for development CI only.""" +import json +import os +import pathlib +import re +import sys +import urllib.parse + + +def resolve(config, production=False): + url, digest = config.get('url'), config.get('sha256') + if url is not None or digest is not None: + parsed = urllib.parse.urlparse(url or '') + if parsed.scheme != 'https' or not parsed.hostname or parsed.username or parsed.password or not re.fullmatch(r'[a-f0-9]{64}', digest or ''): + raise ValueError('A release dependency needs an HTTPS URL and SHA-256') + return {'mode': 'release'} + if production: + raise ValueError('DSP publication requires a published Core platform bundle; pin its URL and SHA-256 in a PR first') + source = config.get('developmentSource') or {} + repository, commit = source.get('repository', ''), source.get('commit', '') + if repository != 'dispatch-core' or not re.fullmatch(r'[a-f0-9]{40}', commit): + raise ValueError('Development CI requires an exact Core repository commit') + return {'mode': 'source', 'repository': repository, 'commit': commit} + + +if __name__ == '__main__': + config = json.loads(pathlib.Path(__file__).with_name('platform-dependencies.json').read_text()) + result = resolve(config, '--production' in sys.argv) + if os.environ.get('GITHUB_OUTPUT'): + with open(os.environ['GITHUB_OUTPUT'], 'a') as output: + for key, value in result.items(): + output.write(f'{key}={value}\n') + print(json.dumps(result)) diff --git a/dsp/tooling/plugin-cli.js b/dsp/tooling/plugin-cli.js new file mode 100644 index 0000000..e2798b9 --- /dev/null +++ b/dsp/tooling/plugin-cli.js @@ -0,0 +1,12 @@ +'use strict'; +const path=require('node:path'); +async function main([command,noun,value,...rest]){ + if(rest.length||!value)throw new Error('plugin_cli_arguments_invalid'); + if(command==='create'&&noun==='plugin')return require('dispatch-sdk/tooling/create-plugin').createPlugin({id:value,directory:path.resolve('plugins',value)}); + if(command!=='plugin')throw new Error('plugin_cli_arguments_invalid'); + const pluginRoot=path.resolve(/^[a-z][a-z0-9-]{0,63}$/.test(value)?path.join('plugins',value):value); + if(noun==='generate'||noun==='check')return require('dispatch-sdk/tooling/plugin-contracts').generateContracts(pluginRoot,{check:noun==='check'}); + if(noun==='dev')throw new Error('Use the Core development runner with this plugin path; see DEVELOPMENT.md.'); + throw new Error('plugin_cli_arguments_invalid'); +} +module.exports={main}; diff --git a/dsp/tooling/project.js b/dsp/tooling/project.js new file mode 100644 index 0000000..4cb9b7c --- /dev/null +++ b/dsp/tooling/project.js @@ -0,0 +1,15 @@ +'use strict'; +const path=require('node:path'),{spawnSync}=require('node:child_process'); +const root=path.resolve(__dirname,'..'); +async function main(args){ + if(args[0]==='bootstrap'){ + if(!args[1])throw new Error('usage: bootstrap VERIFIED_PLATFORM_PACKAGE_BUNDLE'); + const bundle=path.resolve(args[1]); + const receipt=require(path.join(bundle,'install.cjs')).installPlatformPackages(bundle,path.join(root,'node_modules'),require('../package.json').dispatchPackages); + const result=spawnSync('npm',['ci','--ignore-scripts','--no-audit','--no-fund'],{cwd:path.join(root,'tooling/frontend'),stdio:'inherit'}); + if(result.status!==0)throw new Error('dependency_install_failed');return receipt; + } + return require('dispatch-sdk/tooling/project').main(root,args); +} +if(require.main===module)main(process.argv.slice(2)).then(result=>console.log(JSON.stringify(result))).catch(error=>{console.error(error.message);process.exitCode=1;}); +module.exports={main}; diff --git a/dsp/tooling/release.js b/dsp/tooling/release.js new file mode 100644 index 0000000..fd9684d --- /dev/null +++ b/dsp/tooling/release.js @@ -0,0 +1,4 @@ +'use strict'; +try { + console.log(JSON.stringify(require('dispatch-sdk/tooling/release-publication').main(require('node:path').resolve(__dirname, '..'), process.argv.slice(2)))); +} catch (error) { console.error(error.message); process.exitCode = 1; } diff --git a/dsp/tooling/test_platform_lock.py b/dsp/tooling/test_platform_lock.py new file mode 100644 index 0000000..b10bf5e --- /dev/null +++ b/dsp/tooling/test_platform_lock.py @@ -0,0 +1,34 @@ +import importlib.util +import pathlib +import unittest + +spec = importlib.util.spec_from_file_location('platform_lock', pathlib.Path(__file__).with_name('platform-lock.py')) +lock = importlib.util.module_from_spec(spec) +spec.loader.exec_module(lock) + + +class PlatformLockTests(unittest.TestCase): + def test_source_is_exact_and_never_a_production_dependency(self): + config = {'developmentSource': {'repository': 'dispatch-core', 'commit': 'a' * 40}} + self.assertEqual(lock.resolve(config)['mode'], 'source') + with self.assertRaisesRegex(ValueError, 'published Core'): + lock.resolve(config, production=True) + config['developmentSource']['commit'] = 'main' + with self.assertRaises(ValueError): + lock.resolve(config) + + def test_release_needs_both_https_and_digest(self): + config = {'url': 'https://example.com/package.tar.gz', 'sha256': 'a' * 64} + self.assertEqual(lock.resolve(config, production=True), {'mode': 'release'}) + for changes in ({'url': 'http://example.com/package'}, {'sha256': None}, {'url': 'https://user:secret@example.com/package'}): + with self.assertRaises(ValueError): + lock.resolve({**config, **changes}) + + def test_bad_release_does_not_fall_back_to_source(self): + with self.assertRaises(ValueError): + lock.resolve({'url': 'https://example.com/package', 'sha256': None, + 'developmentSource': {'repository': 'dispatch-core', 'commit': 'a' * 40}}) + + +if __name__ == '__main__': + unittest.main() diff --git a/dsp/tooling/tests.json b/dsp/tooling/tests.json new file mode 100644 index 0000000..d410b16 --- /dev/null +++ b/dsp/tooling/tests.json @@ -0,0 +1,61 @@ +{ + "unit": [ + "runtime/auth-broker/tests/amazon-logistics-adapter.test.js", + "runtime/auth-broker/tests/amazon-verification.test.js", + "runtime/auth-broker/tests/attempt-guard.test.js", + "runtime/auth-broker/tests/authentication-diagnostics.test.js", + "runtime/auth-broker/tests/browser-assistance.test.js", + "runtime/auth-broker/tests/browser-client.test.js", + "runtime/auth-broker/tests/cdp-pipe.test.js", + "runtime/auth-broker/tests/connections.test.js", + "runtime/auth-broker/tests/paycom-adapter.test.js", + "runtime/auth-broker/tests/paycom-assistance-continuation.test.js", + "runtime/auth-broker/tests/paycom-credentials-cli.test.js", + "runtime/auth-broker/tests/paycom-native-window.test.js", + "runtime/auth-broker/tests/paycom-profile.test.js", + "runtime/auth-broker/tests/paycom-readiness.test.js", + "runtime/auth-broker/tests/paycom-session-persistence.test.js", + "runtime/auth-broker/tests/paycom-session-start.test.js", + "runtime/auth-broker/tests/security.test.js", + "runtime/auth-broker/tests/server.test.js", + "runtime/auth-broker/tests/session-manager.test.js", + "runtime/auth-broker/tests/vault.test.js", + "runtime/cli/tests/cli.test.js", + "runtime/cli/tests/interactions.test.js", + "runtime/collection-manager/tests/capacity-runner.test.js", + "runtime/collection-manager/tests/cli.test.js", + "runtime/collection-manager/tests/execution.test.js", + "runtime/collection-manager/tests/manager.test.js", + "runtime/collection-manager/tests/plugins.test.js", + "runtime/collection-manager/tests/read-snapshots.test.js", + "runtime/collection-manager/tests/runner.test.js", + "runtime/collection-manager/tests/standard-collections.test.js", + "runtime/collection-manager/tests/store.test.js", + "runtime/collection-manager/tests/syncs.test.js", + "runtime/gateway/tests/gateway.test.js", + "runtime/plugin-host/tests/storage.test.js", + "runtime/sdk/tests/connections.test.js", + "runtime/sdk/tests/local-sync-port.test.js", + "runtime/sdk/tests/sdk.test.js", + "runtime/sdk/tests/setup-auth.test.js", + "runtime/sdk/tests/workforce.test.js", + "runtime/supervisor/tests/execution.test.js", + "runtime/supervisor/tests/paycom-setup.test.js", + "runtime/supervisor/tests/plugins.test.js", + "runtime/workers/tests/authentication.test.js", + "plugins/paycom/backend/tests/collector.test.js", + "plugins/paycom/backend/tests/publication-continuity.test.js", + "plugins/paycom/backend/tests/published.test.js", + "plugins/paycom/backend/tests/staging-cleanup.test.js", + "plugins/paycom/backend/tests/store.test.js", + "plugins/paycom/backend/tests/sync.test.js", + "runtime/workers/tests/single-browser.test.js" + ], + "integration": [ + "runtime/auth-broker/tests/browser-runtime.test.js", + "runtime/gateway/tests/managed-runtime.test.js", + "runtime/supervisor/tests/runtime-container.test.js", + "plugins/paycom/backend/tests/settings.test.js" + ], + "concurrency": 1 +} diff --git a/dsp/tooling/workflow.py b/dsp/tooling/workflow.py new file mode 100644 index 0000000..dc076e0 --- /dev/null +++ b/dsp/tooling/workflow.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Read-only PR/release facts for the human-approved Dispatch workflow.""" +import argparse,json,pathlib,subprocess + +def run(*args): + return subprocess.check_output(args,text=True).strip() + +def main(): + parser=argparse.ArgumentParser() + parser.add_argument('command',choices=['pr-details','release-status']) + parser.add_argument('--repo',required=True,help='OWNER/dispatch-core or OWNER/dispatch-dsp') + parser.add_argument('--pr',type=int) + args=parser.parse_args() + import re + if not re.fullmatch(r'[A-Za-z0-9_.-]+/dispatch-(core|dsp)',args.repo):parser.error('Use the selected Core or DSP repository') + root=pathlib.Path(__file__).resolve().parents[1] + expected=json.loads((root/'package.json').read_text())['name'] + if args.repo.split('/')[1]!=expected:parser.error('Repository must match this project') + if args.command=='pr-details': + if not args.pr:parser.error('--pr is required') + print(run('gh','pr','view',str(args.pr),'--repo',args.repo,'--json','number,title,url,baseRefName,headRefName,headRefOid,isDraft,mergeable,reviewDecision,statusCheckRollup')) + else: + print(run('gh','release','list','--repo',args.repo,'--limit','10','--json','tagName,name,publishedAt,isLatest,isPrerelease')) + print('Report the deployed Core/DSP versions separately. Ask the user for the next version before preparing a release.') + +if __name__=='__main__':main() diff --git a/dsp/tsconfig.json b/dsp/tsconfig.json new file mode 100644 index 0000000..32dee49 --- /dev/null +++ b/dsp/tsconfig.json @@ -0,0 +1,34 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": [ + "ES2022", + "DOM", + "DOM.Iterable" + ], + "module": "ESNext", + "moduleResolution": "Bundler", + "jsx": "react-jsx", + "strict": true, + "skipLibCheck": true, + "noEmit": true, + "esModuleInterop": true, + "paths": { + "dispatch-sdk": [ + "./node_modules/dispatch-sdk/types/index.d.ts" + ], + "dispatch-sdk/*": [ + "./node_modules/dispatch-sdk/types/*.d.ts" + ], + "*": [ + "./tooling/frontend/node_modules/@types/*", + "./tooling/frontend/node_modules/*" + ] + }, + "allowImportingTsExtensions": true + }, + "include": [ + "plugins/*/frontend", + "tooling/frontend/env.d.ts" + ] +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..366f600 --- /dev/null +++ b/package.json @@ -0,0 +1,11 @@ +{ + "name": "dispatch-platform", "private": true, "version": "0.0.0", + "scripts": { + "bootstrap": "node tooling/project.cjs bootstrap", + "check": "node tooling/project.cjs check", + "test": "node tooling/project.cjs test", + "build": "node tooling/project.cjs build", + "test:integration": "node tooling/project.cjs integration" + }, + "engines": {"node": ">=22"} +} diff --git a/plugins/README.md b/plugins/README.md new file mode 100644 index 0000000..2f7ddf8 --- /dev/null +++ b/plugins/README.md @@ -0,0 +1,52 @@ +# Dispatch plugins + +Dispatch distributes reviewed, versioned packages. The platform catalog describes +available plugins; clicking Install copies the selected package and its bundled +SDK into that DSP's `plugins//versions//` directory. DSPs keep separate +files and version receipts. Updating a distribution package does not update +already installed DSPs. Owners explicitly choose Update when a newer version is +available. The current build target is Paycom. + +Source belongs in `plugins//`: `frontend/` contains React pages, `backend/` +contains provider code and login adapters, and `dispatch-plugin.json` declares +permissions, pages, actions, jobs, collectors and schedules. Paycom's +`backend/installed.js` exposes the standalone worker entrypoints. Its +`dashboard/published.js` supplies the package's bounded published-data reader; +legacy HTTP handlers remain for compatibility. Installed entrypoints execute in +DSP-scoped workers, never through a require of DSP code in the Core process. + +`tooling/build-installed-plugin.mjs` builds backend and auth bundles, a separate +frontend bundle, declarative collection definitions, a private SDK copy and a +SHA-256 package inventory. `tooling/distribute-plugin-package.js` verifies a +reviewed digest and adds an immutable version to the private platform catalog. +DSP HTTP requests cannot supply executable paths, archives or arbitrary digests. + +Installation fences the DSP, drains work, copies and verifies code, snapshots +plugin state, initializes/migrates the package in an isolated worker, writes its +receipt and connection grants, applies collection definitions, and then records +Core acknowledgement. Failed initialization restores the snapshot while fenced. +Interrupted acknowledgement retries the same revision. An always-on DSP resumes +after acknowledgement; an on-demand DSP returns to its existing execution queue. + +The owner page supports Install, Update, Disable, Enable and Uninstall. Signed +platform-owner DSP views have the same owner authority and retain actor audit +attribution. Disable and Uninstall immediately close the Core access gate, +revoke sessions and stop schedules. Re-enable preserves the selected version and +prior schedule state. All these actions retain credentials and business data. + +The independently loaded dashboard bundle is selected by acknowledged version +and revision. Its authenticated asset endpoint verifies the installed inventory +and current DSP authority, including after reads. The shell supplies the UI SDK, +React and authenticated HTTP transport. Reviewed frontend JavaScript shares the +shell origin; it is not a sandbox for arbitrary third-party UI code. + +Backend plugins use `dispatch-sdk` for connections, jobs, schedules, storage, +publication, actions and structured progress. The SDK transport is bound to one +DSP/plugin/revision/job; caller-supplied identity cannot widen its authority. +Provider passwords remain in a DSP's encrypted auth vault and are exposed only +to its separately isolated login worker. Plugin executables and private state +are separate. See [the SDK](../sdk/README.md) and the +[storage, operations and migration guide](../docs/plugin-runtime.md). + +Cortex remains automatic sign-in and owner-entered email verification. This +plugin system adds no Cortex collection or synchronization. diff --git a/plugins/paycom/backend/OVERVIEW.md b/plugins/paycom/backend/OVERVIEW.md new file mode 100644 index 0000000..83b0e75 --- /dev/null +++ b/plugins/paycom/backend/OVERVIEW.md @@ -0,0 +1,250 @@ +--- +title: Paycom plugin overview +status: current +last_verified: 2026-09-02 +--- + +# Paycom Collector + +The Paycom collector gathers pay periods, employee rosters, canonical employee-period resource links, and employee timecards. Browser-backed methods authenticate through the Dispatch Auth Broker; deterministic link generation does not open Paycom. The collector is managed through the central Collection Manager and does not contain a scheduler, retry loop, credential vault, Hermes tool, or agent-specific integration. + +Shared provider ownership is described in [`../OVERVIEW.md`](../../README.md); this file covers Paycom-specific behavior. + +## Implemented methods + +| Method | Behavior | +|---|---| +| `collector.health` | Checks collector storage and Auth Broker availability without collecting. | +| `pay-periods.discover` | Publishes the previous, current, and next biweekly periods projected from the approved Sunday anchor. | +| `collection.resolve-targets` | Resolves standard collection dates and ranges into unique exact Paycom pay periods without collecting. | +| `roster.snapshot` | Captures the exact Paycom employee-search API response, validates membership and schema, and publishes a complete roster. | +| `roster.period` | Collects and publishes the authoritative roster for one exact Saturday period end. | +| `resource-links.current-period` | Generates and publishes one canonical timecard-summary link per active current-period employee. | +| `resource-links.period` | Generates the same complete link manifest for an exact roster period. | +| `resource-links.audit` | Recomputes link coverage, URLs, roster binding, and publication integrity without browsing. | +| `timecards.current-period` | Collects all active employees for the current period. | +| `timecards.period` | Captures the requested period's roster, then collects exactly that historical membership for a caller-specified Saturday period end. | +| `timecards.from-published-roster` | Collects an exact period from an already-published authoritative roster and fences activation to that roster revision. | +| `timecards.audit` | Revalidates an exact-period roster/timecard binding, every stored record, complete membership, identities, hashes, and relational projections without browsing. | +| `timecards.incremental` | Fills employees missing from an existing period snapshot and republishes complete current-roster membership. | +| `reconcile.current-period` | Audits current-period timecard membership against the active roster without opening a browser. | +| `sync.current-workforce` | Collects the complete visible current workforce and immediately publishes additions, edits, timecard changes, unknown transitions from absence, and explicit active/inactive transitions without deleting employees. | + +Timecards use the source's bounded `maxConcurrency` value (`1`–`6`, currently `6`) inside one broker-authenticated browser. Each worker now keeps one reusable CDP page target for multiple employees instead of creating and connecting a fresh target for every timecard. Nonessential image, font, and media resources are blocked, while document capture, DOM extraction, exact employee/period validation, complete-membership reconciliation, private staging, and atomic publication remain unchanged. A target is discarded and recreated before retrying a transient page failure. + +Successful timecard receipts include identifier-free performance counters: total browser time, collection time, worker and target-open counts, retry count, item p50/p95/max durations, and p50/p95/max phase timings for target open, navigation, response, load, response-body verification, readiness, extraction, and validation. They never include employee identities, provider URLs, or browser/authentication material. + +### Performance evidence + +Successful receipts provide bounded identifier-free wall-time, phase, worker, target-open, and retry aggregates so an operator can compare repeated supervised runs without exposing workforce data. Repository documentation does not retain one installation's workforce counts, run durations, current publication, or benchmark result. A concurrency or transport optimization is accepted only after repeated local measurements preserve exact target membership, persistence proof, atomic publication, cleanup, and lease release. + +## Standard collection interface + +The `paycom-main` source declares standard scopes `roster`, `links`, `timecards`, and `full`. It supports current, latest-complete, exact date, today/yesterday, inclusive range, rolling days/weeks, and exact-period selectors through the Dispatch SDK and unified CLI. A preview resolves dates to exact period-ending Saturdays without collecting. The standard `timecards` and `full` graphs publish the authoritative period roster once, collect timecards from that exact publication, fence activation to its ID and content hash, and run `timecards.audit` before downstream work. Verify mode audits existing timecard and/or resource-link publications without provider collection. + +See the [Collection Manager overview](../../../runtime/collection-manager/OVERVIEW.md) for collector registration and scheduling. + +For a new managed installation, Provisioner `0.5.0` loads this same reviewed definition, validates the code-owned release-tree/specification/executable digests and version `0.17.2`, fixes source/profile `paycom-main`, derives timezone from Access Control, and requires exact configuration attestation. Activation first starts or reuses one job-bound `paycom-periods` manager run, then runs the standard `current`/`full` workforce graph in `refresh` mode with a separate fenced activation-job idempotency key. The pay-period run and all five roster/timecard/link/audit runs must succeed, the manager database/queue must be clean and idle without critical sync alerts, and the managed sync must remain stopped/idle. The catalog-hashed `dispatch-paycom-activation-evidence` helper privately reads both stores and binds the current preparatory run plus exact workforce verification-run IDs, immutable publication-origin runs, active publication IDs, and content digests while binding the selected target to the preparatory pay-period run; Core invokes it through a bounded no-shell stdin/stdout adapter. Access Control persists the closed evidence bundle/digest atomically with `ready`. A stale worker cannot cancel shared work; only a current deadline holder may cancel and drain its exact run or batch. The managed sync stays stopped until a later explicit action. + +This activation uses the normal plugin publication transactions; there is no alternate activation writer or direct SQLite readiness shortcut. Failed or partial work remains unroutable, preserves prior active pointers, and cannot touch another runtime or the existing local EXMP store. + +Paycom `0.17.2` registers `paycom-main-workforce` with full-workforce `reconcileBatchSize=100` and `publishMode=additions_edits`. The compatibility `lookbackPeriods` setting is accepted only as `1`; larger values fail validation rather than being silently ignored. One public `dispatch sync start paycom-main-workforce` command ensures the managed Auth Broker is ready, advances the lifecycle generation, and queues the first tick immediately. Missing previously active employees publish as `lifecycleStatus=unknown` while retaining the last verified timecard and link; only explicit source evidence publishes `inactive`. Repeated start is idempotent, restart creates a new generation, and stop cancels active or queued sync work before returning. + +Every additions/edit tick performs a persisted read-back proof before commit. The active publication is re-audited, and every timecard selected and observed during the tick must have the same full normalized business hash in SQLite. Because that hash covers the complete timecard record, a new or edited punch, kind, time, allocation, exception, comment, waiver, approval, attestation, or total cannot be acknowledged unless the matching normalized value is active in the database. A mismatch fails the tick with `integrity_failed` and rolls back. `no_change` ticks prove the selected observations equal the existing active publication. Receipts expose explicit source `businessDate`/`businessTimezone`, aggregate `persistence` counts, and a transaction-bound aggregate `delta` covering timecard/day/punch/approval and data-quality transitions. Schema version 5 writes the same privacy-safe evidence to durable `paycom_sync_change_history`; it contains no employee identity, punch time, or record value. Repetitive no-change ledger rows older than 365 days compact to one per source/target/business date; published change rows are preserved. + +Version `0.17.0` added one absolute 120-second deadline per timecard item with worker-target retirement, one bounded retry, and sibling CDP cancellation after the first terminal error. Version `0.17.1` corrects period enforcement: the exact closed roster POST is rewritten to the requested deterministic current period, while the page's original UI default is not misclassified as authoritative provider-calendar evidence. Version `0.17.2` selects the one rendered punch-time child and ignores Paycom's hidden duplicate-time element while retaining fail-closed ambiguity checks. Successful receipts report `requestedPeriodEnforced:true`; `pay-periods.discover` remains deterministic. The collector rejects a source-local midnight rollover as `business_date_changed` before publication. Rendered navigation remains the production path; no fixed authenticated request path is enabled without full normalized parity evidence. + +Timecard collection stores a canonical URL without the browser-only cache buster and separates the normalized business hash from the raw source evidence hash. Existing `sourceSha256` values remain evidence hashes; new rows also carry `businessSha256` and per-row `observedAt`. SQLite schema version 4 extends private `paycom_sync_state` and `paycom_sync_employees` rows with timecard business hashes, oldest-first observation timestamps, full-verification timestamps, and a full-reconciliation cadence anchor. + +### Provider acceptance boundary + +The registered Paycom roster, timecard, and resource-link paths have undergone supervised provider acceptance. Operational run identifiers, business dates, workforce counts, punch aggregates, and current service state are local-only and intentionally excluded from this repository. Absence remains retention-only and is never interpreted as confirmed deletion. + + + +## Security boundary + +The collector receives an opaque loopback CDP endpoint only after the Auth Broker completes Paycom login. The collector never receives the broker credential object, password, or security PIN values. + +The collector has full post-login browser access and is therefore trusted with the resulting authenticated session, Paycom page data, cookies, and authorization state. Those values must never be printed, placed in a Collection Manager specification, or persisted in receipts. + +Browser leases use a short renewable TTL. The wrapper renews while collection is alive, releases normally in `finally`, and attempts immediate release on `SIGTERM` or `SIGINT`. If a collector is killed without cleanup, the lease expires automatically. + +## Authentication input and session reuse + +Paycom first checks the DSP-owned saved browser session in headless Chrome. If a +fresh credential or numbered-PIN form requires input, the adapter asks the broker +to close Chrome gracefully and reopen that same profile in a normal window. The +broker permits one such transition before any credential submission, retaining +exclusive profile ownership and the existing attempt guard. + +The normal window uses a private, authenticated Xvfb display and an explicit +loopback debugging port. PIN fields are checked against their labels and hidden +indices, focused through the operating system, checked for focus, and typed using +X11 keyboard events. The actual Continue button receives an operating-system +mouse click. Credential values remain in memory and the encrypted vault; input +helper commands travel over stdin and are never logged. Primary credential form +handling remains unchanged. Protected application verification and removal of +credential-bearing tabs still precede collector handoff. + +The host must supply Xvfb, Python 3, libX11 and libXtst for fresh authentication; +the managed directory runtime can use its existing read-only host tools. No new +browser framework or continuously running display is required. Ordinary saved +session collection stays headless. Display startup, browser errors, cancellation +and lease release clean up owned processes and private Xauthority files. Browser +profiles, credentials and collected databases remain in the DSP private roots. + +The separately configured [host browser assistance](../../../host/browser-assistance/README.md) +can handle CAPTCHA during sign-in and the initial collection handoff. Unexpected +verification, changed forms and provider rejection preserve verification guards. +Local browser tests verify the input and session lifecycle; reuse of a live saved +session is reported separately from fresh provider acceptance. + +## Temporary collection files + +Parsed publication candidates are private JSON files under +`dsps//staging/plugins/paycom/.attempt-/`. Every collection +removes its staging before reporting success, including folders from earlier +attempts of that same run and a replay of an already committed result. Cleanup +checks that the directories are gone and syncs their parent directory. Partial +staging-write failures also remove their owned files. Other runs' staging is +left to those runs. + +If cleanup cannot complete, the collector reports `stage_cleanup_failed` instead +of acknowledging success. An already committed database update remains valid; +an idempotent retry can finish cleanup. This covers normal completion, handled +errors and retries; an uncatchable process or host failure may leave staging +until that run is retried. Saved authentication state, durable collection receipts, +retained data and SQLite-managed WAL/SHM files are outside staging cleanup. + +## Publication model + +Data is stored in: + +```text +/paycom/paycom.sqlite3 +``` + +Collection follows: + +```text +roster publication -> roster-bound collection -> private staging candidate + -> validate -> SQLite transaction -> activate publication + -> exact-period timecard audit -> remove staging +``` + +`active_publications` and `active_resource_link_publications` point to current versions. Prior publications are retained. Partial collection never advances an active pointer. Canonical timecard business records and user-facing links exclude the browser-only `dispatch_timecards` cache buster; raw HTML hashes remain separate evidence. See `references/data-contract.md`. + +## Collection Manager + +The registered source is `paycom-main`. It refers to Auth Broker profile `paycom-main`; that is only an identifier and contains no credentials. + +Inspect the collector through the existing management tool: + +```bash +CTL=./runtime/collection-manager/bin/dispatch-collectionctl +$CTL collector paycom +$CTL methods paycom +$CTL source paycom-main +$CTL plans +``` + +Run non-authenticated checks: + +```bash +$CTL run paycom-health +$CTL run paycom-periods +$CTL run paycom-current-resource-links +$CTL run paycom-resource-links-audit +$CTL run paycom-period-timecards-audit +$CTL drain 60000 # only when the manager daemon is stopped +``` + +Queue authenticated collection: + +```bash +$CTL run paycom-roster +$CTL run paycom-current-timecards +$CTL run paycom-incremental-timecards +``` + +Inspect or start the managed sync: + +```bash +./bin/dispatch sync status paycom-main-workforce +./bin/dispatch sync history paycom-main-workforce --limit 100 --offset 0 +./bin/dispatch sync start paycom-main-workforce +./bin/dispatch workforce status +./bin/dispatch workforce employees --lifecycle unknown --limit 50 +./bin/dispatch workforce timecards --limit 50 +./bin/dispatch workforce punches --date 2026-08-30 --kind in_day --from-time 10:01 --limit 50 +``` + +The sync schema contains only `reconcileBatchSize`, `fullReconcileMinutes`, `lookbackPeriods`, and `publishMode`; deletion settings are absent and `lookbackPeriods` is fixed to `1` until a real multi-period publication contract exists. Terminal authentication projects `blocked` and uses delayed probes rather than duplicate full attempts. `dispatch workforce` reads the active roster, timecard summaries, canonical links, and protected minimal punch rows under one consistent SQLite snapshot without triggering collection. Live run and workforce evidence remains local-only. + + + +For a historical period, first inspect the method schema, then provide a bounded override file: + +```json +{"periodEnd":"2026-09-05"} +``` + +```bash +$CTL run paycom-period-timecards /absolute/path/to/input.json +$CTL run paycom-period-timecards-from-roster /absolute/path/to/input.json +$CTL run paycom-period-timecards-audit /absolute/path/to/input.json +``` + +The default historical period in the checked-in specification is an example/current bootstrap value. Use an override for another period. + +All collector plans remain manual. The Collection Manager owns the durable `paycom-main-workforce` interval, start/stop/restart/edit lifecycle, retries, coalescing, and run history. + +## Credential enrollment and live prerequisites + +Credentials must be enrolled locally using the Auth Broker's dedicated controlling-terminal wizard. Stop the broker, then run: + +```bash +./runtime/auth-broker/bin/dispatch-paycom-credentials enroll +``` + +The command defaults to profile `paycom-main`, disables terminal echo, and requires every value twice before committing the encrypted profile. It returns metadata only. Never send credentials through chat or place them in argv, environment variables, redirected stdin, logs, tests, Collection Manager inputs, or ordinary files. + +After successful enrollment, restart the Auth Broker and verify the profile without exposing values: + +```bash +./runtime/auth-broker/bin/dispatch-auth-brokerctl status paycom-main +``` + +A real roster or timecard run requires: + +1. A locally enrolled `paycom-main` Auth Broker profile. +2. Current Paycom login, security-PIN, roster API, and timecard DOM layouts matching the fail-closed adapters. +3. A successful `paycom-roster` run before dependency-bound standalone timecard plans can run. + +`dispatch sync start paycom-main-workforce` automatically starts and validates the managed Auth Broker before changing sync state. The broker may advance only the exact allowlisted Paycom security-profile campaign through its fixed three-step `Not Now`/warning/`Continue` sequence. Unknown setup pages, generic continue/skip controls, login-layout drift, CAPTCHA, MFA, lockout, unexpected URLs, malformed timecards, and unexpected roster members fail closed. A returned strict subset is observation-only and retains every absent prior employee. + +Current, incremental, and roster-bound exact-period timecards require an audited active roster whose target is the same period end. The candidate records that roster publication ID and content hash; publication rechecks both inside the activation transaction, so a changed roster fails closed. Standalone `timecards.period` retains its self-contained historical membership capture for callers that have not first published an exact-period roster. + +## Component commands + +```bash +./tooling/build +./tooling/test +./tooling/verify +./tooling/health +``` + +The Collection Manager specification is: + +```text +config/collection-manager.json +``` + +Apply it with: + +```bash +$CTL apply ./plugins/paycom/backend/config/collection-manager.json +``` + +`apply` is an upsert and does not prune unrelated manager records. diff --git a/plugins/paycom/backend/adapters/activation-evidence.js b/plugins/paycom/backend/adapters/activation-evidence.js new file mode 100644 index 0000000..42466bc --- /dev/null +++ b/plugins/paycom/backend/adapters/activation-evidence.js @@ -0,0 +1,62 @@ +'use strict'; + +const path = require('node:path'); +const { runJson } = require('../../../../runtime/adapters/local/process-helper'); + +const EVIDENCE_HELPER = path.resolve(__dirname, "../bin/dispatch-paycom-activation-evidence"); +const HASH_RE = /^[a-f0-9]{64}$/; + +function fail(code = 'first_publication_failed') { + throw Object.assign(new Error(code), { code }); +} +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} +function validResponse(value) { + if (!plain(value) || value.ok !== true || value.status !== 'ok' + || Object.keys(value).sort().join(',') !== 'evidence,ok,status' || !plain(value.evidence)) return false; + const evidence = value.evidence; + return Object.keys(evidence).sort().join(',') + === 'batchId,capturedAt,definitionDigest,preparationRunId,previewDigest,publications,requestDigest,runs,target' + && HASH_RE.test(evidence.definitionDigest) && HASH_RE.test(evidence.requestDigest) + && HASH_RE.test(evidence.previewDigest) && typeof evidence.batchId === 'string' + && typeof evidence.preparationRunId === 'string' + && typeof evidence.target === 'string' && typeof evidence.capturedAt === 'string' + && Array.isArray(evidence.runs) && plain(evidence.publications); +} + +function createLocalPaycomActivationEvidencePort(options) { + if (!plain(options) || Object.keys(options).sort().join(',') !== 'environment' + || !plain(options.environment)) fail('runtime_boundary_violation'); + const environment = Object.freeze({ + ...options.environment, + DISPATCH_MANAGED_RUNTIME: '1', + }); + function verify(input) { + if (!plain(input) || Object.keys(input).sort().join(',') !== 'batchId,definitionDigest,preparationRunId' + || typeof input.batchId !== 'string' || typeof input.preparationRunId !== 'string' + || !HASH_RE.test(input.definitionDigest)) fail(); + try { + return runJson(EVIDENCE_HELPER, [], { + environment, + interpreter: 'node', + input: JSON.stringify(input), + timeout: 30_000, + validate: validResponse, + }).value.evidence; + } catch (error) { + if (error?.code === 'unsafe_executable' || error?.code === 'helper_unavailable') { + fail('runtime_boundary_violation'); + } + fail(); + } + } + return Object.freeze({ verify }); +} + +module.exports = { + EVIDENCE_HELPER, + createLocalPaycomActivationEvidencePort, + validResponse, +}; diff --git a/plugins/paycom/backend/adapters/publication.js b/plugins/paycom/backend/adapters/publication.js new file mode 100644 index 0000000..a388098 --- /dev/null +++ b/plugins/paycom/backend/adapters/publication.js @@ -0,0 +1,35 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); + +const { PaycomStore } = require('../src/store'); +const { TIMECARD_SUMMARY } = require('../src/resource-links'); + +class LocalPaycomPublicationPort { + #database; + #storeFactory; + + constructor({ database = require('../src/paths').DATABASE, storeFactory = (file, options) => new PaycomStore(file, options) } = {}) { + this.#database = database; + this.#storeFactory = storeFactory; + } + + health() { + if (!fs.existsSync(this.#database)) return null; + let store; + try { + store = this.#storeFactory(this.#database, { readOnly: true }); + return { + payPeriods: store.audit('pay_periods'), + roster: store.audit('roster'), + timecards: store.audit('timecards'), + resourceLinks: { kind: 'resource_links', ...store.auditResourceLinks(TIMECARD_SUMMARY) }, + }; + } finally { + try { store?.close(); } catch {} + } + } +} + +module.exports = { LocalPaycomPublicationPort }; diff --git a/plugins/paycom/backend/adapters/published-job.js b/plugins/paycom/backend/adapters/published-job.js new file mode 100644 index 0000000..7ff5bf4 --- /dev/null +++ b/plugins/paycom/backend/adapters/published-job.js @@ -0,0 +1,60 @@ +'use strict'; +const fs = require('node:fs'); +const path = require('node:path'); +const { spawn } = require('node:child_process'); +const { PaycomStore } = require('../src/store'); +const { openDatabase } = require('dispatch-protocol/published/database'); +const completed = new Map(); + +function pendingFingerprint({ database, publishedDatabase, timezone }) { + if (!fs.existsSync(database)) return null; + const source = new PaycomStore(database, { readOnly: true }); + const hash = require('node:crypto').createHash('sha256'); let changed = false; + let published; + try { + published = openDatabase(publishedDatabase); + for (const row of source.db.prepare(`SELECT r.target,r.publication_id roster,t.publication_id timecards,l.publication_id links + FROM active_publications r JOIN active_publications t ON t.kind='timecards' AND t.target=r.target + JOIN active_resource_link_publications l ON l.target=r.target WHERE r.kind='roster' ORDER BY r.target`).iterate()) { + const expected = [row.roster, row.timecards, row.links, timezone].join(':'); + if (!published || published.prepare('SELECT fingerprint FROM periods WHERE target=?').get(row.target)?.fingerprint !== expected) { + changed = true; hash.update(row.target + '\n' + expected + '\n'); + } + } + return changed ? hash.digest('hex') : null; + } finally { source.close(); published?.close(); } +} + +function publish(options) { + const fingerprint = pendingFingerprint(options); + if (!fingerprint || completed.get(options.database) === fingerprint) return Promise.resolve({ changed: 0 }); + // A temporary DSP-local process releases all model-building memory when it + // exits. The gateway remains responsive while a large period is published. + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ['--no-warnings', path.join(__dirname, 'published-job.js'), options.database, options.publishedDatabase, options.timezone], + { stdio: ['ignore', 'pipe', 'ignore'], env: process.env }); + let output = '', settled = false; + const finish = (error, value) => { if (settled) return; settled = true; clearTimeout(timer); error ? reject(error) : resolve(value); }; + const timer = setTimeout(() => { child.kill('SIGKILL'); finish(new Error('published_build_timeout')); }, 300000); + child.stdout.on('data', chunk => { output += chunk; if (output.length > 1024) { child.kill('SIGKILL'); finish(new Error('published_build_failed')); } }); + child.once('error', () => finish(new Error('published_build_failed'))); + child.once('close', code => { + if (code !== 0) return finish(new Error('published_build_failed')); + try { + const value = JSON.parse(output); if (!Number.isInteger(value.changed)) throw new Error(); + // Incomplete source pointers are retried when they change, rather than + // keeping an otherwise idle DSP awake rebuilding the same partial data. + if (completed.size >= 16) completed.delete(completed.keys().next().value); + completed.set(options.database, fingerprint); finish(null, value); + } + catch { finish(new Error('published_build_failed')); } + }); + }); +} +if (require.main === module) { + process.umask(0o077); + if (process.argv.length !== 5) process.exitCode = 1; + else require('./published').publishWorkforce({ database: process.argv[2], publishedDatabase: process.argv[3], timezone: process.argv[4] }) + .then(value => process.stdout.write(JSON.stringify(value) + '\n')).catch(() => { process.exitCode = 1; }); +} +module.exports = { publish, needsPublication: options => Boolean(pendingFingerprint(options)) }; diff --git a/plugins/paycom/backend/adapters/published.js b/plugins/paycom/backend/adapters/published.js new file mode 100644 index 0000000..ab73fd0 --- /dev/null +++ b/plugins/paycom/backend/adapters/published.js @@ -0,0 +1,94 @@ +'use strict'; + +const fs = require('node:fs'); +const { openDatabase, transaction } = require('dispatch-protocol/published/database'); +const { PaycomStore } = require('../src/store'); +const { workforceViews, dailyRow, dailyRowOrder, dailySummary, compareDailyRows } = require('./workforce'); +const { DAY_SORT_KEYS: SORTS } = require('dispatch-protocol/contracts/src/workforce'); + +function schema(db) { + db.exec(`CREATE TABLE IF NOT EXISTS periods(target TEXT PRIMARY KEY,start TEXT NOT NULL,timezone TEXT NOT NULL,fingerprint TEXT NOT NULL,snapshot_json TEXT NOT NULL); + CREATE TABLE IF NOT EXISTS employees(target TEXT NOT NULL REFERENCES periods(target) ON DELETE CASCADE,code TEXT NOT NULL,ordinal INTEGER NOT NULL,lifecycle TEXT NOT NULL,body TEXT NOT NULL,detail TEXT NOT NULL,PRIMARY KEY(target,code)); + CREATE INDEX IF NOT EXISTS employees_page ON employees(target,lifecycle,ordinal); + CREATE TABLE IF NOT EXISTS dates(target TEXT NOT NULL REFERENCES periods(target) ON DELETE CASCADE,date TEXT NOT NULL,summary_json TEXT NOT NULL,PRIMARY KEY(target,date)); + CREATE TABLE IF NOT EXISTS days(target TEXT NOT NULL REFERENCES periods(target) ON DELETE CASCADE,date TEXT NOT NULL,code TEXT NOT NULL,ordinal INTEGER NOT NULL,lifecycle TEXT NOT NULL,condition TEXT NOT NULL,search TEXT NOT NULL,body TEXT NOT NULL, + ${SORTS.flatMap(key => ['asc', 'desc'].map(direction => `rank_${key}_${direction} INTEGER NOT NULL`)).join(',')},PRIMARY KEY(target,date,code)); + CREATE INDEX IF NOT EXISTS days_page ON days(target,date,ordinal); + CREATE TABLE IF NOT EXISTS items(target TEXT NOT NULL REFERENCES periods(target) ON DELETE CASCADE,kind TEXT NOT NULL,ordinal INTEGER NOT NULL,lifecycle TEXT NOT NULL,date TEXT,punch_kind TEXT,time TEXT,body TEXT NOT NULL,PRIMARY KEY(target,kind,ordinal)); + CREATE INDEX IF NOT EXISTS items_page ON items(target,kind,lifecycle,date,ordinal); + PRAGMA user_version=1;`); + for (const key of SORTS) for (const direction of ['asc', 'desc']) { + db.exec(`CREATE INDEX IF NOT EXISTS days_${key}_${direction} ON days(target,date,rank_${key}_${direction});`); + } +} + +function publishPeriod(db, raw, timezone) { + const views = workforceViews(raw); + const fingerprint = [raw.roster.publication.id, raw.timecards.publication.id, raw.resourceLinks.publication.id, timezone].join(':'); + const target = views.snapshot.target; + if (db.prepare('SELECT fingerprint FROM periods WHERE target=?').get(target)?.fingerprint === fingerprint) return false; + const period = views.timecards[0]; + if (!period) throw new Error('workforce_inconsistent'); + transaction(db, () => { + db.prepare('DELETE FROM periods WHERE target=?').run(target); + db.prepare('INSERT INTO periods VALUES(?,?,?,?,?)').run(target, period.periodStart, timezone, fingerprint, JSON.stringify(views.snapshot)); + const employeeInsert = db.prepare('INSERT INTO employees VALUES(?,?,?,?,?,?)'); + const timecardByCode = new Map(views.timecards.map(item => [item.employeeCode, item])); + for (const [ordinal, employee] of views.employees.entries()) { + const timecard = timecardByCode.get(employee.employeeCode) || null; + const record = views.rawTimecardByCode.get(employee.employeeCode); + const detail = { target, collectedAt: views.snapshot.collectedAt.roster, employee, timecard, + days: record ? record.record.days.map(day => dailyRow(employee, record, day.date, views.snapshot.collectedAt.timecards)) : [], businessTimezone: timezone }; + employeeInsert.run(target, employee.employeeCode, ordinal, employee.lifecycleStatus, JSON.stringify(employee), JSON.stringify(detail)); + } + const rowInsert = db.prepare(`INSERT INTO days VALUES(${Array(8 + SORTS.length * 2).fill('?').join(',')})`); + const dates = []; + for (let date = new Date(period.periodStart + 'T12:00:00Z'); date.toISOString().slice(0, 10) <= target; date.setUTCDate(date.getUTCDate() + 1)) dates.push(date.toISOString().slice(0, 10)); + if (dates.length !== 14) throw new Error('workforce_inconsistent'); + for (const date of dates) { + const rows = views.employees.filter(item => item.lifecycleStatus !== 'inactive' || views.rawTimecardByCode.has(item.employeeCode)) + .map(employee => dailyRow(employee, views.rawTimecardByCode.get(employee.employeeCode), date, views.snapshot.collectedAt.timecards)).sort(dailyRowOrder); + db.prepare('INSERT INTO dates VALUES(?,?,?)').run(target, date, JSON.stringify(dailySummary(rows))); + const ranks = SORTS.flatMap(key => ['asc', 'desc'].map(direction => new Map([...rows].sort((a, b) => compareDailyRows(a, b, key, direction)).map((row, index) => [row.employeeCode, index])))); + for (const [ordinal, row] of rows.entries()) rowInsert.run(target, date, row.employeeCode, ordinal, row.lifecycleStatus, row.condition, + `${row.employeeName}\n${row.employeeCode}`.toLocaleLowerCase('en-US'), JSON.stringify(row), ...ranks.map(rank => rank.get(row.employeeCode))); + } + const insert = db.prepare('INSERT INTO items VALUES(?,?,?,?,?,?,?,?)'); + for (const kind of ['timecards', 'resourceLinks', 'punches']) for (const [ordinal, item] of views[kind].entries()) { + const { employeeCode: ignored, ...punch } = item; + insert.run(target, kind, ordinal, item.lifecycleStatus, item.date || null, item.kind || null, item.time || null, JSON.stringify(kind === 'punches' ? punch : item)); + } + }); + return true; +} + +// Runs inside the DSP. Only changed complete periods are rebuilt; retained +// periods remain queryable. Publication failure cannot replace the prior data. +async function publishWorkforce({ database, publishedDatabase, timezone }) { + if (!fs.existsSync(database)) return { changed: 0 }; + new Intl.DateTimeFormat('en-US', { timeZone: timezone }).format(); + const store = new PaycomStore(database, { readOnly: true }); + let db; let changed = 0; + try { + // Read workers mount this projection read-only. Rollback journaling keeps + // reads independent of writable WAL/shared-memory sidecars after a restart. + db = openDatabase(publishedDatabase, { write: true, journalMode: 'DELETE' }); schema(db); + const pointers = store.db.prepare(`SELECT r.target,r.publication_id roster,t.publication_id timecards,l.publication_id links + FROM active_publications r JOIN active_publications t ON t.kind='timecards' AND t.target=r.target + JOIN active_resource_link_publications l ON l.target=r.target + WHERE r.kind='roster' ORDER BY r.target`).all(); + for (const row of pointers) { + const fingerprint = [row.roster, row.timecards, row.links, timezone].join(':'); + if (db.prepare('SELECT fingerprint FROM periods WHERE target=?').get(row.target)?.fingerprint === fingerprint) continue; + // A collection may be between independent first-publication steps. + // Inconsistent periods are not exposed and will be retried next time. + let raw; + try { raw = store.activeWorkforce(row.target); } + catch (error) { if (error.message === 'workforce_inconsistent') continue; throw error; } + if (publishPeriod(db, raw, timezone)) changed++; + await new Promise(resolve => setImmediate(resolve)); + } + return { changed }; + } finally { db?.close(); store.close(); } +} +module.exports = { publishWorkforce, publishPeriod, schema }; diff --git a/plugins/paycom/backend/adapters/workforce.js b/plugins/paycom/backend/adapters/workforce.js new file mode 100644 index 0000000..020bc37 --- /dev/null +++ b/plugins/paycom/backend/adapters/workforce.js @@ -0,0 +1,365 @@ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); + +const { PaycomStore } = require('../src/store'); + +const PUNCH_KIND_MAP = Object.freeze({ + 'IN DAY': 'in_day', + 'OUT LUNCH': 'out_lunch', + 'IN LUNCH': 'in_lunch', + 'OUT DAY': 'out_day', + '': 'unclassified', +}); + +function punchTime(value) { + const match = /^(0[1-9]|1[0-2]):([0-5][0-9]) ([AP])M$/.exec(value || ''); + if (!match) throw Object.assign(new Error('workforce_inconsistent'), { code: 'workforce_inconsistent' }); + let hour = Number(match[1]) % 12; + if (match[3] === 'P') hour += 12; + return `${String(hour).padStart(2, '0')}:${match[2]}`; +} + +function lifecycleStatus(employee) { + return employee.lifecycleStatus || (employee.isActive ? 'active' : 'inactive'); +} + +function employeeView(employee) { + return { + employeeCode: employee.employeeCode, + employeeName: employee.employeeName, + lifecycleStatus: lifecycleStatus(employee), + lastExplicitActive: employee.isActive, + department: { code: employee.departmentCode || '', name: employee.departmentDesc || '' }, + deliveryStation: { code: employee.deliveryStationCode || '', name: employee.deliveryStationDesc || '' }, + positionTitle: employee.positionTitle || '', + payClass: employee.payClass || '', + payType: employee.payType || '', + primarySupervisor: employee.primarySupervisor || '', + isDriver: employee.isDriverDepartment === true || employee.isDriverPosition === true, + }; +} + +function snapshotView(workforce) { + const employees = workforce.roster.employees; + const lifecycleCounts = { active: 0, inactive: 0, unknown: 0 }; + for (const employee of employees) lifecycleCounts[lifecycleStatus(employee)] += 1; + return { + target: workforce.roster.publication.target, + collectedAt: { + roster: workforce.roster.publication.collected_at, + timecards: workforce.timecards.publication.collected_at, + resourceLinks: workforce.resourceLinks.publication.collected_at, + }, + counts: { + employees: employees.length, + timecards: workforce.timecards.rows.length, + resourceLinks: workforce.resourceLinks.rows.length, + }, + lifecycleCounts, + consistent: true, + }; +} + +function paginate(items, query) { + const page = items.slice(query.offset, query.offset + query.limit); + return { + items: page, + total: items.length, + limit: query.limit, + offset: query.offset, + hasMore: query.offset + page.length < items.length, + }; +} + +function dailyPunch(punch) { + const kind = PUNCH_KIND_MAP[punch.kind]; + if (!kind) throw Object.assign(new Error('workforce_inconsistent'), { code: 'workforce_inconsistent' }); + const actual = punch.provenanceAvailable === true && punch.actualTime; + return { + kind, + time: punchTime(actual || punch.displayTime), + timeBasis: actual ? 'actual' : 'displayed', + }; +} + +function dailyRow(employee, timecard, businessDate, collectedAt) { + const day = (timecard.record.days || []).find(item => item.date === businessDate) || null; + const punches = { + inDay: [], outLunch: [], inLunch: [], outDay: [], unclassified: [], + }; + for (const punch of day?.punches || []) { + const projected = dailyPunch(punch); + const key = projected.kind === 'in_day' ? 'inDay' + : projected.kind === 'out_lunch' ? 'outLunch' + : projected.kind === 'in_lunch' ? 'inLunch' + : projected.kind === 'out_day' ? 'outDay' : 'unclassified'; + punches[key].push({ time: projected.time, timeBasis: projected.timeBasis }); + } + const punchCount = Object.values(punches).reduce((total, items) => total + items.length, 0); + const needsReview = day?.missingPunch === true || punches.unclassified.length > 0; + const condition = needsReview ? 'needs_review' + : punchCount === 0 ? 'no_activity' + : punches.inDay.length > 0 && punches.outDay.length > 0 ? 'complete' : 'incomplete'; + return { + employeeCode: employee.employeeCode, + employeeName: employee.employeeName, + lifecycleStatus: employee.lifecycleStatus, + isDriver: employee.isDriver, + department: employee.department, + deliveryStation: employee.deliveryStation, + businessDate, + condition, + missingPunch: day?.missingPunch === true, + totalHours: day?.totalHours === null || day?.totalHours === undefined ? null : String(day.totalHours), + punchCount, + unresolvedSlotCount: Array.isArray(day?.unresolvedSlots) ? day.unresolvedSlots.length : 0, + punches, + observedAt: timecard.observedAt || collectedAt, + }; +} + +const { dailySummary } = require('dispatch-protocol/contracts/src/workforce-summary'); + +function dailyRowOrder(left, right) { + const activityOrder = Number(left.condition === 'no_activity') - Number(right.condition === 'no_activity'); + return activityOrder || left.employeeName.localeCompare(right.employeeName) + || left.employeeCode.localeCompare(right.employeeCode); +} + +function punchStatus(row) { + if (row.missingPunch) return 'Missing punch'; + if (row.condition === 'needs_review') return 'Needs review'; + if (row.condition === 'no_activity') return 'No punches'; + if (row.condition === 'complete') return 'Clocked out'; + if (row.punches.outLunch.length > row.punches.inLunch.length) return 'On lunch'; + return 'Clocked in'; +} + +// Compare the full filtered day before pagination. Empty values stay last in both directions. +function compareDailyRows(left, right, key, direction = 'asc') { + const value = row => key === 'condition' ? punchStatus(row) + : key === 'totalHours' ? (row.totalHours === null || row.totalHours.trim() === '' ? null : Number(row.totalHours)) + : key === 'employeeName' ? row.employeeName : row.punches[key]?.[0]?.time ?? null; + const a = value(left), b = value(right); + const emptyA = a === null || a === '' || typeof a === 'number' && !Number.isFinite(a); + const emptyB = b === null || b === '' || typeof b === 'number' && !Number.isFinite(b); + if (emptyA !== emptyB) return emptyA ? 1 : -1; + const comparison = emptyA ? 0 : typeof a === 'number' ? a - b + : a.localeCompare(b, 'en', { sensitivity: 'base', numeric: true }); + return comparison * (direction === 'desc' ? -1 : 1) + || left.employeeName.localeCompare(right.employeeName, 'en', { sensitivity: 'base' }) + || left.employeeCode.localeCompare(right.employeeCode); +} + +function workforceViews(workforce) { + const snapshot = snapshotView(workforce); + const employees = workforce.roster.employees.map(employeeView); + const employeeByCode = new Map(employees.map(employee => [employee.employeeCode, employee])); + const linkByCode = new Map(workforce.resourceLinks.rows.map(row => [row.employeeCode, row.canonicalUrl])); + const [periodStart, periodEnd] = workforce.resourceLinks.publication.period_key.split('_'); + const resourceLinks = workforce.resourceLinks.rows.map(row => { + const employee = employeeByCode.get(row.employeeCode); + if (!employee) throw Object.assign(new Error('workforce_inconsistent'), { code: 'workforce_inconsistent' }); + return { + employeeCode: row.employeeCode, employeeName: employee.employeeName, + lifecycleStatus: employee.lifecycleStatus, + resourceType: workforce.resourceLinks.publication.resource_type, + periodStart, periodEnd, canonicalUrl: row.canonicalUrl, + }; + }); + const timecards = workforce.timecards.rows.map(row => { + const employee = employeeByCode.get(row.employeeCode); + if (!employee || !linkByCode.has(row.employeeCode)) throw Object.assign(new Error('workforce_inconsistent'), { code: 'workforce_inconsistent' }); + return { + employeeCode: row.employeeCode, + employeeName: row.employeeName, + lifecycleStatus: employee.lifecycleStatus, + periodStart: row.record.periodStart, + periodEnd: row.record.periodEnd, + periodTotalHours: String(row.record.periodTotalHours), + missingDays: row.record.days.filter(day => day.missingPunch).length, + observedAt: row.observedAt || workforce.timecards.publication.collected_at, + canonicalUrl: linkByCode.get(row.employeeCode), + }; + }); + const rawTimecardByCode = new Map(workforce.timecards.rows.map(row => [row.employeeCode, row])); + const punches = []; + for (const row of workforce.timecards.rows) { + const employee = employeeByCode.get(row.employeeCode); + if (!employee) throw Object.assign(new Error('workforce_inconsistent'), { code: 'workforce_inconsistent' }); + for (const day of row.record.days || []) { + for (const punch of day.punches || []) { + const kind = PUNCH_KIND_MAP[punch.kind]; + if (!kind) throw Object.assign(new Error('workforce_inconsistent'), { code: 'workforce_inconsistent' }); + const actual = punch.provenanceAvailable === true && punch.actualTime; + punches.push({ + employeeCode: row.employeeCode, + employeeName: employee.employeeName, + lifecycleStatus: employee.lifecycleStatus, + date: day.date, + kind, + time: punchTime(actual || punch.displayTime), + timeBasis: actual ? 'actual' : 'displayed', + observedAt: row.observedAt || workforce.timecards.publication.collected_at, + }); + } + } + } + punches.sort((left, right) => left.date.localeCompare(right.date) || left.time.localeCompare(right.time) + || left.employeeName.localeCompare(right.employeeName) || left.employeeCode.localeCompare(right.employeeCode)); + return { snapshot, employees, employeeByCode, timecards, rawTimecardByCode, resourceLinks, punches }; +} + +class LocalPaycomWorkforcePort { + #database; + #storeFactory; + #timezone; + + constructor({ + database = require('../src/paths').DATABASE, + timezone = 'America/Los_Angeles', + storeFactory = (file, options) => new PaycomStore(file, options), + } = {}) { + if (typeof timezone !== 'function') { + try { new Intl.DateTimeFormat('en-US', { timeZone: timezone }).format(); } + catch { throw new TypeError('workforce_timezone_invalid'); } + } + this.#database = database; + this.#timezone = timezone; + this.#storeFactory = storeFactory; + } + + #businessTimezone() { + try { + const value = typeof this.#timezone === 'function' ? this.#timezone() : this.#timezone; + if (typeof value !== 'string' || !value || value.length > 64) throw new Error(); + new Intl.DateTimeFormat('en-US', { timeZone: value }).format(); + return value; + } catch { throw Object.assign(new Error('workforce_inconsistent'), { code: 'workforce_inconsistent' }); } + } + + #read(target = null, businessDate = null) { + if (!fs.existsSync(this.#database)) return null; + let store; + try { + store = this.#storeFactory(this.#database, { readOnly: true }); + if (typeof store.active === 'function' && !store.active('roster', target)) return null; + let workforce = store.activeWorkforce(target); + if (businessDate) { + const end = new Date(workforce.roster.publication.target + 'T12:00:00Z'); + const delta = Math.ceil((new Date(businessDate + 'T12:00:00Z') - end) / (14 * 86400000)); + end.setUTCDate(end.getUTCDate() + delta * 14); + const selectedTarget = end.toISOString().slice(0, 10); + if (selectedTarget !== workforce.roster.publication.target && typeof store.active === 'function' && store.active('roster', selectedTarget)) workforce = store.activeWorkforce(selectedTarget); + } + return workforceViews(workforce); + } finally { + try { store?.close(); } catch {} + } + } + + snapshot() { + return this.#read()?.snapshot || null; + } + + employees(query) { + const value = this.#read(); + if (!value) return null; + const items = query.lifecycleStatus === null + ? value.employees + : value.employees.filter(employee => employee.lifecycleStatus === query.lifecycleStatus); + return { target: value.snapshot.target, collectedAt: value.snapshot.collectedAt.roster, ...paginate(items, query) }; + } + + employee(employeeCode) { + const value = this.#read(); + if (!value) return null; + const employee = value.employeeByCode.get(employeeCode); + if (!employee) return { target: value.snapshot.target, collectedAt: value.snapshot.collectedAt.roster, employee: null, timecard: null }; + const timecard = value.timecards.find(row => row.employeeCode === employeeCode) || null; + const raw = value.rawTimecardByCode.get(employeeCode); + const days = raw ? raw.record.days.map(day => dailyRow(employee, raw, day.date, value.snapshot.collectedAt.timecards)) : []; + return { target: value.snapshot.target, collectedAt: value.snapshot.collectedAt.roster, employee, timecard, days, businessTimezone: this.#businessTimezone() }; + } + + timecards(query) { + const value = this.#read(); + if (!value) return null; + const items = query.lifecycleStatus === null + ? value.timecards + : value.timecards.filter(row => row.lifecycleStatus === query.lifecycleStatus); + return { target: value.snapshot.target, collectedAt: value.snapshot.collectedAt.timecards, ...paginate(items, query) }; + } + + punches(query) { + const value = this.#read(); + if (!value) return null; + let items = value.punches.filter(punch => punch.date === query.date); + if (query.kind !== null) items = items.filter(punch => punch.kind === query.kind); + if (query.fromTime !== null) items = items.filter(punch => punch.time >= query.fromTime); + if (query.throughTime !== null) items = items.filter(punch => punch.time <= query.throughTime); + if (query.lifecycleStatus !== null) items = items.filter(punch => punch.lifecycleStatus === query.lifecycleStatus); + items = items.map(({ employeeCode: ignored, ...punch }) => punch); + return { + target: value.snapshot.target, + businessDate: query.date, + businessTimezone: this.#businessTimezone(), + collectedAt: value.snapshot.collectedAt.timecards, + ...paginate(items, query), + }; + } + + day(query) { + const value = this.#read(null, query.date); + if (!value) return null; + const first = value.timecards[0]; + if (!first) throw Object.assign(new Error('workforce_inconsistent'), { code: 'workforce_inconsistent' }); + const available = query.date >= first.periodStart && query.date <= first.periodEnd; + let allItems = []; + if (available) { + allItems = value.employees.filter(employee => employee.lifecycleStatus !== 'inactive' || value.rawTimecardByCode.has(employee.employeeCode)).map(employee => { + const timecard = value.rawTimecardByCode.get(employee.employeeCode); + if (!timecard) throw Object.assign(new Error('workforce_inconsistent'), { code: 'workforce_inconsistent' }); + return dailyRow(employee, timecard, query.date, value.snapshot.collectedAt.timecards); + }).sort(dailyRowOrder); + } + const summary = dailySummary(allItems); + let items = allItems; + if (query.lifecycleStatus !== null) items = items.filter(row => row.lifecycleStatus === query.lifecycleStatus); + if (query.search !== null) { + const search = query.search.toLocaleLowerCase('en-US'); + items = items.filter(row => row.employeeName.toLocaleLowerCase('en-US').includes(search) + || row.employeeCode.toLocaleLowerCase('en-US').includes(search)); + } + if (query.attention === 'needs_review') items = items.filter(row => row.condition === 'needs_review'); + if (query.attention === 'incomplete') items = items.filter(row => ['needs_review', 'incomplete'].includes(row.condition)); + if (query.attention === 'no_activity') items = items.filter(row => row.condition === 'no_activity'); + return { + target: value.snapshot.target, + businessDate: query.date, + businessTimezone: this.#businessTimezone(), + periodStart: first.periodStart, + periodEnd: first.periodEnd, + available, + collectedAt: value.snapshot.collectedAt.timecards, + summary, + ...paginate(query.sort ? [...items].sort((a, b) => compareDailyRows(a, b, query.sort, query.direction)) : items, query), + }; + } + + resourceLinks(query) { + const value = this.#read(); + if (!value) return null; + const items = query.lifecycleStatus === null + ? value.resourceLinks + : value.resourceLinks.filter(row => row.lifecycleStatus === query.lifecycleStatus); + return { target: value.snapshot.target, collectedAt: value.snapshot.collectedAt.resourceLinks, ...paginate(items, query) }; + } +} + +module.exports = { + LocalPaycomWorkforcePort, lifecycleStatus, employeeView, snapshotView, paginate, workforceViews, + compareDailyRows, punchStatus, punchTime, dailyPunch, dailyRow, dailySummary, dailyRowOrder, PUNCH_KIND_MAP, +}; diff --git a/plugins/paycom/backend/auth/adapter.js b/plugins/paycom/backend/auth/adapter.js new file mode 100644 index 0000000..139d736 --- /dev/null +++ b/plugins/paycom/backend/auth/adapter.js @@ -0,0 +1,842 @@ +'use strict'; + +const { CdpConnection, createTarget } = require('dispatch-sdk/node/cdp'); + +const ORIGIN = 'https://www.paycomonline.net'; +const LOGIN_PATH = '/v4/cl/cl-login.php'; +const LOGIN_URL = `${ORIGIN}${LOGIN_PATH}`; +const LOGIN_ACTION_PATH = '/v4/cl/cl-loginproc.php'; +const LOGIN_ACTION_URL = `${ORIGIN}${LOGIN_ACTION_PATH}`; +const AUTH_PATH_PREFIX = '/v4/cl/web.php/'; +const AUTH_PREFIX = `${ORIGIN}${AUTH_PATH_PREFIX}`; +const CLIENT_LANDING_PATH = '/v4/cl/web.php/client-landing/arc'; +const MAIN_MENU_PATH = '/v4/cl/cl-menu.php'; +const SECURITY_QUESTION_PATH = '/v4/cl/web.php/security/security-question/login'; +const TIMECARD_SEARCH_PATH = '/v4/cl/web.php/timecardsearch/index'; +const TIMECARD_SEARCH_URL = `${ORIGIN}${TIMECARD_SEARCH_PATH}?from=main_menu`; +const SECURITY_PROFILE_PATH = '/v4/cl/web.php/two-factor/react/index/preferences/campaign'; +const SECURITY_PROFILE_WARNING = 'You will no longer be prompted at login to verify your info this month. You will continue to use security questions to access your account. You may verify your information at any time from your contact information page.'; +const LOGIN_FIELDS = Object.freeze({ + clientCode: 'input[name="clientcode"]', + username: 'input[name="username"]', + password: 'input[name="password"]', +}); + +class PaycomAuthError extends Error { + constructor(code) { super(code); this.code = code; } +} + +function throwIfAborted(signal) { + if (signal?.aborted) throw new PaycomAuthError('acquisition_cancelled'); +} + +function delay(ms, signal) { + return new Promise((resolve, reject) => { + try { throwIfAborted(signal); } catch (error) { reject(error); return; } + const timer = setTimeout(done, ms); + function done() { cleanup(); resolve(); } + function aborted() { cleanup(); reject(new PaycomAuthError('acquisition_cancelled')); } + function cleanup() { clearTimeout(timer); signal?.removeEventListener('abort', aborted); } + signal?.addEventListener('abort', aborted, { once: true }); + }); +} + +function parsedPaycomUrl(value) { + try { + const url = new URL(value); + if (url.protocol !== 'https:' || url.hostname !== 'www.paycomonline.net' || url.port || url.username || url.password || url.hash) return null; + return url; + } catch { return null; } +} + +function exactLoginUrl(value) { + const url = parsedPaycomUrl(value); + return Boolean(url && url.pathname === LOGIN_PATH && url.search === ''); +} + +function exactLoginActionUrl(value) { + const url = parsedPaycomUrl(value); + return Boolean(url && url.pathname === LOGIN_ACTION_PATH && url.search === ''); +} + +function authenticatedUrl(value) { + const url = parsedPaycomUrl(value); + if (!url) return false; + const keys = [...url.searchParams.keys()]; + const unique = new Set(keys); + const oneEach = unique.size === keys.length; + if (url.pathname.startsWith(AUTH_PATH_PREFIX) + && url.pathname !== CLIENT_LANDING_PATH) return oneEach && keys.every(key => key === 'session_nonce') && keys.length <= 1; + if (![CLIENT_LANDING_PATH, MAIN_MENU_PATH].includes(url.pathname) || !oneEach + || keys.some(key => !['frmlogin', 'session_nonce'].includes(key)) || keys.length > 2) return false; + return !url.searchParams.has('frmlogin') || url.searchParams.get('frmlogin') === '1'; +} + +function exactSecurityQuestionUrl(value) { + const url = parsedPaycomUrl(value); + if (!url || url.pathname !== SECURITY_QUESTION_PATH) return false; + const keys = [...url.searchParams.keys()]; + return keys.length <= 1 && keys.every(key => key === 'session_nonce'); +} + +function exactTimecardSearchUrl(value) { + const url = parsedPaycomUrl(value); + if (!url || url.pathname !== TIMECARD_SEARCH_PATH) return false; + const keys = [...url.searchParams.keys()]; + return new Set(keys).size === keys.length && keys.length <= 2 + && keys.every(key => ['from', 'session_nonce'].includes(key)) + && url.searchParams.get('from') === 'main_menu'; +} + +function securityProfileUrl(value) { + const url = parsedPaycomUrl(value); + if (!url || url.pathname !== SECURITY_PROFILE_PATH) return false; + const keys = [...url.searchParams.keys()]; + return keys.length <= 1 && keys.every(key => key === 'session_nonce'); +} + +function securityProfileState(snapshot) { + if (!securityProfileUrl(snapshot?.url) || !snapshot.securityProfile) return null; + const inputs = JSON.stringify(snapshot.securityProfile.inputNames); + const buttons = JSON.stringify(snapshot.securityProfile.buttonTexts); + const text = String(snapshot.text || '').replace(/\s+/g, ' ').trim(); + if (inputs !== JSON.stringify(['cell-number', 'email', 'work-number']) + || !text.includes('Setup Your Security Profile') || !text.includes('Verify your contact information')) return 'manual_verification_required'; + if (buttons === JSON.stringify(['Continue', 'Not Now', 'Verify', 'Verify', 'Verify']) && !text.includes('Warning')) return 'security_profile_prompt'; + if (buttons === JSON.stringify(['', 'Cancel', 'Continue', 'Continue', 'Not Now', 'Verify', 'Verify', 'Verify']) + && text.includes('Warning') && text.includes(SECURITY_PROFILE_WARNING)) return 'security_profile_confirmation'; + return 'manual_verification_required'; +} + +// Reused immediately before optional-profile clicks to catch verification +// controls that appeared after the last observation. No field values are read. +const VERIFICATION_CONTROLS = ` + const displayed=e=>{ + if(!visible(e))return false; + const style=getComputedStyle(e),rect=e.getBoundingClientRect(); + return style.visibility!=='hidden'&&style.visibility!=='collapse'&&style.opacity!=='0'&&rect.width>0&&rect.height>0; + }; + const otpPresent=Array.from(document.querySelectorAll('input[autocomplete="one-time-code"],input[name="otp"],input[name="verificationCode"],input[name="verification_code"]')).some(displayed); + const captchaPresent=Array.from(document.querySelectorAll('iframe')).some(frame=>{ + if(!displayed(frame))return false; + try{const url=new URL(frame.src);return /(?:^|[./_-])(?:hcaptcha|recaptcha|captcha)(?:[./_-]|$)/i.test(url.hostname+url.pathname)}catch{return false} + }); +`; + +const SNAPSHOT = `(()=>{ + const visible=e=>!!e&&!e.disabled&&e.offsetParent!==null; + ${VERIFICATION_CONTROLS} + const login=['input[name="clientcode"]','input[name="username"]','input[name="password"]']; + const loginFields=login.map(selector=>document.querySelector(selector)); + const challenge=[]; + for(const field of Array.from(document.querySelectorAll('input')).filter(visible)){ + if(login.some(selector=>field.matches(selector)))continue; + const labels=[]; + if(field.id){const label=document.querySelector('label[for="'+CSS.escape(field.id)+'"]');if(label)labels.push(label.innerText||label.textContent||'');} + labels.push(field.getAttribute('aria-label')||'',field.placeholder||'',field.name||'',field.id||''); + const found=[]; + for(const label of labels){ + const text=String(label).trim(); + const match=text.match(/^\\s*(?:(?:enter|unique)\\s+)?(?:paycom\\s+)?(?:security\\s+)?pin(?:\\s*(?:number|no\\.?|#))?\\s*([1-5])\\s*[:?]?\\s*$/i)||text.match(/^(?:security[_-]?)?pin[_-]?([1-5])$/i); + if(match)found.push(Number(match[1])); + } + const unique=Array.from(new Set(found)); + if(unique.length===1)challenge.push({index:unique[0],name:field.name||'',id:field.id||''}); + } + const forms=new Set(loginFields.filter(Boolean).map(field=>field.form)); + const challengeForms=new Set(challenge.map(item=>document.querySelector(item.id?'#'+CSS.escape(item.id):'input[name="'+CSS.escape(item.name)+'"]')?.form).filter(Boolean)); + const profileInputs=Array.from(document.querySelectorAll('input')).filter(visible); + const profileButtons=Array.from(document.querySelectorAll('button,input[type="submit"]')).filter(visible); + const timecardStatus=Array.from(document.querySelectorAll('p,[data-testid="typography"]')).filter(visible) + .map(element=>(element.innerText||element.textContent||'').trim()).filter(text=>/^Employee Status Is .+/.test(text)); + const timecardExports=Array.from(document.querySelectorAll('button')).filter(visible) + .filter(element=>/^Export$/i.test((element.innerText||element.textContent||'').trim())); + return { + url:location.href, + title:String(document.title||'').slice(0,120), + readyState:document.readyState, + otpPresent,captchaPresent, + text:(document.body&&document.body.innerText||'').slice(0,12000), + loginPresent:loginFields.map(Boolean), + loginVisible:loginFields.map(visible), + loginFormCount:forms.size, + loginFormAction:forms.size===1&&loginFields[0]?.form?loginFields[0].form.action:'', + loginFormMethod:forms.size===1&&loginFields[0]?.form?loginFields[0].form.method:'', + challengeFormCount:challengeForms.size, + challengeFormAction:challengeForms.size===1?[...challengeForms][0].action:'', + challengeFormMethod:challengeForms.size===1?[...challengeForms][0].method:'', + authenticated:Boolean(document.querySelector('#mainMenuLink')&&document.querySelector('#clientLogout')&&!document.querySelector('input[name="clientcode"]')), + timecardSearchReady:document.title==='Timecard Search'&&timecardStatus.length===1&&timecardExports.length===1, + securityProfile:{ + inputNames:profileInputs.map(field=>field.name).sort(), + buttonTexts:profileButtons.map(button=>(button.innerText||button.value||'').trim()).sort() + }, + challenge + }; +})()`; + +function verificationTextMatches(text) { + const value = String(text || '').toLowerCase(); + return [ + ['captcha', /captcha/], ['verification_code', /verification code/], + ['verify_identity', /verify your identity/], ['multi_factor', /multi-factor/], ['one_time_code', /one-time code/], + ].filter(([, pattern]) => pattern.test(value)).map(([name]) => name); +} + +function classifyState(snapshot, { phase = 'observation' } = {}) { + const result = (state, reason = state) => ({ state, reason }); + if (!snapshot || typeof snapshot.url !== 'string' || !parsedPaycomUrl(snapshot.url)) return result('manual_verification_required', 'untrusted_url'); + const text = String(snapshot.text || '').toLowerCase(); + const parsed = parsedPaycomUrl(snapshot.url); + if (/account.{0,30}(locked|disabled)|too many.{0,20}attempt/.test(text)) return result('account_locked'); + const primaryRejected = /invalid.{0,30}(client|user|password|credential)|incorrect.{0,20}(password|login)/.test(text); + const securityRejected = /security answers?.{0,40}(not correct|incorrect|invalid)/.test(text); + const exactLoginRoute = exactLoginUrl(snapshot.url) || exactLoginActionUrl(snapshot.url); + const exactSecurityRoute = exactSecurityQuestionUrl(snapshot.url); + if (primaryRejected || securityRejected) { + if (phase === 'primary_login' && primaryRejected && exactLoginRoute) return result('primary_credentials_rejected'); + if (phase === 'security_questions' && securityRejected && (exactSecurityRoute || exactLoginRoute)) return result('security_answers_rejected'); + return result('manual_verification_required', 'ambiguous_rejection'); + } + const manualChallenge = verificationTextMatches(text).length > 0; + // Visible verification controls must never be dismissed as an optional campaign. + if (snapshot.otpPresent === true || snapshot.captchaPresent === true) return result('manual_verification_required', 'additional_verification'); + if (parsed.pathname === SECURITY_PROFILE_PATH) { + if (Array.isArray(snapshot.challenge) && snapshot.challenge.length) return result('manual_verification_required', 'unexpected_challenge'); + if (!securityProfileUrl(snapshot.url)) return result('manual_verification_required', 'unexpected_query'); + if (snapshot.readyState !== 'complete') return result('pending', 'page_loading'); + const state = securityProfileState(snapshot); + // Match the known, optional Not Now flow before interpreting incidental + // identity-verification wording, as the established Paycom provider does. + if (state && state !== 'manual_verification_required') return result(state); + if (manualChallenge) return result('manual_verification_required', 'additional_verification'); + return result('manual_verification_required', 'security_profile_layout_changed'); + } + if (manualChallenge) return result('manual_verification_required', 'additional_verification'); + // Visible fields can precede Paycom's DOMContentLoaded form-token handler. + if (snapshot.readyState !== 'complete') return result('pending', 'page_loading'); + if (parsed.pathname === SECURITY_QUESTION_PATH) { + if (!exactSecurityQuestionUrl(snapshot.url)) return result('manual_verification_required', 'unexpected_query'); + if (!Array.isArray(snapshot.challenge) || !snapshot.challenge.length) return result('pending', 'page_loading'); + const indices = snapshot.challenge.map(item => item.index); + return indices.length === 2 && new Set(indices).size === 2 + && snapshot.challengeFormCount === 1 && exactSecurityQuestionUrl(snapshot.challengeFormAction) + && String(snapshot.challengeFormMethod).toUpperCase() === 'POST' + ? result('security_questions_required') : result('manual_verification_required', 'challenge_layout_changed'); + } + if (parsed.pathname === TIMECARD_SEARCH_PATH) { + if (!exactTimecardSearchUrl(snapshot.url)) return result('manual_verification_required', 'unexpected_query'); + return snapshot.timecardSearchReady === true ? result('timecard_application') : result('pending', 'page_loading'); + } + if (snapshot.authenticated === true && authenticatedUrl(snapshot.url)) return result('authenticated'); + if (Array.isArray(snapshot.challenge) && snapshot.challenge.length) return result('manual_verification_required', 'unexpected_challenge'); + if (Array.isArray(snapshot.loginPresent) && snapshot.loginPresent.length === 3 && snapshot.loginPresent.every(Boolean) + && Array.isArray(snapshot.loginVisible) && snapshot.loginVisible.every(Boolean) + && snapshot.loginFormCount === 1 && exactLoginUrl(snapshot.url) && exactLoginActionUrl(snapshot.loginFormAction) + && String(snapshot.loginFormMethod).toUpperCase() === 'POST') return result('logged_out'); + return result('pending', 'page_unrecognized'); +} + +function classify(snapshot, context) { return classifyState(snapshot, context).state; } + +function classificationDiagnostic(state, phase, snapshot, reason) { + const route = exactSecurityQuestionUrl(snapshot?.url) ? 'security_question' + : (exactLoginUrl(snapshot?.url) || exactLoginActionUrl(snapshot?.url)) ? 'login' + : securityProfileUrl(snapshot?.url) ? 'security_profile' + : authenticatedUrl(snapshot?.url) ? 'application' : 'unknown'; + const evidence = ['primary_credentials_rejected', 'security_answers_rejected'].includes(state) + ? 'provider_rejection' : 'adapter_check'; + return { phase, route, evidence, reason }; +} + +function stateMetadata(snapshot, diagnostic = null) { + const url = parsedPaycomUrl(snapshot?.url); + let formActionPath = null; + try { + const action = new URL(snapshot?.challengeFormAction || ''); + if (action.origin === ORIGIN) formActionPath = action.pathname; + } catch {} + const profileInputs = Array.isArray(snapshot?.securityProfile?.inputNames) + ? snapshot.securityProfile.inputNames.filter(value => typeof value === 'string' && value.length <= 64).slice(0, 16) : []; + const profileActions = Array.isArray(snapshot?.securityProfile?.buttonTexts) + ? snapshot.securityProfile.buttonTexts.filter(value => typeof value === 'string' && value.length <= 64).slice(0, 16) : []; + return { + origin: url?.origin || null, + path: url?.pathname || null, + queryKeys: url ? [...url.searchParams.keys()].sort() : [], + title: typeof snapshot?.title === 'string' ? snapshot.title.slice(0, 120) : null, + readyState: ['loading', 'interactive', 'complete'].includes(snapshot?.readyState) ? snapshot.readyState : null, + otpPresent: typeof snapshot?.otpPresent === 'boolean' ? snapshot.otpPresent : null, + captchaPresent: typeof snapshot?.captchaPresent === 'boolean' ? snapshot.captchaPresent : null, + verificationTextMatches: verificationTextMatches(snapshot?.text), + loginFormCount: Number.isInteger(snapshot?.loginFormCount) ? snapshot.loginFormCount : null, + challengeFormCount: Number.isInteger(snapshot?.challengeFormCount) ? snapshot.challengeFormCount : null, + profileInputNames: url?.pathname === SECURITY_PROFILE_PATH ? profileInputs : [], + profileActionLabels: url?.pathname === SECURITY_PROFILE_PATH ? profileActions : [], + challengeIndices: Array.isArray(snapshot?.challenge) ? snapshot.challenge.map(item => item.index).sort() : [], + challengeFormActionPath: formActionPath, + challengeFormMethod: ['GET', 'POST'].includes(String(snapshot?.challengeFormMethod).toUpperCase()) + ? String(snapshot.challengeFormMethod).toUpperCase() : null, + diagnostic, + }; +} + +function loginExpression(credentials) { + const values = JSON.stringify({ clientCode: credentials.clientCode, username: credentials.username, password: credentials.password }).replace(/{ + const values=${values}; + const safe=(u,path)=>{try{const x=new URL(u);return x.protocol==='https:'&&x.hostname==='www.paycomonline.net'&&!x.port&&!x.username&&!x.password&&!x.hash&&x.pathname===path&&!x.search}catch{return false}}; + if(!safe(location.href,'/v4/cl/cl-login.php')||document.readyState!=='complete')return {status:'login_layout_changed'}; + const visible=e=>!!e&&!e.disabled&&e.offsetParent!==null; + const selectors=${JSON.stringify(LOGIN_FIELDS)}; + const fields={}; + for(const key of Object.keys(selectors)){fields[key]=document.querySelector(selectors[key]);if(!visible(fields[key]))return {status:'login_layout_changed'}} + const forms=new Set(Object.values(fields).map(field=>field.form)); + if(forms.size!==1||!fields.password.form||String(fields.password.form.method).toUpperCase()!=='POST'||!safe(fields.password.form.action,'/v4/cl/cl-loginproc.php'))return {status:'login_layout_changed'}; + const setter=Object.getOwnPropertyDescriptor(HTMLInputElement.prototype,'value').set; + for(const key of Object.keys(fields)){setter.call(fields[key],values[key]);fields[key].dispatchEvent(new Event('input',{bubbles:true}));fields[key].dispatchEvent(new Event('change',{bubbles:true}));} + fields.password.form.requestSubmit(); + return {status:'submitted'}; + })()`; +} + +function challengeIndices(challenge) { + if (!Array.isArray(challenge) || challenge.length !== 2) throw new PaycomAuthError('manual_verification_required'); + const indices = challenge.map(item => item?.index).sort((a, b) => a - b); + if (indices.some(index => !Number.isInteger(index) || index < 1 || index > 5) || new Set(indices).size !== 2) { + throw new PaycomAuthError('manual_verification_required'); + } + return indices; +} + +function challengeFocusExpression(challenge, targetIndex, { verifyFocus = false } = {}) { + const indices = challengeIndices(challenge); + if (!indices.includes(targetIndex)) throw new PaycomAuthError('manual_verification_required'); + return `(()=>{ + const safe=u=>{try{const x=new URL(u),keys=Array.from(x.searchParams.keys());return x.protocol==='https:'&&x.hostname==='www.paycomonline.net'&&!x.port&&!x.username&&!x.password&&!x.hash&&x.pathname==='/v4/cl/web.php/security/security-question/login'&&keys.length<=1&&keys.every(key=>key==='session_nonce')}catch{return false}}; + if(!safe(location.href)||document.readyState!=='complete')return {status:'challenge_layout_changed'}; + const expected=${JSON.stringify(indices)},targetIndex=${targetIndex},verifyFocus=${JSON.stringify(verifyFocus)}; + const visible=e=>!!e&&!e.disabled&&e.offsetParent!==null; + const fields=[]; + for(const field of Array.from(document.querySelectorAll('input')).filter(visible)){ + const labels=[];if(field.id){const label=document.querySelector('label[for="'+CSS.escape(field.id)+'"]');if(label)labels.push(label.innerText||label.textContent||'')} + labels.push(field.getAttribute('aria-label')||'',field.placeholder||'',field.name||'',field.id||''); + const found=[];for(const label of labels){const text=String(label).trim();const match=text.match(/^\\s*(?:(?:enter|unique)\\s+)?(?:paycom\\s+)?(?:security\\s+)?pin(?:\\s*(?:number|no\\.?|#))?\\s*([1-5])\\s*[:?]?\\s*$/i)||text.match(/^(?:security[_-]?)?pin[_-]?([1-5])$/i);if(match)found.push(Number(match[1]))} + const unique=Array.from(new Set(found));if(unique.length===1)fields.push({field,index:unique[0]}); + } + const actual=fields.map(item=>item.index).sort((a,b)=>a-b),names=fields.map(item=>item.field.name).sort(),types=fields.map(item=>String(item.field.type||'').toLowerCase()); + const hiddenOk=fields.every(item=>{const name=item.field.name==='firstSecurityQuestion'?'firstIndex':item.field.name==='secondSecurityQuestion'?'secondIndex':null;if(!name||!item.field.form)return false;const hidden=Array.from(item.field.form.querySelectorAll('input[type="hidden"][name="'+name+'"]'));return hidden.length===1&&hidden[0].value===String(item.index)}); + if(JSON.stringify(actual)!==JSON.stringify(expected)||fields.length!==2||JSON.stringify(names)!==JSON.stringify(['firstSecurityQuestion','secondSecurityQuestion'])||types.some(type=>type!=='password')||new Set(fields.map(item=>item.field.form)).size!==1||!hiddenOk)return {status:'challenge_layout_changed'}; + const target=fields.find(item=>item.index===targetIndex)?.field; + if(!target||!target.form||String(target.form.method).toUpperCase()!=='POST'||!safe(target.form.action)||target.value!=='')return {status:'challenge_layout_changed'}; + if(verifyFocus)return document.activeElement===target?{status:'native_challenge_field_focused'}:{status:'challenge_layout_changed'}; + target.scrollIntoView({block:'center',inline:'center'});const rect=target.getBoundingClientRect(); + return rect.width>0&&rect.height>0?{status:'native_challenge_field_ready',x:rect.x+rect.width/2,y:rect.y+rect.height/2}:{status:'challenge_layout_changed'}; + })()`; +} + +function challengeExpression(credentials, challenge, { retainValues = false } = {}) { + const indices = challengeIndices(challenge); + const values = retainValues ? {} : Object.fromEntries(indices.map(index => [String(index), credentials[`pin${index}`]])); + const encoded = JSON.stringify(values).replace(/{ + const safe=u=>{try{const x=new URL(u),keys=Array.from(x.searchParams.keys());return x.protocol==='https:'&&x.hostname==='www.paycomonline.net'&&!x.port&&!x.username&&!x.password&&!x.hash&&x.pathname==='/v4/cl/web.php/security/security-question/login'&&keys.length<=1&&keys.every(key=>key==='session_nonce')}catch{return false}}; + if(!safe(location.href)||document.readyState!=='complete')return {status:'challenge_layout_changed'}; + const expected=${JSON.stringify(indices)},values=${encoded},retainValues=${JSON.stringify(retainValues)}; + const visible=e=>!!e&&!e.disabled&&e.offsetParent!==null; + if(retainValues){${VERIFICATION_CONTROLS}if(otpPresent||captchaPresent)return {status:'challenge_layout_changed'}} + const fields=[]; + for(const field of Array.from(document.querySelectorAll('input')).filter(visible)){ + const labels=[];if(field.id){const label=document.querySelector('label[for="'+CSS.escape(field.id)+'"]');if(label)labels.push(label.innerText||label.textContent||'')} + labels.push(field.getAttribute('aria-label')||'',field.placeholder||'',field.name||'',field.id||''); + const found=[];for(const label of labels){const text=String(label).trim();const match=text.match(/^\\s*(?:(?:enter|unique)\\s+)?(?:paycom\\s+)?(?:security\\s+)?pin(?:\\s*(?:number|no\\.?|#))?\\s*([1-5])\\s*[:?]?\\s*$/i)||text.match(/^(?:security[_-]?)?pin[_-]?([1-5])$/i);if(match)found.push(Number(match[1]))} + const unique=Array.from(new Set(found));if(unique.length===1)fields.push({field,index:unique[0]}); + } + const actual=fields.map(item=>item.index).sort((a,b)=>a-b); + const form=fields[0]?.field.form,names=fields.map(item=>item.field.name).sort(),types=fields.map(item=>String(item.field.type||'').toLowerCase()); + const buttons=form?Array.from(form.querySelectorAll('button,input[type="submit"]')).filter(visible):[]; + const submitters=buttons.filter(button=>String(button.type||'').toLowerCase()==='submit'&&button.name==='continue'&&(button.innerText||button.value||'').trim()==='Continue'); + const hiddenOk=fields.every(item=>{const name=item.field.name==='firstSecurityQuestion'?'firstIndex':item.field.name==='secondSecurityQuestion'?'secondIndex':null;if(!name||!form)return false;const hidden=Array.from(form.querySelectorAll('input[type="hidden"][name="'+name+'"]'));return hidden.length===1&&hidden[0].value===String(item.index)}); + if(JSON.stringify(actual)!==JSON.stringify(expected)||fields.length!==2||JSON.stringify(names)!==JSON.stringify(['firstSecurityQuestion','secondSecurityQuestion'])||types.some(type=>type!=='password')||fields.some(item=>retainValues?!item.field.value:item.field.value!==values[String(item.index)])||new Set(fields.map(item=>item.field.form)).size!==1||!form||String(form.method).toUpperCase()!=='POST'||!safe(form.action)||!hiddenOk||submitters.length!==1)return {status:'challenge_layout_changed'}; + const button=submitters[0];button.scrollIntoView({block:'center',inline:'center'});const rect=button.getBoundingClientRect(),x=rect.left+rect.width/2,y=rect.top+rect.height/2; + return rect.width>0&&rect.height>0&&Number.isFinite(x)&&Number.isFinite(y)&&x>=0&&y>=0&&x<=10000&&y<=10000?{status:'native_challenge_ready',x,y}:{status:'challenge_layout_changed'}; + })()`; +} + +function requireNativeInput(browser) { + if (!browser?.nativeInput || typeof browser.nativeInput.click !== 'function' || typeof browser.nativeInput.type !== 'function') { + throw new PaycomAuthError('browser_interaction_required'); + } + return browser.nativeInput; +} + +async function submitNativeChallenge(connection, credentials, challenge, browser, signal) { + const input = requireNativeInput(browser); + const indices = challengeIndices(challenge); + // Validate both values before typing either one. Never trim or reinterpret a PIN. + if (indices.some(index => typeof credentials[`pin${index}`] !== 'string' + || !/^[\x20-\x7e]{1,64}$/.test(credentials[`pin${index}`]))) throw new PaycomAuthError('manual_verification_required'); + for (const index of indices) { + throwIfAborted(signal); + const field = await connection.evaluate(challengeFocusExpression(challenge, index)); + if (field?.status !== 'native_challenge_field_ready') throw new PaycomAuthError('manual_verification_required'); + await input.click(connection, field.x, field.y, signal); + const focused = await connection.evaluate(challengeFocusExpression(challenge, index, { verifyFocus: true })); + if (focused?.status !== 'native_challenge_field_focused') throw new PaycomAuthError('manual_verification_required'); + await input.type(credentials[`pin${index}`], signal); + } + const ready = await connection.evaluate(challengeExpression(credentials, challenge)); + if (ready?.status !== 'native_challenge_ready') throw new PaycomAuthError('manual_verification_required'); + throwIfAborted(signal); + await input.click(connection, ready.x, ready.y, signal); +} + +function securityProfileDismissExpression() { + return `(()=>{ + const url=new URL(location.href),keys=Array.from(url.searchParams.keys()); + if(url.protocol!=='https:'||url.hostname!=='www.paycomonline.net'||url.port||url.username||url.password||url.hash||url.pathname!==${JSON.stringify(SECURITY_PROFILE_PATH)}||keys.length>1||keys.some(key=>key!=='session_nonce'))return {status:'manual_verification_required'}; + const visible=e=>!!e&&!e.disabled&&e.offsetParent!==null; + ${VERIFICATION_CONTROLS} + if(otpPresent||captchaPresent)return {status:'manual_verification_required'}; + const inputs=Array.from(document.querySelectorAll('input')).filter(visible),names=inputs.map(e=>e.name).sort(); + if(JSON.stringify(names)!==JSON.stringify(['cell-number','email','work-number']))return {status:'manual_verification_required'}; + const buttons=Array.from(document.querySelectorAll('button,input[type="submit"]')).filter(visible),texts=buttons.map(e=>(e.innerText||e.value||'').trim()).sort(); + if(JSON.stringify(texts)!==JSON.stringify(['Continue','Not Now','Verify','Verify','Verify']))return {status:'manual_verification_required'}; + const button=buttons.find(e=>(e.innerText||e.value||'').trim()==='Not Now'); + if(!button)return {status:'manual_verification_required'}; + button.scrollIntoView({block:'center',inline:'center'}); + const rect=button.getBoundingClientRect(),x=rect.left+rect.width/2,y=rect.top+rect.height/2; + return rect.width>0&&rect.height>0&&Number.isFinite(x)&&Number.isFinite(y)&&x>=0&&y>=0 + ?{status:'security_profile_dismiss_ready',x,y}:{status:'manual_verification_required'}; + })()`; +} + +function securityProfileConfirmationExpression() { + return `(()=>{ + const url=new URL(location.href),keys=Array.from(url.searchParams.keys()); + if(url.protocol!=='https:'||url.hostname!=='www.paycomonline.net'||url.port||url.username||url.password||url.hash||url.pathname!==${JSON.stringify(SECURITY_PROFILE_PATH)}||keys.length>1||keys.some(key=>key!=='session_nonce'))return {status:'manual_verification_required'}; + const visible=e=>!!e&&!e.disabled&&e.offsetParent!==null; + ${VERIFICATION_CONTROLS} + if(otpPresent||captchaPresent)return {status:'manual_verification_required'}; + const inputs=Array.from(document.querySelectorAll('input')).filter(visible),names=inputs.map(e=>e.name).sort(); + if(JSON.stringify(names)!==JSON.stringify(['cell-number','email','work-number']))return {status:'manual_verification_required'}; + const buttons=Array.from(document.querySelectorAll('button,input[type="submit"]')).filter(visible),texts=buttons.map(e=>(e.innerText||e.value||'').trim()).sort(); + if(JSON.stringify(texts)!==JSON.stringify(['','Cancel','Continue','Continue','Not Now','Verify','Verify','Verify']))return {status:'manual_verification_required'}; + const warning=${JSON.stringify(SECURITY_PROFILE_WARNING)},body=(document.body&&document.body.innerText||'').replace(/\\s+/g,' ').trim(); + if(!body.includes('Setup Your Security Profile')||!body.includes('Verify your contact information')||!body.includes('Warning')||!body.includes(warning))return {status:'manual_verification_required'}; + let button=null; + for(const candidate of buttons.filter(e=>(e.innerText||e.value||'').trim()==='Continue')){ + for(let node=candidate,depth=0;depth<8&&node;node=node.parentElement,depth++){ + const local=Array.from(node.querySelectorAll('button,input[type="submit"]')).filter(visible),localTexts=local.map(e=>(e.innerText||e.value||'').trim()).sort(),localText=(node.innerText||'').replace(/\\s+/g,' ').trim(); + if(JSON.stringify(localTexts)===JSON.stringify(['','Cancel','Continue'])&&localText.includes('Warning')&&localText.includes(warning)){button=candidate;break} + } + if(button)break; + } + if(!button)return {status:'manual_verification_required'}; + button.scrollIntoView({block:'center',inline:'center'}); + const rect=button.getBoundingClientRect(),x=rect.left+rect.width/2,y=rect.top+rect.height/2; + return rect.width>0&&rect.height>0&&Number.isFinite(x)&&Number.isFinite(y)&&x>=0&&y>=0 + ?{status:'security_profile_confirmation_ready',x,y}:{status:'manual_verification_required'}; + })()`; +} + +function securityProfileProceedExpression() { + return `(()=>{ + const url=new URL(location.href),keys=Array.from(url.searchParams.keys()); + if(url.protocol!=='https:'||url.hostname!=='www.paycomonline.net'||url.port||url.username||url.password||url.hash||url.pathname!==${JSON.stringify(SECURITY_PROFILE_PATH)}||keys.length>1||keys.some(key=>key!=='session_nonce'))return {status:'manual_verification_required'}; + const visible=e=>!!e&&!e.disabled&&e.offsetParent!==null; + ${VERIFICATION_CONTROLS} + if(otpPresent||captchaPresent)return {status:'manual_verification_required'}; + const inputs=Array.from(document.querySelectorAll('input')).filter(visible),names=inputs.map(e=>e.name).sort(); + if(JSON.stringify(names)!==JSON.stringify(['cell-number','email','work-number']))return {status:'manual_verification_required'}; + const buttons=Array.from(document.querySelectorAll('button,input[type="submit"]')).filter(visible),texts=buttons.map(e=>(e.innerText||e.value||'').trim()).sort(); + const body=(document.body&&document.body.innerText||'').replace(/\\s+/g,' ').trim(); + if(JSON.stringify(texts)!==JSON.stringify(['Continue','Not Now','Verify','Verify','Verify'])||!body.includes('Setup Your Security Profile')||!body.includes('Verify your contact information')||body.includes('Warning'))return {status:'manual_verification_required'}; + const candidates=buttons.filter(e=>(e.innerText||e.value||'').trim()==='Continue'); + if(candidates.length!==1)return {status:'manual_verification_required'}; + const button=candidates[0];button.scrollIntoView({block:'center',inline:'center'}); + const rect=button.getBoundingClientRect(),x=rect.left+rect.width/2,y=rect.top+rect.height/2; + return rect.width>0&&rect.height>0&&Number.isFinite(x)&&Number.isFinite(y)&&x>=0&&y>=0 + ?{status:'security_profile_proceed_ready',x,y}:{status:'manual_verification_required'}; + })()`; +} + +function awaitingProfileControls(snapshot, reason) { + return securityProfileUrl(snapshot?.url) && snapshot.otpPresent !== true && snapshot.captchaPresent !== true + && ['security_profile_layout_changed', 'additional_verification'].includes(reason); +} + +async function waitForState(connection, timeoutMs, accepted, signal, context = {}) { + const deadline = Date.now() + timeoutMs; + let last; + while (Date.now() < deadline) { + throwIfAborted(signal); + try { + last = await connection.evaluate(SNAPSHOT); + if (last?.url === 'about:blank') { + await delay(50, signal); + continue; + } + const { state, reason } = classifyState(last, context); + const profileRendering = awaitingProfileControls(last, reason); + if (accepted.has(state) && !profileRendering) return { + state, snapshot: last, diagnostic: classificationDiagnostic(state, context.phase || 'observation', last, reason), + }; + } catch (error) { + if (signal?.aborted) throw new PaycomAuthError('acquisition_cancelled'); + } + await delay(200, signal); + } + if (securityProfileUrl(last?.url)) return { + state: 'manual_verification_required', snapshot: last, + diagnostic: classificationDiagnostic('manual_verification_required', context.phase || 'observation', last, 'security_profile_response_timeout'), + }; + throw new PaycomAuthError('authentication_timeout'); +} + +const SECURITY_PROFILE_STATES = new Set(['security_profile_prompt', 'security_profile_confirmation']); + +async function waitForStateChange(connection, previous, signal, context = {}) { + const deadline = Date.now() + 15_000; + let lastSnapshot; + while (Date.now() < deadline) { + throwIfAborted(signal); + try { + const snapshot = await connection.evaluate(SNAPSHOT); + lastSnapshot = snapshot; + const { state, reason } = classifyState(snapshot, context); + // React can temporarily remove the campaign controls after a click. Wait + // without clicking again; never grant this grace to another route or to + // visible verification controls or a provider rejection. + const profileRendering = awaitingProfileControls(snapshot, reason); + if (state !== 'pending' && state !== previous && !profileRendering) return { + state, snapshot, diagnostic: classificationDiagnostic(state, context.phase || 'observation', snapshot, reason), + }; + } catch (error) { + if (signal?.aborted) throw new PaycomAuthError('acquisition_cancelled'); + } + await delay(200, signal); + } + if (securityProfileUrl(lastSnapshot?.url)) return { + state: 'manual_verification_required', snapshot: lastSnapshot, + diagnostic: classificationDiagnostic('manual_verification_required', context.phase || 'observation', lastSnapshot, 'security_profile_response_timeout'), + }; + if (context.phase === 'security_questions' && lastSnapshot) return { + state: 'manual_verification_required', snapshot: lastSnapshot, + diagnostic: classificationDiagnostic('manual_verification_required', context.phase, lastSnapshot, 'challenge_response_timeout'), + }; + if (context.phase === 'security_profile' && lastSnapshot) return { + state: 'manual_verification_required', snapshot: lastSnapshot, + diagnostic: classificationDiagnostic('manual_verification_required', context.phase, lastSnapshot, 'security_profile_response_timeout'), + }; + throw new PaycomAuthError('manual_verification_required'); +} + +async function nativeClick(connection, prepared, expected) { + if (prepared?.status !== expected || !Number.isFinite(prepared.x) || !Number.isFinite(prepared.y) + || prepared.x < 0 || prepared.y < 0 || prepared.x > 10_000 || prepared.y > 10_000) throw new PaycomAuthError('manual_verification_required'); + await connection.command('Input.dispatchMouseEvent', { type: 'mouseMoved', x: prepared.x, y: prepared.y }); + await connection.command('Input.dispatchMouseEvent', { type: 'mousePressed', x: prepared.x, y: prepared.y, button: 'left', clickCount: 1 }); + await connection.command('Input.dispatchMouseEvent', { type: 'mouseReleased', x: prepared.x, y: prepared.y, button: 'left', clickCount: 1 }); +} + +async function resolveSecurityProfile(connection, initial, signal) { + let current = initial; + let confirmationAccepted = false; + for (let phase = 0; phase < 3 && SECURITY_PROFILE_STATES.has(current.state); phase++) { + const before = current.state; + const proceed = before === 'security_profile_prompt' && confirmationAccepted; + const expression = before === 'security_profile_confirmation' + ? securityProfileConfirmationExpression() + : proceed ? securityProfileProceedExpression() : securityProfileDismissExpression(); + const expected = before === 'security_profile_confirmation' + ? 'security_profile_confirmation_ready' + : proceed ? 'security_profile_proceed_ready' : 'security_profile_dismiss_ready'; + let prepared; + try { prepared = await connection.evaluate(expression); } catch { throw new PaycomAuthError('manual_verification_required'); } + try { await nativeClick(connection, prepared, expected); } catch (error) { + throw error instanceof PaycomAuthError ? error : new PaycomAuthError('manual_verification_required'); + } + if (before === 'security_profile_confirmation') confirmationAccepted = true; + current = await waitForStateChange(connection, before, signal, { phase: 'security_profile' }); + } + if (SECURITY_PROFILE_STATES.has(current.state)) throw new PaycomAuthError('manual_verification_required'); + return current; +} + +async function verifyTimecardApplication(connection, initial, credentials, ensureSubmitted, signal, observe = value => value, loginOnly = false, browser = null, observationOnly = false) { + let current = initial; + let challengeSubmitted = false; + for (let phase = 0; phase < 8; phase++) { + if (SECURITY_PROFILE_STATES.has(current.state)) { + current = observe(await resolveSecurityProfile(connection, current, signal)); + continue; + } + if (current.state === 'security_questions_required') { + if (challengeSubmitted) throw new PaycomAuthError('manual_verification_required'); + throwIfAborted(signal); + if (observationOnly) ensureSubmitted('security_questions'); + requireNativeInput(browser); + ensureSubmitted('security_questions'); + try { await submitNativeChallenge(connection, credentials, current.snapshot.challenge, browser, signal); } + catch (error) { + observe({ state: 'manual_verification_required', snapshot: current.snapshot, + diagnostic: classificationDiagnostic('manual_verification_required', 'security_questions', current.snapshot, 'challenge_entry_failed') }); + throw error; + } + challengeSubmitted = true; + current = observe(await waitForStateChange(connection, 'security_questions_required', signal, { phase: 'security_questions' })); + continue; + } + if (current.state === 'timecard_application') return current; + if (current.state === 'authenticated') { + if (loginOnly) return current; + throwIfAborted(signal); + const navigation = await connection.command('Page.navigate', { url: TIMECARD_SEARCH_URL }); + if (navigation?.errorText) throw new PaycomAuthError('manual_verification_required'); + current = observe(await waitForState(connection, 45_000, new Set([ + 'timecard_application', 'security_questions_required', 'security_profile_prompt', + 'security_profile_confirmation', 'manual_verification_required', 'account_locked', 'logged_out', + ]), signal, { phase: 'application_navigation' })); + continue; + } + throw new PaycomAuthError(current.state); + } + throw new PaycomAuthError('manual_verification_required'); +} + +async function prepareHandoff(browser, sourceTarget, sourceConnection, signal, onState, observedUrl = null) { + const observed = state => { if (typeof onState === 'function') try { onState(state); } catch {} }; + throwIfAborted(signal); + const currentUrl = observedUrl || await sourceConnection.evaluate('location.href'); + if (!exactTimecardSearchUrl(currentUrl)) throw new PaycomAuthError('manual_verification_required'); + observed('handoff_source_verified'); + if (typeof browser.browserWebSocketUrl !== 'string') throw new PaycomAuthError('browser_protocol_failed'); + const cleanTarget = await createTarget(browser.endpoint, currentUrl); + observed('handoff_target_created'); + const cleanConnection = await CdpConnection.connect(cleanTarget.webSocketDebuggerUrl); + try { + const clean = await waitForState(cleanConnection, 45_000, new Set(['timecard_application', 'security_questions_required', 'security_profile_prompt', 'security_profile_confirmation', 'manual_verification_required']), signal); + if (clean.state !== 'timecard_application') throw new PaycomAuthError('manual_verification_required'); + observed('handoff_target_verified'); + } finally { + cleanConnection.close(); + } + const browserConnection = await CdpConnection.connect(browser.browserWebSocketUrl); + try { + for (let pass = 0; pass < 20; pass++) { + throwIfAborted(signal); + const targets = await browserConnection.command('Target.getTargets'); + const unwanted = (targets.targetInfos || []).filter(info => info.targetId !== cleanTarget.id && info.type === 'page'); + if (!unwanted.length) break; + for (const info of unwanted) await browserConnection.command('Target.closeTarget', { targetId: info.targetId }); + await delay(50, signal); + if (pass === 19) throw new PaycomAuthError('manual_verification_required'); + } + observed('handoff_targets_closed'); + } finally { + browserConnection.close(); + } + return { status: 'authenticated', targetId: cleanTarget.id, replacedCredentialPage: sourceTarget.id !== cleanTarget.id }; +} + +// Only a digest leaves the page, kept in broker memory for this handoff. It +// proves the already-entered PINs were not changed while browser control moved. +const RETAINED_PIN_FINGERPRINT = `(async()=>{ + const fields=['firstSecurityQuestion','secondSecurityQuestion'].map(name=>document.querySelector('input[name="'+name+'"]')); + if(fields.some(field=>!field||field.type!=='password'||!field.value))return null; + const bytes=new TextEncoder().encode(JSON.stringify(fields.map(field=>field.value))); + const digest=await crypto.subtle.digest('SHA-256',bytes); + return Array.from(new Uint8Array(digest),value=>value.toString(16).padStart(2,'0')).join(''); +})()`; + +const paycomAdapter = Object.freeze({ + provider: 'paycom', + nativeInteraction: true, + async prepareBrowserAssistance(browser, { signal } = {}) { + throwIfAborted(signal); + if (!browser.nativeInput) return null; + const { boundedJson } = require('dispatch-sdk/node/cdp'); + const targets = await boundedJson(`${browser.endpoint}/json/list`, { signal }); + let selected = null; + for (const target of targets.filter(item => item.type === 'page' && parsedPaycomUrl(item.url))) { + const connection = await CdpConnection.connect(target.webSocketDebuggerUrl, { signal }); + try { + const snapshot = await connection.evaluate(SNAPSHOT); + if (classifyState(snapshot).reason === 'additional_verification' && snapshot.captchaPresent === true && snapshot.otpPresent !== true) { + if (selected) return null; + const frame = (await connection.command('Page.getFrameTree')).frameTree.frame; + const pinFingerprint = await connection.evaluate(RETAINED_PIN_FINGERPRINT); + selected = { ...target, loaderId: frame.loaderId, challenge: snapshot.challenge, pinFingerprint, + resumeLogin: !pinFingerprint && (snapshot.loginPresent?.every(Boolean) || snapshot.challenge?.length === 2) }; + } + } finally { connection.close(); } + } + if (!selected) return null; + // This profile belongs exclusively to Paycom. Ensure the browser tool starts + // on the challenge instead of a blank or stale sign-in tab. + for (const target of targets.filter(item => item.type === 'page' && item.id !== selected.id)) { + await boundedJson(`${browser.endpoint}/json/close/${target.id}`, { signal }).catch(() => {}); + } + return { pluginId: 'paycom', type: 'captcha', targetId: selected.id, + loaderId: selected.loaderId, challenge: selected.challenge, pinFingerprint: selected.pinFingerprint, resumeLogin: selected.resumeLogin }; + }, + async completeBrowserAssistance(browser, context, { signal, onState, loginOnly = false, resumeAuthentication } = {}) { + const { boundedJson } = require('dispatch-sdk/node/cdp'); + const targets = await boundedJson(`${browser.endpoint}/json/list`, { signal }); + const target = targets.find(item => item.type === 'page' && item.id === context.targetId && parsedPaycomUrl(item.url)); + if (!target) throw new PaycomAuthError('manual_verification_required'); + const connection = await CdpConnection.connect(target.webSocketDebuggerUrl, { signal }); + const observed = value => { + if (typeof onState === 'function') try { onState(value.state, stateMetadata(value.snapshot, value.diagnostic)); } catch {} + return value; + }; + const refuseSubmission = () => { throw new PaycomAuthError('manual_verification_required'); }; + try { + await connection.command('Page.enable'); + let current = observed(await waitForState(connection, 15_000, new Set([ + 'authenticated', 'timecard_application', 'security_questions_required', 'security_profile_prompt', + 'security_profile_confirmation', 'manual_verification_required', 'account_locked', 'logged_out', + ]), signal, { phase: 'recovery_observation' })); + if (context.resumeLogin && ['logged_out', 'security_questions_required'].includes(current.state) + && typeof resumeAuthentication === 'function') { + const frame = (await connection.command('Page.getFrameTree')).frameTree.frame; + if (frame.loaderId !== context.loaderId) refuseSubmission(); + // The CAPTCHA is gone and this original form had no retained PINs. + // Return control to the credential-owning broker to finish normal login. + return await resumeAuthentication(); + } + if (current.state === 'security_questions_required') { + const frame = (await connection.command('Page.getFrameTree')).frameTree.frame; + if (!context.pinFingerprint || frame.loaderId !== context.loaderId + || await connection.evaluate(RETAINED_PIN_FINGERPRINT) !== context.pinFingerprint) refuseSubmission(); + const ready = await connection.evaluate(challengeExpression(Object.freeze({}), context.challenge, { retainValues: true })); + if (ready?.status !== 'native_challenge_ready') refuseSubmission(); + throwIfAborted(signal); + // Complete the same pending form once. Never retype PINs or submit a + // new/reloaded form after the challenge; ordinary provider handlers run. + await requireNativeInput(browser).click(connection, ready.x, ready.y, signal); + current = observed(await waitForStateChange(connection, 'security_questions_required', signal, { phase: 'security_questions' })); + } + const verified = await verifyTimecardApplication(connection, current, Object.freeze({}), refuseSubmission, + signal, observed, loginOnly, browser, true); + if (loginOnly) return { status: 'authenticated' }; + return prepareHandoff(browser, target, connection, signal, onState, verified.snapshot.url); + } finally { connection.close(); } + }, + async inspect(browser, { signal, onState } = {}) { + throwIfAborted(signal); + // Reuse a valid session; Paycom redirects expired sessions to its login form. + const target = await createTarget(browser.endpoint, `${ORIGIN}${CLIENT_LANDING_PATH}`); + const connection = await CdpConnection.connect(target.webSocketDebuggerUrl, { commandTimeoutMs: 10_000 }); + try { + await connection.command('Page.enable'); + const deadline = Date.now() + 15_000; + let snapshot = null; + let state = 'pending'; + while (Date.now() < deadline) { + throwIfAborted(signal); + try { + snapshot = await connection.evaluate(SNAPSHOT); + if (snapshot?.url !== 'about:blank') { + state = classify(snapshot, { phase: 'inspection' }); + if (state !== 'pending' || snapshot.readyState === 'complete') break; + } + } catch (error) { + if (signal?.aborted) throw new PaycomAuthError('acquisition_cancelled'); + } + await delay(200, signal); + } + if (!snapshot || snapshot.url === 'about:blank') throw new PaycomAuthError('authentication_timeout'); + if (state === 'pending' && snapshot.readyState === 'complete') state = 'manual_verification_required'; + const metadata = stateMetadata(snapshot); + if (typeof onState === 'function') try { onState(state, metadata); } catch {} + return { state, observedAt: new Date().toISOString(), metadata }; + } finally { + connection.close(); + } + }, + async recover(browser, { signal, onState, loginOnly = false } = {}) { + const observed = value => { + if (typeof onState === 'function') try { onState(value.state, stateMetadata(value.snapshot, value.diagnostic)); } catch {} + return value; + }; + const refuseSubmission = () => { throw new PaycomAuthError('manual_verification_required'); }; + throwIfAborted(signal); + // Reuse a valid session; Paycom redirects expired sessions to its login form. + const target = await createTarget(browser.endpoint, `${ORIGIN}${CLIENT_LANDING_PATH}`); + const connection = await CdpConnection.connect(target.webSocketDebuggerUrl, { commandTimeoutMs: 10_000 }); + try { + await connection.command('Page.enable'); + const current = observed(await waitForState(connection, 15_000, new Set([ + 'authenticated', 'timecard_application', 'logged_out', 'security_questions_required', + 'security_profile_prompt', 'security_profile_confirmation', 'manual_verification_required', + 'account_locked', + ]), signal, { phase: 'recovery_observation' })); + if (['logged_out', 'security_questions_required'].includes(current.state)) refuseSubmission(); + if (['manual_verification_required', 'account_locked'].includes(current.state)) throw new PaycomAuthError(current.state); + const verified = await verifyTimecardApplication( + connection, current, Object.freeze({}), refuseSubmission, signal, observed, loginOnly, browser, true, + ); + // Setup stops at authenticated login; only collectors need application access and handoff. + if (loginOnly) return { status: 'authenticated' }; + return prepareHandoff(browser, target, connection, signal, onState, verified.snapshot.url); + } finally { + connection.close(); + } + }, + async authenticate(browser, credentials, { signal, onSubmit, onState, loginOnly = false } = {}) { + const observed = value => { + if (typeof onState === 'function') try { onState(value.state, stateMetadata(value.snapshot, value.diagnostic)); } catch {} + return value; + }; + let submissionLatched = false; + const ensureSubmitted = kind => { + if (submissionLatched) return; + if (typeof onSubmit !== 'function') throw new PaycomAuthError('authentication_failed'); + onSubmit(kind); + submissionLatched = true; + }; + throwIfAborted(signal); + // Reuse a valid session; Paycom redirects expired sessions to its login form. + const target = await createTarget(browser.endpoint, `${ORIGIN}${CLIENT_LANDING_PATH}`); + const connection = await CdpConnection.connect(target.webSocketDebuggerUrl, { commandTimeoutMs: 10_000 }); + try { + await connection.command('Page.enable'); + let current = observed(await waitForState(connection, 15_000, new Set([ + 'authenticated', 'timecard_application', 'logged_out', 'security_questions_required', + 'security_profile_prompt', 'security_profile_confirmation', 'manual_verification_required', + 'account_locked', + ]), signal, { phase: 'observation' })); + if (current.snapshot?.captchaPresent === true && current.snapshot?.otpPresent !== true) requireNativeInput(browser); + if (current.state === 'logged_out') { + throwIfAborted(signal); + requireNativeInput(browser); + ensureSubmitted('credentials'); + const submitted = await connection.evaluate(loginExpression(credentials)); + if (submitted?.status !== 'submitted') throw new PaycomAuthError('manual_verification_required'); + current = observed(await waitForState(connection, 45_000, new Set([ + 'authenticated', 'timecard_application', 'security_questions_required', 'security_profile_prompt', + 'security_profile_confirmation', 'manual_verification_required', 'primary_credentials_rejected', 'account_locked', + ]), signal, { phase: 'primary_login' })); + } + const verified = await verifyTimecardApplication(connection, current, credentials, ensureSubmitted, signal, observed, loginOnly, browser); + // Setup stops at authenticated login; only collectors need application access and handoff. + if (loginOnly) return { status: 'authenticated' }; + return prepareHandoff(browser, target, connection, signal, onState, verified.snapshot.url); + } finally { + connection.close(); + } + }, +}); + +module.exports = { + paycomAdapter, PaycomAuthError, LOGIN_URL, LOGIN_ACTION_URL, AUTH_PREFIX, LOGIN_FIELDS, SNAPSHOT, + SECURITY_QUESTION_PATH, TIMECARD_SEARCH_PATH, TIMECARD_SEARCH_URL, CLIENT_LANDING_PATH, MAIN_MENU_PATH, + SECURITY_PROFILE_PATH, SECURITY_PROFILE_WARNING, classify, classifyState, loginExpression, challengeExpression, + challengeFocusExpression, submitNativeChallenge, + securityProfileUrl, securityProfileState, securityProfileDismissExpression, + securityProfileConfirmationExpression, securityProfileProceedExpression, resolveSecurityProfile, + verifyTimecardApplication, waitForState, prepareHandoff, exactLoginUrl, exactLoginActionUrl, + exactSecurityQuestionUrl, exactTimecardSearchUrl, authenticatedUrl, +}; diff --git a/plugins/paycom/backend/bin/dispatch-paycom-activation-evidence b/plugins/paycom/backend/bin/dispatch-paycom-activation-evidence new file mode 100755 index 0000000..7f23663 --- /dev/null +++ b/plugins/paycom/backend/bin/dispatch-paycom-activation-evidence @@ -0,0 +1,56 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; + +const { parseStrictJson } = require('../../../../runtime/auth-broker/src/strict-json'); +const { managedRuntimeEnvironmentFromProcess } = require('dispatch-protocol/paths/runtime-paths'); +const { verifyActivationEvidence } = require('../src/activation-evidence'); + +const MAX_INPUT_BYTES = 4096; +let chunks = []; +let size = 0; +let failed = false; + +function reject() { + if (failed) return; + failed = true; + chunks = []; + process.stdout.write('{"ok":false,"status":"first_publication_failed"}\n'); + process.exitCode = 1; +} + +process.stdin.on('data', chunk => { + size += chunk.length; + if (size > MAX_INPUT_BYTES) { + reject(); + process.stdin.destroy(); + return; + } + chunks.push(chunk); +}); +process.stdin.on('error', reject); +process.stdin.on('end', () => { + if (failed) return; + try { + const text = Buffer.concat(chunks).toString('utf8'); + chunks = []; + if (!text.endsWith('\n') || text.slice(0, -1).includes('\n')) throw new Error('invalid_input'); + const input = parseStrictJson(text.slice(0, -1)); + if (!input || typeof input !== 'object' || Array.isArray(input) + || Object.keys(input).sort().join(',') !== 'batchId,definitionDigest,preparationRunId') throw new Error('invalid_input'); + managedRuntimeEnvironmentFromProcess(); + const { defaultPaths: collectionPaths } = require('../../../../runtime/collection-manager/src/paths'); + const { DATABASE: paycomDatabase } = require('../src/paths'); + const evidence = verifyActivationEvidence({ + collectionPaths: collectionPaths(), + paycomDatabase, + batchId: input.batchId, + preparationRunId: input.preparationRunId, + definitionDigest: input.definitionDigest, + clock: Date.now, + }); + process.stdout.write(`${JSON.stringify({ ok: true, status: 'ok', evidence })}\n`); + } catch { + reject(); + } +}); +process.stdin.resume(); diff --git a/plugins/paycom/backend/bin/dispatch-paycom-collector b/plugins/paycom/backend/bin/dispatch-paycom-collector new file mode 100755 index 0000000..90ae257 --- /dev/null +++ b/plugins/paycom/backend/bin/dispatch-paycom-collector @@ -0,0 +1,42 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; + +process.umask(0o077); + +const { execute, safeFailure } = require('../src/collector'); +const { parseStrictJson } = require('../../../../runtime/auth-broker/src/strict-json'); + +const MAX_INPUT_BYTES = 65_536; +let chunks = []; +let bytes = 0; +let finished = false; + +function respond(value) { + if (finished) return; + finished = true; + process.stdout.write(`${JSON.stringify(value)}\n`); +} + +process.stdin.on('data', chunk => { + bytes += chunk.length; + if (bytes > MAX_INPUT_BYTES) { + chunks = []; + respond(safeFailure(Object.assign(new Error('invalid_request'), { code: 'invalid_request' }))); + process.stdin.destroy(); + } else chunks.push(chunk); +}); +process.stdin.on('error', () => respond(safeFailure(Object.assign(new Error('invalid_request'), { code: 'invalid_request' })))); +process.stdin.on('end', async () => { + if (finished) return; + try { + const text = Buffer.concat(chunks).toString('utf8'); + chunks = []; + if (!text.endsWith('\n') || text.includes('\r') || text.slice(0, -1).includes('\n')) throw Object.assign(new Error('invalid_request'), { code: 'invalid_request' }); + let request; + try { request = parseStrictJson(text.slice(0, -1)); } + catch { throw Object.assign(new Error('invalid_request'), { code: 'invalid_request' }); } + respond(await execute(request)); + } catch (error) { + respond(safeFailure(error)); + } +}); diff --git a/plugins/paycom/backend/bin/dispatch-paycom-publication-continuity b/plugins/paycom/backend/bin/dispatch-paycom-publication-continuity new file mode 100755 index 0000000..aa4c09e --- /dev/null +++ b/plugins/paycom/backend/bin/dispatch-paycom-publication-continuity @@ -0,0 +1,27 @@ +#!/usr/bin/env -S node --no-warnings +'use strict'; +const { parseStrictJson } = require('../../../../runtime/auth-broker/src/strict-json'); +const { managedRuntimeEnvironmentFromProcess } = require('dispatch-protocol/paths/runtime-paths'); +const { verifyPublicationContinuity } = require('../src/publication-continuity'); +let size = 0, chunks = [], rejected = false; +function reject() { + if (rejected) return; + rejected = true; chunks = []; + process.stdout.write('{"status":"first_publication_failed"}\n'); process.exitCode = 1; +} +if (process.argv.length !== 2) reject(); +process.stdin.on('error', reject); +process.stdin.on('data', chunk => { + size += chunk.length; + if (size > 8192) { reject(); process.stdin.destroy(); } else if (!rejected) chunks.push(chunk); +}); +process.stdin.on('end', () => { + if (rejected) return; + try { + managedRuntimeEnvironmentFromProcess(); + const raw = Buffer.concat(chunks).toString('utf8'); chunks = []; + if (!raw.endsWith('\n') || raw.slice(0, -1).includes('\n')) throw new Error(); + const { DATABASE } = require('../src/paths'); + process.stdout.write(JSON.stringify(verifyPublicationContinuity(DATABASE, parseStrictJson(raw.slice(0, -1)))) + '\n'); + } catch { reject(); } +}); diff --git a/plugins/paycom/backend/config/activation.json b/plugins/paycom/backend/config/activation.json new file mode 100644 index 0000000..02f9f97 --- /dev/null +++ b/plugins/paycom/backend/config/activation.json @@ -0,0 +1,7 @@ +{ + "schemaVersion": 1, + "status": "source-development", + "version": "0.18.8", + "executable": "${DISPATCH_PROJECT_ROOT}/plugins/paycom/backend/bin/dispatch-paycom-collector", + "managedBy": "collection-manager" +} diff --git a/plugins/paycom/backend/config/collection-manager.json b/plugins/paycom/backend/config/collection-manager.json new file mode 100644 index 0000000..2e4eb94 --- /dev/null +++ b/plugins/paycom/backend/config/collection-manager.json @@ -0,0 +1,303 @@ +{ + "schemaVersion": 1, + "collectors": [ + { + "id": "paycom", + "version": "0.18.8", + "description": "Paycom roster, resource-link, and timecard collector using broker-authenticated browser sessions", + "command": "${DISPATCH_PROJECT_ROOT}/plugins/paycom/backend/bin/dispatch-paycom-collector", + "sourceSchema": { + "type": "object", + "properties": { + "timezone": { "type": "string", "maxLength": 64 }, + "maxConcurrency": { "type": "integer", "minimum": 1, "maximum": 6 } + }, + "required": ["timezone", "maxConcurrency"], + "additionalProperties": false + }, + "methods": { + "collection.resolve-targets": { + "description": "Resolve standard collection selectors into exact Paycom pay-period targets without collecting", + "inputSchema": { + "type": "object", + "properties": { + "selectorKind": { "type": "string", "enum": ["date", "latest-complete", "date-range", "exact-target"] }, + "date": { "type": "string", "maxLength": 10, "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "start": { "type": "string", "maxLength": 10, "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "end": { "type": "string", "maxLength": 10, "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "key": { "type": "string", "maxLength": 128, "pattern": "^[A-Za-z0-9_.:-]+$" } + }, + "required": ["selectorKind"], + "additionalProperties": false + }, + "timeoutSeconds": 30, + "maxAttempts": 1, + "backoffSeconds": [], + "concurrencyKeys": [] + }, + "collector.health": { + "description": "Inspect Paycom collector storage and Auth Broker readiness without collecting", + "inputSchema": { "type": "object", "properties": {}, "required": [], "additionalProperties": false }, + "timeoutSeconds": 30, + "maxAttempts": 1, + "backoffSeconds": [], + "concurrencyKeys": ["health:paycom"] + }, + "pay-periods.discover": { + "description": "Calculate and publish the previous, current, and next biweekly Paycom periods", + "inputSchema": { "type": "object", "properties": {}, "required": [], "additionalProperties": false }, + "timeoutSeconds": 30, + "maxAttempts": 1, + "backoffSeconds": [], + "concurrencyKeys": ["publish:paycom-periods"] + }, + "roster.snapshot": { + "description": "Collect, validate, and atomically publish the current Paycom employee roster", + "inputSchema": { "type": "object", "properties": {}, "required": [], "additionalProperties": false }, + "timeoutSeconds": 300, + "maxAttempts": 3, + "backoffSeconds": [30, 120], + "concurrencyKeys": ["auth:{authProfile}", "publish:paycom-roster"] + }, + "roster.period": { + "description": "Collect and publish the authoritative active roster for one Paycom period", + "inputSchema": { + "type": "object", + "properties": { "periodEnd": { "type": "string", "maxLength": 10, "pattern": "^\\d{4}-\\d{2}-\\d{2}$" } }, + "required": ["periodEnd"], + "additionalProperties": false + }, + "timeoutSeconds": 300, + "maxAttempts": 3, + "backoffSeconds": [30, 120], + "concurrencyKeys": ["auth:{authProfile}", "publish:paycom-roster"] + }, + "resource-links.current-period": { + "description": "Generate and atomically publish one canonical link per active current-period employee", + "inputSchema": { + "type": "object", + "properties": { "resourceType": { "type": "string", "enum": ["paycom.timecard.summary"] } }, + "required": ["resourceType"], + "additionalProperties": false + }, + "timeoutSeconds": 60, + "maxAttempts": 1, + "backoffSeconds": [], + "concurrencyKeys": ["publish:paycom-resource-links"] + }, + "resource-links.period": { + "description": "Generate and atomically publish one canonical link per active employee for an exact Paycom period", + "inputSchema": { + "type": "object", + "properties": { + "resourceType": { "type": "string", "enum": ["paycom.timecard.summary"] }, + "periodEnd": { "type": "string", "maxLength": 10, "pattern": "^\\d{4}-\\d{2}-\\d{2}$" } + }, + "required": ["resourceType", "periodEnd"], + "additionalProperties": false + }, + "timeoutSeconds": 60, + "maxAttempts": 1, + "backoffSeconds": [], + "concurrencyKeys": ["publish:paycom-resource-links"] + }, + "resource-links.audit": { + "description": "Audit canonical resource-link coverage against the exact active period roster", + "inputSchema": { + "type": "object", + "properties": { + "resourceType": { "type": "string", "enum": ["paycom.timecard.summary"] }, + "periodEnd": { "type": "string", "maxLength": 10, "pattern": "^\\d{4}-\\d{2}-\\d{2}$" } + }, + "required": ["resourceType"], + "additionalProperties": false + }, + "timeoutSeconds": 60, + "maxAttempts": 1, + "backoffSeconds": [], + "concurrencyKeys": ["publish:paycom-resource-links"] + }, + "timecards.current-period": { + "description": "Collect every active employee timecard for the current Paycom period and atomically publish it", + "inputSchema": { "type": "object", "properties": {}, "required": [], "additionalProperties": false }, + "timeoutSeconds": 3600, + "maxAttempts": 2, + "backoffSeconds": [120], + "concurrencyKeys": ["auth:{authProfile}", "publish:paycom-timecards"] + }, + "timecards.period": { + "description": "Collect every active employee timecard for one bounded Paycom period ending on a Saturday", + "inputSchema": { + "type": "object", + "properties": { "periodEnd": { "type": "string", "maxLength": 10, "pattern": "^\\d{4}-\\d{2}-\\d{2}$" } }, + "required": ["periodEnd"], + "additionalProperties": false + }, + "timeoutSeconds": 3600, + "maxAttempts": 2, + "backoffSeconds": [120], + "concurrencyKeys": ["auth:{authProfile}", "publish:paycom-timecards"] + }, + "timecards.from-published-roster": { + "description": "Collect an exact period using the already-published authoritative roster and fence publication to that roster revision", + "inputSchema": { + "type": "object", + "properties": { "periodEnd": { "type": "string", "maxLength": 10, "pattern": "^\\d{4}-\\d{2}-\\d{2}$" } }, + "required": ["periodEnd"], + "additionalProperties": false + }, + "timeoutSeconds": 3600, + "maxAttempts": 2, + "backoffSeconds": [120], + "concurrencyKeys": ["auth:{authProfile}", "publish:paycom-roster", "publish:paycom-timecards"] + }, + "timecards.audit": { + "description": "Audit exact-period timecard integrity, roster binding, identities, membership, and relational projections without browsing", + "inputSchema": { + "type": "object", + "properties": { "periodEnd": { "type": "string", "maxLength": 10, "pattern": "^\\d{4}-\\d{2}-\\d{2}$" } }, + "required": ["periodEnd"], + "additionalProperties": false + }, + "timeoutSeconds": 60, + "maxAttempts": 1, + "backoffSeconds": [], + "concurrencyKeys": ["publish:paycom-roster", "publish:paycom-timecards"] + }, + "timecards.incremental": { + "description": "Fill missing active-employee timecards for the current period and republish a complete snapshot", + "inputSchema": { "type": "object", "properties": {}, "required": [], "additionalProperties": false }, + "timeoutSeconds": 3600, + "maxAttempts": 2, + "backoffSeconds": [120], + "concurrencyKeys": ["auth:{authProfile}", "publish:paycom-timecards"] + }, + "reconcile.current-period": { + "description": "Audit current-period timecard membership against the active roster without browsing", + "inputSchema": { "type": "object", "properties": {}, "required": [], "additionalProperties": false }, + "timeoutSeconds": 60, + "maxAttempts": 1, + "backoffSeconds": [], + "concurrencyKeys": ["publish:paycom-timecards"] + }, + "sync.current-workforce": { + "description": "Immediately publish complete visible workforce additions, edits, and explicit active-status transitions; retain every absent employee", + "inputSchema": { + "type": "object", + "properties": { + "reconcileBatchSize": { "type": "integer", "minimum": 1, "maximum": 500 }, + "fullReconcileMinutes": { "type": "integer", "minimum": 60, "maximum": 10080 }, + "lookbackPeriods": { "type": "integer", "enum": [1] }, + "publishMode": { "type": "string", "enum": ["shadow", "additions_edits_preview", "additions_edits"] } + }, + "required": ["reconcileBatchSize", "fullReconcileMinutes", "lookbackPeriods", "publishMode"], + "additionalProperties": false + }, + "timeoutSeconds": 3600, + "maxAttempts": 2, + "backoffSeconds": [120], + "concurrencyKeys": ["auth:{authProfile}", "observe:paycom-workforce", "publish:paycom-roster", "publish:paycom-timecards", "publish:paycom-resource-links"] + } + } + } + ], + "sources": [ + { + "id": "paycom-main", + "collector": "paycom", + "authProfile": "paycom-main", + "config": { "timezone": "America/Los_Angeles", "maxConcurrency": 6 }, + "collection": { + "targetType": "pay-period", + "resolverMethod": "collection.resolve-targets", + "selectors": ["current", "latest-complete", "date", "relative-date", "date-range", "last-duration", "exact-target"], + "targetFields": ["periodEnd"], + "scopes": { + "roster": { + "description": "Authoritative active roster for each resolved pay period", + "tasks": [ + { "id": "roster", "plan": "paycom-period-roster", "input": {}, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": [] } + ] + }, + "links": { + "description": "Period roster, canonical timecard links, and link audit", + "tasks": [ + { "id": "roster", "plan": "paycom-period-roster", "input": {}, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": [] }, + { "id": "links", "plan": "paycom-period-resource-links", "input": { "resourceType": "paycom.timecard.summary" }, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": ["roster"] }, + { "id": "links-audit", "plan": "paycom-period-resource-links-audit", "input": { "resourceType": "paycom.timecard.summary" }, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": ["links"] } + ], + "auditTasks": [ + { "id": "links-audit", "plan": "paycom-period-resource-links-audit", "input": { "resourceType": "paycom.timecard.summary" }, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": [] } + ] + }, + "timecards": { + "description": "Authoritative roster-bound exact-period timecards with an explicit post-publication audit", + "tasks": [ + { "id": "roster", "plan": "paycom-period-roster", "input": {}, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": [] }, + { "id": "timecards", "plan": "paycom-period-timecards-from-roster", "input": {}, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": ["roster"] }, + { "id": "timecards-audit", "plan": "paycom-period-timecards-audit", "input": {}, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": ["timecards"] } + ], + "auditTasks": [ + { "id": "timecards-audit", "plan": "paycom-period-timecards-audit", "input": {}, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": [] } + ] + }, + "full": { + "description": "Roster, roster-bound timecards, canonical links, and explicit publication audits", + "tasks": [ + { "id": "roster", "plan": "paycom-period-roster", "input": {}, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": [] }, + { "id": "timecards", "plan": "paycom-period-timecards-from-roster", "input": {}, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": ["roster"] }, + { "id": "timecards-audit", "plan": "paycom-period-timecards-audit", "input": {}, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": ["timecards"] }, + { "id": "links", "plan": "paycom-period-resource-links", "input": { "resourceType": "paycom.timecard.summary" }, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": ["timecards-audit"] }, + { "id": "links-audit", "plan": "paycom-period-resource-links-audit", "input": { "resourceType": "paycom.timecard.summary" }, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": ["links"] } + ], + "auditTasks": [ + { "id": "timecards-audit", "plan": "paycom-period-timecards-audit", "input": {}, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": [] }, + { "id": "links-audit", "plan": "paycom-period-resource-links-audit", "input": { "resourceType": "paycom.timecard.summary" }, "targetInput": { "periodEnd": "periodEnd" }, "dependsOn": [] } + ] + } + }, + "limits": { "maxTargets": 64, "maxRangeDays": 730 } + }, + "enabled": true + } + ], + "plans": [ + { "id": "paycom-health", "source": "paycom-main", "method": "collector.health", "schedule": { "type": "manual" }, "input": {}, "dependsOn": [], "enabled": true }, + { "id": "paycom-periods", "source": "paycom-main", "method": "pay-periods.discover", "schedule": { "type": "manual" }, "input": {}, "dependsOn": [], "enabled": true }, + { "id": "paycom-roster", "source": "paycom-main", "method": "roster.snapshot", "schedule": { "type": "manual" }, "input": {}, "dependsOn": [], "enabled": true }, + { "id": "paycom-period-roster", "source": "paycom-main", "method": "roster.period", "schedule": { "type": "manual" }, "input": { "periodEnd": "2026-09-05" }, "dependsOn": [], "enabled": true }, + { "id": "paycom-current-resource-links", "source": "paycom-main", "method": "resource-links.current-period", "schedule": { "type": "manual" }, "input": { "resourceType": "paycom.timecard.summary" }, "dependsOn": [{ "plan": "paycom-roster", "maxAgeSeconds": 86400 }], "enabled": true }, + { "id": "paycom-period-resource-links", "source": "paycom-main", "method": "resource-links.period", "schedule": { "type": "manual" }, "input": { "resourceType": "paycom.timecard.summary", "periodEnd": "2026-09-05" }, "dependsOn": [{ "plan": "paycom-period-roster", "maxAgeSeconds": 86400 }], "enabled": true }, + { "id": "paycom-period-resource-links-audit", "source": "paycom-main", "method": "resource-links.audit", "schedule": { "type": "manual" }, "input": { "resourceType": "paycom.timecard.summary", "periodEnd": "2026-09-05" }, "dependsOn": [], "enabled": true }, + { "id": "paycom-resource-links-audit", "source": "paycom-main", "method": "resource-links.audit", "schedule": { "type": "manual" }, "input": { "resourceType": "paycom.timecard.summary" }, "dependsOn": [{ "plan": "paycom-current-resource-links", "maxAgeSeconds": 86400 }], "enabled": true }, + { "id": "paycom-current-timecards", "source": "paycom-main", "method": "timecards.current-period", "schedule": { "type": "manual" }, "input": {}, "dependsOn": [{ "plan": "paycom-roster", "maxAgeSeconds": 86400 }], "enabled": true }, + { "id": "paycom-period-timecards", "source": "paycom-main", "method": "timecards.period", "schedule": { "type": "manual" }, "input": { "periodEnd": "2026-09-05" }, "dependsOn": [], "enabled": true }, + { "id": "paycom-period-timecards-from-roster", "source": "paycom-main", "method": "timecards.from-published-roster", "schedule": { "type": "manual" }, "input": { "periodEnd": "2026-09-05" }, "dependsOn": [], "enabled": true }, + { "id": "paycom-period-timecards-audit", "source": "paycom-main", "method": "timecards.audit", "schedule": { "type": "manual" }, "input": { "periodEnd": "2026-09-05" }, "dependsOn": [], "enabled": true }, + { "id": "paycom-incremental-timecards", "source": "paycom-main", "method": "timecards.incremental", "schedule": { "type": "manual" }, "input": {}, "dependsOn": [{ "plan": "paycom-roster", "maxAgeSeconds": 86400 }], "enabled": true }, + { "id": "paycom-reconcile", "source": "paycom-main", "method": "reconcile.current-period", "schedule": { "type": "manual" }, "input": {}, "dependsOn": [{ "plan": "paycom-current-timecards", "maxAgeSeconds": 86400 }], "enabled": true }, + { "id": "paycom-current-workforce-sync", "source": "paycom-main", "method": "sync.current-workforce", "schedule": { "type": "manual" }, "input": { "reconcileBatchSize": 100, "fullReconcileMinutes": 1440, "lookbackPeriods": 1, "publishMode": "additions_edits" }, "dependsOn": [], "enabled": true } + ], + "syncs": [ + { + "id": "paycom-main-workforce", + "plan": "paycom-current-workforce-sync", + "intervalSeconds": 3600, + "jitterSeconds": 300, + "overlap": "coalesce", + "settingsSchema": { + "type": "object", + "properties": { + "reconcileBatchSize": { "type": "integer", "minimum": 1, "maximum": 500 }, + "fullReconcileMinutes": { "type": "integer", "minimum": 60, "maximum": 10080 }, + "lookbackPeriods": { "type": "integer", "enum": [1] }, + "publishMode": { "type": "string", "enum": ["shadow", "additions_edits_preview", "additions_edits"] } + }, + "required": ["reconcileBatchSize", "fullReconcileMinutes", "lookbackPeriods"], + "additionalProperties": false + }, + "settings": { "reconcileBatchSize": 100, "fullReconcileMinutes": 1440, "lookbackPeriods": 1, "publishMode": "additions_edits" }, + "desiredState": "stopped" + } + ] +} diff --git a/plugins/paycom/backend/dispatch-plugin.yaml b/plugins/paycom/backend/dispatch-plugin.yaml new file mode 100644 index 0000000..d099a7d --- /dev/null +++ b/plugins/paycom/backend/dispatch-plugin.yaml @@ -0,0 +1,41 @@ +schema_version: 1 +id: paycom +display_name: Paycom Collector +version: 0.18.8 +summary: Paycom full-workforce immediate sync with retained unknown absences, persisted detail proof, and privacy-safe aggregate deltas. +owner: + data: paycom + team: Dispatch Operations +paths: + source: src + tests: tests + database: /paycom + state: /paycom + staging: + locks: /paycom/locks + receipts: /paycom/receipts +commands: + test: ./tooling/test + build: ./tooling/build + verify: ./tooling/verify + health: ./tooling/health +components: + - id: paycom-collector + kind: collector + source: src + capabilities: + read_local_data: true + mutate_data: true + collect: true + network: true + authentication: true + direct_delivery: false + long_running: true + coordinator_mode: paycom + runtime: + releases: runtime/paycom-collector + activation_record: config/activation.json +retention: + keep_current: true + keep_rollback: 1 + preserve_pinned: true diff --git a/plugins/paycom/backend/installed.js b/plugins/paycom/backend/installed.js new file mode 100644 index 0000000..859e74b --- /dev/null +++ b/plugins/paycom/backend/installed.js @@ -0,0 +1,76 @@ +'use strict'; +const path = require('node:path'); +const { WorkforceClient } = require('dispatch-protocol/contracts/src/workforce-client'); +const { LocalPaycomWorkforcePort } = require('./adapters/workforce'); +const { exactObject, invalid } = require('dispatch-protocol/contracts/src/input'); + +// This entrypoint is packaged independently of the platform. Host capabilities +// arrive through dispatch-sdk; the package owns its database and projections. +function createPlugin({ dispatch }) { + if (typeof dispatch?.storage?.directory !== 'function') throw new TypeError('plugin_storage_required'); + const database = path.join(dispatch.storage.directory('database'), 'paycom.sqlite3'); + async function invoke(action, input) { + if (action === 'workforce.day') { + exactObject(input, ['query'], ['query']); + return read({ dispatch, request: { view: 'day', query: input.query } }); + } + if (action.startsWith('workforce.')) { + const config = await dispatch.schedules.status('paycom-main-workforce'); + const workforce = new WorkforceClient({ port: new LocalPaycomWorkforcePort({ database, timezone: config.timezone }) }); + if (action === 'workforce.employees') { exactObject(input, ['query'], ['query']); return workforce.employees(input.query); } + if (action === 'workforce.employee') { exactObject(input, ['code'], ['code']); return workforce.employee(input.code); } + } + if (action === 'sync.status' || action === 'sync.run_now') { + exactObject(input, action === 'sync.status' ? ['id'] : ['id', 'options'], ['id']); + if (input.id !== 'paycom-main-workforce') invalid(); + return action === 'sync.status' ? (await dispatch.schedules.status(input.id)).result + : dispatch.schedules.run(input.id, input.options || {}); + } + invalid(); + } + return Object.freeze({ invoke }); +} + +async function initialize({ dispatch, timezone }) { + const { PaycomStore } = require('./src/store'); + const store = new PaycomStore(path.join(dispatch.storage.directory('database'), 'paycom.sqlite3')); + try { if (store.db.prepare('PRAGMA quick_check').get().quick_check !== 'ok') throw new Error('plugin_initialization_failed'); } + finally { store.close(); } + // Rebuild the package-owned read model from retained business publications + // before acknowledging an existing DSP's migration or package upgrade. + await publish({ dispatch, timezone }); + return true; +} +async function collect({ dispatch, request, signal }) { + return require('./src/collector-core').execute(request, { + database: path.join(dispatch.storage.directory('database'), 'paycom.sqlite3'), + stagingRoot: dispatch.storage.directory('staging'), + authentication: async () => (await dispatch.connections.status('paycom', { signal })).state, + browserRunner: (_request, useBrowser) => dispatch.connections.withSession({ connection: 'paycom', signal }, useBrowser), + }); +} +async function publish({ dispatch, timezone }) { + return require('./adapters/published').publishWorkforce({ + database: path.join(dispatch.storage.directory('database'), 'paycom.sqlite3'), + publishedDatabase: path.join(dispatch.storage.directory('published'), 'paycom.sqlite3'), timezone, + }); +} +async function read({ dispatch, request }) { + const settings = (await dispatch.settings.get()).values; + const client = require('../dashboard/published').createPublishedClient({ directory: dispatch.storage.directory('published'), settings }); + if (request.view === 'settings-options') return client.settingsOptions(); + if (!['day', 'employees', 'employee', 'snapshot', 'timecards', 'punches', 'resourceLinks'].includes(request.view)) invalid(); + return client.workforce[request.view](request.view === 'employee' ? request.query.code : request.query); +} +async function inspect({ dispatch }) { + return new (require('./adapters/publication').LocalPaycomPublicationPort)({ database: path.join(dispatch.storage.directory('database'), 'paycom.sqlite3') }).health(); +} +async function evidence({ dispatch, request }) { + return require('./src/activation-evidence-core').verifyActivationEvidence({ + batchId: request.batchId, preparationRunId: request.preparationRunId, definitionDigest: request.definitionDigest, + clock: Date.now, paycomDatabase: path.join(dispatch.storage.directory('database'), 'paycom.sqlite3'), + manager: { batch: id => { if (id !== request.batch.id) invalid(); return request.batch; }, + run: id => { const run = request.runs.find(item => item.id === id); if (!run) invalid(); return run; } }, + }); +} +module.exports = { createPlugin, initialize, collect, publish, read, inspect, evidence }; diff --git a/plugins/paycom/backend/package-paths.js b/plugins/paycom/backend/package-paths.js new file mode 100644 index 0000000..62cb8ee --- /dev/null +++ b/plugins/paycom/backend/package-paths.js @@ -0,0 +1,12 @@ +'use strict'; +// The package builder selects this module for the legacy default-path import. +// Its directories are mounted by the worker host and come only from the SDK. +const path = require('node:path'); +const { createWorkerClient } = require('dispatch-sdk/node'); +const storage = createWorkerClient().storage; +module.exports = { + DATA_ROOT: storage.directory('database'), + DATABASE: path.join(storage.directory('database'), 'paycom.sqlite3'), + STAGING_ROOT: storage.directory('staging'), + AUTH_SOCKET: null, +}; diff --git a/plugins/paycom/backend/package.json b/plugins/paycom/backend/package.json new file mode 100644 index 0000000..7a474e7 --- /dev/null +++ b/plugins/paycom/backend/package.json @@ -0,0 +1,16 @@ +{ + "name": "dispatch-paycom-plugin", + "version": "0.18.8", + "private": true, + "description": "Paycom full-workforce immediate sync with retained unknown lifecycle state, persisted detail proof, and privacy-safe aggregate deltas", + "type": "commonjs", + "scripts": { + "build": "./scripts/build", + "test": "./scripts/test", + "verify": "./scripts/verify", + "health": "./scripts/health" + }, + "engines": { + "node": ">=22" + } +} diff --git a/plugins/paycom/backend/plugin.js b/plugins/paycom/backend/plugin.js new file mode 100644 index 0000000..660dbb3 --- /dev/null +++ b/plugins/paycom/backend/plugin.js @@ -0,0 +1,40 @@ +'use strict'; + +// The runtime owns storage, authorization and worker coordination. Paycom only +// contributes its own lifecycle behavior and existing public workforce adapter. +function createPlugin({ configuration, client }) { + const setup = require('./runtime/setup') + .createContainerPaycomSetup(configuration, client); + async function invoke(action, value) { + const input = require('dispatch-protocol/gateway/protocol').validateActionInput(action, value); + if (action.startsWith('sync.') && input.id !== 'paycom-main-workforce') throw Object.assign(new Error('invalid_input'), { code: 'invalid_input' }); + if (action === 'workforce.day') return client.workforce.day(input.query); + if (action === 'workforce.employees') return client.workforce.employees(input.query); + if (action === 'workforce.employee') return client.workforce.employee(input.code); + if (action === 'sync.status') return client.sync.status(input.id); + if (action === 'sync.run_now') return client.sync.runNow(input.id, input.options); + throw Object.assign(new Error('invalid_input'), { code: 'invalid_input' }); + } + // The host gates requests, stops schedules and drains cancelled collectors. + // Installation changes must never lock/unlock the credential profile: doing + // so would replace or clear an independent provider verification guard. + return { setup, invoke, busy: setup.busy }; +} +function createAuthAdapters() { + return { paycom: require('./auth/adapter').paycomAdapter }; +} + +function createClientPorts({ paths, collectionPort }) { + return { + paycom: new (require('./adapters/publication').LocalPaycomPublicationPort)({ database: paths.paycom.database }), + workforce: new (require('./adapters/workforce').LocalPaycomWorkforcePort)({ database: paths.paycom.database, + ...(collectionPort ? { timezone: () => collectionPort.source('paycom-main').config.timezone } : {}) }), + }; +} + +function publish({ paths, directory, timezone }) { + return require('./adapters/published-job').publish({ database: paths.paycom.database, + publishedDatabase: require('node:path').join(directory, 'paycom.sqlite3'), timezone }); +} + +module.exports = { createPlugin, createAuthAdapters, createClientPorts, publish }; diff --git a/plugins/paycom/backend/references/data-contract.md b/plugins/paycom/backend/references/data-contract.md new file mode 100644 index 0000000..bf2bcb3 --- /dev/null +++ b/plugins/paycom/backend/references/data-contract.md @@ -0,0 +1,74 @@ +--- +title: Paycom collector data contract +status: current +last_verified: 2026-09-02 +--- + +# Paycom collector data contract + +The collector owns `/paycom/paycom.sqlite3`. + +Each collection creates a private staging candidate, validates it, publishes it in one SQLite transaction, advances the appropriate active pointer, verifies row counts and database integrity, and then removes staging. Prior publications remain available for rollback or audit. Read interfaces use `PaycomStore.activeWorkforce()` in read-only mode: roster, timecards, and resource links are read under one SQLite snapshot and their roster publication bindings are checked before any public DTO is returned. + +Published kinds: + +- `pay_periods`: previous/current/next biweekly period boundaries. +- `roster`: complete employee records, including active and active-driver flags. +- `timecards`: one complete active-employee snapshot for a period; legacy rows retain validated Paycom DOM projections and raw source hashes, while new semantic rows also store canonical cache-buster-free URLs, normalized business hashes, and per-row observation times. +- `resource_links`: one canonical URL per employee who is active in the exact bound roster publication and period. Link publications are stored in dedicated tables, use a separate active pointer, and contain no credentials, sessions, cookies, or browser-only cache-busting parameters. + +## Stored business fields + +Normalized roster records may contain: + +- employee code and name; +- canonical lifecycle status plus explicit Paycom active state; +- department and delivery-station codes/descriptions; +- position title, pay class, terminal group, pay type, and primary supervisor; +- roster-summary missing-punch, total-hours, overtime-hours, employee-approval, and supervisor-approval values; +- driver-department, driver-position, and active-driver classifications. + +A private timecard row contains employee identity, source/business hashes, observation time, and a validated normalized record. The normalized record contains: + +- period start/end/key, weekly totals, and period total hours; +- exactly 14 day rows with date/weekday, pay code, allocations, hours, total hours, dollar amount, exceptions, waiver state, comments, unresolved slots, and missing-punch state; +- zero or more additional rows for the same period; +- punches with slot/type, displayed/actual/rounded time, clock code/name, comments, provenance availability, approval/change-request status, current/requested direction, operation, and change note; +- approval, attestation, and meal-waiver table projections when present; +- a fixed canonical Paycom source route and extraction format/version. + +Resource-link rows contain employee code and one canonical `paycom.timecard.summary` HTTPS URL for the bound period. Pay-period rows contain start, end, key, and `previous`/`current`/`next` relation. + +## Public workforce boundary + +Applications must use `dispatch.workforce` rather than reading these tables. The public employee DTO intentionally exposes only identity, lifecycle, department, delivery station, position, pay class/type, supervisor, and driver classification. The public timecard DTO exposes period bounds, total hours, missing-day count, observation time, lifecycle, and canonical link. Raw punches, comments, allocations, exceptions, approvals, attestations, waivers, source/business hashes, publication IDs, and record JSON remain protected plugin data. + +## Sync persistence proof + +For `additions_edits`, the collector does not treat browser collection or a successful SQLite write call as proof that detailed timecards were saved. Before the transaction commits, it: + +1. reloads and fully audits the active roster-bound timecard publication; +2. recalculates its content hash from persisted `record_json` values and relational projections; +3. compares every timecard selected and observed during the tick with the active row using the full normalized business hash; +4. requires the configured source-timezone date to exist in every active timecard; +5. emits aggregate daily punch coverage only after all checks pass. + +The business hash includes the entire normalized record except the browser-only cache-buster in the source URL. Punch additions/edits, kinds, displayed/actual/rounded times, clock provenance, change-request details, allocations, exceptions, comments, waivers, approvals, attestations, and totals therefore participate in equality. A mismatch raises `integrity_failed`; the encompassing SQLite transaction rolls back active pointers and sync observation state. A valid `no_change` tick performs the same selected-row comparison against the already-active publication. + +The receipt's `persistence` object is aggregate-only: `verified`, coverage date, publication metadata, active timecard/date-row counts, selected/persisted/mismatch counts, total punches, each normalized punch-kind count, and timecards containing `IN DAY`. It contains no employee code, name, punch time, clock, comment, allocation, or exception value. Shadow/baseline-required/preview modes do not claim persistence because they do not activate business data. + +The same transaction computes a `delta` by comparing the prior active timecard graph with the candidate graph before commit. The closed aggregate shape counts roster and timecard additions/changes/removals, changed day sections, punch additions/edits/removals, added/removed punch kinds, missing-punch and unresolved-slot transitions, and changed approval, attestation, meal-waiver, additional-row, comment, and total sections. Punch identity is used only inside the transaction; the result contains no employee code, date-row key, slot, time, or business value. `no_change` produces a zero mutation delta while still reporting the unchanged active timecard count. + +## Publication and retention metadata + +The database also stores publication/run identity, collected/activated timestamps, row counts, metadata, content hashes, active pointers, attempt fences, and roster bindings. The active publication plus one prior publication is retained per kind/target for rollback and audit, so physical table row counts can exceed the current workforce count. Private sync tables retain one source/target checkpoint plus per-employee profile, summary, and timecard fingerprints and observation/full-verification timestamps for change detection. Schema version 5 adds `paycom_sync_change_history`: one idempotent privacy-safe row per committed ready run containing source/target, business date/timezone, observed time, disposition, validated aggregate delta JSON, and persistence JSON. It deliberately has no employee identity or punch-detail columns and survives full-publication pruning. Bounded post-commit housekeeping compacts only `no_change` rows older than 365 days to one row per source/target/business date; published/change rows and recent rows are retained. + +Schema version 4 introduced private `paycom_sync_state` and `paycom_sync_employees` observation tables. They hold source, profile, summary, normalized timecard fingerprints and observation/full-verification times. Legacy removal columns remain only for schema compatibility and are always cleared to zero/null; sync never confirms or publishes deletions. Published roster records carry canonical `lifecycleStatus`: visible positive status maps to `active` or `inactive`, while a previously active employee absent from the active-only source becomes `unknown`. Unknown records retain their last verified timecard/link state. A visible return maps unknown→active; only positive inactive evidence removes active-only dependent membership. + +The public protected Workforce punch query does not expose these private tables. It derives a minimum DTO from one coherent `activeWorkforce()` read transaction and emits employee name, lifecycle, date, normalized kind, local `HH:MM`, actual/displayed basis, and observation/collection context only. Employee code, source URL, punch comments, allocations, exceptions, approvals, publication identity, and hashes remain private. + +The authenticated dashboard also uses the protected Workforce daily projection. Daily rows include employee identity, date, grouped punch times and their actual/displayed basis, reported hours, missing-punch state, a derived condition, and observation time. An employee lookup may return up to 14 of these same projected rows alongside the existing employee profile and period summary. It never exposes raw record JSON, comments, clocks, allocations, approval tables, attestations, waivers, hashes, or publication identifiers. Both dashboard employee endpoints require `workforce.read` and use the session-selected DSP runtime. + +Daily date lookup selects the saved roster-bound period containing the requested date when available, including previously collected periods. It never initiates collection. Uncollected dates return `available: false` with no employee rows; dates with no activity in a saved period remain distinguishable. Explicit daily sorting occurs before pagination; time columns compare the first source-order punch, hours compare numerically, and empty values stay last for ascending and descending orders. + +No password, PIN, cookie, authorization header, CDP endpoint, lease token, or browser profile path may be stored in this database or emitted in a receipt. diff --git a/plugins/paycom/backend/runtime/activation.js b/plugins/paycom/backend/runtime/activation.js new file mode 100644 index 0000000..b31c351 --- /dev/null +++ b/plugins/paycom/backend/runtime/activation.js @@ -0,0 +1,402 @@ +'use strict'; + +const path = require('node:path'); +const { + isResult, + serverInstallationManifest, +} = require('dispatch-protocol/contracts/src'); +const { + managedPaycomDefinition, + managedPaycomFirstPublicationRequest, + PAYCOM_FIRST_PUBLICATION_TASKS, + PAYCOM_PROFILE_ID, + PAYCOM_SYNC_ID, +} = require('./definition'); + +const DEFAULT_PUBLICATION_TIMEOUT_MS = 2 * 60 * 60 * 1000; +const DEFAULT_PUBLICATION_POLL_MS = 1000; +const MAX_PAY_PERIOD_PREPARATION_MS = 5 * 60 * 1000; +const PAYCOM_PERIODS_PLAN = 'paycom-periods'; +const EXPECTED_FIRST_PUBLICATION_PLANS = Object.freeze(Object.fromEntries( + Object.entries(PAYCOM_FIRST_PUBLICATION_TASKS).map(([plan, definition]) => [plan, definition.method]), +)); + +function fail(code) { + throw Object.assign(new Error(code), { code }); +} + +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exact(value, fields, code = 'runtime_boundary_violation') { + if (!plain(value) || Object.keys(value).sort().join(',') !== [...fields].sort().join(',')) fail(code); + return value; +} + +function same(left, right) { + return JSON.stringify(left) === JSON.stringify(right); +} + +function successful(result, statuses, code) { + if (!isResult(result) || !result.ok || !statuses.includes(result.status)) fail(code); + return result; +} + +function createManagedPaycomActivationRuntime(options) { + const optionFields = [ + 'manifest', 'manifestAuthority', 'layout', 'serviceManager', 'supervisor', 'client', + 'collectionAdmin', 'gateway', 'evidenceVerifier', 'clock', 'delay', 'publicationTimeoutMs', 'publicationPollMs', + 'projectRoot', 'infrastructureVerifier', + ]; + if (!plain(options) || Object.keys(options).some(key => !optionFields.includes(key)) + || !['manifest', 'manifestAuthority', 'client', + 'collectionAdmin', 'gateway', 'evidenceVerifier'].every(key => Object.hasOwn(options, key))) { + fail('runtime_boundary_violation'); + } + const manifest = serverInstallationManifest(options.manifest, options.manifestAuthority); + const layout = options.layout; + const serviceManager = options.serviceManager; + const supervisor = options.supervisor; + const client = options.client; + const collectionAdmin = options.collectionAdmin; + const gateway = options.gateway; + const evidenceVerifier = options.evidenceVerifier; + if ((typeof options.infrastructureVerifier !== 'function' && (!layout || typeof layout.inspect !== 'function' + || !serviceManager || typeof serviceManager.plan !== 'function' || typeof serviceManager.inspectInstalled !== 'function' + || !supervisor || typeof supervisor.inspect !== 'function' || typeof supervisor.health !== 'function')) + || !client?.auth || !client?.collections || !client?.sync || !client?.paycom + || !collectionAdmin || !['preview', 'apply', 'inspect', 'attest'] + .every(method => typeof collectionAdmin[method] === 'function') + || !gateway || typeof gateway.health !== 'function' + || !evidenceVerifier || typeof evidenceVerifier.verify !== 'function') fail('runtime_boundary_violation'); + const clock = options.clock === undefined ? Date.now : options.clock; + const delay = options.delay === undefined ? ms => new Promise(resolve => setTimeout(resolve, ms)) : options.delay; + const publicationTimeoutMs = options.publicationTimeoutMs === undefined + ? DEFAULT_PUBLICATION_TIMEOUT_MS : options.publicationTimeoutMs; + const publicationPollMs = options.publicationPollMs === undefined + ? DEFAULT_PUBLICATION_POLL_MS : options.publicationPollMs; + if (typeof clock !== 'function' || typeof delay !== 'function' + || !Number.isSafeInteger(publicationTimeoutMs) || publicationTimeoutMs < 1000 || publicationTimeoutMs > DEFAULT_PUBLICATION_TIMEOUT_MS + || !Number.isSafeInteger(publicationPollMs) || publicationPollMs < 10 || publicationPollMs > 10_000) { + fail('runtime_boundary_violation'); + } + const projectRoot = options.projectRoot; + const definitionOptions = projectRoot === undefined ? {} : { projectRoot }; + const expectedDefinition = () => managedPaycomDefinition(manifest, options.manifestAuthority, definitionOptions); + + function selectedManifest(value) { + const selected = serverInstallationManifest(value, options.manifestAuthority); + if (!same(selected, manifest)) fail('runtime_identity_mismatch'); + return selected; + } + + async function verifyInfrastructure(manifestValue) { + const selected = selectedManifest(manifestValue); + if (options.infrastructureVerifier) return options.infrastructureVerifier(selected); + const selectedLayout = layout.inspect(selected, options.manifestAuthority); + const plan = serviceManager.plan(selected, options.manifestAuthority, selectedLayout); + serviceManager.inspectInstalled(plan); + supervisor.inspect(plan); + supervisor.health(plan); + const auth = successful(await client.auth.health(), ['ready'], 'runtime_health_failed'); + const collections = successful(await client.collections.health(), ['ready'], 'runtime_health_failed'); + if (auth.data?.vault?.verified !== true || collections.data?.manager?.running !== true + || collections.data?.databaseIntegrity !== 'ok' + || collections.data?.syncAlerts?.critical !== 0) fail('runtime_health_failed'); + const gatewayHealth = successful(await gateway.health(), ['ready'], 'runtime_health_failed'); + if (gatewayHealth.data?.runtimeIdentity !== 'matched') fail('runtime_identity_mismatch'); + return Object.freeze({ + runtimeKey: manifest.runtime.key, + runtime_layout: true, + service_supervision: true, + auth_broker: true, + collection_manager: true, + runtime_gateway: true, + }); + } + + async function configure(definition) { + const expected = expectedDefinition(); + if (!definition || definition.digest !== expected.digest + || !same(definition.specification, expected.specification)) fail('runtime_boundary_violation'); + const preview = collectionAdmin.preview(expected.specification); + if (!plain(preview) || preview.valid !== true) fail('runtime_health_failed'); + const applied = collectionAdmin.apply(expected.specification); + const expectedCounts = { collectors: 1, sources: 1, plans: 15, syncs: 1 }; + if (!same(applied, expectedCounts)) fail('runtime_health_failed'); + const inspected = collectionAdmin.inspect(); + if (inspected?.initialized !== true || !same(inspected.counts, expectedCounts)) fail('runtime_health_failed'); + const attested = collectionAdmin.attest(expected.specification); + if (!plain(attested) || Object.keys(attested).length !== 1 || attested.matched !== true) { + fail('runtime_health_failed'); + } + const [source, sync] = await Promise.all([ + client.collections.source('paycom-main'), + client.sync.status('paycom-main-workforce'), + ]); + successful(source, ['found'], 'runtime_health_failed'); + successful(sync, ['found'], 'runtime_health_failed'); + if (source.data?.authProfile !== PAYCOM_PROFILE_ID || sync.data?.desiredState !== 'stopped') { + fail('runtime_health_failed'); + } + return Object.freeze({ digest: expected.digest, ...expectedCounts }); + } + + async function startWorkforceSync() { + const expected = expectedDefinition(); + let current = await client.sync.status(PAYCOM_SYNC_ID); + if (!current.ok && current.status === 'sync_not_found') { + await configure(expected); + current = await client.sync.status(PAYCOM_SYNC_ID); + } + successful(current, ['found'], 'runtime_health_failed'); + if (!['running', 'stopped'].includes(current.data?.desiredState)) fail('runtime_health_failed'); + // Reconnection must not reapply definitions or reset an existing hourly window. + const spec = structuredClone(expected.specification); + const sync = spec.syncs.find(item => item.id === PAYCOM_SYNC_ID); + sync.desiredState = current.data.desiredState; + sync.intervalSeconds = current.data.intervalSeconds; + sync.jitterSeconds = current.data.jitterSeconds; + if (collectionAdmin.attest(spec)?.matched !== true) fail('runtime_health_failed'); + let automatic = true, intervalSeconds = current.data.intervalSeconds; + if (process.env.DISPATCH_PLUGIN_BACKEND === 'core_v1') { + const settings = await require('dispatch-sdk/runtime').createFrameworkClient().request('plugin.settings', {pluginId:'paycom',request:{action:'get'}}); + automatic = settings.values.automatic_sync; + intervalSeconds = settings.values.sync_interval_seconds; + } + if (current.data.intervalSeconds !== intervalSeconds) { + successful(await client.sync.edit(PAYCOM_SYNC_ID, { intervalSeconds, jitterSeconds: Math.min(current.data.jitterSeconds,intervalSeconds-1) }), ['updated'], 'runtime_health_failed'); + } + if (automatic) successful(await client.sync.start(PAYCOM_SYNC_ID), ['started'], 'runtime_health_failed'); + return Object.freeze({ syncId: PAYCOM_SYNC_ID, intervalSeconds, desiredState: automatic ? 'running' : 'stopped' }); + } + + async function testProvider(profileId) { + if (profileId !== PAYCOM_PROFILE_ID) fail('provider_auth_required'); + const status = successful(await client.auth.profileStatus(profileId), ['configured'], 'provider_auth_required'); + if (status.data?.profile?.configured !== true || status.data.profile.provider !== 'paycom') { + fail('provider_auth_required'); + } + const response = await client.auth.testProfile(profileId); + if (!response?.ok) fail(require('dispatch-protocol/contracts/src/paycom-setup').setupFailure(response?.status)); + const tested = successful(response, ['authenticated'], 'provider_auth_required'); + if (tested.data?.profile !== profileId || tested.data?.provider !== 'paycom' + || typeof tested.data?.testedAt !== 'string' || Number.isNaN(Date.parse(tested.data.testedAt))) { + fail('provider_auth_required'); + } + return Object.freeze({ + profileId, + provider: 'paycom', + status: 'authenticated', + testedAt: tested.data.testedAt, + }); + } + + async function batch(batchId) { + return successful(await client.collections.batchStatus(batchId, { limit: 50, offset: 0 }), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + } + + function publicationFromBatch(result) { + const value = result.data; + if (!value || typeof value.id !== 'string' || !value.counts || value.runPage?.hasMore !== false + || value.runCount !== value.runPage.total || value.runCount !== value.runPage.items.length + || value.runCount !== Object.keys(EXPECTED_FIRST_PUBLICATION_PLANS).length) { + fail('first_publication_failed'); + } + const plans = value.runPage.items.map(item => item?.run?.plan).sort(); + if (!same(plans, Object.keys(EXPECTED_FIRST_PUBLICATION_PLANS).sort()) + || value.runPage.items.some(item => item?.run?.source !== 'paycom-main' + || item.run.method !== EXPECTED_FIRST_PUBLICATION_PLANS[item.run.plan] + || item.taskId !== PAYCOM_FIRST_PUBLICATION_TASKS[item.run.plan]?.taskId + || item.targetKey !== value.runPage.items[0]?.targetKey)) fail('first_publication_failed'); + return Object.freeze({ + batchId: value.id, + status: value.status, + runCount: value.runCount, + succeededRuns: value.counts.succeeded, + failedRuns: value.counts.failed, + cancelledRuns: value.counts.cancelled, + }); + } + + async function cancelAndDrain(batchId) { + let result; + try { + result = successful(await client.collections.cancelBatch(batchId, { limit: 50, offset: 0 }), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + } catch { + fail('installation_operation_in_progress'); + } + const deadline = clock() + Math.min(60_000, publicationTimeoutMs); + while (['queued', 'running'].includes(result.status)) { + if (clock() >= deadline) fail('installation_operation_in_progress'); + await delay(publicationPollMs); + try { result = await batch(batchId); } + catch { fail('installation_operation_in_progress'); } + } + return result; + } + + async function cancelRunAndDrain(runId) { + let result; + try { + result = successful(await client.collections.cancelRun(runId), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + } catch { + fail('installation_operation_in_progress'); + } + const deadline = clock() + Math.min(60_000, publicationTimeoutMs); + while (['queued', 'running'].includes(result.status)) { + if (clock() >= deadline) fail('installation_operation_in_progress'); + await delay(publicationPollMs); + try { + result = successful(await client.collections.runStatus(runId), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + } catch { fail('installation_operation_in_progress'); } + } + return result; + } + + async function ensurePayPeriodBaseline(operationOptions) { + const idempotencyKey = `${operationOptions.idempotencyKey}:periods`; + let result = successful(await client.collections.startRun( + PAYCOM_PERIODS_PLAN, {}, { idempotencyKey }, + ), ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + const runId = result.data?.id; + if (typeof runId !== 'string') fail('first_publication_failed'); + const deadline = clock() + Math.min(MAX_PAY_PERIOD_PREPARATION_MS, publicationTimeoutMs); + for (;;) { + await operationOptions.heartbeat(); + if (result.status === 'succeeded') return runId; + if (['failed', 'cancelled'].includes(result.status)) fail('first_publication_failed'); + if (clock() >= deadline) { + await cancelRunAndDrain(runId); + fail('first_publication_failed'); + } + await delay(publicationPollMs); + result = successful(await client.collections.runStatus(runId), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + } + } + + async function publishFirst(request, operationOptions) { + if (!same(request, managedPaycomFirstPublicationRequest())) fail('runtime_boundary_violation'); + exact(operationOptions, ['idempotencyKey', 'heartbeat']); + if (typeof operationOptions.idempotencyKey !== 'string' + || !/^activation:[a-z][a-z0-9_-]{2,95}$/.test(operationOptions.idempotencyKey) + || typeof operationOptions.heartbeat !== 'function') { + fail('runtime_boundary_violation'); + } + const preparationRunId = await ensurePayPeriodBaseline(operationOptions); + let result = successful(await client.collections.enqueue(request, { idempotencyKey: operationOptions.idempotencyKey }), + ['queued', 'running', 'succeeded', 'failed', 'cancelled'], 'first_publication_failed'); + const batchId = result.data?.id; + if (typeof batchId !== 'string') fail('first_publication_failed'); + const deadline = clock() + publicationTimeoutMs; + for (;;) { + await operationOptions.heartbeat(); + result = await batch(batchId); + if (['succeeded', 'failed', 'cancelled'].includes(result.status)) { + return Object.freeze({ ...publicationFromBatch(result), preparationRunId }); + } + if (clock() >= deadline) { + await cancelAndDrain(batchId); + fail('first_publication_failed'); + } + await delay(publicationPollMs); + } + } + + async function verifyPublication(batchId, preparationRunId) { + if (typeof preparationRunId !== 'string') fail('first_publication_failed'); + const batchResult = await batch(batchId); + const terminal = publicationFromBatch(batchResult); + if (terminal.status !== 'succeeded' || terminal.succeededRuns !== terminal.runCount + || terminal.failedRuns !== 0 || terminal.cancelledRuns !== 0) fail('first_publication_failed'); + const manager = successful(await client.collections.health(), ['ready'], 'first_publication_failed'); + if (manager.data?.manager?.running !== true || manager.data?.databaseIntegrity !== 'ok' + || manager.data?.counts?.queued !== 0 + || manager.data?.counts?.running !== 0 + || manager.data?.syncAlerts?.critical !== 0) fail('first_publication_failed'); + const sync = successful(await client.sync.status(PAYCOM_SYNC_ID), ['found'], 'first_publication_failed'); + if (sync.data?.desiredState !== 'stopped' || sync.data?.activity !== 'idle' + || sync.data?.activeRun !== null || sync.data?.queuedRunCount !== 0) { + fail('first_publication_failed'); + } + const health = successful(await client.paycom.health(), ['ready'], 'first_publication_failed'); + const data = health.data; + if (data?.ready !== true || data.publicationStatus !== 'ready' + || data.payPeriods?.projectionValid !== true + || ![data.payPeriods, data.roster, data.timecards, data.resourceLinks] + .every(value => value?.verified === true)) fail('first_publication_failed'); + const target = data.roster.target; + if (typeof target !== 'string' || data.timecards.target !== target || data.resourceLinks.target !== target) { + fail('first_publication_failed'); + } + if (batchResult.data.runPage.items[0].targetKey !== target) fail('first_publication_failed'); + const evidence = await evidenceVerifier.verify({ + batchId, + preparationRunId, + definitionDigest: expectedDefinition().digest, + }); + exact(evidence, [ + 'definitionDigest', 'requestDigest', 'previewDigest', 'batchId', 'preparationRunId', 'target', 'runs', + 'publications', 'capturedAt', + ], 'first_publication_failed'); + if (evidence.batchId !== batchId || evidence.preparationRunId !== preparationRunId + || evidence.target !== target) fail('first_publication_failed'); + return Object.freeze({ ...evidence }); + } + + async function inspectSchedule() { + const sync = successful(await client.sync.status(PAYCOM_SYNC_ID), ['found'], 'runtime_health_failed'); + if (!['running', 'stopped'].includes(sync.data?.desiredState)) fail('runtime_health_failed'); + return Object.freeze({ syncWasRunning: sync.data.desiredState === 'running' }); + } + + async function quiesceSchedule(syncWasRunning) { + if (typeof syncWasRunning !== 'boolean') fail('runtime_boundary_violation'); + const before = successful(await client.sync.status(PAYCOM_SYNC_ID), ['found'], 'runtime_health_failed'); + if (before.data?.desiredState === 'running') { + const stopped = successful(await client.sync.stop(PAYCOM_SYNC_ID, { drain: true }), ['stopped'], 'runtime_health_failed'); + if (stopped.data?.desiredState !== 'stopped' || stopped.data?.activity !== 'idle' + || stopped.data?.activeRun !== null || stopped.data?.queuedRunCount !== 0) fail('runtime_health_failed'); + } else if (before.data?.desiredState !== 'stopped') fail('runtime_health_failed'); + const manager = successful(await client.collections.health(), ['ready'], 'runtime_health_failed'); + if (manager.data?.counts?.queued !== 0 || manager.data?.counts?.running !== 0) fail('runtime_health_failed'); + return Object.freeze({ syncWasRunning }); + } + + async function restoreSchedule(syncWasRunning) { + if (typeof syncWasRunning !== 'boolean') fail('runtime_boundary_violation'); + const before = successful(await client.sync.status(PAYCOM_SYNC_ID), ['found'], 'runtime_health_failed'); + if (syncWasRunning && before.data?.desiredState === 'stopped') { + const started = successful(await client.sync.start(PAYCOM_SYNC_ID), ['started'], 'runtime_health_failed'); + if (started.data?.sync?.desiredState !== 'running') fail('runtime_health_failed'); + } else if (!syncWasRunning && before.data?.desiredState === 'running') { + const stopped = successful(await client.sync.stop(PAYCOM_SYNC_ID, { drain: true }), ['stopped'], 'runtime_health_failed'); + if (stopped.data?.desiredState !== 'stopped' || stopped.data?.activity !== 'idle' + || stopped.data?.activeRun !== null || stopped.data?.queuedRunCount !== 0) fail('runtime_health_failed'); + } else if (before.data?.desiredState !== (syncWasRunning ? 'running' : 'stopped')) { + fail('runtime_health_failed'); + } + return Object.freeze({ syncWasRunning }); + } + + return Object.freeze({ + verifyInfrastructure, + configure, + startWorkforceSync, + testProvider, + publishFirst, + verifyPublication, + inspectSchedule, + quiesceSchedule, + restoreSchedule, + }); +} + +module.exports = { DEFAULT_PUBLICATION_TIMEOUT_MS, DEFAULT_PUBLICATION_POLL_MS, MAX_PAY_PERIOD_PREPARATION_MS, PAYCOM_PERIODS_PLAN, EXPECTED_FIRST_PUBLICATION_PLANS, createManagedPaycomActivationRuntime }; diff --git a/plugins/paycom/backend/runtime/definition.js b/plugins/paycom/backend/runtime/definition.js new file mode 100644 index 0000000..cdcfe2c --- /dev/null +++ b/plugins/paycom/backend/runtime/definition.js @@ -0,0 +1,186 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { PAYCOM_PROFILE_ID, PAYCOM_SOURCE_ID, PAYCOM_SYNC_ID, PAYCOM_COLLECTION_SCOPE, managedPaycomFirstPublicationRequest } = require('dispatch-protocol/paycom-activation'); +const fs = require('node:fs'); +const path = require('node:path'); +const { PROJECT_ROOT } = require('dispatch-protocol/paths/runtime-paths'); +const { serverInstallationManifest } = require('dispatch-protocol/contracts/src'); +const { validateSpec } = require('dispatch-runtime-kit/collection-manager/src/validation'); + +const PAYCOM_SPEC_RELATIVE_PATH = 'plugins/paycom/backend/config/collection-manager.json'; +const PAYCOM_EXECUTABLE_RELATIVE_PATH = 'plugins/paycom/backend/bin/dispatch-paycom-collector'; +const MANAGED_PAYCOM_CATALOG = Object.freeze({ + pluginVersion: '0.18.8', + specificationSha256: 'e80bb94afb06b90f80908341c9b11622f166fdeb05152fcf288fba719efbd73e', + executableSha256: '1a7653b184ec9b986a1449f212657fb5ff254e6246625200b9715ac3b67b70ef', + sourceTreeSha256: '0616622c895f3eb6654cf69a1894a672e9797d880208a1ed1d8fc7bd2b8e6c57', +}); +const PAYCOM_FIRST_PUBLICATION_TASKS = Object.freeze({ + 'paycom-period-roster': Object.freeze({ taskId: 'roster', method: 'roster.period', publication: 'roster' }), + 'paycom-period-timecards-from-roster': Object.freeze({ + taskId: 'timecards', method: 'timecards.from-published-roster', publication: 'timecards', + }), + 'paycom-period-timecards-audit': Object.freeze({ + taskId: 'timecards-audit', method: 'timecards.audit', publication: null, + }), + 'paycom-period-resource-links': Object.freeze({ + taskId: 'links', method: 'resource-links.period', publication: 'resourceLinks', + }), + 'paycom-period-resource-links-audit': Object.freeze({ + taskId: 'links-audit', method: 'resource-links.audit', publication: null, + }), +}); + +function fail(code = 'runtime_boundary_violation') { + throw Object.assign(new Error(code), { code }); +} + +function canonicalProjectRoot(value) { + if (typeof value !== 'string' || !path.isAbsolute(value) || path.resolve(value) !== value || /[\0\r\n]/.test(value)) fail(); + let info; + let canonical; + try { + info = fs.lstatSync(value); + canonical = fs.realpathSync(value); + } catch { fail(); } + if (!info.isDirectory() || info.isSymbolicLink() || canonical !== value) fail(); + for (let current = value; ; current = path.dirname(current)) { + const currentInfo = fs.lstatSync(current); + if (!currentInfo.isDirectory() || currentInfo.isSymbolicLink() + || ![0, process.geteuid()].includes(currentInfo.uid) || (currentInfo.mode & 0o022) !== 0) fail(); + if (path.dirname(current) === current) break; + } + return value; +} + +function fileSha256(file) { + return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex'); +} + +function ownedSourceFile(projectRoot, relative, { executable = false } = {}) { + const selected = path.resolve(projectRoot, relative); + if (!selected.startsWith(`${projectRoot}${path.sep}`)) fail(); + let info; + let canonical; + try { + info = fs.lstatSync(selected); + canonical = fs.realpathSync(selected); + } catch { fail(); } + if (!info.isFile() || info.isSymbolicLink() || ![0, process.geteuid()].includes(info.uid) + || info.nlink !== 1 || (info.mode & 0o022) !== 0 || executable && (info.mode & 0o111) === 0 + || canonical !== selected) fail(); + for (let current = path.dirname(selected); current !== projectRoot; current = path.dirname(current)) { + const directory = fs.lstatSync(current); + if (!directory.isDirectory() || directory.isSymbolicLink() + || ![0, process.geteuid()].includes(directory.uid) || (directory.mode & 0o022) !== 0) fail(); + } + return selected; +} + +function releaseSourceFiles(projectRoot) { + const relatives = [ + 'plugins/paycom/backend/package.json', + 'plugins/paycom/backend/dispatch-plugin.yaml', + PAYCOM_SPEC_RELATIVE_PATH, + PAYCOM_EXECUTABLE_RELATIVE_PATH, + 'plugins/paycom/backend/bin/dispatch-paycom-activation-evidence', + 'plugins/paycom/backend/bin/dispatch-paycom-publication-continuity', + ]; + const visit = relative => { + const directory = path.join(projectRoot, relative); + let entries; + try { entries = fs.readdirSync(directory, { withFileTypes: true }); } catch { fail(); } + for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { + const child = path.join(relative, entry.name); + if (entry.isDirectory()) visit(child); + else if (entry.isFile()) relatives.push(child); + else fail(); + } + }; + visit('plugins/paycom/backend/src'); + return relatives.sort(); +} + +function releaseSourceDigest(projectRoot) { + const hash = crypto.createHash('sha256'); + for (const relative of releaseSourceFiles(projectRoot)) { + const file = ownedSourceFile(projectRoot, relative, { + executable: relative === PAYCOM_EXECUTABLE_RELATIVE_PATH + || relative === 'plugins/paycom/backend/bin/dispatch-paycom-activation-evidence' + || relative === 'plugins/paycom/backend/bin/dispatch-paycom-publication-continuity', + }); + hash.update(relative); + hash.update('\0'); + hash.update(fs.readFileSync(file)); + hash.update('\0'); + } + return hash.digest('hex'); +} + +function readDefinition(projectRoot) { + const file = ownedSourceFile(projectRoot, PAYCOM_SPEC_RELATIVE_PATH); + if (fileSha256(file) !== MANAGED_PAYCOM_CATALOG.specificationSha256) fail(); + let value; + try { value = JSON.parse(fs.readFileSync(file, 'utf8')); } + catch { fail(); } + if (!value || typeof value !== 'object' || Array.isArray(value)) fail(); + return value; +} + +function deepFreeze(value) { + if (!value || typeof value !== 'object' || Object.isFrozen(value)) return value; + for (const child of Object.values(value)) deepFreeze(child); + return Object.freeze(value); +} + +function verifyManagedPaycomSource(projectRoot = PROJECT_ROOT) { + const root = canonicalProjectRoot(projectRoot); + if (releaseSourceDigest(root) !== MANAGED_PAYCOM_CATALOG.sourceTreeSha256) fail(); + if (fileSha256(ownedSourceFile(root, PAYCOM_SPEC_RELATIVE_PATH)) !== MANAGED_PAYCOM_CATALOG.specificationSha256 + || fileSha256(ownedSourceFile(root, PAYCOM_EXECUTABLE_RELATIVE_PATH, { executable: true })) !== MANAGED_PAYCOM_CATALOG.executableSha256) fail(); + return root; +} + +function managedPaycomDefinition(manifestValue, authorityValue, { projectRoot = PROJECT_ROOT, container = false } = {}) { + if (typeof container !== 'boolean') fail(); + const manifest = serverInstallationManifest(manifestValue, authorityValue); + const root = verifyManagedPaycomSource(projectRoot); + const definition = readDefinition(root); + const collector = definition.collectors?.find(item => item?.id === 'paycom'); + const source = definition.sources?.find(item => item?.id === PAYCOM_SOURCE_ID); + const sync = definition.syncs?.find(item => item?.id === PAYCOM_SYNC_ID); + if (definition.collectors?.length !== 1 || definition.sources?.length !== 1 + || !collector || !source || !sync || collector.version !== MANAGED_PAYCOM_CATALOG.pluginVersion + || collector.command !== '${DISPATCH_PROJECT_ROOT}/plugins/paycom/backend/bin/dispatch-paycom-collector' + || source.collector !== 'paycom' || source.authProfile !== PAYCOM_PROFILE_ID + || sync.plan !== 'paycom-current-workforce-sync') fail(); + const executable = ownedSourceFile(root, PAYCOM_EXECUTABLE_RELATIVE_PATH, { executable: true }); + if (fileSha256(executable) !== MANAGED_PAYCOM_CATALOG.executableSha256) fail(); + collector.command = container ? `/opt/dispatch/${PAYCOM_EXECUTABLE_RELATIVE_PATH}` : executable; + source.config.timezone = manifest.organization.timezone; + validateSpec(definition); + const encoded = JSON.stringify(definition); + const digest = crypto.createHash('sha256').update(encoded).digest('hex'); + return deepFreeze({ + profileId: PAYCOM_PROFILE_ID, + sourceId: PAYCOM_SOURCE_ID, + syncId: PAYCOM_SYNC_ID, + specification: definition, + digest, + }); +} + +module.exports = { + PAYCOM_PROFILE_ID, + PAYCOM_SOURCE_ID, + PAYCOM_SYNC_ID, + PAYCOM_COLLECTION_SCOPE, + PAYCOM_SPEC_RELATIVE_PATH, + PAYCOM_EXECUTABLE_RELATIVE_PATH, + MANAGED_PAYCOM_CATALOG, + PAYCOM_FIRST_PUBLICATION_TASKS, + managedPaycomDefinition, + managedPaycomFirstPublicationRequest, + verifyManagedPaycomSource, +}; diff --git a/plugins/paycom/backend/runtime/setup.js b/plugins/paycom/backend/runtime/setup.js new file mode 100644 index 0000000..df1a11d --- /dev/null +++ b/plugins/paycom/backend/runtime/setup.js @@ -0,0 +1,135 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { success, failure } = require('dispatch-protocol/contracts/src/result'); +const { setupRequest, paycomReadiness } = require('dispatch-protocol/contracts/src/paycom-setup'); +const { request: brokerRequest } = require('dispatch-runtime-kit/auth-broker/src/client'); +const { ensurePrivateDirectory } = require('../../../../runtime/auth-broker/src/vault'); +const { managedInstallationRuntimeEnvironment } = require('dispatch-protocol/paths/runtime-paths'); +const { createManagedPaycomActivationRuntime } = require('./activation'); +const { managedPaycomDefinition, managedPaycomFirstPublicationRequest } = require('./definition'); +const { LocalCollectionAdminPort } = require('../../../../runtime/adapters/local/collection-admin-port'); +const { createLocalPaycomActivationEvidencePort } = require('../adapters/activation-evidence'); +const { configuration, assertMountBoundary } = require('../../../../runtime/supervisor/src/supervisor'); + +function fail(code = 'runtime_boundary_violation') { throw Object.assign(new Error(code), { code }); } +const SAFE_ERRORS = new Set(['provider_auth_required', 'first_publication_failed', 'runtime_health_failed', + 'profile_exists', 'profile_not_configured', 'invalid_input', 'setup_interrupted', 'setup_busy', + 'mfa_required', 'captcha_required', 'account_locked', 'invalid_credentials', 'primary_credentials_rejected', + 'security_answers_rejected', 'manual_verification_required', 'attempt_cooldown', 'profile_locked']); +function errorCode(error) { return SAFE_ERRORS.has(error?.code) ? error.code : 'runtime_boundary_violation'; } + +function createContainerPaycomSetup(config, client) { + const root = require('dispatch-protocol/paths/feature-paths').featurePaths(config.paths, 'paycom').stateRoot; + // Fresh DSPs have no feature state yet. Validate/create each private level + // rather than bypassing the storage boundary with recursive directory creation. + ensurePrivateDirectory(path.dirname(root)); + ensurePrivateDirectory(root); + const running = new Map(); + function file(key) { return path.join(root, `${key}.json`); } + function read(key) { + ensurePrivateDirectory(root); + let info; + try { info = fs.lstatSync(file(key)); } catch (error) { if (error.code === 'ENOENT') return null; throw error; } + if (!info.isFile() || info.isSymbolicLink() || info.nlink !== 1 || info.uid !== process.geteuid() + || (info.mode & 0o7777) !== 0o600 || info.size > 128 * 1024) fail(); + const value = JSON.parse(fs.readFileSync(file(key), 'utf8')); + if (!value || !['running', 'succeeded', 'failed'].includes(value.status) + || Object.keys(value).sort().join(',') !== 'data,error,status') fail(); + return value; + } + function write(key, value) { + read(key); + if (fs.readdirSync(root).length > 256) fail(); + const candidate = `${file(key)}.${crypto.randomBytes(8).toString('hex')}.tmp`; + const fd = fs.openSync(candidate, fs.constants.O_WRONLY | fs.constants.O_CREAT | fs.constants.O_EXCL | fs.constants.O_NOFOLLOW, 0o600); + try { fs.writeFileSync(fd, `${JSON.stringify(value)}\n`); fs.fsyncSync(fd); } finally { fs.closeSync(fd); } + fs.renameSync(candidate, file(key)); + const parent = fs.openSync(root, fs.constants.O_RDONLY | fs.constants.O_DIRECTORY | fs.constants.O_NOFOLLOW); + try { fs.fsyncSync(parent); } finally { fs.closeSync(parent); } + } + function runtime(input) { + const environment = managedInstallationRuntimeEnvironment(config.layout); + return createManagedPaycomActivationRuntime({ + manifest: input.manifest, manifestAuthority: input.manifestAuthority, client, + projectRoot: '/opt/dispatch', + gateway: { health: async () => success('ready', { runtimeIdentity: 'matched' }) }, + collectionAdmin: new LocalCollectionAdminPort({ paths: config.paths.collection }), + evidenceVerifier: createLocalPaycomActivationEvidencePort({ environment }), + infrastructureVerifier: async manifest => { + assertMountBoundary(configuration()); + const [auth, manager] = await Promise.all([client.auth.health(), client.collections.health()]); + if (!auth.ok || auth.data?.vault?.verified !== true || !manager.ok + || manager.data?.manager?.running !== true || manager.data?.databaseIntegrity !== 'ok') fail('runtime_health_failed'); + return { runtimeKey: manifest.runtime.key, runtime_layout: true, service_supervision: true, + auth_broker: true, collection_manager: true, runtime_gateway: true }; + }, + }); + } + async function execute(input) { + if (input.command === 'enroll') { + if (input.expiresAt < Date.now() || input.expiresAt > Date.now() + 60_000) fail('invalid_input'); + let result = await brokerRequest(config.paths.auth.socket, { + action: 'enroll-paycom', credentials: input.credentials, intent: input.intent, + }); + // A failed first delivery may leave nothing to replace. Only the broker's + // explicit missing-profile response permits saving these credentials anew. + if (!result.ok && result.status === 'profile_not_configured' && input.intent === 'replace') { + result = await brokerRequest(config.paths.auth.socket, { + action: 'enroll-paycom', credentials: input.credentials, intent: 'create', + }); + } + if (!result.ok) fail(result.status); + return { configured: true }; + } + const selected = runtime(input); + if (input.step === 'test') return selected.testProvider('paycom-main'); + await selected.verifyInfrastructure(input.manifest); + if (input.step === 'sync') return selected.startWorkforceSync(); + if (input.step === 'configure') return selected.configure(managedPaycomDefinition(input.manifest, input.manifestAuthority, { projectRoot: '/opt/dispatch' })); + if (input.step === 'publish') return selected.publishFirst(managedPaycomFirstPublicationRequest(), { + idempotencyKey: `activation:${input.parameters.jobId}`, heartbeat: async () => {}, + }); + if (input.step === 'verify') return selected.verifyPublication(input.parameters.batchId, input.parameters.preparationRunId); + fail('invalid_input'); + } + const handle = async value => { + try { + const input = setupRequest(value, config.runtimeKey); + if (input.step === 'readiness') { + // Always read the broker's current guard, never a cached setup receipt. + const response = await brokerRequest(config.paths.auth.socket, { action: 'profile-readiness', profile: 'paycom-main' }); + if (!response.ok || response.status !== 'found') fail('runtime_health_failed'); + return success('succeeded', paycomReadiness(response.readiness)); + } + if (input.step === 'infrastructure') return success('succeeded', await runtime(input).verifyInfrastructure(input.manifest)); + const enrollment = input.command === 'enroll'; + const identity = enrollment ? { requestId: input.requestId, intent: input.intent } + : { requestId: input.requestId, step: input.step, manifest: input.manifest, parameters: input.parameters }; + const key = crypto.createHash('sha256').update(JSON.stringify(identity)).digest('hex'); + let saved = read(key); + if (saved?.status === 'running' && !running.has(key)) { + saved = { status: 'failed', data: null, error: 'setup_interrupted' }; write(key, saved); + } + if (!saved && input.command === 'status') return success('not_started', null); + if (!saved) { + if (running.size) return failure('setup_busy'); + saved = { status: 'running', data: null, error: null }; write(key, saved); + const work = Promise.resolve().then(() => execute(input)).then( + data => write(key, { status: 'succeeded', data, error: null }), + error => write(key, { status: 'failed', data: null, error: errorCode(error) }), + ).finally(() => running.delete(key)); + running.set(key, work); + // Enrollment is short and never survives in the control-plane database. + if (enrollment) { await work; saved = read(key); } + else work.catch(() => {}); + } + return saved.status === 'failed' ? failure(saved.error) : success(saved.status, saved.data); + } catch (error) { return failure(errorCode(error)); } + }; + handle.busy = () => running.size > 0; + return handle; +} +module.exports = { createContainerPaycomSetup }; diff --git a/plugins/paycom/backend/scripts/build b/plugins/paycom/backend/scripts/build new file mode 100755 index 0000000..dfb6b41 --- /dev/null +++ b/plugins/paycom/backend/scripts/build @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +set -euo pipefail +shopt -s nullglob +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +files=("$ROOT"/src/*.js "$ROOT"/tests/*.js "$ROOT"/bin/dispatch-paycom-collector "$ROOT"/bin/dispatch-paycom-activation-evidence "$ROOT"/bin/dispatch-paycom-publication-continuity) +for file in "${files[@]}"; do node --check "$file" >/dev/null; done +printf '%s\n' '{"ok":true,"status":"built"}' diff --git a/plugins/paycom/backend/scripts/health b/plugins/paycom/backend/scripts/health new file mode 100755 index 0000000..ef1039e --- /dev/null +++ b/plugins/paycom/backend/scripts/health @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +cd "$ROOT" +node --no-warnings - <<'NODE' +process.umask(0o077); +const fs = require('node:fs'); +const { PaycomStore } = require('./src/store'); +const { DATABASE } = require('./src/paths'); +if (!fs.existsSync(DATABASE)) { + console.log(JSON.stringify({ ok: true, status: 'not_initialized', database: 'missing' })); +} else { +const store = new PaycomStore(DATABASE, { readOnly: true }); +try { + console.log(JSON.stringify({ + ok: true, + status: 'ready', + database: fs.existsSync(DATABASE) ? 'ready' : 'missing', + payPeriods: store.audit('pay_periods'), + roster: store.audit('roster'), + timecards: store.audit('timecards'), + resourceLinks: store.auditResourceLinks(), + })); +} finally { store.close(); } +} +NODE diff --git a/plugins/paycom/backend/scripts/test b/plugins/paycom/backend/scripts/test new file mode 100755 index 0000000..4caa848 --- /dev/null +++ b/plugins/paycom/backend/scripts/test @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +node --no-warnings --test "$ROOT"/tests/*.test.js diff --git a/plugins/paycom/backend/scripts/verify b/plugins/paycom/backend/scripts/verify new file mode 100755 index 0000000..ae89b20 --- /dev/null +++ b/plugins/paycom/backend/scripts/verify @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +cd "$ROOT" +"$ROOT/tooling/build" >/dev/null +node --no-warnings - <<'NODE' +process.umask(0o077); +const { PaycomStore } = require('./src/store'); +const { DATABASE } = require('./src/paths'); +const fs = require('node:fs'); +if (!fs.existsSync(DATABASE)) { + console.log(JSON.stringify({ok:false,status:'not_initialized'})); + process.exit(1); +} +const store = new PaycomStore(DATABASE, { readOnly: true }); +try { + const quick = store.db.prepare('PRAGMA quick_check').get()?.quick_check; + const foreignKeyErrors = store.db.prepare('PRAGMA foreign_key_check').all().length; + const active = store.db.prepare('SELECT kind,target FROM active_publications ORDER BY kind,target').all(); + const failed = active.map(row => store.audit(row.kind, row.target)).filter(result => !result.verified); + const activeLinks = store.db.prepare('SELECT resource_type resourceType,target FROM active_resource_link_publications ORDER BY resource_type,target').all(); + const failedLinks = activeLinks.map(row => store.auditResourceLinks(row.resourceType, row.target)).filter(result => !result.verified); + const schemaVersion = store.db.prepare('SELECT version FROM schema_meta').get()?.version; + if (schemaVersion !== 5 || quick !== 'ok' || foreignKeyErrors !== 0 || failed.length !== 0 || failedLinks.length !== 0) process.exit(1); + console.log(JSON.stringify({ok:true,status:'verified',schemaVersion,quickCheck:quick,foreignKeyErrors,activePublications:active.length,activeResourceLinkPublications:activeLinks.length})); +} finally { store.close(); } +NODE diff --git a/plugins/paycom/backend/src/activation-evidence-core.js b/plugins/paycom/backend/src/activation-evidence-core.js new file mode 100644 index 0000000..2a1bb7f --- /dev/null +++ b/plugins/paycom/backend/src/activation-evidence-core.js @@ -0,0 +1,169 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { PaycomStore } = require('./store'); +const { TIMECARD_SUMMARY } = require('./resource-links'); + +const HASH_RE = /^[a-f0-9]{64}$/; +const SOURCE_ID = 'paycom-main'; +const EXPECTED_REQUEST = Object.freeze({ + source: SOURCE_ID, + scope: 'full', + selector: Object.freeze({ kind: 'current' }), + mode: 'refresh', +}); +const EXPECTED_TASKS = Object.freeze({ + 'paycom-period-roster': Object.freeze({ taskId: 'roster', method: 'roster.period', publication: 'roster' }), + 'paycom-period-timecards-from-roster': Object.freeze({ + taskId: 'timecards', method: 'timecards.from-published-roster', publication: 'timecards', + }), + 'paycom-period-timecards-audit': Object.freeze({ taskId: 'timecards-audit', method: 'timecards.audit' }), + 'paycom-period-resource-links': Object.freeze({ + taskId: 'links', method: 'resource-links.period', publication: 'resourceLinks', + }), + 'paycom-period-resource-links-audit': Object.freeze({ taskId: 'links-audit', method: 'resource-links.audit' }), +}); + +function fail(code = 'first_publication_failed') { + throw Object.assign(new Error(code), { code }); +} +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} +function same(left, right) { return JSON.stringify(left) === JSON.stringify(right); } +function digest(value) { return crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex'); } +function publicationRecord(publication, runId) { + if (!publication || typeof publication.id !== 'string' || typeof publication.run_id !== 'string' + || typeof publication.content_sha256 !== 'string' || !HASH_RE.test(publication.content_sha256)) fail(); + return Object.freeze({ + id: publication.id, + runId, + originRunId: publication.run_id, + contentSha256: publication.content_sha256, + batchBound: true, + }); +} + +function verifyActivationEvidence(options) { + if (!plain(options) || Object.keys(options).sort().join(',') + !== 'batchId,clock,definitionDigest,manager,paycomDatabase,preparationRunId' + || typeof options.manager?.batch !== 'function' || typeof options.manager?.run !== 'function' || typeof options.paycomDatabase !== 'string' + || typeof options.clock !== 'function' || typeof options.batchId !== 'string' + || typeof options.preparationRunId !== 'string' + || typeof options.definitionDigest !== 'string' || !HASH_RE.test(options.definitionDigest)) { + fail('runtime_boundary_violation'); + } + const { batchId, definitionDigest, preparationRunId } = options; + let manager; + let paycom; + try { + manager = options.manager; + paycom = new PaycomStore(options.paycomDatabase, { readOnly: true }); + const batch = manager.batch(batchId); + if (batch.status !== 'succeeded' || batch.source !== SOURCE_ID || batch.scope !== 'full' + || !same(batch.request, EXPECTED_REQUEST) || typeof batch.previewHash !== 'string' + || !HASH_RE.test(batch.previewHash) || batch.runs.length !== Object.keys(EXPECTED_TASKS).length) fail(); + const selectedRuns = []; + let target = null; + const byPublication = {}; + const auditRuns = {}; + for (const item of batch.runs) { + const expected = EXPECTED_TASKS[item.run.plan]; + const run = manager.run(item.run.id); + const runTarget = expected?.publication ? run.receipt?.data?.target + : run.receipt?.data?.audit?.periodEnd || run.receipt?.data?.audit?.target; + if (!expected || item.taskId !== expected.taskId || item.targetKey !== runTarget + || run.status !== 'succeeded' || run.source !== SOURCE_ID || run.plan !== item.run.plan + || run.method !== expected.method || run.receipt?.ok !== true + || run.receipt.data?.method !== expected.method) fail(); + if (target === null) target = item.targetKey; + if (item.targetKey !== target) fail(); + selectedRuns.push(Object.freeze({ + id: run.id, + taskId: item.taskId, + plan: run.plan, + method: run.method, + })); + if (expected.publication) { + const receipt = run.receipt.data; + if (typeof receipt.publicationId !== 'string' || typeof receipt.contentSha256 !== 'string' + || !HASH_RE.test(receipt.contentSha256) + || !['published', 'no_change', 'reactivated'].includes(receipt.disposition)) fail(); + byPublication[expected.publication] = { run, receipt }; + } else { + const audit = run.receipt.data.audit; + if (!plain(audit) || audit.verified !== true) fail(); + auditRuns[run.method] = audit; + } + } + if (typeof target !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(target) + || Object.keys(byPublication).sort().join(',') !== 'resourceLinks,roster,timecards' + || Object.keys(auditRuns).sort().join(',') !== 'resource-links.audit,timecards.audit') fail(); + + const roster = paycom.active('roster', target); + const timecards = paycom.active('timecards', target); + const resourceLinks = paycom.activeResourceLinks(TIMECARD_SUMMARY, target)?.publication; + const rosterAudit = roster && paycom.auditPublication(roster.id); + const timecardAudit = paycom.auditTimecards(target); + const resourceLinkAudit = paycom.auditResourceLinks(TIMECARD_SUMMARY, target); + const payPeriodAudit = paycom.auditPayPeriodTarget(target); + const preparationRun = manager.run(preparationRunId); + if (!rosterAudit?.verified || !timecardAudit?.verified || !resourceLinkAudit?.verified + || !payPeriodAudit?.verified + || preparationRun.status !== 'succeeded' || preparationRun.plan !== 'paycom-periods' + || preparationRun.source !== SOURCE_ID || preparationRun.method !== 'pay-periods.discover' + || preparationRun.receipt?.ok !== true + || preparationRun.receipt.data?.method !== 'pay-periods.discover' + || !['published', 'no_change', 'reactivated'].includes(preparationRun.receipt.data?.disposition) + || preparationRun.receipt.data?.publicationId !== payPeriodAudit.publicationId + || preparationRun.receipt.data?.contentSha256 !== payPeriodAudit.contentSha256 + || byPublication.roster.receipt.publicationId !== roster.id + || byPublication.roster.receipt.contentSha256 !== roster.content_sha256 + || byPublication.timecards.receipt.publicationId !== timecards.id + || byPublication.timecards.receipt.contentSha256 !== timecards.content_sha256 + || byPublication.resourceLinks.receipt.publicationId !== resourceLinks.id + || byPublication.resourceLinks.receipt.contentSha256 !== resourceLinks.content_sha256 + || auditRuns['timecards.audit'].rosterPublicationId !== roster.id + || auditRuns['timecards.audit'].timecardPublicationId !== timecards.id + || auditRuns['resource-links.audit'].publicationId !== resourceLinks.id + || auditRuns['resource-links.audit'].rosterPublicationId !== roster.id) fail(); + + const captured = options.clock(); + if (!Number.isSafeInteger(captured) || captured < 0) fail('runtime_boundary_violation'); + return Object.freeze({ + definitionDigest, + requestDigest: digest(EXPECTED_REQUEST), + previewDigest: batch.previewHash, + batchId: batch.id, + preparationRunId, + target, + runs: Object.freeze(selectedRuns.sort((left, right) => left.plan.localeCompare(right.plan))), + publications: Object.freeze({ + payPeriods: Object.freeze({ + id: payPeriodAudit.publicationId, + runId: preparationRunId, + originRunId: payPeriodAudit.runId, + contentSha256: payPeriodAudit.contentSha256, + batchBound: false, + }), + roster: publicationRecord(roster, byPublication.roster.run.id), + timecards: publicationRecord(timecards, byPublication.timecards.run.id), + resourceLinks: publicationRecord(resourceLinks, byPublication.resourceLinks.run.id), + }), + capturedAt: new Date(captured).toISOString(), + }); + } catch (error) { + if (error?.code === 'runtime_boundary_violation') throw error; + fail(); + } finally { + try { paycom?.close(); } catch {} + try { manager?.close(); } catch {} + } +} + +module.exports = { + EXPECTED_REQUEST, + EXPECTED_TASKS, + verifyActivationEvidence, +}; diff --git a/plugins/paycom/backend/src/activation-evidence.js b/plugins/paycom/backend/src/activation-evidence.js new file mode 100644 index 0000000..456393a --- /dev/null +++ b/plugins/paycom/backend/src/activation-evidence.js @@ -0,0 +1,170 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { CollectionStore } = require('dispatch-runtime-kit/collection-manager/src/store'); +const { PaycomStore } = require('./store'); +const { TIMECARD_SUMMARY } = require('./resource-links'); + +const HASH_RE = /^[a-f0-9]{64}$/; +const SOURCE_ID = 'paycom-main'; +const EXPECTED_REQUEST = Object.freeze({ + source: SOURCE_ID, + scope: 'full', + selector: Object.freeze({ kind: 'current' }), + mode: 'refresh', +}); +const EXPECTED_TASKS = Object.freeze({ + 'paycom-period-roster': Object.freeze({ taskId: 'roster', method: 'roster.period', publication: 'roster' }), + 'paycom-period-timecards-from-roster': Object.freeze({ + taskId: 'timecards', method: 'timecards.from-published-roster', publication: 'timecards', + }), + 'paycom-period-timecards-audit': Object.freeze({ taskId: 'timecards-audit', method: 'timecards.audit' }), + 'paycom-period-resource-links': Object.freeze({ + taskId: 'links', method: 'resource-links.period', publication: 'resourceLinks', + }), + 'paycom-period-resource-links-audit': Object.freeze({ taskId: 'links-audit', method: 'resource-links.audit' }), +}); + +function fail(code = 'first_publication_failed') { + throw Object.assign(new Error(code), { code }); +} +function plain(value) { + return Boolean(value) && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} +function same(left, right) { return JSON.stringify(left) === JSON.stringify(right); } +function digest(value) { return crypto.createHash('sha256').update(JSON.stringify(value)).digest('hex'); } +function publicationRecord(publication, runId) { + if (!publication || typeof publication.id !== 'string' || typeof publication.run_id !== 'string' + || typeof publication.content_sha256 !== 'string' || !HASH_RE.test(publication.content_sha256)) fail(); + return Object.freeze({ + id: publication.id, + runId, + originRunId: publication.run_id, + contentSha256: publication.content_sha256, + batchBound: true, + }); +} + +function verifyActivationEvidence(options) { + if (!plain(options) || Object.keys(options).sort().join(',') + !== 'batchId,clock,collectionPaths,definitionDigest,paycomDatabase,preparationRunId' + || !plain(options.collectionPaths) || typeof options.paycomDatabase !== 'string' + || typeof options.clock !== 'function' || typeof options.batchId !== 'string' + || typeof options.preparationRunId !== 'string' + || typeof options.definitionDigest !== 'string' || !HASH_RE.test(options.definitionDigest)) { + fail('runtime_boundary_violation'); + } + const { batchId, definitionDigest, preparationRunId } = options; + let manager; + let paycom; + try { + manager = new CollectionStore(options.collectionPaths, { readOnly: true }); + paycom = new PaycomStore(options.paycomDatabase, { readOnly: true }); + const batch = manager.batch(batchId); + if (batch.status !== 'succeeded' || batch.source !== SOURCE_ID || batch.scope !== 'full' + || !same(batch.request, EXPECTED_REQUEST) || typeof batch.previewHash !== 'string' + || !HASH_RE.test(batch.previewHash) || batch.runs.length !== Object.keys(EXPECTED_TASKS).length) fail(); + const selectedRuns = []; + let target = null; + const byPublication = {}; + const auditRuns = {}; + for (const item of batch.runs) { + const expected = EXPECTED_TASKS[item.run.plan]; + const run = manager.run(item.run.id); + const runTarget = expected?.publication ? run.receipt?.data?.target + : run.receipt?.data?.audit?.periodEnd || run.receipt?.data?.audit?.target; + if (!expected || item.taskId !== expected.taskId || item.targetKey !== runTarget + || run.status !== 'succeeded' || run.source !== SOURCE_ID || run.plan !== item.run.plan + || run.method !== expected.method || run.receipt?.ok !== true + || run.receipt.data?.method !== expected.method) fail(); + if (target === null) target = item.targetKey; + if (item.targetKey !== target) fail(); + selectedRuns.push(Object.freeze({ + id: run.id, + taskId: item.taskId, + plan: run.plan, + method: run.method, + })); + if (expected.publication) { + const receipt = run.receipt.data; + if (typeof receipt.publicationId !== 'string' || typeof receipt.contentSha256 !== 'string' + || !HASH_RE.test(receipt.contentSha256) + || !['published', 'no_change', 'reactivated'].includes(receipt.disposition)) fail(); + byPublication[expected.publication] = { run, receipt }; + } else { + const audit = run.receipt.data.audit; + if (!plain(audit) || audit.verified !== true) fail(); + auditRuns[run.method] = audit; + } + } + if (typeof target !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(target) + || Object.keys(byPublication).sort().join(',') !== 'resourceLinks,roster,timecards' + || Object.keys(auditRuns).sort().join(',') !== 'resource-links.audit,timecards.audit') fail(); + + const roster = paycom.active('roster', target); + const timecards = paycom.active('timecards', target); + const resourceLinks = paycom.activeResourceLinks(TIMECARD_SUMMARY, target)?.publication; + const rosterAudit = roster && paycom.auditPublication(roster.id); + const timecardAudit = paycom.auditTimecards(target); + const resourceLinkAudit = paycom.auditResourceLinks(TIMECARD_SUMMARY, target); + const payPeriodAudit = paycom.auditPayPeriodTarget(target); + const preparationRun = manager.run(preparationRunId); + if (!rosterAudit?.verified || !timecardAudit?.verified || !resourceLinkAudit?.verified + || !payPeriodAudit?.verified + || preparationRun.status !== 'succeeded' || preparationRun.plan !== 'paycom-periods' + || preparationRun.source !== SOURCE_ID || preparationRun.method !== 'pay-periods.discover' + || preparationRun.receipt?.ok !== true + || preparationRun.receipt.data?.method !== 'pay-periods.discover' + || !['published', 'no_change', 'reactivated'].includes(preparationRun.receipt.data?.disposition) + || preparationRun.receipt.data?.publicationId !== payPeriodAudit.publicationId + || preparationRun.receipt.data?.contentSha256 !== payPeriodAudit.contentSha256 + || byPublication.roster.receipt.publicationId !== roster.id + || byPublication.roster.receipt.contentSha256 !== roster.content_sha256 + || byPublication.timecards.receipt.publicationId !== timecards.id + || byPublication.timecards.receipt.contentSha256 !== timecards.content_sha256 + || byPublication.resourceLinks.receipt.publicationId !== resourceLinks.id + || byPublication.resourceLinks.receipt.contentSha256 !== resourceLinks.content_sha256 + || auditRuns['timecards.audit'].rosterPublicationId !== roster.id + || auditRuns['timecards.audit'].timecardPublicationId !== timecards.id + || auditRuns['resource-links.audit'].publicationId !== resourceLinks.id + || auditRuns['resource-links.audit'].rosterPublicationId !== roster.id) fail(); + + const captured = options.clock(); + if (!Number.isSafeInteger(captured) || captured < 0) fail('runtime_boundary_violation'); + return Object.freeze({ + definitionDigest, + requestDigest: digest(EXPECTED_REQUEST), + previewDigest: batch.previewHash, + batchId: batch.id, + preparationRunId, + target, + runs: Object.freeze(selectedRuns.sort((left, right) => left.plan.localeCompare(right.plan))), + publications: Object.freeze({ + payPeriods: Object.freeze({ + id: payPeriodAudit.publicationId, + runId: preparationRunId, + originRunId: payPeriodAudit.runId, + contentSha256: payPeriodAudit.contentSha256, + batchBound: false, + }), + roster: publicationRecord(roster, byPublication.roster.run.id), + timecards: publicationRecord(timecards, byPublication.timecards.run.id), + resourceLinks: publicationRecord(resourceLinks, byPublication.resourceLinks.run.id), + }), + capturedAt: new Date(captured).toISOString(), + }); + } catch (error) { + if (error?.code === 'runtime_boundary_violation') throw error; + fail(); + } finally { + try { paycom?.close(); } catch {} + try { manager?.close(); } catch {} + } +} + +module.exports = { + EXPECTED_REQUEST, + EXPECTED_TASKS, + verifyActivationEvidence, +}; diff --git a/plugins/paycom/backend/src/authenticated-browser.js b/plugins/paycom/backend/src/authenticated-browser.js new file mode 100644 index 0000000..f2ba48a --- /dev/null +++ b/plugins/paycom/backend/src/authenticated-browser.js @@ -0,0 +1,70 @@ +'use strict'; + +const { acquireServiceBrowser } = require('../../../../runtime/auth-broker/src/service-client'); +const { acquireAuthenticatedBrowser } = require('../../../../runtime/auth-broker/src/browser-client'); + +const DEFAULT_TTL_SECONDS = 90; + +async function acquirePaycomBrowser({ authProfile, runId, ttlSeconds = DEFAULT_TTL_SECONDS, socketPath } = {}) { + if (!authProfile || authProfile === 'paycom-main') return acquireServiceBrowser({ + service: 'paycom', feature: 'paycom', runId, ttlSeconds, socketPath, + }); + return acquireAuthenticatedBrowser({ + profile: authProfile, + collector: 'paycom', + runId, + ttlSeconds, + ...(socketPath ? { socketPath } : {}), + }); +} + +async function withPaycomBrowser(options, useBrowser) { + if (typeof useBrowser !== 'function') throw new TypeError('useBrowser must be a function'); + const ttlSeconds = options?.ttlSeconds ?? DEFAULT_TTL_SECONDS; + const lease = await acquirePaycomBrowser({ ...options, ttlSeconds }); + let useError = null; + let renewError = null; + let renewPromise = null; + const heartbeat = setInterval(async () => { + if (renewPromise || renewError) return; + renewPromise = lease.renew(ttlSeconds) + .catch(error => { renewError = error; }) + .finally(() => { renewPromise = null; }); + await renewPromise; + }, Math.max(10_000, Math.floor(ttlSeconds * 1000 / 3))); + heartbeat.unref?.(); + let terminating = false; + const terminate = () => { + if (terminating) return; + terminating = true; + clearInterval(heartbeat); + Promise.resolve(renewPromise).catch(() => {}).then(() => lease.release()).catch(() => {}).finally(() => process.exit(143)); + }; + process.once('SIGTERM', terminate); + process.once('SIGINT', terminate); + try { + const result = await useBrowser({ + protocol: lease.protocol, + endpoint: lease.endpoint, + access: lease.access, + expiresAt: lease.expiresAt, + renew: seconds => lease.renew(seconds), + status: () => lease.status(), + }); + if (renewError) throw renewError; + return result; + } catch (error) { + useError = error; + throw error; + } finally { + clearInterval(heartbeat); + process.removeListener('SIGTERM', terminate); + process.removeListener('SIGINT', terminate); + if (renewPromise) await renewPromise.catch(() => {}); + try { await lease.release(); } + catch (releaseError) { if (!useError) throw releaseError; } + if (!useError && renewError) throw renewError; + } +} + +module.exports = { acquirePaycomBrowser, withPaycomBrowser, DEFAULT_TTL_SECONDS }; diff --git a/plugins/paycom/backend/src/browser.js b/plugins/paycom/backend/src/browser.js new file mode 100644 index 0000000..ade0df1 --- /dev/null +++ b/plugins/paycom/backend/src/browser.js @@ -0,0 +1,534 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { + CdpConnection, CdpError, createTarget, boundedJson, +} = require('dispatch-sdk/node/cdp'); +const { + buildTimecardUrl, canonicalTimecardUrl, isCapturedTimecardUrl, parsePeriodKey, +} = require('./timecard-period'); +const { buildExtractionExpression, validateTimecardRecord } = require('./timecard-dom'); +const { canonicalBusinessTimecard, timecardBusinessSha256 } = require('./fingerprints'); + +const TIMECARD_SEARCH_URL = 'https://www.paycomonline.net/v4/cl/web.php/timecardsearch/index?from=main_menu'; +const ROSTER_API = 'https://time-and-attendance.paycomonline.net/api/cl/timecard-search/employees'; +const MAX_SOURCE_BYTES = 2_097_152; +const TIMECARD_BLOCKED_URLS = Object.freeze([ + '*.png', '*.jpg', '*.jpeg', '*.gif', '*.webp', '*.svg', '*.ico', + '*.woff', '*.woff2', '*.ttf', '*.otf', '*.mp3', '*.mp4', '*.webm', +]); +const TIMECARD_PHASES = Object.freeze(['navigate', 'response', 'load', 'body', 'ready', 'extract', 'validate']); +const DEFAULT_ITEM_TIMEOUT_MS = 120_000; +const ROSTER_REQUEST_FIELDS = Object.freeze([ + 'allocationCategories', 'approvalMode', 'eeCodes', 'endDate', 'getCount', 'highlighting', + 'isAdvancedFilterApplied', 'loadTotals', 'minWageUrl', 'onlyBorrowedEmployees', 'payClassCodes', + 'q', 'selectedColumns', 'selectedEarnings', 'skip', 'sortParams', 'startDate', 'take', +]); + +function fail(code) { const error = new Error(code); error.code = code; throw error; } +function sha256(bytes) { return crypto.createHash('sha256').update(bytes).digest('hex'); } +function delay(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } + +function rosterApiUrl(value) { + try { + const url = new URL(value); + return url.protocol === 'https:' && url.hostname === 'time-and-attendance.paycomonline.net' + && !url.port && !url.username && !url.password && !url.hash + && url.pathname === '/api/cl/timecard-search/employees' ? url : null; + } catch { return null; } +} + +function rosterRequest(value, period) { + try { + const request = value?.request; + const url = rosterApiUrl(request.url); + if (request.method !== 'POST' || !url || url.search !== '' || typeof value.requestId !== 'string') return null; + const body = JSON.parse(request.postData); + if (body.startDate !== period.start || body.endDate !== period.end || !Array.isArray(body.eeCodes) + || body.eeCodes.length < 1 || body.eeCodes.length > 5000 || new Set(body.eeCodes).size !== body.eeCodes.length + || body.eeCodes.some(code => typeof code !== 'string' || !/^[A-Za-z0-9]{4}$/.test(code))) return null; + return { requestId: value.requestId, codes: body.eeCodes.map(code => code.toUpperCase()).sort() }; + } catch { return null; } +} + +function emptyFilter(value) { + return value === null || value === '' || (Array.isArray(value) && value.length === 0); +} + +function rosterAuthorityAssessment(body) { + if (!body || Object.getPrototypeOf(body) !== Object.prototype) { + return { observable: false, authoritative: false, code: 'roster_source_not_authoritative' }; + } + if (![false, true].includes(body.isAdvancedFilterApplied)) { + return { + observable: false, + authoritative: false, + code: body.isAdvancedFilterApplied === null + ? 'roster_filter_advanced_null' : 'roster_filter_advanced_invalid', + }; + } + const hardChecks = [ + [body.q === null || body.q === '', 'roster_filter_search'], + [body.onlyBorrowedEmployees === false, 'roster_filter_borrowed'], + [body.skip === null || body.skip === 0, 'roster_filter_page_offset'], + [body.take === null || (Number.isInteger(body.take) && body.take >= body.eeCodes.length), 'roster_filter_page_size'], + [body.getCount === null || body.getCount === true, 'roster_filter_count'], + ]; + const failed = hardChecks.find(([accepted]) => !accepted); + if (failed) return { observable: false, authoritative: false, code: failed[1] }; + const softFiltersPresent = body.isAdvancedFilterApplied + || !emptyFilter(body.payClassCodes) || !emptyFilter(body.selectedEarnings) || !emptyFilter(body.approvalMode); + if (softFiltersPresent) { + return { observable: true, authoritative: false, code: 'roster_filters_present' }; + } + return { observable: true, authoritative: true, code: null }; +} + +function authoritativeRosterBody(body) { + return rosterAuthorityAssessment(body).authoritative; +} + +function fetchRosterRequest(value, period, { unfiltered = false } = {}) { + try { + const request = value?.request; + const url = rosterApiUrl(request?.url); + if (!url || url.search !== '' || request.method !== 'POST' || typeof value.requestId !== 'string' + || Number.isInteger(value.responseStatusCode)) return null; + const body = JSON.parse(request.postData); + if (!body || Object.getPrototypeOf(body) !== Object.prototype + || Object.keys(body).sort().join(',') !== [...ROSTER_REQUEST_FIELDS].sort().join(',') + || typeof body.startDate !== 'string' || typeof body.endDate !== 'string' + || !Array.isArray(body.eeCodes) || body.eeCodes.length < 1 || body.eeCodes.length > 5000 + || new Set(body.eeCodes).size !== body.eeCodes.length + || body.eeCodes.some(code => typeof code !== 'string' || !/^[A-Za-z0-9]{4}$/.test(code))) return null; + if (typeof unfiltered !== 'boolean') return null; + // The saved Timecard Search view may apply pay-class/approval filters. + // Collection requests the entire supplied employee list without changing + // those saved UI preferences. Search/pagination/borrowed-only views still fail. + const originalAuthority = rosterAuthorityAssessment(body); + const selectedBody = unfiltered && originalAuthority.observable ? { + ...body, isAdvancedFilterApplied: false, selectedEarnings: [], approvalMode: null, + } : body; + const authority = rosterAuthorityAssessment(selectedBody); + const uiPeriod = parsePeriodKey(`${body.startDate}_${body.endDate}`); + return { + requestId: value.requestId, + codes: body.eeCodes.map(code => code.toUpperCase()).sort(), + observable: authority.observable, + // Paycom requires the supplied pay-class list even when all requested + // employees are wanted. Complete mode additionally requires an exact + // response for every supplied employee code before it can be published. + authoritative: authority.authoritative || unfiltered && originalAuthority.observable, + authorityCode: authority.code, + uiPeriod: { start: uiPeriod.start, end: uiPeriod.end }, + postData: Buffer.from(JSON.stringify({ ...selectedBody, startDate: period.start, endDate: period.end })).toString('base64'), + }; + } catch { return null; } +} + +function fetchRosterResponse(value) { + try { + const url = rosterApiUrl(value?.request?.url); + const contentTypes = (value.responseHeaders || []).filter(header => String(header.name).toLowerCase() === 'content-type'); + return Boolean(url && url.search === '' && value.request.method === 'POST' && Number.isInteger(value.responseStatusCode) + && value.responseStatusCode === 200 && contentTypes.length === 1 + && /^application\/json(?:;|$)/i.test(String(contentTypes[0].value))); + } catch { return false; } +} + +function rosterResponse(value, requestId) { + try { + const url = rosterApiUrl(value?.response?.url); + return value.requestId === requestId && value.response.status === 200 + && String(value.response.mimeType || '').toLowerCase() === 'application/json' + && url && url.search === ''; + } catch { return false; } +} + +function rosterMembershipAssessment(bytes, codes) { + try { + const raw = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + if (!Array.isArray(raw.eeCodes) || raw.eeCodes.length < 1 || raw.eeCodes.length > 5000) return null; + const returned = raw.eeCodes.map(code => String(code).toUpperCase()).sort(); + if (new Set(returned).size !== returned.length || returned.some(code => !/^[A-Z0-9]{4}$/.test(code))) return null; + const requested = new Set(codes); + if (returned.some(code => !requested.has(code))) return null; + return { + exact: returned.length === codes.length && returned.every((code, index) => code === codes[index]), + returnedCount: returned.length, + }; + } catch { return null; } +} + +function rosterMembership(bytes, codes) { + return rosterMembershipAssessment(bytes, codes)?.exact === true; +} + +function rosterReadExpression({ headers, body }) { + // The destination is fixed; redirects cannot forward captured session headers. + return `(${async function read(input) { + try { + const response = await fetch(input.url, { method: 'POST', credentials: 'include', + redirect: 'error', cache: 'no-store', headers: input.headers, body: input.body, + signal: AbortSignal.timeout(55_000) }); + if (response.status !== 200 || !/^application\/json(?:;|$)/i.test(response.headers.get('content-type') || '') + || !response.body) return { status: response.status, error: 'invalid_response' }; + const reader = response.body.getReader(); + const decoder = new TextDecoder('utf-8', { fatal: true }); + let bytes = 0, text = ''; + for (;;) { + const part = await reader.read(); + if (part.done) break; + bytes += part.value.byteLength; + if (bytes > input.maximum) { await reader.cancel(); return { status: 0, error: 'too_large' }; } + text += decoder.decode(part.value, { stream: true }); + } + text += decoder.decode(); + return { status: response.status, text }; + } catch (error) { + return { status: 0, error: ['TimeoutError', 'AbortError'].includes(error.name) ? 'timeout' : 'request_failed' }; + } + }})(${JSON.stringify({ url: ROSTER_API, headers, body, maximum: MAX_SOURCE_BYTES })})`; +} + +async function collectRoster(endpoint, periodValue, options = {}) { + const period = parsePeriodKey(periodValue.key); + // The handoff page can still issue startup requests. Intercepting it before + // navigating can select a request from the discarded document. A blank page + // shares this lease's authentication while keeping capture scoped to its own navigation. + const target = await createTarget(endpoint, 'about:blank'); + let connection = null; + let requestPause = null; + let selectedRequestId = null; + const continueOtherRequests = event => { + let value; + try { value = JSON.parse(event.data); } catch { return; } + const paused = value.params; + if (value.method !== 'Fetch.requestPaused' || !rosterApiUrl(paused?.request?.url) + || Number.isInteger(paused.responseStatusCode)) return; + if (paused.request.method === 'POST' && selectedRequestId === null) { + selectedRequestId = paused.requestId; + return; + } + // Other startup/preflight requests can be dependencies of the selected call. + if (paused.requestId !== selectedRequestId) + connection.command('Fetch.continueRequest', { requestId: paused.requestId }).catch(() => {}); + }; + try { + connection = await CdpConnection.connect(target.webSocketDebuggerUrl, { commandTimeoutMs: 60_000 }); + connection.socket.addEventListener('message', continueOtherRequests); + await connection.command('Page.enable'); + await connection.command('Fetch.enable', { patterns: [{ urlPattern: ROSTER_API, requestStage: 'Request' }] }); + const requestPromise = connection.waitFor('Fetch.requestPaused', value => value?.request?.method === 'POST' + && rosterApiUrl(value.request.url) !== null && !Number.isInteger(value.responseStatusCode), 60_000); + requestPromise.catch(() => {}); + const navigation = await connection.command('Page.navigate', { url: TIMECARD_SEARCH_URL }); + if (navigation.errorText) fail('navigation_failed'); + try { requestPause = await requestPromise; } + catch (error) { if (error instanceof CdpError && error.code === 'browser_timeout') fail('roster_request_timeout'); throw error; } + const request = fetchRosterRequest(requestPause, period, options); + if (!request) { + if (rosterApiUrl(requestPause?.request?.url)?.search) fail('roster_request_url_mismatch'); + fail('roster_period_mismatch'); + } + const headers = Object.fromEntries(Object.entries(requestPause.request.headers || {}) + .filter(([key]) => /^(accept|authorization|content-type|x-xsrf-token|x-csrf-token|x-requested-with)$/i.test(key))); + if (!Object.values(headers).every(value => typeof value === 'string' && value.length <= 16_384 + && !/[\r\n]/.test(value))) fail('roster_request_url_mismatch'); + // Release the page's original request, then make our own bounded read using + // its session headers. Startup requests can otherwise replace the captured + // period or return a cached response for the saved UI period. + await connection.command('Fetch.continueRequest', { requestId: requestPause.requestId }); + requestPause = null; + await connection.command('Fetch.disable'); + const result = await connection.evaluate(rosterReadExpression({ + headers, body: Buffer.from(request.postData, 'base64').toString('utf8'), + })); + if (result?.error === 'timeout') fail('roster_response_timeout'); + if (result?.status !== 200 || typeof result.text !== 'string') fail('roster_response_invalid'); + const bytes = Buffer.from(result.text, 'utf8'); + const membership = bytes.length >= 32 && bytes.length <= MAX_SOURCE_BYTES + ? rosterMembershipAssessment(bytes, request.codes) : null; + if (!membership) fail('roster_membership_mismatch'); + return { + bytes, + sourceSha256: sha256(bytes), + completeness: { + observable: request.observable, + authoritative: request.authoritative && membership.exact, + authorityCode: membership.exact ? (request.authoritative ? null : request.authorityCode) : 'roster_membership_subset', + requestedCount: request.codes.length, + returnedCount: membership.returnedCount, + }, + uiPeriod: request.uiPeriod, + }; + } catch (error) { + if (error instanceof CdpError && error.code === 'browser_timeout') fail('paycom_timeout'); + throw error; + } finally { + connection?.socket.removeEventListener('message', continueOtherRequests); + if (requestPause) try { await connection.command('Fetch.continueRequest', { requestId: requestPause.requestId }); } catch {} + if (connection) try { await connection.command('Fetch.disable'); } catch {} + connection?.close(); + try { await boundedJson(`${endpoint}/json/close/${encodeURIComponent(target.id)}`); } catch {} + } +} + +function timecardResponse(value, targetUrl) { + return Boolean(value?.type === 'Document' && value.response?.url === targetUrl && value.response.status === 200 + && String(value.response.mimeType || '').toLowerCase() === 'text/html'); +} + +async function openTimecardSession(endpoint, clock = () => performance.now(), signal = null) { + if (typeof clock !== 'function' || signal !== null && !(signal instanceof AbortSignal)) fail('invalid_collection'); + const target = await createTarget(endpoint, 'about:blank'); + let connection = null; + try { + connection = await CdpConnection.connect(target.webSocketDebuggerUrl, { commandTimeoutMs: 60_000, signal }); + await connection.command('Page.enable'); + await connection.command('Page.setLifecycleEventsEnabled', { enabled: true }); + await connection.command('Network.enable'); + try { await connection.command('Network.setBlockedURLs', { urls: [...TIMECARD_BLOCKED_URLS] }); } catch {} + return { + async collect(employee, period, variant = 1, onTiming = () => {}) { + if (typeof onTiming !== 'function') fail('invalid_collection'); + const targetUrl = buildTimecardUrl(employee.employeeCode, period, variant); + if (!isCapturedTimecardUrl(targetUrl, { employeeCode: employee.employeeCode, period })) fail('navigation_policy_violation'); + try { + const responsePromise = connection.waitFor('Network.responseReceived', value => timecardResponse(value, targetUrl), 120_000); + responsePromise.catch(() => {}); + const navigateStarted = clock(); + const navigation = await connection.command('Page.navigate', { url: targetUrl }); + const navigateFinished = clock(); + if (navigation.errorText || typeof navigation.loaderId !== 'string') fail('navigation_failed'); + const response = await responsePromise; + const responseFinished = clock(); + await connection.waitFor('Network.loadingFinished', value => value.requestId === response.requestId, 120_000); + await connection.waitFor('Page.lifecycleEvent', value => value.loaderId === navigation.loaderId && value.name === 'load', 60_000); + const loadFinished = clock(); + const body = await connection.command('Network.getResponseBody', { requestId: response.requestId }); + if (typeof body?.body !== 'string') fail('timecard_body_unavailable'); + const sourceHtml = body.base64Encoded ? Buffer.from(body.body, 'base64') : Buffer.from(body.body, 'utf8'); + if (sourceHtml.length < 1024 || sourceHtml.length > MAX_SOURCE_BYTES + || !sourceHtml.includes(Buffer.from('id="tbltimesheet"')) || !sourceHtml.includes(Buffer.from('id="periodtotals"'))) fail('timecard_html_invalid'); + const bodyFinished = clock(); + let ready = false; + for (let index = 0; index < 60; index++) { + ready = await connection.evaluate("document.readyState==='complete'&&!!document.querySelector('#tbltimesheet')&&!!document.querySelector('#periodtotals')"); + if (ready) break; + await delay(100); + } + if (!ready) fail('timecard_page_timeout'); + const readyFinished = clock(); + const capturedRecord = await connection.evaluate(buildExtractionExpression({ employeeCode: employee.employeeCode, period, sourceUrl: targetUrl })); + const extractFinished = clock(); + validateTimecardRecord(capturedRecord, { employeeCode: employee.employeeCode, period, sourceUrl: targetUrl }); + const record = canonicalBusinessTimecard(capturedRecord); + const sourceUrl = canonicalTimecardUrl(employee.employeeCode, period); + validateTimecardRecord(record, { employeeCode: employee.employeeCode, period, sourceUrl }); + const sourceSha256 = sha256(sourceHtml); + const businessSha256 = timecardBusinessSha256(record); + const observedAt = new Date().toISOString(); + const validateFinished = clock(); + onTiming({ + navigateMs: navigateFinished - navigateStarted, + responseMs: responseFinished - navigateFinished, + loadMs: loadFinished - responseFinished, + bodyMs: bodyFinished - loadFinished, + readyMs: readyFinished - bodyFinished, + extractMs: extractFinished - readyFinished, + validateMs: validateFinished - extractFinished, + }); + return { + employeeCode: employee.employeeCode, + employeeName: employee.employeeName, + record, + sourceSha256, + businessSha256, + observedAt, + }; + } catch (error) { + if (error instanceof CdpError && error.code === 'browser_timeout') fail('paycom_timeout'); + throw error; + } + }, + async close() { + try { await connection.command('Network.disable'); } catch {} + connection.close(); + try { await boundedJson(`${endpoint}/json/close/${encodeURIComponent(target.id)}`); } catch {} + }, + }; + } catch (error) { + connection?.close(); + try { await boundedJson(`${endpoint}/json/close/${encodeURIComponent(target.id)}`); } catch {} + throw error; + } +} + +async function collectOneTimecard(endpoint, employee, period, variant = 1) { + const session = await openTimecardSession(endpoint); + try { return await session.collect(employee, period, variant); } + finally { await session.close(); } +} + +function percentile(values, ratio) { + if (!values.length) return 0; + const ordered = [...values].sort((left, right) => left - right); + return Math.round(ordered[Math.max(0, Math.ceil(ordered.length * ratio) - 1)]); +} + +function phaseMetrics(samples) { + const result = {}; + for (const phase of ['targetOpen', ...TIMECARD_PHASES]) { + const values = samples[phase]; + result[`${phase}P50Ms`] = percentile(values, 0.5); + result[`${phase}P95Ms`] = percentile(values, 0.95); + result[`${phase}MaxMs`] = percentile(values, 1); + } + return result; +} + +async function collectTimecards(endpoint, employees, periodValue, concurrency = 3, onProgress = () => {}, options = {}) { + const period = parsePeriodKey(periodValue.key); + if (!Array.isArray(employees) || employees.length < 1 || employees.length > 5000 || !Number.isInteger(concurrency) || concurrency < 1 || concurrency > 6 + || typeof onProgress !== 'function' || !options || typeof options !== 'object' || Array.isArray(options)) fail('invalid_collection'); + const clock = options.clock || (() => performance.now()); + const itemTimeoutMs = options.itemTimeoutMs ?? DEFAULT_ITEM_TIMEOUT_MS; + const openSession = options.openSession || ((workerIndex, signal) => openTimecardSession(endpoint, clock, signal)); + if (typeof openSession !== 'function' || typeof clock !== 'function' + || !Number.isInteger(itemTimeoutMs) || itemTimeoutMs < 10 || itemTimeoutMs > DEFAULT_ITEM_TIMEOUT_MS) fail('invalid_collection'); + const started = clock(); + const results = Array(employees.length); + const itemDurations = []; + const phaseSamples = Object.fromEntries(['targetOpen', ...TIMECARD_PHASES].map(phase => [phase, []])); + let next = 0; + let completed = 0; + let retries = 0; + let openedTargets = 0; + let cancelled = false; + let firstError = null; + const activeControllers = new Set(); + const cancelWorkers = () => { + cancelled = true; + for (const controller of activeControllers) controller.abort(); + }; + async function worker(workerIndex) { + let session = null; + let sessionController = null; + const closeSession = async () => { + if (!session) { + if (sessionController) activeControllers.delete(sessionController); + sessionController = null; + return; + } + const current = session; + const controller = sessionController; + session = null; + sessionController = null; + if (controller) activeControllers.delete(controller); + try { await current.close(); } catch {} + }; + try { + while (!cancelled) { + const index = next++; + if (index >= employees.length) return; + const itemStarted = clock(); + let last; + for (let attempt = 0; attempt < 2; attempt++) { + try { + if (!session) { + const openStarted = clock(); + sessionController = new AbortController(); + activeControllers.add(sessionController); + try { session = await openSession(workerIndex, sessionController.signal); } + catch (error) { + activeControllers.delete(sessionController); + sessionController = null; + throw error; + } + phaseSamples.targetOpen.push(Math.max(0, clock() - openStarted)); + openedTargets += 1; + } + let timeout = null; + let itemTimedOut = false; + const collected = session.collect(employees[index], period, (workerIndex + attempt) % 2 + 1, timing => { + if (!timing || typeof timing !== 'object' || Array.isArray(timing) + || Object.keys(timing).sort().join(',') !== TIMECARD_PHASES.map(phase => `${phase}Ms`).sort().join(',')) fail('invalid_collection'); + const values = {}; + for (const phase of TIMECARD_PHASES) { + const value = timing[`${phase}Ms`]; + if (typeof value !== 'number' || !Number.isFinite(value) || value < 0 || value > 120_000) fail('invalid_collection'); + values[phase] = value; + } + for (const phase of TIMECARD_PHASES) phaseSamples[phase].push(values[phase]); + }); + collected.catch(() => {}); + try { + const timedOut = new Promise((resolve, reject) => { + timeout = setTimeout(() => { + itemTimedOut = true; + sessionController?.abort(); + reject(Object.assign(new Error('timecard_item_timeout'), { code: 'timecard_item_timeout' })); + }, itemTimeoutMs); + }); + try { results[index] = await Promise.race([collected, timedOut]); } + catch (error) { + if (itemTimedOut) fail('timecard_item_timeout'); + throw error; + } + } finally { + clearTimeout(timeout); + } + completed += 1; + itemDurations.push(Math.max(0, clock() - itemStarted)); + onProgress(completed, employees.length); + last = null; + break; + } catch (error) { + last = error; + await closeSession(); + const code = error?.code || error?.message; + if (!['paycom_timeout', 'timecard_body_unavailable', 'timecard_page_timeout', 'timecard_item_timeout'].includes(code) || attempt === 1 || cancelled) { + if (!firstError) firstError = error; + cancelWorkers(); + return; + } + retries += 1; + await delay(250 * (attempt + 1)); + } + } + if (last) { + if (!firstError) firstError = last; + cancelWorkers(); + return; + } + } + } finally { + await closeSession(); + } + } + const workerCount = Math.min(concurrency, employees.length); + await Promise.allSettled(Array.from({ length: workerCount }, (_, index) => worker(index))); + if (firstError) throw firstError; + return { + rows: results, + performance: { + totalMs: Math.round(Math.max(0, clock() - started)), + itemCount: results.length, + workerCount, + openedTargets, + retryCount: retries, + itemP50Ms: percentile(itemDurations, 0.5), + itemP95Ms: percentile(itemDurations, 0.95), + itemMaxMs: percentile(itemDurations, 1), + ...phaseMetrics(phaseSamples), + }, + }; +} + +module.exports = { + TIMECARD_SEARCH_URL, ROSTER_API, ROSTER_REQUEST_FIELDS, rosterReadExpression, + rosterApiUrl, rosterRequest, rosterAuthorityAssessment, authoritativeRosterBody, fetchRosterRequest, fetchRosterResponse, rosterResponse, + rosterMembershipAssessment, rosterMembership, + collectRoster, collectOneTimecard, collectTimecards, +}; diff --git a/plugins/paycom/backend/src/collector-core.js b/plugins/paycom/backend/src/collector-core.js new file mode 100644 index 0000000..dfc4dae --- /dev/null +++ b/plugins/paycom/backend/src/collector-core.js @@ -0,0 +1,554 @@ +'use strict'; + +const fs = require('node:fs'); +const { collectRoster, collectTimecards } = require('./browser'); +const { parseRosterSource } = require('./roster-parser'); +const { ANCHOR_START, periodContaining, periodFromEnd, previousPeriod, nextPeriod } = require('./timecard-period'); +const { PaycomStore, stageCandidate, cleanupStage, cleanupRunStages } = require('./store'); +const { TIMECARD_SUMMARY, RESOURCE_TYPES, ROUTE_VERSION, linkRows } = require('./resource-links'); +const { planWorkforceMirror } = require('./sync-publication'); + +const METHODS = new Set([ + 'collection.resolve-targets', + 'collector.health', 'pay-periods.discover', 'roster.snapshot', 'roster.period', + 'resource-links.current-period', 'resource-links.period', 'resource-links.audit', + 'timecards.current-period', 'timecards.period', 'timecards.from-published-roster', 'timecards.audit', 'timecards.incremental', + 'reconcile.current-period', 'sync.current-workforce', +]); +const SAFE_ERRORS = new Set([ + 'invalid_request', 'deadline_exceeded', 'profile_not_configured', 'profile_locked', 'session_busy', + 'adapter_unavailable', 'browser_unavailable', 'unsafe_browser', 'browser_start_failed', 'browser_profile_busy', + 'browser_protocol_failed', 'browser_timeout', + 'authentication_timeout', 'primary_credentials_rejected', 'security_answers_rejected', 'invalid_credentials', 'account_locked', 'manual_verification_required', + 'authentication_failed', 'acquisition_cancelled', 'broker_closing', 'broker_unavailable', + 'attempt_cooldown', 'attempt_state_invalid', 'session_revoked', + 'lease_not_found', 'lease_not_ready', 'browser_lost', 'browser_cleanup_failed', + 'paycom_page_unavailable', 'cdp_invalid_targets', 'navigation_failed', 'navigation_policy_violation', + 'paycom_timeout', 'roster_request_timeout', 'roster_response_timeout', 'roster_loading_timeout', + 'roster_request_url_mismatch', 'roster_response_invalid', 'roster_period_mismatch', 'roster_body_unavailable', + 'roster_membership_mismatch', 'roster_source_not_authoritative', + 'roster_filter_search', 'roster_filter_advanced_null', 'roster_filter_advanced_enabled', + 'roster_filter_advanced_invalid', 'roster_filter_borrowed', 'roster_filter_pay_class', + 'roster_filter_earnings', 'roster_filter_approval', 'roster_filter_page_offset', + 'roster_filter_page_size', 'roster_filter_count', + 'api_invalid', 'csv_invalid', 'roster_not_loaded', 'roster_invalid', + 'timecards_not_loaded', 'timecards_invalid', 'timecard_body_unavailable', 'timecard_html_invalid', + 'timecard_page_timeout', 'timecard_item_timeout', 'candidate_invalid', 'candidate_too_large', 'unsafe_storage', + 'publication_verification_failed', 'publication_base_changed', 'business_delta_invalid', 'business_date_changed', + 'stage_cleanup_failed', 'integrity_failed', 'membership_mismatch', + 'stale_collection_attempt', 'invalid_period', 'invalid_employee_code', 'invalid_collection', + 'resource_type_invalid', 'resource_links_invalid', +]); + +function fail(code) { const error = new Error(code); error.code = code; throw error; } +function plain(value) { return value && typeof value === 'object' && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype; } +function exactKeys(value, allowed, required = allowed) { + return plain(value) && Object.keys(value).every(key => allowed.includes(key)) && required.every(key => Object.hasOwn(value, key)); +} +function dateInTimezone(timezone, now = new Date()) { + const parts = new Intl.DateTimeFormat('en-CA', { timeZone: timezone, year: 'numeric', month: '2-digit', day: '2-digit' }).formatToParts(now); + const get = type => parts.find(part => part.type === type)?.value; + return `${get('year')}-${get('month')}-${get('day')}`; +} + +function validateRequest(value) { + if (!exactKeys(value, ['protocolVersion', 'runId', 'plan', 'source', 'method', 'input', 'attempt', 'deadline']) + || value.protocolVersion !== 1 || typeof value.runId !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/.test(value.runId) + || typeof value.plan !== 'string' || !METHODS.has(value.method) || !plain(value.input) + || !Number.isInteger(value.attempt) || value.attempt < 1 || typeof value.deadline !== 'string' || Number.isNaN(Date.parse(value.deadline)) + || !exactKeys(value.source, ['id', 'collector', 'authProfile', 'config']) || value.source.collector !== 'paycom' + || typeof value.source.id !== 'string' || typeof value.source.authProfile !== 'string' || !plain(value.source.config) + || !exactKeys(value.source.config, ['timezone', 'maxConcurrency']) + || typeof value.source.config.timezone !== 'string' || value.source.config.timezone.length > 64 + || !Number.isInteger(value.source.config.maxConcurrency) || value.source.config.maxConcurrency < 1 || value.source.config.maxConcurrency > 6) fail('invalid_request'); + try { new Intl.DateTimeFormat('en-US', { timeZone: value.source.config.timezone }).format(); } catch { fail('invalid_request'); } + const schemas = { + 'collection.resolve-targets': [['selectorKind', 'date', 'start', 'end', 'key'], ['selectorKind']], + 'collector.health': [[], []], + 'pay-periods.discover': [[], []], + 'roster.snapshot': [[], []], + 'roster.period': [['periodEnd'], ['periodEnd']], + 'resource-links.current-period': [['resourceType'], ['resourceType']], + 'resource-links.period': [['resourceType', 'periodEnd'], ['resourceType', 'periodEnd']], + 'resource-links.audit': [['resourceType', 'periodEnd'], ['resourceType']], + 'timecards.current-period': [[], []], + 'timecards.period': [['periodEnd'], ['periodEnd']], + 'timecards.from-published-roster': [['periodEnd'], ['periodEnd']], + 'timecards.audit': [['periodEnd'], ['periodEnd']], + 'timecards.incremental': [[], []], + 'reconcile.current-period': [[], []], + 'sync.current-workforce': [ + ['reconcileBatchSize', 'fullReconcileMinutes', 'lookbackPeriods', 'publishMode'], + ['reconcileBatchSize', 'fullReconcileMinutes', 'lookbackPeriods'], + ], + }; + const [allowed, required] = schemas[value.method]; + if (!exactKeys(value.input, allowed, required)) fail('invalid_request'); + if ('periodEnd' in value.input && (typeof value.input.periodEnd !== 'string' || !/^\d{4}-\d{2}-\d{2}$/.test(value.input.periodEnd))) fail('invalid_request'); + if ('resourceType' in value.input && !RESOURCE_TYPES.includes(value.input.resourceType)) fail('invalid_request'); + if (value.method === 'sync.current-workforce') { + const ranges = { + reconcileBatchSize: [1, 500], + fullReconcileMinutes: [60, 10_080], + lookbackPeriods: [1, 1], + }; + for (const [key, [minimum, maximum]] of Object.entries(ranges)) { + if (!Number.isInteger(value.input[key]) || value.input[key] < minimum || value.input[key] > maximum) fail('invalid_request'); + } + if ('publishMode' in value.input && !['shadow', 'additions_edits_preview', 'additions_edits'].includes(value.input.publishMode)) fail('invalid_request'); + } + if (value.method === 'collection.resolve-targets') { + const expected = { + date: ['selectorKind', 'date'], + 'latest-complete': ['selectorKind', 'date'], + 'date-range': ['selectorKind', 'start', 'end'], + 'exact-target': ['selectorKind', 'key'], + }[value.input.selectorKind]; + if (!expected || !exactKeys(value.input, expected) + || expected.filter(key => key !== 'selectorKind').some(key => typeof value.input[key] !== 'string')) fail('invalid_request'); + } + if (Date.now() >= Date.parse(value.deadline)) fail('deadline_exceeded'); + return value; +} + +function periodsFor(date) { + const current = periodContaining(date); + return [ + { ...previousPeriod(current), relation: 'previous' }, + { ...current, relation: 'current' }, + { ...nextPeriod(current), relation: 'next' }, + ].map(({ start, end, key, relation }) => ({ start, end, key, relation })); +} + +function resolvedTargets(input) { + let periods; + if (input.selectorKind === 'date') periods = [periodContaining(input.date)]; + else if (input.selectorKind === 'latest-complete') periods = [previousPeriod(periodContaining(input.date))]; + else if (input.selectorKind === 'exact-target') periods = [periodFromEnd(input.key)]; + else { + if (input.start > input.end) fail('invalid_period'); + periods = []; + let period = periodContaining(input.start); + while (period.start <= input.end) { + periods.push(period); + if (periods.length > 512) fail('invalid_period'); + period = nextPeriod(period); + } + } + return { + targetType: 'pay-period', + targets: periods.map(period => ({ + key: period.end, start: period.start, end: period.end, values: { periodEnd: period.end }, + })), + }; +} + +function candidateBase(request, kind, target, metadata, rows) { + return { kind, target, runId: request.runId, attempt: request.attempt, collectedAt: new Date().toISOString(), metadata, rows }; +} + +function publish(store, request, candidate, stagingRoot) { + const stage = stageCandidate(stagingRoot, candidate); + let result; + try { + result = store.publish(stage); + } finally { + cleanupStage(stage, stagingRoot); + } + return { + ok: true, + status: result.disposition === 'no_change' ? 'no_change' : 'published', + data: { + method: request.method, target: candidate.target, publicationId: result.publicationId, + disposition: result.disposition, rowCount: result.rowCount, contentSha256: result.contentSha256, + }, + }; +} + +async function execute(request, { + database, + stagingRoot, + rosterCollector = collectRoster, + timecardCollector = collectTimecards, + browserRunner, + authentication, + businessClock = () => new Date(), +} = {}) { + validateRequest(request); + if (typeof rosterCollector !== 'function' || typeof timecardCollector !== 'function' + || typeof browserRunner !== 'function' || typeof businessClock !== 'function') fail('invalid_request'); + const startedAt = businessClock(); + if (!(startedAt instanceof Date) || Number.isNaN(startedAt.valueOf())) fail('invalid_request'); + const today = dateInTimezone(request.source.config.timezone, startedAt); + const current = periodContaining(today); + if (request.method === 'collection.resolve-targets') { + return { ok: true, status: 'succeeded', data: resolvedTargets(request.input) }; + } + const store = new PaycomStore(database); + const publishCandidate = candidate => publish(store, request, candidate, stagingRoot); + try { + if (request.method === 'collector.health') { + let authenticationState = 'unavailable'; + try { + authenticationState = await authentication(); + } catch {} + const payPeriods = store.audit('pay_periods'); + const roster = store.audit('roster'); + const activeTimecards = store.active('timecards'); + const timecards = activeTimecards ? store.auditTimecards(activeTimecards.target) : store.audit('timecards'); + const resourceLinks = store.auditResourceLinks(TIMECARD_SUMMARY); + return { ok: true, status: 'succeeded', data: { method: request.method, database: fs.existsSync(database) ? 'ready' : 'missing', authentication: authenticationState, payPeriods, roster, timecards, resourceLinks } }; + } + if (request.method === 'pay-periods.discover') { + const rows = periodsFor(today); + return publishCandidate(candidateBase(request, 'pay_periods', today, { + timezone: request.source.config.timezone, basis: 'biweekly_anchor', anchorStart: ANCHOR_START, + }, rows)); + } + if (request.method === 'sync.current-workforce') { + const replay = store.shadowReceiptForRun(request.source.id, current.end, request.runId); + if (replay) { + const outcome = replay.syncOutcome || { + mode: 'shadow', disposition: 'no_change', publicationStatus: 'shadow', + wouldPublish: false, mirror: null, publications: null, + }; + const { syncOutcome: ignored, ...observation } = replay; + return { + ok: true, + status: outcome.disposition, + data: { + method: request.method, + target: current.end, + businessDate: outcome.businessDate || outcome.persistence?.date || today, + businessTimezone: outcome.businessTimezone || request.source.config.timezone, + mode: outcome.mode, + publicationStatus: outcome.publicationStatus, + wouldPublish: outcome.wouldPublish, + sourceCompleteness: replay.sourceCompleteness || 'observation_only', + ...observation, + ...(outcome.mirror ? { mirror: outcome.mirror } : {}), + ...(outcome.delta ? { delta: outcome.delta } : {}), + ...(outcome.persistence ? { persistence: outcome.persistence } : {}), + ...(outcome.publications ? { publications: outcome.publications } : {}), + replayed: true, + }, + }; + } + const publishMode = request.input.publishMode || 'shadow'; + let publicationContext = null; + if (['additions_edits_preview', 'additions_edits'].includes(publishMode)) { + const rosterPublication = store.active('roster', current.end); + const timecardPublication = store.active('timecards', current.end); + if (rosterPublication && timecardPublication) { + const rosterAudit = store.audit('roster', current.end); + const timecardAudit = store.auditTimecards(current.end); + if (!rosterAudit.verified || !timecardAudit.verified) fail('integrity_failed'); + const priorRoster = store.activeRoster(current.end); + const priorTimecards = store.activeTimecards(current.end); + const priorLinks = store.activeResourceLinks(TIMECARD_SUMMARY, current.end); + if (priorLinks && !store.auditResourceLinks(TIMECARD_SUMMARY, current.end).verified) fail('integrity_failed'); + publicationContext = { + priorRoster, + priorTimecards, + resourceLinksLoaded: Boolean(priorLinks), + base: { + rosterPublicationId: rosterPublication.id, + rosterContentSha256: rosterPublication.content_sha256, + timecardPublicationId: timecardPublication.id, + timecardContentSha256: timecardPublication.content_sha256, + resourceLinkPublicationId: priorLinks?.publication.id || null, + resourceLinkContentSha256: priorLinks?.publication.content_sha256 || null, + }, + }; + } else if (!rosterPublication && !timecardPublication + && !store.activeResourceLinks(TIMECARD_SUMMARY, current.end)) { + publicationContext = { + priorRoster: { employees: [] }, priorTimecards: { rows: [] }, + resourceLinksLoaded: false, base: null, + }; + } else fail('integrity_failed'); + } + const browserStarted = performance.now(); + const shadow = await browserRunner(request, async ({ endpoint }) => { + const captured = await rosterCollector(endpoint, current, { unfiltered: true }); + if (!captured?.completeness?.observable) { + fail(captured?.completeness?.authorityCode || 'roster_source_not_authoritative'); + } + const requestedPeriodEnforced = captured.uiPeriod !== undefined; + if (requestedPeriodEnforced && (!captured.uiPeriod + || Object.keys(captured.uiPeriod).sort().join(',') !== 'end,start')) fail('roster_period_mismatch'); + const parsed = parseRosterSource(captured.bytes); + if (parsed.employeeCount !== captured.completeness.returnedCount) fail('roster_membership_mismatch'); + const bootstrap = publicationContext?.base === null; + if (bootstrap && !captured.completeness.authoritative) fail('roster_source_not_authoritative'); + const observedDate = businessClock(); + if (!(observedDate instanceof Date) || Number.isNaN(observedDate.valueOf())) fail('invalid_request'); + const observedAt = observedDate.toISOString(); + const selection = store.planWorkforceShadow({ + sourceId: request.source.id, + target: current.end, + observedAt, + employees: parsed.employees, + reconcileBatchSize: request.input.reconcileBatchSize, + fullReconcileMinutes: request.input.fullReconcileMinutes, + fullCollection: bootstrap, + }); + const collection = await timecardCollector( + endpoint, selection.selectedEmployees, current, request.source.config.maxConcurrency, + ); + const expected = selection.selectedEmployees.map(employee => employee.employeeCode).sort(); + const actual = collection.rows.map(row => row.employeeCode).sort(); + if (expected.length !== actual.length + || expected.some((code, index) => code !== actual[index])) fail('membership_mismatch'); + return { captured, parsed, observedAt, selection, collection, requestedPeriodEnforced }; + }); + if (dateInTimezone(request.source.config.timezone, new Date(shadow.observedAt)) !== today) { + fail('business_date_changed'); + } + const observationInput = { + sourceId: request.source.id, + target: current.end, + runId: request.runId, + observedAt: shadow.observedAt, + sourceSha256: shadow.captured.sourceSha256, + employees: shadow.parsed.employees, + sourceCompleteness: shadow.captured.completeness.authoritative ? 'authoritative' : 'observation_only', + timecardRows: shadow.collection.rows, + reconcileBatchSize: request.input.reconcileBatchSize, + fullReconcileMinutes: request.input.fullReconcileMinutes, + fullCollection: publicationContext?.base === null, + }; + let disposition = 'no_change'; + let publicationStatus = publishMode === 'shadow' ? 'shadow' : 'baseline_required'; + let wouldPublish = false; + let observation; + let mirror = null; + let delta = null; + let persistence = null; + let publications = null; + if (['additions_edits_preview', 'additions_edits'].includes(publishMode) && publicationContext) { + const mirrorPlan = planWorkforceMirror({ + period: current, + priorRosterRows: publicationContext.priorRoster.employees, + priorTimecardRows: publicationContext.priorTimecards.rows, + sourceEmployees: shadow.parsed.employees, + collectedTimecardRows: shadow.collection.rows, + }); + if (!publicationContext.resourceLinksLoaded) mirrorPlan.hasChanges = true; + const operationInput = { + runId: request.runId, + attempt: request.attempt, + collectedAt: shadow.observedAt, + coverageDate: today, + businessTimezone: request.source.config.timezone, + period: current, + sourceSha256: shadow.captured.sourceSha256, + sourceFormat: shadow.parsed.sourceFormat, + sourceEmployees: shadow.parsed.employees, + mirrorPlan, + observation: observationInput, + stagingRoot, + base: publicationContext.base, + }; + const published = publishMode === 'additions_edits_preview' + ? store.previewWorkforceSync(operationInput) + : store.publishWorkforceSync(operationInput); + disposition = published.disposition; + publicationStatus = publishMode === 'additions_edits_preview' ? 'preview' : 'ready'; + wouldPublish = published.wouldPublish; + observation = published.observation; + mirror = published.counts; + delta = published.delta || null; + persistence = published.persistence || null; + if (published.rosterPublicationId) { + publications = { + rosterPublicationId: published.rosterPublicationId, + timecardPublicationId: published.timecardPublicationId, + resourceLinkPublicationId: published.resourceLinkPublicationId, + }; + } + } else { + const syncOutcome = publishMode === 'shadow' + ? { + mode: 'shadow', disposition: 'no_change', publicationStatus: 'shadow', + wouldPublish: false, mirror: null, publications: null, + businessDate: today, businessTimezone: request.source.config.timezone, + } + : { + mode: publishMode, disposition: 'no_change', publicationStatus: 'baseline_required', + wouldPublish: false, mirror: null, publications: null, + businessDate: today, businessTimezone: request.source.config.timezone, + }; + observation = store.observeWorkforceShadow({ ...observationInput, syncOutcome }); + } + const { syncOutcome: ignored, ...observationData } = observation; + return { + ok: true, + status: disposition, + data: { + method: request.method, + target: current.end, + businessDate: today, + businessTimezone: request.source.config.timezone, + mode: publishMode, + publicationStatus, + wouldPublish, + sourceCompleteness: observation.sourceCompleteness, + disposition, + ...observationData, + ...(mirror ? { mirror } : {}), + ...(delta ? { delta } : {}), + ...(persistence ? { persistence } : {}), + ...(publications ? { publications } : {}), + performance: { + ...shadow.collection.performance, + requestedPeriodEnforced: shadow.requestedPeriodEnforced, + browserMs: Math.round(performance.now() - browserStarted), + }, + }, + }; + } + if (['roster.snapshot', 'roster.period'].includes(request.method)) { + const rosterPeriod = request.method === 'roster.period' ? periodFromEnd(request.input.periodEnd) : current; + const captured = await browserRunner(request, ({ endpoint }) => rosterCollector(endpoint, rosterPeriod)); + const parsed = parseRosterSource(captured.bytes); + const metadata = { + periodKey: rosterPeriod.key, sourceSha256: captured.sourceSha256, + employeeCount: parsed.employeeCount, activeEmployeeCount: parsed.activeEmployeeCount, + activeDriverCount: parsed.activeDriverCount, sourceFormat: parsed.sourceFormat, + }; + return publishCandidate(candidateBase(request, 'roster', rosterPeriod.end, metadata, parsed.employees)); + } + if (request.method.startsWith('resource-links.')) { + const linkPeriod = 'periodEnd' in request.input ? periodFromEnd(request.input.periodEnd) : current; + if (request.method === 'resource-links.audit') { + const audit = store.auditResourceLinks(request.input.resourceType, linkPeriod.end); + if (!audit.verified) return { ok: false, status: 'failed', data: { method: request.method, audit }, error: { code: audit.code } }; + return { ok: true, status: 'succeeded', data: { method: request.method, audit } }; + } + const roster = store.activeRoster(linkPeriod.end); + const activeEmployees = roster.employees.filter(employee => employee.isActive); + const rows = linkRows(request.input.resourceType, activeEmployees, linkPeriod); + const receipt = publishCandidate({ + ...candidateBase(request, 'resource_links', linkPeriod.end, { + resourceType: request.input.resourceType, + periodStart: linkPeriod.start, + periodEnd: linkPeriod.end, + rosterPublicationId: roster.publication.id, + rosterContentSha256: roster.publication.content_sha256, + routeVersion: ROUTE_VERSION, + }, rows), + periodKey: linkPeriod.key, + }); + const audit = store.auditResourceLinks(request.input.resourceType, linkPeriod.end); + if (!audit.verified || audit.rowCount !== activeEmployees.length) fail('integrity_failed'); + receipt.data.audit = { verified: true, activeEmployees: activeEmployees.length, links: audit.rowCount }; + return receipt; + } + if (request.method === 'timecards.audit') { + const auditPeriod = periodFromEnd(request.input.periodEnd); + const audit = store.auditTimecards(auditPeriod.end); + if (!audit.verified) return { ok: false, status: 'failed', data: { method: request.method, audit }, error: { code: audit.code } }; + return { ok: true, status: 'succeeded', data: { method: request.method, audit } }; + } + if (request.method === 'reconcile.current-period') { + const audit = store.auditTimecards(current.end); + if (!audit.verified) return { ok: false, status: 'failed', data: { method: request.method, audit }, error: { code: audit.code } }; + return { ok: true, status: 'succeeded', data: { method: request.method, audit } }; + } + const period = ['timecards.period', 'timecards.from-published-roster'].includes(request.method) + ? periodFromEnd(request.input.periodEnd) : current; + if (request.method === 'timecards.period') { + const browserStarted = performance.now(); + const historical = await browserRunner(request, async ({ endpoint }) => { + const captured = await rosterCollector(endpoint, period); + const parsed = parseRosterSource(captured.bytes); + const employees = parsed.employees.filter(employee => employee.isActive) + .map(employee => ({ employeeCode: employee.employeeCode, employeeName: employee.employeeName })); + if (!employees.length) fail('roster_invalid'); + const collection = await timecardCollector(endpoint, employees, period, request.source.config.maxConcurrency); + const rows = collection.rows; + const expected = [...employees.map(employee => employee.employeeCode)].sort(); + const actual = [...rows.map(row => row.employeeCode)].sort(); + if (expected.length !== actual.length || expected.some((code, index) => code !== actual[index])) fail('membership_mismatch'); + return { captured, parsed, rows, performance: collection.performance }; + }); + const receipt = publishCandidate({ + ...candidateBase(request, 'timecards', period.end, { + periodStart: period.start, periodEnd: period.end, + rosterSourceSha256: historical.captured.sourceSha256, + rosterEmployeeCount: historical.parsed.employeeCount, + activeEmployeeCount: historical.rows.length, + mode: 'historical_period_membership', + }, historical.rows), + periodKey: period.key, + }); + receipt.data.performance = { ...historical.performance, browserMs: Math.round(performance.now() - browserStarted) }; + return receipt; + } + const roster = store.activeRoster(period.end); + const rosterAudit = store.audit('roster', period.end); + if (!rosterAudit.verified) fail('integrity_failed'); + if (roster.publication.target !== period.end) fail('roster_period_mismatch'); + const activeEmployees = roster.employees.filter(employee => employee.isActive) + .map(employee => ({ employeeCode: employee.employeeCode, employeeName: employee.employeeName })); + if (!activeEmployees.length) fail('roster_invalid'); + let rows; + let collectionPerformance = null; + let mode = request.method === 'timecards.from-published-roster' ? 'published_roster' : 'full'; + if (request.method === 'timecards.incremental') { + mode = 'incremental'; + const prior = store.activeTimecards(period.end); + const activeByCode = new Map(activeEmployees.map(employee => [employee.employeeCode, employee])); + const retained = (prior?.rows || []).filter(row => activeByCode.has(row.employeeCode)) + .map(row => ({ ...row, employeeName: activeByCode.get(row.employeeCode).employeeName })); + const present = new Set(retained.map(row => row.employeeCode)); + const missing = activeEmployees.filter(employee => !present.has(employee.employeeCode)); + if (!missing.length && retained.length === activeEmployees.length) { + const audit = store.auditTimecards(period.end); + if (!audit.verified) fail(audit.code); + return { ok: true, status: 'no_change', data: { method: request.method, target: period.end, publicationId: prior.publication.id, disposition: 'complete', rowCount: retained.length, audit } }; + } + let collected = []; + if (missing.length) { + const browserStarted = performance.now(); + const collection = await browserRunner(request, ({ endpoint }) => timecardCollector(endpoint, missing, period, request.source.config.maxConcurrency)); + collected = collection.rows; + collectionPerformance = { ...collection.performance, browserMs: Math.round(performance.now() - browserStarted) }; + } + rows = [...retained, ...collected].sort((left, right) => left.employeeCode.localeCompare(right.employeeCode)); + } else { + const browserStarted = performance.now(); + const collection = await browserRunner(request, ({ endpoint }) => timecardCollector(endpoint, activeEmployees, period, request.source.config.maxConcurrency)); + rows = collection.rows; + collectionPerformance = { ...collection.performance, browserMs: Math.round(performance.now() - browserStarted) }; + } + const expectedCodes = [...activeEmployees.map(employee => employee.employeeCode)].sort(); + const actualCodes = [...rows.map(row => row.employeeCode)].sort(); + if (expectedCodes.length !== actualCodes.length || expectedCodes.some((code, index) => code !== actualCodes[index])) fail('membership_mismatch'); + const candidate = { + ...candidateBase(request, 'timecards', period.end, { + periodStart: period.start, periodEnd: period.end, rosterPublicationId: roster.publication.id, + rosterContentSha256: roster.publication.content_sha256, mode, + }, rows), + periodKey: period.key, + }; + const receipt = publishCandidate(candidate); + const audit = store.auditTimecards(period.end); + if (!audit.verified) fail(audit.code); + receipt.data.audit = audit; + if (collectionPerformance) receipt.data.performance = collectionPerformance; + return receipt; + } finally { + try { store.close(); } + finally { cleanupRunStages(stagingRoot, request.runId); } + } +} + +function safeFailure(error) { + const candidate = error?.code || error?.message; + const code = SAFE_ERRORS.has(candidate) || /^timecard_[a-z0-9_]{1,55}$/.test(candidate || '') ? candidate : 'collection_failed'; + return { ok: false, status: 'failed', data: null, error: { code } }; +} + +module.exports = { METHODS, SAFE_ERRORS, validateRequest, dateInTimezone, periodsFor, resolvedTargets, execute, safeFailure }; diff --git a/plugins/paycom/backend/src/collector.js b/plugins/paycom/backend/src/collector.js new file mode 100644 index 0000000..aacaaa5 --- /dev/null +++ b/plugins/paycom/backend/src/collector.js @@ -0,0 +1,18 @@ +'use strict'; +// Compatibility entry for existing standalone collector commands. Installed +// workers call collector-core with the scoped dispatch-sdk capabilities. +const core = require('./collector-core'); +const { DATABASE, STAGING_ROOT, AUTH_SOCKET } = require('./paths'); +const { withPaycomBrowser } = require('./authenticated-browser'); +function execute(request, options = {}) { + return core.execute(request, { + database: DATABASE, stagingRoot: STAGING_ROOT, + browserRunner: (input, callback) => withPaycomBrowser({ authProfile: input.source.authProfile, + runId: input.runId, ttlSeconds: 90, socketPath: AUTH_SOCKET }, callback), + authentication: async () => { + const health = await require('dispatch-runtime-kit/auth-broker/src/client').request(AUTH_SOCKET, { action: 'health' }); + return health.ok && health.status === 'ready' ? 'ready' : health.status || 'unavailable'; + }, ...options, + }); +} +module.exports = { ...core, execute }; diff --git a/plugins/paycom/backend/src/csv.js b/plugins/paycom/backend/src/csv.js new file mode 100644 index 0000000..f370ccc --- /dev/null +++ b/plugins/paycom/backend/src/csv.js @@ -0,0 +1,8 @@ +'use strict'; +const HEADERS=['Employee Code','Employee Name','Status','Department Code','Department Desc','Delivery Station Code Code','Delivery Station Code Desc','Position Title','Pay Class','Terminal Group','Pay Type','Primary Supervisor','Missing Punches','Total Hours','Total Overtime Hours','Percent Approved (EE)','Percent Approved (SUP)']; +function fail(){throw new Error('csv_invalid');} +function parseRecords(text){const rows=[];let row=[],field='',quoted=false;for(let i=0;iv===HEADERS[i]))fail();const employees=[];let totals=null;const seen=new Set();for(const values of records.slice(1)){if(values.length!==HEADERS.length)fail();const raw=Object.fromEntries(HEADERS.map((h,i)=>[h,values[i]]));if(raw['Employee Code']==='Grand Totals'&&raw['Employee Name']===''){if(totals)fail();totals=raw;continue;}const code=raw['Employee Code'];if(!/^[A-Za-z0-9]{4}$/.test(code)||!raw['Employee Name'].trim()||seen.has(code.toUpperCase()))fail();seen.add(code.toUpperCase());if(!['A','I'].includes(raw.Status))fail();numberText(raw['Missing Punches'],{integer:true});for(const h of ['Total Hours','Total Overtime Hours','Percent Approved (EE)','Percent Approved (SUP)'])numberText(raw[h]);const active=raw.Status==='A';const driverDepartment=new Set(['Driver','Driver- Step Van','Driver-Step Van']).has(raw['Department Desc']);const driverPosition=/driver|step\s*van/i.test(raw['Position Title']);employees.push({employeeCode:code,employeeName:raw['Employee Name'],status:raw.Status,lifecycleStatus:active?'active':'inactive',departmentCode:raw['Department Code'],departmentDesc:raw['Department Desc'],deliveryStationCode:raw['Delivery Station Code Code'],deliveryStationDesc:raw['Delivery Station Code Desc'],positionTitle:raw['Position Title'],payClass:raw['Pay Class'],terminalGroup:raw['Terminal Group'],payType:raw['Pay Type'],primarySupervisor:raw['Primary Supervisor'],missingPunches:raw['Missing Punches'],totalHours:raw['Total Hours'],totalOvertimeHours:raw['Total Overtime Hours'],employeeApprovalPercentage:raw['Percent Approved (EE)'],supervisorApprovalPercentage:raw['Percent Approved (SUP)'],isActive:active,isDriverDepartment:driverDepartment,isDriverPosition:driverPosition,isActiveDriver:active&&driverDepartment});} +if(!employees.length)fail();return {sourceFormat:'paycom-timecard-csv.v1',headers:[...HEADERS],rowCount:records.length-1,employeeCount:employees.length,activeEmployeeCount:employees.filter(x=>x.isActive).length,activeDriverCount:employees.filter(x=>x.isActiveDriver).length,sourceBytes:source.length,totals,employees};} +module.exports={HEADERS,parseRosterCsv}; diff --git a/plugins/paycom/backend/src/fingerprints.js b/plugins/paycom/backend/src/fingerprints.js new file mode 100644 index 0000000..b8f0a24 --- /dev/null +++ b/plugins/paycom/backend/src/fingerprints.js @@ -0,0 +1,68 @@ +'use strict'; + +const crypto = require('node:crypto'); +const { canonicalTimecardUrl, parsePeriodKey } = require('./timecard-period'); + +const PROFILE_FIELDS = Object.freeze([ + 'employeeCode', 'employeeName', 'status', 'lifecycleStatus', + 'departmentCode', 'departmentDesc', 'deliveryStationCode', 'deliveryStationDesc', + 'positionTitle', 'payClass', 'terminalGroup', 'payType', 'primarySupervisor', + 'isActive', 'isDriverDepartment', 'isDriverPosition', 'isActiveDriver', +]); +const SUMMARY_FIELDS = Object.freeze([ + 'missingPunches', 'totalHours', 'totalOvertimeHours', + 'employeeApprovalPercentage', 'supervisorApprovalPercentage', +]); + +function plain(value) { + return value && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function canonicalStringify(value) { + if (Array.isArray(value)) return `[${value.map(canonicalStringify).join(',')}]`; + if (plain(value)) { + return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalStringify(value[key])}`).join(',')}}`; + } + return JSON.stringify(value); +} + +function sha256(value) { + return crypto.createHash('sha256').update(Buffer.from(canonicalStringify(value))).digest('hex'); +} + +function projection(value, fields) { + return Object.fromEntries(fields.map(field => [field, Object.hasOwn(value, field) ? value[field] : null])); +} + +function rosterProfileSha256(employee) { + if (!plain(employee)) throw new Error('api_invalid'); + return sha256(projection(employee, PROFILE_FIELDS)); +} + +function rosterSummarySha256(employee) { + if (!plain(employee)) throw new Error('api_invalid'); + return sha256(projection(employee, SUMMARY_FIELDS)); +} + +function canonicalBusinessTimecard(record) { + if (!plain(record) || typeof record.employeeCode !== 'string' || typeof record.periodKey !== 'string') { + throw new Error('timecard_identity_invalid'); + } + const period = parsePeriodKey(record.periodKey); + return { ...record, sourceUrl: canonicalTimecardUrl(record.employeeCode, period) }; +} + +function timecardBusinessSha256(record) { + return sha256(canonicalBusinessTimecard(record)); +} + +module.exports = { + PROFILE_FIELDS, + SUMMARY_FIELDS, + canonicalStringify, + canonicalBusinessTimecard, + rosterProfileSha256, + rosterSummarySha256, + timecardBusinessSha256, +}; diff --git a/plugins/paycom/backend/src/paths.js b/plugins/paycom/backend/src/paths.js new file mode 100644 index 0000000..bc2b690 --- /dev/null +++ b/plugins/paycom/backend/src/paths.js @@ -0,0 +1,15 @@ +'use strict'; + +const path = require('node:path'); +const { PROJECT_ROOT: SOURCE_ROOT, assertExternalRuntimePaths, resolveLocalRuntimePaths } = require('dispatch-protocol/paths/runtime-paths'); + +const resolved = resolveLocalRuntimePaths(); +const PROJECT_ROOT = resolved.projectRoot; +const PLUGIN_ROOT = path.resolve(__dirname, ".."); +const DATA_ROOT = process.env.DISPATCH_PAYCOM_DATA_ROOT || resolved.paycom.dataRoot; +const DATABASE = path.join(DATA_ROOT, 'paycom.sqlite3'); +const STAGING_ROOT = process.env.DISPATCH_PAYCOM_STAGING_ROOT || resolved.paycom.stagingRoot; +const AUTH_SOCKET = process.env.DISPATCH_AUTH_SOCKET || resolved.paycom.authSocket; +assertExternalRuntimePaths(SOURCE_ROOT, [DATA_ROOT, DATABASE, STAGING_ROOT, AUTH_SOCKET]); + +module.exports = { PROJECT_ROOT, PLUGIN_ROOT, DATA_ROOT, DATABASE, STAGING_ROOT, AUTH_SOCKET }; diff --git a/plugins/paycom/backend/src/publication-continuity.js b/plugins/paycom/backend/src/publication-continuity.js new file mode 100644 index 0000000..bf68dfb --- /dev/null +++ b/plugins/paycom/backend/src/publication-continuity.js @@ -0,0 +1,38 @@ +'use strict'; +const { PaycomStore } = require('./store'); +const { TIMECARD_SUMMARY } = require('./resource-links'); +const { publicationBaseline, createPublicationBaseline } = require('dispatch-protocol/contracts/src/publication-baseline'); +function fail() { throw Object.assign(new Error('first_publication_failed'), { code: 'first_publication_failed' }); } +function verifyPublicationContinuity(database, input) { + if (!input || Object.getPrototypeOf(input) !== Object.prototype + || !['capture', 'verify'].includes(input.mode) + || Object.keys(input).sort().join(',') !== (input.mode === 'capture' ? 'mode' : 'baseline,mode')) fail(); + const expected = input.mode === 'verify' ? publicationBaseline(input.baseline) : null; + const store = new PaycomStore(database, { readOnly: true }); + try { + store.db.exec('BEGIN'); + const periods = store.active('pay_periods'); + const current = periods && store.db.prepare("SELECT period_end FROM pay_periods WHERE publication_id=? AND relation='current'").all(periods.id); + if (current?.length !== 1) fail(); + const target = current[0].period_end; + const values = { + payPeriods: [periods, store.auditPayPeriodTarget(target)], + roster: [store.active('roster', target), store.audit('roster', target)], + timecards: [store.active('timecards', target), store.audit('timecards', target)], + resourceLinks: [store.activeResourceLinks(TIMECARD_SUMMARY, target)?.publication, + store.auditResourceLinks(TIMECARD_SUMMARY, target)], + }; + const publications = {}; + for (const [name, [publication, audit]] of Object.entries(values)) { + if (!publication || !audit.verified || audit.contentSha256 !== publication.content_sha256) fail(); + publications[name] = { id: publication.id, originRunId: publication.run_id, contentSha256: publication.content_sha256 }; + } + if (!store.auditTimecards(target).verified) fail(); + const baseline = createPublicationBaseline(target, publications); + if (expected && expected.digest !== baseline.digest) fail(); + store.db.exec('COMMIT'); + return expected ? Object.freeze({ status: 'verified', publicationBaselineDigest: baseline.digest }) + : Object.freeze({ status: 'verified', publicationBaseline: baseline }); + } finally { store.close(); } +} +module.exports = { verifyPublicationContinuity }; diff --git a/plugins/paycom/backend/src/resource-links.js b/plugins/paycom/backend/src/resource-links.js new file mode 100644 index 0000000..c083848 --- /dev/null +++ b/plugins/paycom/backend/src/resource-links.js @@ -0,0 +1,89 @@ +'use strict'; + +const { canonicalTimecardUrl, parsePeriodKey, validateCode } = require('./timecard-period'); + +const TIMECARD_SUMMARY = 'paycom.timecard.summary'; +const RESOURCE_TYPES = Object.freeze([TIMECARD_SUMMARY]); +const ROUTE_VERSION = 1; + +function plain(value) { + return value && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} + +function exactKeys(value, keys) { + return plain(value) && Object.keys(value).length === keys.length + && keys.every(key => Object.hasOwn(value, key)); +} + +function buildResourceLink(resourceType, employeeCode, period) { + if (resourceType !== TIMECARD_SUMMARY) throw new Error('resource_type_invalid'); + validateCode(employeeCode); + return canonicalTimecardUrl(employeeCode.toUpperCase(), parsePeriodKey(period.key)); +} + +function isResourceLink(value, { resourceType, employeeCode, period }) { + try { + return typeof value === 'string' + && value === buildResourceLink(resourceType, employeeCode, period); + } catch { + return false; + } +} + +function linkRows(resourceType, employees, period) { + if (!RESOURCE_TYPES.includes(resourceType) || !Array.isArray(employees) + || employees.length < 1 || employees.length > 5000) throw new Error('resource_links_invalid'); + const seen = new Set(); + return employees.map(employee => { + if (!plain(employee) || typeof employee.employeeCode !== 'string' + || !/^[A-Za-z0-9]{4}$/.test(employee.employeeCode) || employee.isActive !== true) { + throw new Error('resource_links_invalid'); + } + const employeeCode = employee.employeeCode.toUpperCase(); + if (seen.has(employeeCode)) throw new Error('resource_links_invalid'); + seen.add(employeeCode); + return { employeeCode, canonicalUrl: buildResourceLink(resourceType, employeeCode, period) }; + }).sort((left, right) => left.employeeCode.localeCompare(right.employeeCode)); +} + +function validateResourceLinkCandidate(candidate) { + const period = parsePeriodKey(candidate.periodKey); + const metadataKeys = [ + 'resourceType', 'periodStart', 'periodEnd', 'rosterPublicationId', + 'rosterContentSha256', 'routeVersion', + ]; + if (candidate.target !== period.end || !exactKeys(candidate.metadata, metadataKeys) + || !RESOURCE_TYPES.includes(candidate.metadata.resourceType) + || candidate.metadata.periodStart !== period.start || candidate.metadata.periodEnd !== period.end + || typeof candidate.metadata.rosterPublicationId !== 'string' + || !/^[a-f0-9-]{36}$/.test(candidate.metadata.rosterPublicationId) + || !/^[a-f0-9]{64}$/.test(candidate.metadata.rosterContentSha256) + || candidate.metadata.routeVersion !== ROUTE_VERSION + || !Array.isArray(candidate.rows) || candidate.rows.length < 1 || candidate.rows.length > 5000) { + throw new Error('candidate_invalid'); + } + const seen = new Set(); + for (const row of candidate.rows) { + if (!exactKeys(row, ['employeeCode', 'canonicalUrl']) + || typeof row.employeeCode !== 'string' || !/^[A-Z0-9]{4}$/.test(row.employeeCode) + || seen.has(row.employeeCode) + || !isResourceLink(row.canonicalUrl, { + resourceType: candidate.metadata.resourceType, + employeeCode: row.employeeCode, + period, + })) throw new Error('candidate_invalid'); + seen.add(row.employeeCode); + } + return candidate; +} + +module.exports = { + TIMECARD_SUMMARY, + RESOURCE_TYPES, + ROUTE_VERSION, + buildResourceLink, + isResourceLink, + linkRows, + validateResourceLinkCandidate, +}; diff --git a/plugins/paycom/backend/src/roster-parser.js b/plugins/paycom/backend/src/roster-parser.js new file mode 100644 index 0000000..67946f1 --- /dev/null +++ b/plugins/paycom/backend/src/roster-parser.js @@ -0,0 +1,8 @@ +'use strict'; +const {HEADERS}=require('./csv'); +const REQUIRED=['employeeCode','fullName','eestatus','allocation','position','payClassCode','terminalCode','payType','primarySupervisor','missingPunches','totals','approvalPercentages']; +function invalid(){throw new Error('api_invalid');} +function finiteText(value){if(typeof value!=='number'||!Number.isFinite(value))invalid();return String(value);} +function parseRosterApi(source){if(!Buffer.isBuffer(source)||source.length<32||source.length>2097152)invalid();let value;try{value=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(source));}catch{invalid();}if(!value||typeof value!=='object'||Array.isArray(value)||!Array.isArray(value.eeCodes)||!Array.isArray(value.employees)||value.employees.length<1||value.employees.length>5000||value.eeCodes.length!==value.employees.length)invalid();const seen=new Set(),employees=[];for(const raw of value.employees){if(!raw||typeof raw!=='object'||Array.isArray(raw)||REQUIRED.some(key=>!(key in raw)))invalid();const code=raw.employeeCode;if(typeof code!=='string'||!/^[A-Za-z0-9]{4}$/.test(code)||seen.has(code.toUpperCase())||typeof raw.fullName!=='string'||!raw.fullName.trim()||raw.eestatus!=='A')invalid();seen.add(code.toUpperCase());const selections=raw.allocation&&raw.allocation.selections;if(!Array.isArray(selections)||selections.length!==2)invalid();const byName=Object.fromEntries(selections.map(item=>[item&&item.categoryName,item]));const department=byName.Department,station=byName['Delivery Station Code'];if(!department||!station||department.isDepartment!==true||station.isDepartment!==false)invalid();for(const item of [department,station])if(typeof item.code!=='string'||typeof item.description!=='string')invalid();for(const key of ['position','payClassCode','terminalCode','payType','primarySupervisor'])if(typeof raw[key]!=='string')invalid();if(typeof raw.missingPunches!=='number'||!Number.isInteger(raw.missingPunches)||!raw.totals||!raw.approvalPercentages)invalid();const driverDepartment=new Set(['Driver','Driver- Step Van','Driver-Step Van']).has(department.description),driverPosition=/driver|step\s*van/i.test(raw.position);employees.push({employeeCode:code,employeeName:raw.fullName,status:'A',lifecycleStatus:'active',departmentCode:department.code,departmentDesc:department.description,deliveryStationCode:station.code,deliveryStationDesc:station.description,positionTitle:raw.position,payClass:raw.payClassCode,terminalGroup:raw.terminalCode,payType:raw.payType,primarySupervisor:raw.primarySupervisor,missingPunches:String(raw.missingPunches),totalHours:finiteText(raw.totals.totalHours),totalOvertimeHours:finiteText(raw.totals.otHours),employeeApprovalPercentage:finiteText(raw.approvalPercentages.employee),supervisorApprovalPercentage:finiteText(raw.approvalPercentages.supervisor),isActive:true,isDriverDepartment:driverDepartment,isDriverPosition:driverPosition,isActiveDriver:driverDepartment});}const eeCodes=new Set(value.eeCodes.map(item=>typeof item==='string'?item.toUpperCase():invalid()));if(eeCodes.size!==seen.size||[...seen].some(code=>!eeCodes.has(code)))invalid();return {sourceFormat:'paycom-employees-json.v1',headers:[...HEADERS],rowCount:employees.length,employeeCount:employees.length,activeEmployeeCount:employees.length,activeDriverCount:employees.filter(item=>item.isActiveDriver).length,sourceBytes:source.length,totals:null,employees};} +function parseRosterSource(source){const first=source.subarray(0,64).toString('utf8').trimStart();return first.startsWith('{')?parseRosterApi(source):require('./csv').parseRosterCsv(source);} +module.exports={parseRosterApi,parseRosterSource}; diff --git a/plugins/paycom/backend/src/store.js b/plugins/paycom/backend/src/store.js new file mode 100644 index 0000000..4871d33 --- /dev/null +++ b/plugins/paycom/backend/src/store.js @@ -0,0 +1,1514 @@ +'use strict'; + +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { DatabaseSync } = require('node:sqlite'); +const { validateTimecardRecord } = require('./timecard-dom'); +const { isCapturedTimecardUrl, isCanonicalTimecardUrl, parsePeriodKey, periodFromEnd, + previousPeriod, nextPeriod, ANCHOR_START } = require('./timecard-period'); +const { rosterProfileSha256, rosterSummarySha256, timecardBusinessSha256 } = require('./fingerprints'); +const { TIMECARD_SUMMARY, ROUTE_VERSION, validateResourceLinkCandidate } = require('./resource-links'); +const { validateBusinessDelta, computeBusinessDelta } = require('./sync-delta'); + +const KINDS = new Set(['pay_periods', 'roster', 'timecards', 'resource_links']); +const RUN_RE = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$/; +const DATE_RE = /^\d{4}-\d{2}-\d{2}$/; +const UUID_RE = /^[a-f0-9-]{36}$/; +const SHA256_RE = /^[a-f0-9]{64}$/; +const MAX_STAGE_BYTES = 134_217_728; +const NO_CHANGE_HISTORY_RETENTION_MS = 365 * 24 * 60 * 60 * 1000; +const MAX_HISTORY_COMPACTION_DELETE = 100; + +function fail(code) { const error = new Error(code); error.code = code; throw error; } +function plain(value) { return value && typeof value === 'object' && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype; } +function sha256(bytes) { return crypto.createHash('sha256').update(bytes).digest('hex'); } +function canonicalStringify(value) { + if (Array.isArray(value)) return `[${value.map(canonicalStringify).join(',')}]`; + if (plain(value)) return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonicalStringify(value[key])}`).join(',')}}`; + return JSON.stringify(value); +} +function canonicalRows(candidate) { + const rows = [...candidate.rows]; + const key = candidate.kind === 'pay_periods' ? row => row.key : row => row.employeeCode.toUpperCase(); + return rows.sort((left, right) => key(left).localeCompare(key(right))); +} +function contentSha256(candidate) { + return sha256(Buffer.from(canonicalStringify({ + kind: candidate.kind, + target: candidate.target, + ...(candidate.periodKey ? { periodKey: candidate.periodKey } : {}), + metadata: candidate.metadata, + rows: canonicalRows(candidate), + }))); +} + +function privateDirectory(directory, { create = true } = {}) { + const resolved = path.resolve(directory); + if (!fs.existsSync(resolved)) { + if (!create) fail('unsafe_storage'); + fs.mkdirSync(resolved, { recursive: true, mode: 0o700 }); + } + const info = fs.lstatSync(resolved); + if (!info.isDirectory() || info.isSymbolicLink() || info.uid !== process.geteuid() || fs.realpathSync(resolved) !== resolved) fail('unsafe_storage'); + if ((info.mode & 0o077) !== 0) fail('unsafe_storage'); + return resolved; +} + +function privateFile(file, { create = false } = {}) { + const resolved = path.resolve(file); + if (!fs.existsSync(resolved) && create) { + const descriptor = fs.openSync(resolved, 'wx', 0o600); + fs.closeSync(descriptor); + } + const info = fs.lstatSync(resolved); + if (!info.isFile() || info.isSymbolicLink() || info.uid !== process.geteuid() || info.nlink !== 1 || fs.realpathSync(resolved) !== resolved) fail('unsafe_storage'); + if ((info.mode & 0o177) !== 0) fail('unsafe_storage'); + return resolved; +} + +function exactKeys(value, keys) { + return plain(value) && Object.keys(value).length === keys.length && keys.every(key => Object.hasOwn(value, key)); +} + +const MIRROR_COUNT_KEYS = [ + 'rosterAddedCount', 'rosterProfileChangedCount', 'rosterSummaryChangedCount', + 'rosterRecordChangedCount', 'timecardAddedCount', 'timecardChangedCount', + 'retainedMissingCount', 'becameUnknownCount', 'returnedFromUnknownCount', 'unknownEmployeeCount', + 'deactivatedCount', 'reactivatedCount', 'activeEmployeeCount', +]; +const PERSISTENCE_COUNT_KEYS = [ + 'timecardCount', 'dateRowCount', 'selectedTimecardCount', 'persistedSelectedTimecardCount', + 'selectedMismatchCount', 'punchCount', 'inDayPunchCount', 'inDayTimecardCount', + 'outLunchPunchCount', 'inLunchPunchCount', 'outDayPunchCount', 'unclassifiedPunchCount', +]; +const PERSISTENCE_KEYS = [ + 'verified', 'code', 'date', 'timecardPublicationId', 'publicationCollectedAt', + ...PERSISTENCE_COUNT_KEYS, +]; + +function validateTimecardPersistence(value) { + if (!exactKeys(value, PERSISTENCE_KEYS) || value.verified !== true || value.code !== 'verified' + || typeof value.date !== 'string' || !DATE_RE.test(value.date) + || typeof value.timecardPublicationId !== 'string' || !UUID_RE.test(value.timecardPublicationId) + || typeof value.publicationCollectedAt !== 'string' || Number.isNaN(Date.parse(value.publicationCollectedAt)) + || PERSISTENCE_COUNT_KEYS.some(key => !Number.isInteger(value[key]) || value[key] < 0) + || value.dateRowCount !== value.timecardCount + || value.persistedSelectedTimecardCount + value.selectedMismatchCount !== value.selectedTimecardCount + || value.inDayTimecardCount > value.inDayPunchCount + || value.inDayPunchCount + value.outLunchPunchCount + value.inLunchPunchCount + + value.outDayPunchCount + value.unclassifiedPunchCount !== value.punchCount) fail('invalid_request'); + return value; +} + +function validTimezone(value) { + if (typeof value !== 'string' || value.length < 1 || value.length > 64) return false; + try { new Intl.DateTimeFormat('en-US', { timeZone: value }).format(); return true; } catch { return false; } +} + +function validateSyncOutcome(value) { + const baseKeys = ['mode', 'disposition', 'publicationStatus', 'wouldPublish', 'mirror', 'publications']; + const optionalKeys = ['businessDate', 'businessTimezone', 'delta', 'persistence']; + const shapeValid = plain(value) + && baseKeys.every(key => Object.hasOwn(value, key)) + && Object.keys(value).every(key => baseKeys.includes(key) || optionalKeys.includes(key)); + const hasBusinessDate = Object.hasOwn(value, 'businessDate'); + const hasBusinessTimezone = Object.hasOwn(value, 'businessTimezone'); + if (!shapeValid || hasBusinessDate !== hasBusinessTimezone + || (hasBusinessDate && (!DATE_RE.test(value.businessDate) || !validTimezone(value.businessTimezone))) + || !['shadow', 'additions_edits_preview', 'additions_edits'].includes(value.mode) + || !['no_change', 'published'].includes(value.disposition) + || !['shadow', 'baseline_required', 'preview', 'ready'].includes(value.publicationStatus) + || typeof value.wouldPublish !== 'boolean' + || (value.mirror !== null && (!exactKeys(value.mirror, MIRROR_COUNT_KEYS) + || MIRROR_COUNT_KEYS.some(key => !Number.isInteger(value.mirror[key]) || value.mirror[key] < 0))) + || (value.publications !== null && (!exactKeys(value.publications, [ + 'rosterPublicationId', 'timecardPublicationId', 'resourceLinkPublicationId', + ]) || Object.values(value.publications).some(id => typeof id !== 'string' || !UUID_RE.test(id))))) { + fail('invalid_request'); + } + if (Object.hasOwn(value, 'delta')) validateBusinessDelta(value.delta); + if (Object.hasOwn(value, 'persistence')) validateTimecardPersistence(value.persistence); + if ((value.mode === 'shadow' && (value.publicationStatus !== 'shadow' || value.wouldPublish + || value.mirror !== null || value.publications !== null)) + || (value.publicationStatus === 'baseline_required' && (!['additions_edits_preview', 'additions_edits'].includes(value.mode) + || value.disposition !== 'no_change' || value.wouldPublish || value.mirror !== null || value.publications !== null)) + || (value.publicationStatus === 'preview' && (value.mode !== 'additions_edits_preview' + || value.disposition !== 'no_change' || value.mirror === null || value.publications !== null)) + || (Object.hasOwn(value, 'persistence') + && (value.mode !== 'additions_edits' || value.publicationStatus !== 'ready')) + || (Object.hasOwn(value, 'delta') + && (!hasBusinessDate || value.mode !== 'additions_edits' || value.publicationStatus !== 'ready')) + || (value.disposition === 'published' && (value.mode !== 'additions_edits' || !value.wouldPublish + || value.publicationStatus !== 'ready' || value.mirror === null || value.publications === null))) { + fail('invalid_request'); + } + return value; +} + +function isoDate(value) { + if (typeof value !== 'string' || !DATE_RE.test(value)) fail('candidate_invalid'); + const date = new Date(`${value}T00:00:00Z`); + if (Number.isNaN(date.valueOf()) || date.toISOString().slice(0, 10) !== value) fail('candidate_invalid'); + return date; +} + +function validatePeriods(candidate) { + if (!Array.isArray(candidate.rows) || candidate.rows.length < 1 || candidate.rows.length > 64) fail('candidate_invalid'); + const seen = new Set(); + for (const row of candidate.rows) { + if (!exactKeys(row, ['start', 'end', 'key', 'relation']) || !['previous', 'current', 'next'].includes(row.relation)) fail('candidate_invalid'); + const period = parsePeriodKey(row.key); + if (period.start !== row.start || period.end !== row.end || seen.has(row.key)) fail('candidate_invalid'); + seen.add(row.key); + } + if (candidate.rows.filter(row => row.relation === 'current').length !== 1) fail('candidate_invalid'); +} + +function validateRoster(candidate) { + if (!Array.isArray(candidate.rows) || candidate.rows.length < 1 || candidate.rows.length > 5000) fail('candidate_invalid'); + const seen = new Set(); + for (const row of candidate.rows) { + if (!plain(row) || typeof row.employeeCode !== 'string' || !/^[A-Za-z0-9]{4}$/.test(row.employeeCode) + || typeof row.employeeName !== 'string' || !row.employeeName.trim() || row.employeeName.length > 300 + || typeof row.isActive !== 'boolean' || typeof row.isActiveDriver !== 'boolean' + || (row.lifecycleStatus !== undefined && !['active', 'inactive', 'unknown'].includes(row.lifecycleStatus)) + || seen.has(row.employeeCode.toUpperCase())) fail('candidate_invalid'); + seen.add(row.employeeCode.toUpperCase()); + } +} + +function validateTimecards(candidate) { + const period = parsePeriodKey(candidate.periodKey); + if (candidate.metadata.periodStart !== period.start || candidate.metadata.periodEnd !== period.end) fail('candidate_invalid'); + if (candidate.target !== period.end || !Array.isArray(candidate.rows) || candidate.rows.length < 1 || candidate.rows.length > 5000) fail('candidate_invalid'); + const seen = new Set(); + for (const row of candidate.rows) { + const legacy = exactKeys(row, ['employeeCode', 'employeeName', 'record', 'sourceSha256']); + const semantic = exactKeys(row, ['employeeCode', 'employeeName', 'record', 'sourceSha256', 'businessSha256', 'observedAt']); + if ((!legacy && !semantic) + || typeof row.employeeCode !== 'string' || !/^[A-Z0-9]{4}$/.test(row.employeeCode) + || typeof row.employeeName !== 'string' || !row.employeeName.trim() || row.employeeName.length > 300 + || !/^[a-f0-9]{64}$/.test(row.sourceSha256) || seen.has(row.employeeCode.toUpperCase())) fail('candidate_invalid'); + const sourceUrlValid = legacy + ? isCapturedTimecardUrl(row.record?.sourceUrl, { employeeCode: row.employeeCode, period }) + : isCanonicalTimecardUrl(row.record?.sourceUrl, { employeeCode: row.employeeCode, period }); + if (!sourceUrlValid) fail('candidate_invalid'); + if (semantic && (!/^[a-f0-9]{64}$/.test(row.businessSha256) + || row.businessSha256 !== timecardBusinessSha256(row.record) + || typeof row.observedAt !== 'string' || Number.isNaN(Date.parse(row.observedAt)))) fail('candidate_invalid'); + validateTimecardRecord(row.record, { employeeCode: row.employeeCode, period, sourceUrl: row.record?.sourceUrl }); + seen.add(row.employeeCode.toUpperCase()); + } +} + +function validateCandidate(candidate) { + if (!plain(candidate) || !KINDS.has(candidate.kind) || !RUN_RE.test(candidate.runId) + || !Number.isInteger(candidate.attempt) || candidate.attempt < 1 || candidate.attempt > 10 + || typeof candidate.collectedAt !== 'string' || Number.isNaN(Date.parse(candidate.collectedAt)) + || !plain(candidate.metadata) || !Array.isArray(candidate.rows)) fail('candidate_invalid'); + const targetDate = isoDate(candidate.target); + if (['roster', 'resource_links'].includes(candidate.kind) && targetDate.getUTCDay() !== 6) fail('candidate_invalid'); + if (candidate.kind === 'pay_periods') validatePeriods(candidate); + else if (candidate.kind === 'roster') validateRoster(candidate); + else if (candidate.kind === 'timecards') { + if (typeof candidate.periodKey !== 'string') fail('candidate_invalid'); + validateTimecards(candidate); + } else { + if (typeof candidate.periodKey !== 'string') fail('candidate_invalid'); + try { validateResourceLinkCandidate(candidate); } catch { fail('candidate_invalid'); } + } + return candidate; +} + +function validateWorkforcePreviewMembership(rosterRows, timecardRows, resourceLinkRows) { + const expected = new Map(rosterRows.filter(row => row.isActive === true) + .map(row => [row.employeeCode.toUpperCase(), row.employeeName])); + const timecards = new Map(timecardRows.map(row => [row.employeeCode, row.employeeName])); + const links = new Set(resourceLinkRows.map(row => row.employeeCode)); + if (expected.size < 1 || expected.size !== timecards.size || expected.size !== links.size + || [...expected].some(([code, name]) => timecards.get(code) !== name || !links.has(code))) { + fail('membership_mismatch'); + } +} + +function stageCandidate(stagingRoot, candidate) { + validateCandidate(candidate); + const bytes = Buffer.from(`${JSON.stringify(candidate)}\n`); + if (bytes.length > MAX_STAGE_BYTES) fail('candidate_too_large'); + const root = privateDirectory(stagingRoot); + if (!RUN_RE.test(candidate.runId)) fail('candidate_invalid'); + const directory = path.join(root, `${candidate.runId}.attempt-${candidate.attempt}`); + if (fs.existsSync(directory)) cleanupStage({ directory }, root); + fs.mkdirSync(directory, { mode: 0o700 }); + try { + const temporary = path.join(directory, '.candidate.tmp'); + const destination = path.join(directory, 'candidate.json'); + const descriptor = fs.openSync(temporary, 'wx', 0o600); + try { fs.writeFileSync(descriptor, bytes); fs.fsyncSync(descriptor); } finally { fs.closeSync(descriptor); } + fs.renameSync(temporary, destination); + const directoryFd = fs.openSync(directory, 'r'); + try { fs.fsyncSync(directoryFd); } finally { fs.closeSync(directoryFd); } + return { directory, file: destination, bytes: bytes.length, sha256: sha256(bytes) }; + } catch (error) { + // A failed write never reaches the caller's publication finally block. + cleanupStage({ directory }, root); + throw error; + } +} + +function readStaged(stage) { + privateDirectory(stage.directory, { create: false }); + privateFile(stage.file); + const bytes = fs.readFileSync(stage.file); + if (bytes.length > MAX_STAGE_BYTES || sha256(bytes) !== stage.sha256 || !bytes.toString('utf8').endsWith('\n')) fail('candidate_invalid'); + let candidate; + try { candidate = JSON.parse(bytes.subarray(0, bytes.length - 1).toString('utf8')); } catch { fail('candidate_invalid'); } + return validateCandidate(candidate); +} + +function cleanupStage(stage, stagingRoot) { + const root = privateDirectory(stagingRoot, { create: false }); + const directory = path.resolve(stage.directory); + if (path.dirname(directory) !== root || directory === root) fail('stage_cleanup_failed'); + try { + privateDirectory(directory, { create: false }); + fs.rmSync(directory, { recursive: true, force: false, maxRetries: 3, retryDelay: 50 }); + const rootFd = fs.openSync(root, 'r'); + try { fs.fsyncSync(rootFd); } finally { fs.closeSync(rootFd); } + try { fs.lstatSync(directory); } catch (error) { if (error.code === 'ENOENT') return; throw error; } + } catch { fail('stage_cleanup_failed'); } + fail('stage_cleanup_failed'); +} + +function cleanupRunStages(stagingRoot, runId) { + if (!RUN_RE.test(runId)) fail('invalid_request'); + try { fs.lstatSync(stagingRoot); } catch (error) { if (error.code === 'ENOENT') return; throw error; } + const root = privateDirectory(stagingRoot, { create: false }); + const prefix = `${runId}.attempt-`; + // Collection Manager serializes attempts of a run. Remove only that run's + // staging, including an interrupted earlier attempt or a committed replay. + for (const name of fs.readdirSync(root)) { + if (name.startsWith(prefix) && /^(?:[1-9]|10)$/.test(name.slice(prefix.length))) { + cleanupStage({ directory: path.join(root, name) }, root); + } + } +} + +function ensurePaycomSchema(db, { readOnly = false } = {}) { + let version = db.prepare('SELECT version FROM schema_meta').get()?.version; + if (version === 5) return; + if (readOnly || ![1, 2, 3, 4].includes(version)) fail('schema_invalid'); + if (version === 1) { + db.exec('BEGIN IMMEDIATE'); + try { + db.exec(` + CREATE TABLE IF NOT EXISTS resource_link_publications( + id TEXT PRIMARY KEY, + resource_type TEXT NOT NULL CHECK(resource_type='paycom.timecard.summary'), + target TEXT NOT NULL, + period_key TEXT NOT NULL, + run_id TEXT NOT NULL, + collected_at TEXT NOT NULL, + roster_publication_id TEXT NOT NULL REFERENCES publications(id) ON DELETE CASCADE, + roster_content_sha256 TEXT NOT NULL, + route_version INTEGER NOT NULL CHECK(route_version=1), + content_sha256 TEXT NOT NULL, + row_count INTEGER NOT NULL, + UNIQUE(resource_type,target,content_sha256) + ); + CREATE TABLE IF NOT EXISTS resource_links( + publication_id TEXT NOT NULL REFERENCES resource_link_publications(id) ON DELETE CASCADE, + employee_code TEXT NOT NULL, + canonical_url TEXT NOT NULL, + PRIMARY KEY(publication_id,employee_code) + ); + CREATE TABLE IF NOT EXISTS active_resource_link_publications( + resource_type TEXT NOT NULL, + target TEXT NOT NULL, + publication_id TEXT NOT NULL UNIQUE REFERENCES resource_link_publications(id) ON DELETE CASCADE, + activated_at TEXT NOT NULL, + PRIMARY KEY(resource_type,target) + ); + CREATE INDEX IF NOT EXISTS resource_link_publications_by_type_target + ON resource_link_publications(resource_type,target,collected_at DESC); + DROP TABLE schema_meta; + CREATE TABLE schema_meta(version INTEGER NOT NULL CHECK(version=2)); + INSERT INTO schema_meta(version) VALUES(2); + `); + db.exec('COMMIT'); + version = 2; + } catch (error) { + try { db.exec('ROLLBACK'); } catch {} + throw error; + } + } + if (version === 2) { + db.exec('BEGIN IMMEDIATE'); + try { + db.exec(` + ALTER TABLE timecards ADD COLUMN business_sha256 TEXT; + ALTER TABLE timecards ADD COLUMN observed_at TEXT; + CREATE TABLE paycom_sync_state( + source_id TEXT NOT NULL, + target TEXT NOT NULL, + last_run_id TEXT NOT NULL, + checked_at TEXT NOT NULL, + source_sha256 TEXT NOT NULL, + employee_count INTEGER NOT NULL, + pending_removal_sha256 TEXT, + pending_removal_count INTEGER NOT NULL DEFAULT 0, + last_receipt_json TEXT NOT NULL, + PRIMARY KEY(source_id,target) + ); + CREATE TABLE paycom_sync_employees( + source_id TEXT NOT NULL, + target TEXT NOT NULL, + employee_code TEXT NOT NULL, + profile_sha256 TEXT NOT NULL, + summary_sha256 TEXT NOT NULL, + last_observed_at TEXT NOT NULL, + missing_observations INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY(source_id,target,employee_code) + ); + CREATE INDEX paycom_sync_employees_by_target + ON paycom_sync_employees(source_id,target,last_observed_at); + DROP TABLE schema_meta; + CREATE TABLE schema_meta(version INTEGER NOT NULL CHECK(version=3)); + INSERT INTO schema_meta(version) VALUES(3); + `); + db.exec('COMMIT'); + version = 3; + } catch (error) { + try { db.exec('ROLLBACK'); } catch {} + throw error; + } + } + if (version === 3) { + db.exec('BEGIN IMMEDIATE'); + try { + db.exec(` + ALTER TABLE paycom_sync_state ADD COLUMN last_full_reconciled_at TEXT; + ALTER TABLE paycom_sync_state ADD COLUMN full_reconcile_anchor_at TEXT; + UPDATE paycom_sync_state SET full_reconcile_anchor_at=checked_at WHERE full_reconcile_anchor_at IS NULL; + ALTER TABLE paycom_sync_employees ADD COLUMN timecard_business_sha256 TEXT; + ALTER TABLE paycom_sync_employees ADD COLUMN last_timecard_observed_at TEXT; + ALTER TABLE paycom_sync_employees ADD COLUMN last_full_verified_at TEXT; + DROP TABLE schema_meta; + CREATE TABLE schema_meta(version INTEGER NOT NULL CHECK(version=4)); + INSERT INTO schema_meta(version) VALUES(4); + `); + db.exec('COMMIT'); + version = 4; + } catch (error) { + try { db.exec('ROLLBACK'); } catch {} + throw error; + } + } + if (version === 4) { + db.exec('BEGIN IMMEDIATE'); + try { + db.exec(` + CREATE TABLE paycom_sync_change_history( + run_id TEXT PRIMARY KEY, + source_id TEXT NOT NULL, + target TEXT NOT NULL, + business_date TEXT NOT NULL, + business_timezone TEXT NOT NULL, + observed_at TEXT NOT NULL, + disposition TEXT NOT NULL CHECK(disposition IN('published','no_change')), + delta_json TEXT NOT NULL, + persistence_json TEXT NOT NULL + ); + CREATE INDEX paycom_sync_change_history_by_source_target + ON paycom_sync_change_history(source_id,target,observed_at DESC,run_id DESC); + DROP TABLE schema_meta; + CREATE TABLE schema_meta(version INTEGER NOT NULL CHECK(version=5)); + INSERT INTO schema_meta(version) VALUES(5); + `); + db.exec('COMMIT'); + version = 5; + } catch (error) { + try { db.exec('ROLLBACK'); } catch {} + throw error; + } + } +} + +class PaycomStore { + constructor(file, { readOnly = false } = {}) { + this.file = path.resolve(file); + privateDirectory(path.dirname(this.file), { create: !readOnly }); + if (!fs.existsSync(this.file) && readOnly) fail('not_initialized'); + if (!fs.existsSync(this.file)) privateFile(this.file, { create: true }); + else privateFile(this.file); + this.db = new DatabaseSync(this.file, { readOnly }); + if (readOnly) { + this.db.exec('PRAGMA query_only=ON; PRAGMA foreign_keys=ON; PRAGMA trusted_schema=OFF; PRAGMA busy_timeout=5000;'); + ensurePaycomSchema(this.db, { readOnly: true }); + } else { + this.db.exec(` + PRAGMA foreign_keys=ON; + PRAGMA journal_mode=WAL; + PRAGMA synchronous=FULL; + PRAGMA busy_timeout=5000; + CREATE TABLE IF NOT EXISTS schema_meta(version INTEGER NOT NULL CHECK(version=1)); + INSERT INTO schema_meta(version) SELECT 1 WHERE NOT EXISTS(SELECT 1 FROM schema_meta); + CREATE TABLE IF NOT EXISTS publications( + id TEXT PRIMARY KEY, kind TEXT NOT NULL CHECK(kind IN('pay_periods','roster','timecards')), + target TEXT NOT NULL, run_id TEXT NOT NULL, collected_at TEXT NOT NULL, + content_sha256 TEXT NOT NULL, row_count INTEGER NOT NULL, metadata_json TEXT NOT NULL, + UNIQUE(kind,target,content_sha256) + ); + CREATE TABLE IF NOT EXISTS active_publications( + kind TEXT NOT NULL, target TEXT NOT NULL, publication_id TEXT NOT NULL UNIQUE REFERENCES publications(id), + activated_at TEXT NOT NULL, PRIMARY KEY(kind,target) + ); + CREATE TABLE IF NOT EXISTS publication_fences( + run_id TEXT PRIMARY KEY, max_attempt INTEGER NOT NULL CHECK(max_attempt >= 1) + ); + CREATE TABLE IF NOT EXISTS pay_periods( + publication_id TEXT NOT NULL REFERENCES publications(id), period_start TEXT NOT NULL, + period_end TEXT NOT NULL, period_key TEXT NOT NULL, relation TEXT NOT NULL, + PRIMARY KEY(publication_id,period_key) + ); + CREATE TABLE IF NOT EXISTS roster_employees( + publication_id TEXT NOT NULL REFERENCES publications(id), employee_code TEXT NOT NULL, + employee_name TEXT NOT NULL, is_active INTEGER NOT NULL, is_active_driver INTEGER NOT NULL, + record_json TEXT NOT NULL, PRIMARY KEY(publication_id,employee_code) + ); + CREATE TABLE IF NOT EXISTS timecards( + publication_id TEXT NOT NULL REFERENCES publications(id), employee_code TEXT NOT NULL, + employee_name TEXT NOT NULL, period_total_hours REAL NOT NULL, missing_days INTEGER NOT NULL, + source_sha256 TEXT NOT NULL, record_json TEXT NOT NULL, + PRIMARY KEY(publication_id,employee_code) + ); + CREATE INDEX IF NOT EXISTS publications_by_kind_target ON publications(kind,target,collected_at DESC); + `); + ensurePaycomSchema(this.db); + } + privateFile(this.file); + } + + validateBoundTimecardMembership(candidate) { + if (candidate.kind !== 'timecards' || candidate.metadata.mode === 'historical_period_membership') return null; + if (!['full', 'incremental', 'published_roster', 'sync_merge'].includes(candidate.metadata.mode)) fail('membership_mismatch'); + const roster = this.active('roster', candidate.target); + if (!roster || candidate.metadata.rosterPublicationId !== roster.id + || candidate.metadata.rosterContentSha256 !== roster.content_sha256) fail('membership_mismatch'); + const expected = this.db.prepare(`SELECT employee_code employeeCode,employee_name employeeName + FROM roster_employees WHERE publication_id=? AND is_active=1 ORDER BY employee_code`).all(roster.id); + const actual = [...candidate.rows].sort((left, right) => left.employeeCode.localeCompare(right.employeeCode)); + if (expected.length !== actual.length || expected.some((row, index) => row.employeeCode !== actual[index].employeeCode + || row.employeeName !== actual[index].employeeName)) fail('membership_mismatch'); + return { rosterPublicationId: roster.id, activeEmployees: expected.length, timecards: actual.length }; + } + + validateResourceLinkMembership(candidate) { + const roster = this.active('roster', candidate.target); + if (!roster || candidate.metadata.rosterPublicationId !== roster.id + || candidate.metadata.rosterContentSha256 !== roster.content_sha256) fail('membership_mismatch'); + const expected = this.db.prepare(`SELECT employee_code employeeCode FROM roster_employees + WHERE publication_id=? AND is_active=1 ORDER BY employee_code`).all(roster.id); + const actual = [...candidate.rows].sort((left, right) => left.employeeCode.localeCompare(right.employeeCode)); + if (expected.length !== actual.length + || expected.some((row, index) => row.employeeCode !== actual[index].employeeCode)) fail('membership_mismatch'); + return { rosterPublicationId: roster.id, activeEmployees: expected.length, links: actual.length }; + } + + auditResourceLinkPublication(publicationId, { databaseChecks = true } = {}) { + const publication = this.db.prepare('SELECT * FROM resource_link_publications WHERE id=?').get(publicationId); + if (!publication) return { verified: false, code: 'not_loaded', publicationId }; + const metadata = { + resourceType: publication.resource_type, + periodStart: publication.period_key.slice(0, 10), + periodEnd: publication.period_key.slice(11), + rosterPublicationId: publication.roster_publication_id, + rosterContentSha256: publication.roster_content_sha256, + routeVersion: publication.route_version, + }; + const rows = this.db.prepare(`SELECT employee_code employeeCode,canonical_url canonicalUrl + FROM resource_links WHERE publication_id=? ORDER BY employee_code`).all(publication.id) + .map(row => ({ employeeCode: row.employeeCode, canonicalUrl: row.canonicalUrl })); + const candidate = { + kind: 'resource_links', target: publication.target, periodKey: publication.period_key, + runId: publication.run_id, attempt: 1, collectedAt: publication.collected_at, metadata, rows, + }; + let projectionValid = true; + try { validateResourceLinkCandidate(candidate); } catch { projectionValid = false; } + const calculatedContentSha256 = projectionValid ? contentSha256(candidate) : null; + const roster = this.db.prepare(`SELECT p.id,p.content_sha256 FROM active_publications a + JOIN publications p ON p.id=a.publication_id WHERE a.kind='roster' AND a.target=?`).get(publication.target); + const rosterValid = roster?.id === publication.roster_publication_id + && roster?.content_sha256 === publication.roster_content_sha256; + const quick = databaseChecks ? this.db.prepare('PRAGMA quick_check').get()?.quick_check : 'ok'; + const foreignKeyErrors = databaseChecks ? this.db.prepare('PRAGMA foreign_key_check').all().length : 0; + const verified = quick === 'ok' && foreignKeyErrors === 0 && projectionValid && rosterValid + && rows.length === publication.row_count && calculatedContentSha256 === publication.content_sha256; + return { + verified, code: verified ? 'verified' : 'integrity_failed', + kind: 'resource_links', resourceType: publication.resource_type, target: publication.target, + publicationId: publication.id, rosterPublicationId: publication.roster_publication_id, + rowCount: rows.length, contentSha256: publication.content_sha256, calculatedContentSha256, + collectedAt: publication.collected_at, quickCheck: quick, foreignKeyErrors, projectionValid, rosterValid, + }; + } + + publishResourceLinks(candidate, contentHash, { transaction = true } = {}) { + if (typeof transaction !== 'boolean') fail('invalid_request'); + let existing; + let wasActive = false; + let publicationId; + let membership; + if (transaction) this.db.exec('BEGIN IMMEDIATE'); + try { + const fence = this.db.prepare('SELECT max_attempt FROM publication_fences WHERE run_id=?').get(candidate.runId); + if (fence && candidate.attempt < fence.max_attempt) fail('stale_collection_attempt'); + this.db.prepare(`INSERT INTO publication_fences(run_id,max_attempt) VALUES(?,?) + ON CONFLICT(run_id) DO UPDATE SET max_attempt=MAX(max_attempt,excluded.max_attempt)`).run(candidate.runId, candidate.attempt); + membership = this.validateResourceLinkMembership(candidate); + existing = this.db.prepare(`SELECT id FROM resource_link_publications + WHERE resource_type=? AND target=? AND content_sha256=?`) + .get(candidate.metadata.resourceType, candidate.target, contentHash); + const active = this.db.prepare(`SELECT publication_id FROM active_resource_link_publications + WHERE resource_type=? AND target=?`).get(candidate.metadata.resourceType, candidate.target); + wasActive = Boolean(existing && active?.publication_id === existing.id); + publicationId = existing?.id || crypto.randomUUID(); + if (!existing) { + this.db.prepare('INSERT INTO resource_link_publications VALUES(?,?,?,?,?,?,?,?,?,?,?)').run( + publicationId, candidate.metadata.resourceType, candidate.target, candidate.periodKey, + candidate.runId, candidate.collectedAt, candidate.metadata.rosterPublicationId, + candidate.metadata.rosterContentSha256, candidate.metadata.routeVersion, contentHash, candidate.rows.length, + ); + const insert = this.db.prepare('INSERT INTO resource_links VALUES(?,?,?)'); + for (const row of candidate.rows) insert.run(publicationId, row.employeeCode, row.canonicalUrl); + } + const verified = this.auditResourceLinkPublication(publicationId, { databaseChecks: false }); + if (!verified.verified || verified.rowCount !== candidate.rows.length + || verified.contentSha256 !== contentHash) fail('publication_verification_failed'); + this.db.prepare(`INSERT INTO active_resource_link_publications(resource_type,target,publication_id,activated_at) + VALUES(?,?,?,?) ON CONFLICT(resource_type,target) DO UPDATE SET + publication_id=excluded.publication_id,activated_at=excluded.activated_at`) + .run(candidate.metadata.resourceType, candidate.target, publicationId, candidate.collectedAt); + const rollback = this.db.prepare(`SELECT id FROM resource_link_publications + WHERE resource_type=? AND target=? AND id<>? ORDER BY collected_at DESC,id DESC`) + .all(candidate.metadata.resourceType, candidate.target, publicationId); + const remove = this.db.prepare('DELETE FROM resource_link_publications WHERE id=?'); + for (const row of rollback.slice(1)) remove.run(row.id); + if (transaction) this.db.exec('COMMIT'); + } catch (error) { + if (transaction) try { this.db.exec('ROLLBACK'); } catch {} + throw error; + } + return { + disposition: wasActive ? 'no_change' : existing ? 'reactivated' : 'published', + publicationId, rowCount: candidate.rows.length, contentSha256: contentHash, membership, + }; + } + + publish(stage, { transaction = true } = {}) { + if (typeof transaction !== 'boolean') fail('invalid_request'); + const candidate = readStaged(stage); + const contentHash = contentSha256(candidate); + if (candidate.kind === 'resource_links') return this.publishResourceLinks(candidate, contentHash, { transaction }); + let existing; + let wasActive = false; + let publicationId; + let membership = null; + if (transaction) this.db.exec('BEGIN IMMEDIATE'); + try { + const fence = this.db.prepare('SELECT max_attempt FROM publication_fences WHERE run_id=?').get(candidate.runId); + if (fence && candidate.attempt < fence.max_attempt) fail('stale_collection_attempt'); + this.db.prepare(`INSERT INTO publication_fences(run_id,max_attempt) VALUES(?,?) + ON CONFLICT(run_id) DO UPDATE SET max_attempt=MAX(max_attempt,excluded.max_attempt)`).run(candidate.runId, candidate.attempt); + membership = this.validateBoundTimecardMembership(candidate); + existing = this.db.prepare('SELECT id FROM publications WHERE kind=? AND target=? AND content_sha256=?') + .get(candidate.kind, candidate.target, contentHash); + const active = this.db.prepare('SELECT publication_id FROM active_publications WHERE kind=? AND target=?') + .get(candidate.kind, candidate.target); + wasActive = Boolean(existing && active?.publication_id === existing.id); + publicationId = existing?.id || crypto.randomUUID(); + if (!existing) { + this.db.prepare('INSERT INTO publications VALUES(?,?,?,?,?,?,?,?)').run( + publicationId, candidate.kind, candidate.target, candidate.runId, candidate.collectedAt, + contentHash, candidate.rows.length, JSON.stringify(candidate.metadata), + ); + if (candidate.kind === 'pay_periods') { + const insert = this.db.prepare('INSERT INTO pay_periods VALUES(?,?,?,?,?)'); + for (const row of candidate.rows) insert.run(publicationId, row.start, row.end, row.key, row.relation); + } else if (candidate.kind === 'roster') { + const insert = this.db.prepare('INSERT INTO roster_employees VALUES(?,?,?,?,?,?)'); + for (const row of candidate.rows) insert.run(publicationId, row.employeeCode, row.employeeName, Number(row.isActive), Number(row.isActiveDriver), JSON.stringify(row)); + } else { + const insert = this.db.prepare(`INSERT INTO timecards( + publication_id,employee_code,employee_name,period_total_hours,missing_days, + source_sha256,record_json,business_sha256,observed_at + ) VALUES(?,?,?,?,?,?,?,?,?)`); + for (const row of candidate.rows) insert.run( + publicationId, row.employeeCode, row.employeeName, row.record.periodTotalHours, + row.record.days.filter(day => day.missingPunch).length, row.sourceSha256, JSON.stringify(row.record), + row.businessSha256 || null, row.observedAt || null, + ); + } + } + const verified = this.auditPublication(publicationId, { databaseChecks: false }); + if (!verified.verified || verified.rowCount !== candidate.rows.length || verified.contentSha256 !== contentHash) fail('publication_verification_failed'); + this.db.prepare(`INSERT INTO active_publications(kind,target,publication_id,activated_at) VALUES(?,?,?,?) + ON CONFLICT(kind,target) DO UPDATE SET publication_id=excluded.publication_id,activated_at=excluded.activated_at`) + .run(candidate.kind, candidate.target, publicationId, candidate.collectedAt); + if (candidate.kind === 'roster') { + this.db.prepare(`DELETE FROM active_resource_link_publications WHERE target=? AND publication_id IN( + SELECT id FROM resource_link_publications WHERE roster_publication_id<>? + )`).run(candidate.target, publicationId); + } + const rollback = this.db.prepare('SELECT id FROM publications WHERE kind=? AND target=? AND id<>? ORDER BY collected_at DESC,id DESC').all(candidate.kind, candidate.target, publicationId); + const childTable = candidate.kind === 'pay_periods' ? 'pay_periods' : candidate.kind === 'roster' ? 'roster_employees' : 'timecards'; + const removeRows = this.db.prepare(`DELETE FROM ${childTable} WHERE publication_id=?`); + const removePublication = this.db.prepare('DELETE FROM publications WHERE id=?'); + for (const row of rollback.slice(1)) { + removeRows.run(row.id); + removePublication.run(row.id); + } + if (transaction) this.db.exec('COMMIT'); + } catch (error) { + if (transaction) try { this.db.exec('ROLLBACK'); } catch {} + throw error; + } + return { + disposition: wasActive ? 'no_change' : existing ? 'reactivated' : 'published', + publicationId, rowCount: candidate.rows.length, contentSha256: contentHash, + ...(membership ? { membership } : {}), + }; + } + + publishWorkforceSync({ + runId, attempt, collectedAt, coverageDate, businessTimezone, period, sourceSha256, sourceFormat, + sourceEmployees, mirrorPlan, observation, stagingRoot, base, preview = false, + }) { + const businessDate = coverageDate === undefined + ? (typeof collectedAt === 'string' ? collectedAt.slice(0, 10) : '') + : coverageDate; + const persistenceDate = businessDate; + const baseKeys = [ + 'rosterPublicationId', 'rosterContentSha256', 'timecardPublicationId', + 'timecardContentSha256', 'resourceLinkPublicationId', 'resourceLinkContentSha256', + ]; + const bootstrap = base === null; + const baseValid = bootstrap || exactKeys(base, baseKeys) + && UUID_RE.test(base.rosterPublicationId) && SHA256_RE.test(base.rosterContentSha256) + && UUID_RE.test(base.timecardPublicationId) && SHA256_RE.test(base.timecardContentSha256) + && ((base.resourceLinkPublicationId === null && base.resourceLinkContentSha256 === null) + || (UUID_RE.test(base.resourceLinkPublicationId) && SHA256_RE.test(base.resourceLinkContentSha256))); + if (!RUN_RE.test(runId) || !Number.isInteger(attempt) || attempt < 1 || attempt > 10 + || typeof collectedAt !== 'string' || Number.isNaN(Date.parse(collectedAt)) + || typeof persistenceDate !== 'string' || !DATE_RE.test(persistenceDate) + || !validTimezone(businessTimezone) + || !period || typeof period.start !== 'string' || typeof period.end !== 'string' || typeof period.key !== 'string' + || persistenceDate < period.start || persistenceDate > period.end + || !SHA256_RE.test(sourceSha256) || typeof sourceFormat !== 'string' + || !Array.isArray(sourceEmployees) || !mirrorPlan || typeof mirrorPlan.hasChanges !== 'boolean' + || !Array.isArray(mirrorPlan.rosterRows) || !Array.isArray(mirrorPlan.timecardRows) + || !Array.isArray(mirrorPlan.resourceLinkRows) || !plain(mirrorPlan.counts) + || !plain(observation) || observation.runId !== runId || observation.target !== period.end + || observation.observedAt !== collectedAt || observation.sourceSha256 !== sourceSha256 + || canonicalStringify(observation.employees) !== canonicalStringify(sourceEmployees) + || typeof stagingRoot !== 'string' || !baseValid || typeof preview !== 'boolean') fail('candidate_invalid'); + const replay = this.shadowReceiptForRun(observation.sourceId, observation.target, runId); + if (replay?.syncOutcome) { + const outcome = validateSyncOutcome(replay.syncOutcome); + return { + disposition: outcome.disposition, + wouldPublish: outcome.wouldPublish, + businessDate: outcome.businessDate || businessDate, + businessTimezone: outcome.businessTimezone || businessTimezone, + counts: outcome.mirror, + observation: replay, + ...(outcome.delta ? { delta: outcome.delta } : {}), + ...(outcome.persistence ? { persistence: outcome.persistence } : {}), + ...(outcome.publications || {}), + }; + } + const publishCandidate = candidate => { + const stage = stageCandidate(stagingRoot, candidate); + try { return this.publish(stage, { transaction: false }); } + finally { cleanupStage(stage, stagingRoot); } + }; + const previewCandidate = candidate => { + const stage = stageCandidate(stagingRoot, candidate); + try { + const validated = readStaged(stage); + return { candidate: validated, contentSha256: contentSha256(validated) }; + } finally { cleanupStage(stage, stagingRoot); } + }; + let rosterResult = null; + let timecardResult = null; + let linkResult = null; + let observationResult; + let outcome; + let delta = null; + let persistence = null; + this.db.exec('BEGIN IMMEDIATE'); + try { + const currentRoster = this.active('roster', period.end); + const currentTimecards = this.active('timecards', period.end); + const priorTimecardRows = this.activeTimecards(period.end)?.rows || []; + const currentLinks = this.activeResourceLinks(TIMECARD_SUMMARY, period.end)?.publication || null; + if (bootstrap ? Boolean(currentRoster || currentTimecards || currentLinks) + : !currentRoster || currentRoster.id !== base.rosterPublicationId + || currentRoster.content_sha256 !== base.rosterContentSha256 + || !currentTimecards || currentTimecards.id !== base.timecardPublicationId + || currentTimecards.content_sha256 !== base.timecardContentSha256 + || (currentLinks?.id || null) !== base.resourceLinkPublicationId + || (currentLinks?.content_sha256 || null) !== base.resourceLinkContentSha256) { + fail('publication_base_changed'); + } + if (bootstrap && (observation.sourceCompleteness !== 'authoritative' || !mirrorPlan.hasChanges)) { + fail('roster_source_not_authoritative'); + } + if (bootstrap && !preview) publishCandidate({ + kind: 'pay_periods', target: businessDate, runId, attempt, collectedAt, + metadata: { timezone: businessTimezone, basis: 'biweekly_anchor', anchorStart: ANCHOR_START }, + rows: [ + { ...previousPeriod(period), relation: 'previous' }, + { ...period, relation: 'current' }, + { ...nextPeriod(period), relation: 'next' }, + ].map(({ start, end, key, relation }) => ({ start, end, key, relation })), + }); + const fence = this.db.prepare('SELECT max_attempt FROM publication_fences WHERE run_id=?').get(runId); + if (fence && attempt < fence.max_attempt) fail('stale_collection_attempt'); + this.db.prepare(`INSERT INTO publication_fences(run_id,max_attempt) VALUES(?,?) + ON CONFLICT(run_id) DO UPDATE SET max_attempt=MAX(max_attempt,excluded.max_attempt)`).run(runId, attempt); + if (mirrorPlan.hasChanges || preview) { + const rosterCandidate = { + kind: 'roster', target: period.end, runId, attempt, collectedAt, + metadata: { + periodKey: period.key, + sourceSha256, + sourceFormat, + employeeCount: sourceEmployees.length, + activeEmployeeCount: sourceEmployees.filter(row => row.isActive).length, + activeDriverCount: sourceEmployees.filter(row => row.isActiveDriver).length, + mode: 'sync_merge', + absencePolicy: 'retain', + retainedMissingCount: mirrorPlan.counts.retainedMissingCount, + }, + rows: mirrorPlan.rosterRows, + }; + if (preview) { + const inspected = previewCandidate(rosterCandidate); + rosterResult = { publicationId: crypto.randomUUID(), contentSha256: inspected.contentSha256 }; + } else rosterResult = publishCandidate(rosterCandidate); + const timecardCandidate = { + kind: 'timecards', target: period.end, periodKey: period.key, runId, attempt, collectedAt, + metadata: { + periodStart: period.start, + periodEnd: period.end, + rosterPublicationId: rosterResult.publicationId, + rosterContentSha256: rosterResult.contentSha256, + mode: 'sync_merge', + }, + rows: mirrorPlan.timecardRows, + }; + const linkCandidate = { + kind: 'resource_links', target: period.end, periodKey: period.key, runId, attempt, collectedAt, + metadata: { + resourceType: TIMECARD_SUMMARY, + periodStart: period.start, + periodEnd: period.end, + rosterPublicationId: rosterResult.publicationId, + rosterContentSha256: rosterResult.contentSha256, + routeVersion: ROUTE_VERSION, + }, + rows: mirrorPlan.resourceLinkRows, + }; + if (preview) { + previewCandidate(timecardCandidate); + previewCandidate(linkCandidate); + validateWorkforcePreviewMembership( + rosterCandidate.rows, timecardCandidate.rows, linkCandidate.rows, + ); + } else { + timecardResult = publishCandidate(timecardCandidate); + linkResult = publishCandidate(linkCandidate); + } + } + const publications = !preview && rosterResult ? { + rosterPublicationId: rosterResult.publicationId, + timecardPublicationId: timecardResult.publicationId, + resourceLinkPublicationId: linkResult.publicationId, + } : null; + if (!preview) { + delta = computeBusinessDelta(priorTimecardRows, mirrorPlan.timecardRows, mirrorPlan.counts); + persistence = this.auditTimecardPersistence(period.end, persistenceDate, observation.timecardRows); + if (!persistence.verified) fail('integrity_failed'); + } + outcome = validateSyncOutcome({ + mode: preview ? 'additions_edits_preview' : 'additions_edits', + disposition: !preview && mirrorPlan.hasChanges ? 'published' : 'no_change', + publicationStatus: preview ? 'preview' : 'ready', + wouldPublish: mirrorPlan.hasChanges, + mirror: { ...mirrorPlan.counts }, + publications, + ...(!preview ? { businessDate, businessTimezone, delta } : {}), + ...(persistence ? { persistence } : {}), + }); + observationResult = this.observeWorkforceShadow({ ...observation, syncOutcome: outcome }, { transaction: false }); + if (!preview) { + this.db.prepare(`INSERT INTO paycom_sync_change_history( + run_id,source_id,target,business_date,business_timezone,observed_at,disposition,delta_json,persistence_json + ) VALUES(?,?,?,?,?,?,?,?,?)`).run( + runId, observation.sourceId, period.end, businessDate, businessTimezone, collectedAt, + outcome.disposition, JSON.stringify(delta), JSON.stringify(persistence), + ); + } + this.db.exec('COMMIT'); + } catch (error) { + try { this.db.exec('ROLLBACK'); } catch {} + throw error; + } + if (!preview) { + try { + const cutoff = new Date(Date.parse(collectedAt) - NO_CHANGE_HISTORY_RETENTION_MS).toISOString(); + this.compactSyncChangeHistory(cutoff, MAX_HISTORY_COMPACTION_DELETE); + } catch {} + } + return { + disposition: outcome.disposition, + wouldPublish: outcome.wouldPublish, + businessDate: outcome.businessDate || businessDate, + businessTimezone: outcome.businessTimezone || businessTimezone, + counts: outcome.mirror, + observation: observationResult, + ...(outcome.delta ? { delta: outcome.delta } : {}), + ...(outcome.persistence ? { persistence: outcome.persistence } : {}), + ...(outcome.publications || {}), + }; + } + + previewWorkforceSync(input) { + if (!plain(input) || Object.hasOwn(input, 'preview')) fail('candidate_invalid'); + return this.publishWorkforceSync({ ...input, preview: true }); + } + + active(kind, target = null) { + if (!KINDS.has(kind) || kind === 'resource_links') fail('invalid_query'); + const sql = target + ? 'SELECT p.* FROM active_publications a JOIN publications p ON p.id=a.publication_id WHERE a.kind=? AND a.target=?' + : 'SELECT p.* FROM active_publications a JOIN publications p ON p.id=a.publication_id WHERE a.kind=? ORDER BY p.target DESC LIMIT 1'; + return target ? this.db.prepare(sql).get(kind, target) : this.db.prepare(sql).get(kind); + } + + activeResourceLinks(resourceType = TIMECARD_SUMMARY, target = null) { + if (resourceType !== TIMECARD_SUMMARY) fail('invalid_query'); + const sql = target + ? `SELECT p.* FROM active_resource_link_publications a JOIN resource_link_publications p + ON p.id=a.publication_id WHERE a.resource_type=? AND a.target=?` + : `SELECT p.* FROM active_resource_link_publications a JOIN resource_link_publications p + ON p.id=a.publication_id WHERE a.resource_type=? ORDER BY p.target DESC LIMIT 1`; + const publication = target ? this.db.prepare(sql).get(resourceType, target) : this.db.prepare(sql).get(resourceType); + if (!publication) return null; + const rows = this.db.prepare(`SELECT employee_code employeeCode,canonical_url canonicalUrl + FROM resource_links WHERE publication_id=? ORDER BY employee_code`).all(publication.id) + .map(row => ({ employeeCode: row.employeeCode, canonicalUrl: row.canonicalUrl })); + if (rows.length !== publication.row_count) fail('resource_links_invalid'); + return { publication, rows }; + } + + auditResourceLinks(resourceType = TIMECARD_SUMMARY, target = null) { + const active = this.activeResourceLinks(resourceType, target); + if (!active) return { verified: false, code: 'not_loaded', resourceType, target }; + return this.auditResourceLinkPublication(active.publication.id); + } + + activeRoster(target = null) { + const publication = this.active('roster', target); + if (!publication) fail('roster_not_loaded'); + const employees = this.db.prepare('SELECT employee_code employeeCode,employee_name employeeName,is_active isActive,is_active_driver isActiveDriver,record_json recordJson FROM roster_employees WHERE publication_id=? ORDER BY employee_code').all(publication.id) + .map(row => ({ ...JSON.parse(row.recordJson), isActive: Boolean(row.isActive), isActiveDriver: Boolean(row.isActiveDriver) })); + if (employees.length !== publication.row_count) fail('roster_invalid'); + return { publication, employees }; + } + + activeTimecards(periodEnd) { + isoDate(periodEnd); + const publication = this.active('timecards', periodEnd); + if (!publication) return null; + const rows = this.db.prepare(`SELECT employee_code employeeCode,employee_name employeeName, + source_sha256 sourceSha256,business_sha256 businessSha256,observed_at observedAt,record_json recordJson + FROM timecards WHERE publication_id=? ORDER BY employee_code`).all(publication.id) + .map(row => ({ + employeeCode: row.employeeCode, + employeeName: row.employeeName, + sourceSha256: row.sourceSha256, + record: JSON.parse(row.recordJson), + ...(row.businessSha256 ? { businessSha256: row.businessSha256, observedAt: row.observedAt } : {}), + })); + if (rows.length !== publication.row_count) fail('timecards_invalid'); + return { publication, rows }; + } + + activeWorkforce(target = null) { + this.db.exec('BEGIN'); + try { + const roster = this.activeRoster(target); + const periodEnd = roster.publication.target; + const timecards = this.activeTimecards(periodEnd); + const resourceLinks = this.activeResourceLinks(TIMECARD_SUMMARY, periodEnd); + let timecardMetadata = null; + try { timecardMetadata = timecards ? JSON.parse(timecards.publication.metadata_json) : null; } catch {} + if (!timecards || !resourceLinks || !timecardMetadata + || timecardMetadata.rosterPublicationId !== roster.publication.id + || timecardMetadata.rosterContentSha256 !== roster.publication.content_sha256 + || resourceLinks.publication.roster_publication_id !== roster.publication.id + || resourceLinks.publication.roster_content_sha256 !== roster.publication.content_sha256) { + fail('workforce_inconsistent'); + } + this.db.exec('COMMIT'); + return { roster, timecards, resourceLinks }; + } catch (error) { + try { this.db.exec('ROLLBACK'); } catch {} + throw error; + } + } + + auditPublication(publicationId, { databaseChecks = true } = {}) { + const publication = this.db.prepare('SELECT * FROM publications WHERE id=?').get(publicationId); + if (!publication) return { verified: false, code: 'not_loaded', publicationId }; + const kind = publication.kind; + let rows = []; + let metadata; + let calculatedContentSha256 = null; + let projectionValid = true; + try { + metadata = JSON.parse(publication.metadata_json); + if (kind === 'pay_periods') { + rows = this.db.prepare('SELECT period_start start,period_end end,period_key key,relation FROM pay_periods WHERE publication_id=? ORDER BY period_key').all(publication.id) + .map(row => ({ start: row.start, end: row.end, key: row.key, relation: row.relation })); + } else if (kind === 'roster') { + rows = this.db.prepare('SELECT employee_code,employee_name,is_active,is_active_driver,record_json FROM roster_employees WHERE publication_id=? ORDER BY employee_code').all(publication.id).map(row => { + const record = JSON.parse(row.record_json); + if (row.employee_code !== record.employeeCode || row.employee_name !== record.employeeName + || Boolean(row.is_active) !== record.isActive || Boolean(row.is_active_driver) !== record.isActiveDriver) projectionValid = false; + return record; + }); + } else { + rows = this.db.prepare(`SELECT employee_code employeeCode,employee_name employeeName, + period_total_hours periodTotalHours,missing_days missingDays,source_sha256 sourceSha256, + business_sha256 businessSha256,observed_at observedAt,record_json recordJson + FROM timecards WHERE publication_id=? ORDER BY employee_code`).all(publication.id) + .map(row => { + const record = JSON.parse(row.recordJson); + if (row.employeeCode !== record.employeeCode || row.periodTotalHours !== record.periodTotalHours + || row.missingDays !== record.days.filter(day => day.missingPunch).length + || (row.businessSha256 && row.businessSha256 !== timecardBusinessSha256(record))) projectionValid = false; + return { + employeeCode: row.employeeCode, + employeeName: row.employeeName, + record, + sourceSha256: row.sourceSha256, + ...(row.businessSha256 ? { businessSha256: row.businessSha256, observedAt: row.observedAt } : {}), + }; + }); + } + if (kind === 'timecards') { + validateTimecards({ kind, target: publication.target, metadata, rows, periodKey: `${metadata.periodStart}_${metadata.periodEnd}` }); + } + calculatedContentSha256 = contentSha256({ + kind, target: publication.target, metadata, rows, + ...(kind === 'timecards' ? { periodKey: `${metadata.periodStart}_${metadata.periodEnd}` } : {}), + }); + } catch { projectionValid = false; } + const count = rows.length; + const quick = databaseChecks ? this.db.prepare('PRAGMA quick_check').get()?.quick_check : 'ok'; + const foreignKeyErrors = databaseChecks ? this.db.prepare('PRAGMA foreign_key_check').all().length : 0; + const verified = quick === 'ok' && foreignKeyErrors === 0 && projectionValid && count === publication.row_count + && calculatedContentSha256 === publication.content_sha256; + return { + verified, + code: verified ? 'verified' : 'integrity_failed', + kind, target: publication.target, publicationId: publication.id, rowCount: count, + contentSha256: publication.content_sha256, calculatedContentSha256, collectedAt: publication.collected_at, + quickCheck: quick, foreignKeyErrors, projectionValid, + }; + } + + auditPayPeriodTarget(periodEnd) { + isoDate(periodEnd); + const publication = this.active('pay_periods'); + if (!publication) return { verified: false, code: 'not_loaded', target: periodEnd }; + const audit = this.auditPublication(publication.id); + const period = this.db.prepare(`SELECT period_end,relation FROM pay_periods + WHERE publication_id=? AND period_end=?`).get(publication.id, periodEnd); + const verified = audit.verified && period?.relation === 'current'; + return { + verified, + code: verified ? 'verified' : audit.verified ? 'target_mismatch' : audit.code, + target: periodEnd, + publicationId: publication.id, + runId: publication.run_id, + contentSha256: publication.content_sha256, + collectedAt: publication.collected_at, + }; + } + + audit(kind, target = null) { + const publication = this.active(kind, target); + if (!publication) return { verified: false, code: 'not_loaded', kind, target }; + return this.auditPublication(publication.id); + } + + auditTimecards(periodEnd) { + isoDate(periodEnd); + const roster = this.active('roster', periodEnd); + const timecards = this.active('timecards', periodEnd); + if (!roster || !timecards) { + return { verified: false, code: !roster ? 'roster_not_loaded' : 'timecards_not_loaded', periodEnd }; + } + const rosterAudit = this.auditPublication(roster.id, { databaseChecks: false }); + const timecardAudit = this.auditPublication(timecards.id); + let metadata = null; + try { metadata = JSON.parse(timecards.metadata_json); } catch {} + const rosterBindingValid = Boolean(metadata + && ['full', 'incremental', 'published_roster', 'sync_merge'].includes(metadata.mode) + && metadata.periodEnd === periodEnd + && metadata.rosterPublicationId === roster.id + && metadata.rosterContentSha256 === roster.content_sha256); + const expected = this.db.prepare(`SELECT employee_code employeeCode,employee_name employeeName + FROM roster_employees WHERE publication_id=? AND is_active=1 ORDER BY employee_code`).all(roster.id); + const actual = this.db.prepare(`SELECT employee_code employeeCode,employee_name employeeName + FROM timecards WHERE publication_id=? ORDER BY employee_code`).all(timecards.id); + const expectedByCode = new Map(expected.map(row => [row.employeeCode, row.employeeName])); + const actualByCode = new Map(actual.map(row => [row.employeeCode, row.employeeName])); + const missingCount = expected.reduce((count, row) => count + Number(!actualByCode.has(row.employeeCode)), 0); + const unexpectedCount = actual.reduce((count, row) => count + Number(!expectedByCode.has(row.employeeCode)), 0); + const identityMismatchCount = actual.reduce((count, row) => count + + Number(expectedByCode.has(row.employeeCode) && expectedByCode.get(row.employeeCode) !== row.employeeName), 0); + const verified = rosterAudit.verified && timecardAudit.verified && rosterBindingValid + && missingCount === 0 && unexpectedCount === 0 && identityMismatchCount === 0 + && expected.length === actual.length; + const code = verified ? 'verified' + : !rosterAudit.verified || !timecardAudit.verified ? 'integrity_failed' : 'membership_mismatch'; + return { + verified, code, periodEnd, + rosterPublicationId: roster.id, timecardPublicationId: timecards.id, + activeEmployees: expected.length, timecards: actual.length, + missingCount, unexpectedCount, duplicateCount: 0, identityMismatchCount, + rosterBindingValid, rosterProjectionValid: rosterAudit.projectionValid, + timecardProjectionValid: timecardAudit.projectionValid, + quickCheck: timecardAudit.quickCheck, foreignKeyErrors: timecardAudit.foreignKeyErrors, + }; + } + + auditTimecardPersistence(periodEnd, date, observedRows = []) { + isoDate(periodEnd); + isoDate(date); + const period = periodFromEnd(periodEnd); + if (!period.dates.includes(date) || !Array.isArray(observedRows) || observedRows.length > 5000) fail('invalid_query'); + const audit = this.auditTimecards(periodEnd); + if (!audit.verified) return { verified: false, code: audit.code, date }; + const active = this.activeTimecards(periodEnd); + const activeByCode = new Map(active.rows.map(row => [row.employeeCode, row])); + const observedCodes = new Set(); + let persistedSelectedTimecardCount = 0; + for (const row of observedRows) { + if (typeof row?.employeeCode !== 'string' || observedCodes.has(row.employeeCode)) fail('invalid_query'); + observedCodes.add(row.employeeCode); + const persisted = activeByCode.get(row.employeeCode); + let matches = Boolean(persisted && persisted.employeeName === row.employeeName); + try { + matches = matches + && timecardBusinessSha256(row.record) === row.businessSha256 + && timecardBusinessSha256(persisted.record) === row.businessSha256; + } catch { matches = false; } + persistedSelectedTimecardCount += Number(matches); + } + let dateRowCount = 0; + let punchCount = 0; + let inDayPunchCount = 0; + let inDayTimecardCount = 0; + let outLunchPunchCount = 0; + let inLunchPunchCount = 0; + let outDayPunchCount = 0; + let unclassifiedPunchCount = 0; + for (const row of active.rows) { + const day = row.record.days.find(item => item.date === date); + if (!day) continue; + dateRowCount += 1; + let hasInDay = false; + for (const punch of day.punches) { + punchCount += 1; + if (punch.kind === 'IN DAY') { inDayPunchCount += 1; hasInDay = true; } + else if (punch.kind === 'OUT LUNCH') outLunchPunchCount += 1; + else if (punch.kind === 'IN LUNCH') inLunchPunchCount += 1; + else if (punch.kind === 'OUT DAY') outDayPunchCount += 1; + else unclassifiedPunchCount += 1; + } + inDayTimecardCount += Number(hasInDay); + } + const result = { + verified: dateRowCount === active.rows.length + && persistedSelectedTimecardCount === observedRows.length, + code: dateRowCount === active.rows.length + && persistedSelectedTimecardCount === observedRows.length ? 'verified' : 'integrity_failed', + date, + timecardPublicationId: active.publication.id, + publicationCollectedAt: active.publication.collected_at, + timecardCount: active.rows.length, + dateRowCount, + selectedTimecardCount: observedRows.length, + persistedSelectedTimecardCount, + selectedMismatchCount: observedRows.length - persistedSelectedTimecardCount, + punchCount, + inDayPunchCount, + inDayTimecardCount, + outLunchPunchCount, + inLunchPunchCount, + outDayPunchCount, + unclassifiedPunchCount, + }; + if (result.verified) validateTimecardPersistence(result); + return result; + } + + reconcileCurrent(periodEnd) { + isoDate(periodEnd); + const roster = this.active('roster'); + const timecards = this.active('timecards', periodEnd); + if (!roster || !timecards) return { verified: false, code: !roster ? 'roster_not_loaded' : 'timecards_not_loaded', periodEnd }; + if (roster.target !== periodEnd) return { verified: false, code: 'roster_period_mismatch', periodEnd, rosterTarget: roster.target }; + const rosterCodes = new Set(this.db.prepare('SELECT employee_code FROM roster_employees WHERE publication_id=? AND is_active=1').all(roster.id).map(row => row.employee_code)); + const timecardCodes = new Set(this.db.prepare('SELECT employee_code FROM timecards WHERE publication_id=?').all(timecards.id).map(row => row.employee_code)); + const missing = [...rosterCodes].filter(code => !timecardCodes.has(code)).sort(); + const unexpected = [...timecardCodes].filter(code => !rosterCodes.has(code)).sort(); + return { + verified: missing.length === 0 && unexpected.length === 0, + code: missing.length === 0 && unexpected.length === 0 ? 'verified' : 'membership_mismatch', + periodEnd, rosterPublicationId: roster.id, timecardPublicationId: timecards.id, + activeEmployees: rosterCodes.size, timecards: timecardCodes.size, + missing: missing.slice(0, 100), unexpected: unexpected.slice(0, 100), + omitted: Math.max(0, missing.length - 100) + Math.max(0, unexpected.length - 100), + }; + } + + planWorkforceShadow({ + sourceId, target, observedAt, employees, reconcileBatchSize, fullReconcileMinutes, fullCollection = false, + }) { + if (typeof fullCollection !== 'boolean' || typeof sourceId !== 'string' || !/^[a-z][a-z0-9_.-]{0,63}$/.test(sourceId) + || !DATE_RE.test(target) || typeof observedAt !== 'string' || Number.isNaN(Date.parse(observedAt)) + || !Array.isArray(employees) || employees.length < 1 || employees.length > 5000 + || !Number.isInteger(reconcileBatchSize) || reconcileBatchSize < 1 || reconcileBatchSize > 500 + || !Number.isInteger(fullReconcileMinutes) || fullReconcileMinutes < 60 || fullReconcileMinutes > 10_080) { + fail('invalid_request'); + } + const current = new Map(); + for (const employee of employees) { + if (typeof employee?.employeeCode !== 'string' || !/^[A-Za-z0-9]{4}$/.test(employee.employeeCode) + || typeof employee.employeeName !== 'string' || !employee.employeeName.trim() + || typeof employee.isActive !== 'boolean') fail('api_invalid'); + const code = employee.employeeCode.toUpperCase(); + if (current.has(code)) fail('api_invalid'); + current.set(code, { + employeeCode: code, + employeeName: employee.employeeName, + isActive: employee.isActive, + profileSha256: rosterProfileSha256(employee), + summarySha256: rosterSummarySha256(employee), + }); + } + const priorRows = this.db.prepare(`SELECT employee_code employeeCode,profile_sha256 profileSha256, + summary_sha256 summarySha256,last_timecard_observed_at lastTimecardObservedAt + FROM paycom_sync_employees WHERE source_id=? AND target=?`).all(sourceId, target); + const prior = new Map(priorRows.map(row => [row.employeeCode, row])); + const baseline = prior.size === 0; + const obvious = new Set(); + for (const [code, value] of current) { + const previous = prior.get(code); + if (!previous) { + if (!baseline && value.isActive) obvious.add(code); + } else if (value.isActive && (previous.profileSha256 !== value.profileSha256 + || previous.summarySha256 !== value.summarySha256)) obvious.add(code); + } + const state = this.db.prepare(`SELECT last_full_reconciled_at lastFullReconciledAt, + full_reconcile_anchor_at fullReconcileAnchorAt + FROM paycom_sync_state WHERE source_id=? AND target=?`).get(sourceId, target); + const reference = state?.lastFullReconciledAt || state?.fullReconcileAnchorAt || null; + const fullReconciliation = fullCollection || Boolean(reference + && Date.parse(observedAt) - Date.parse(reference) >= fullReconcileMinutes * 60_000); + const selected = new Set(fullReconciliation + ? [...current.values()].filter(value => value.isActive).map(value => value.employeeCode) + : obvious); + let rotationCount = 0; + if (!fullReconciliation) { + const rotation = [...current.values()] + .filter(value => value.isActive && !obvious.has(value.employeeCode)) + .sort((left, right) => { + const leftTime = prior.get(left.employeeCode)?.lastTimecardObservedAt || ''; + const rightTime = prior.get(right.employeeCode)?.lastTimecardObservedAt || ''; + return leftTime.localeCompare(rightTime) || left.employeeCode.localeCompare(right.employeeCode); + }) + .slice(0, reconcileBatchSize); + for (const value of rotation) selected.add(value.employeeCode); + rotationCount = rotation.length; + } + return { + baseline, + fullReconciliation, + obviousCandidateCount: obvious.size, + rotationCount, + selectedEmployees: [...selected].sort().map(code => ({ + employeeCode: code, + employeeName: current.get(code).employeeName, + })), + }; + } + + observeWorkforceShadow({ + sourceId, target, runId, observedAt, sourceSha256, employees, + sourceCompleteness = 'observation_only', + timecardRows = null, reconcileBatchSize = null, fullReconcileMinutes = null, + syncOutcome = null, fullCollection = false, + }, { transaction = true } = {}) { + if (typeof transaction !== 'boolean') fail('invalid_request'); + if (syncOutcome !== null) validateSyncOutcome(syncOutcome); + if (typeof sourceId !== 'string' || !/^[a-z][a-z0-9_.-]{0,63}$/.test(sourceId) + || !RUN_RE.test(runId) || !DATE_RE.test(target) + || typeof observedAt !== 'string' || Number.isNaN(Date.parse(observedAt)) + || !/^[a-f0-9]{64}$/.test(sourceSha256) + || !Array.isArray(employees) || employees.length < 1 || employees.length > 5000 + || !['authoritative', 'observation_only'].includes(sourceCompleteness) + || (timecardRows !== null && (!Array.isArray(timecardRows) + || !Number.isInteger(reconcileBatchSize) || reconcileBatchSize < 1 || reconcileBatchSize > 500 + || !Number.isInteger(fullReconcileMinutes) || fullReconcileMinutes < 60 || fullReconcileMinutes > 10_080))) { + fail('invalid_request'); + } + const current = new Map(); + for (const employee of employees) { + if (typeof employee?.employeeCode !== 'string' || !/^[A-Za-z0-9]{4}$/.test(employee.employeeCode)) fail('api_invalid'); + const code = employee.employeeCode.toUpperCase(); + if (current.has(code)) fail('api_invalid'); + current.set(code, { + code, + profileSha256: rosterProfileSha256(employee), + summarySha256: rosterSummarySha256(employee), + }); + } + let receipt; + if (transaction) this.db.exec('BEGIN IMMEDIATE'); + try { + const state = this.db.prepare('SELECT * FROM paycom_sync_state WHERE source_id=? AND target=?').get(sourceId, target); + if (state?.last_run_id === runId) { + receipt = JSON.parse(state.last_receipt_json); + if (transaction) this.db.exec('COMMIT'); + return receipt; + } + let selection = null; + const timecardsByCode = new Map(); + if (timecardRows !== null) { + selection = this.planWorkforceShadow({ + sourceId, target, observedAt, employees, reconcileBatchSize, fullReconcileMinutes, fullCollection, + }); + for (const row of timecardRows) { + if (typeof row?.employeeCode !== 'string' || !/^[A-Z0-9]{4}$/.test(row.employeeCode) + || typeof row.employeeName !== 'string' || !row.employeeName.trim() + || row.record?.employeeCode !== row.employeeCode || row.record?.periodEnd !== target + || !/^[a-f0-9]{64}$/.test(row.businessSha256) + || row.businessSha256 !== timecardBusinessSha256(row.record) + || typeof row.observedAt !== 'string' || Number.isNaN(Date.parse(row.observedAt)) + || timecardsByCode.has(row.employeeCode)) fail('timecards_invalid'); + try { + const period = parsePeriodKey(row.record.periodKey); + validateTimecardRecord(row.record, { + employeeCode: row.employeeCode, period, sourceUrl: row.record.sourceUrl, + }); + } catch { fail('timecards_invalid'); } + timecardsByCode.set(row.employeeCode, row); + } + const expected = selection.selectedEmployees; + if (expected.length !== timecardsByCode.size || expected.some(employee => { + const row = timecardsByCode.get(employee.employeeCode); + return !row || row.employeeName !== employee.employeeName; + })) fail('membership_mismatch'); + } + const priorRows = this.db.prepare(`SELECT employee_code employeeCode,profile_sha256 profileSha256, + summary_sha256 summarySha256,missing_observations missingObservations, + timecard_business_sha256 timecardBusinessSha256 + FROM paycom_sync_employees WHERE source_id=? AND target=?`).all(sourceId, target); + const prior = new Map(priorRows.map(row => [row.employeeCode, row])); + const baseline = prior.size === 0; + let addedCount = 0; + let profileChangedCount = 0; + let summaryChangedCount = 0; + const changedEmployees = new Set(); + for (const [code, value] of current) { + const previous = prior.get(code); + if (!previous) { + if (!baseline) { + addedCount += 1; + changedEmployees.add(code); + } + } else { + const profileChanged = previous.profileSha256 !== value.profileSha256; + const summaryChanged = previous.summarySha256 !== value.summarySha256; + profileChangedCount += Number(profileChanged); + summaryChangedCount += Number(summaryChanged); + if (profileChanged || summaryChanged) changedEmployees.add(code); + } + } + const missing = priorRows.filter(row => !current.has(row.employeeCode)); + const pendingRemovalSha256 = null; + const pendingRemovalCount = 0; + const clearMissing = this.db.prepare(`UPDATE paycom_sync_employees SET missing_observations=0 + WHERE source_id=? AND target=? AND employee_code=?`); + for (const row of missing) clearMissing.run(sourceId, target, row.employeeCode); + const upsert = this.db.prepare(`INSERT INTO paycom_sync_employees( + source_id,target,employee_code,profile_sha256,summary_sha256,last_observed_at,missing_observations + ) VALUES(?,?,?,?,?,?,0) ON CONFLICT(source_id,target,employee_code) DO UPDATE SET + profile_sha256=excluded.profile_sha256,summary_sha256=excluded.summary_sha256, + last_observed_at=excluded.last_observed_at,missing_observations=0`); + for (const value of current.values()) { + upsert.run(sourceId, target, value.code, value.profileSha256, value.summarySha256, observedAt); + } + let timecardAddedCount = 0; + let timecardChangedCount = 0; + let timecardUnchangedCount = 0; + let timecardBaselineCount = 0; + if (selection) { + const updateTimecard = this.db.prepare(`UPDATE paycom_sync_employees SET + timecard_business_sha256=?,last_timecard_observed_at=?, + last_full_verified_at=CASE WHEN ?=1 THEN ? ELSE last_full_verified_at END + WHERE source_id=? AND target=? AND employee_code=?`); + for (const [code, row] of timecardsByCode) { + const previous = prior.get(code); + if (!previous) { + if (baseline) timecardBaselineCount += 1; + else timecardAddedCount += 1; + } else if (!previous.timecardBusinessSha256) timecardBaselineCount += 1; + else if (previous.timecardBusinessSha256 !== row.businessSha256) timecardChangedCount += 1; + else timecardUnchangedCount += 1; + updateTimecard.run( + row.businessSha256, row.observedAt, Number(selection.fullReconciliation), row.observedAt, + sourceId, target, code, + ); + } + } + const candidateCount = changedEmployees.size; + receipt = { + baseline, + absencePolicy: 'retain', + sourceCompleteness, + observedCount: current.size, + addedCount, + profileChangedCount, + summaryChangedCount, + missingCount: missing.length, + candidateCount, + ...(selection ? { + selectedTimecardCount: timecardsByCode.size, + obviousCandidateCount: selection.obviousCandidateCount, + rotationCount: selection.rotationCount, + fullReconciliation: selection.fullReconciliation, + timecardAddedCount, + timecardChangedCount, + timecardUnchangedCount, + timecardBaselineCount, + } : {}), + ...(syncOutcome ? { syncOutcome } : {}), + }; + this.db.prepare(`INSERT INTO paycom_sync_state( + source_id,target,last_run_id,checked_at,source_sha256,employee_count, + pending_removal_sha256,pending_removal_count,last_receipt_json, + last_full_reconciled_at,full_reconcile_anchor_at + ) VALUES(?,?,?,?,?,?,?,?,?,?,?) ON CONFLICT(source_id,target) DO UPDATE SET + last_run_id=excluded.last_run_id,checked_at=excluded.checked_at,source_sha256=excluded.source_sha256, + employee_count=excluded.employee_count,pending_removal_sha256=excluded.pending_removal_sha256, + pending_removal_count=excluded.pending_removal_count,last_receipt_json=excluded.last_receipt_json, + last_full_reconciled_at=COALESCE(excluded.last_full_reconciled_at,paycom_sync_state.last_full_reconciled_at), + full_reconcile_anchor_at=CASE WHEN excluded.last_full_reconciled_at IS NOT NULL + THEN excluded.full_reconcile_anchor_at ELSE paycom_sync_state.full_reconcile_anchor_at END`) + .run(sourceId, target, runId, observedAt, sourceSha256, current.size, + pendingRemovalSha256, pendingRemovalCount, JSON.stringify(receipt), + selection?.fullReconciliation ? observedAt : null, observedAt); + if (transaction) this.db.exec('COMMIT'); + } catch (error) { + if (transaction) try { this.db.exec('ROLLBACK'); } catch {} + throw error; + } + return receipt; + } + + shadowReceiptForRun(sourceId, target, runId) { + if (typeof sourceId !== 'string' || !/^[a-z][a-z0-9_.-]{0,63}$/.test(sourceId) + || !DATE_RE.test(target) || !RUN_RE.test(runId)) fail('invalid_query'); + const row = this.db.prepare(`SELECT last_receipt_json receiptJson FROM paycom_sync_state + WHERE source_id=? AND target=? AND last_run_id=?`).get(sourceId, target, runId); + return row ? JSON.parse(row.receiptJson) : null; + } + + syncState(sourceId, target) { + if (typeof sourceId !== 'string' || !/^[a-z][a-z0-9_.-]{0,63}$/.test(sourceId) || !DATE_RE.test(target)) fail('invalid_query'); + const state = this.db.prepare(`SELECT checked_at checkedAt,employee_count employeeCount, + pending_removal_count pendingRemovalCount,last_full_reconciled_at lastFullReconciledAt, + full_reconcile_anchor_at fullReconcileAnchorAt,last_receipt_json receiptJson + FROM paycom_sync_state WHERE source_id=? AND target=?`).get(sourceId, target); + if (!state) return null; + return { + checkedAt: state.checkedAt, + employeeCount: state.employeeCount, + pendingRemovalCount: state.pendingRemovalCount, + lastFullReconciledAt: state.lastFullReconciledAt, + fullReconcileAnchorAt: state.fullReconcileAnchorAt, + receipt: JSON.parse(state.receiptJson), + }; + } + + compactSyncChangeHistory(beforeObservedAt, maxDelete = MAX_HISTORY_COMPACTION_DELETE) { + if (typeof beforeObservedAt !== 'string' || Number.isNaN(Date.parse(beforeObservedAt)) + || new Date(beforeObservedAt).toISOString() !== beforeObservedAt + || !Number.isInteger(maxDelete) || maxDelete < 1 || maxDelete > 1000) fail('invalid_query'); + const candidates = this.db.prepare(`SELECT run_id runId,source_id sourceId,target,business_date businessDate + FROM paycom_sync_change_history WHERE disposition='no_change' AND observed_at= maxDelete) break; + } + const remove = this.db.prepare('DELETE FROM paycom_sync_change_history WHERE run_id=?'); + let deleted = 0; + for (const runId of deletions) deleted += remove.run(runId).changes; + return { scanned: candidates.length, deleted }; + } + + syncChangeHistory(sourceId, target, limit = 50, offset = 0) { + if (typeof sourceId !== 'string' || !/^[a-z][a-z0-9_.-]{0,63}$/.test(sourceId) + || !DATE_RE.test(target) || !Number.isInteger(limit) || limit < 1 || limit > 100 + || !Number.isInteger(offset) || offset < 0) fail('invalid_query'); + const rows = this.db.prepare(`SELECT run_id runId,business_date businessDate, + business_timezone businessTimezone,observed_at observedAt,disposition,delta_json deltaJson, + persistence_json persistenceJson FROM paycom_sync_change_history + WHERE source_id=? AND target=? ORDER BY observed_at DESC,run_id DESC LIMIT ? OFFSET ?`) + .all(sourceId, target, limit, offset).map(row => ({ + runId: row.runId, + target, + businessDate: row.businessDate, + businessTimezone: row.businessTimezone, + observedAt: row.observedAt, + disposition: row.disposition, + delta: validateBusinessDelta(JSON.parse(row.deltaJson)), + persistence: validateTimecardPersistence(JSON.parse(row.persistenceJson)), + })); + const total = this.db.prepare(`SELECT COUNT(*) count FROM paycom_sync_change_history + WHERE source_id=? AND target=?`).get(sourceId, target).count; + return { items: rows, total, limit, offset, hasMore: offset + rows.length < total }; + } + + close() { this.db.close(); } +} + +module.exports = { + PaycomStore, validateCandidate, stageCandidate, readStaged, cleanupStage, cleanupRunStages, + privateDirectory, privateFile, sha256, contentSha256, +}; diff --git a/plugins/paycom/backend/src/sync-delta.js b/plugins/paycom/backend/src/sync-delta.js new file mode 100644 index 0000000..3e2d790 --- /dev/null +++ b/plugins/paycom/backend/src/sync-delta.js @@ -0,0 +1,233 @@ +'use strict'; + +const { timecardBusinessSha256 } = require('./fingerprints'); + +const PUNCH_KINDS = Object.freeze({ + 'IN DAY': 'inDayCount', + 'OUT LUNCH': 'outLunchCount', + 'IN LUNCH': 'inLunchCount', + 'OUT DAY': 'outDayCount', +}); +const KIND_KEYS = Object.freeze(['inDayCount', 'outLunchCount', 'inLunchCount', 'outDayCount', 'unclassifiedCount']); + +function plain(value) { + return value && typeof value === 'object' && !Array.isArray(value) + && Object.getPrototypeOf(value) === Object.prototype; +} +function canonical(value) { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (plain(value)) return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`; + return JSON.stringify(value); +} +function equal(left, right) { return canonical(left) === canonical(right); } +function counts() { return Object.fromEntries(KIND_KEYS.map(key => [key, 0])); } +function kindKey(punch) { return PUNCH_KINDS[punch?.kind] || 'unclassifiedCount'; } +function exact(value, keys) { + return plain(value) && Object.keys(value).length === keys.length && keys.every(key => Object.hasOwn(value, key)); +} +function nonnegative(value) { return Number.isInteger(value) && value >= 0; } +function keyed(rows, key, code) { + const result = new Map(); + for (const row of rows || []) { + const value = key(row); + if (typeof value !== 'string' || result.has(value)) throw Object.assign(new Error(code), { code }); + result.set(value, row); + } + return result; +} +function recordHash(row) { + return row?.businessSha256 || timecardBusinessSha256(row?.record); +} +function punchMap(record) { + const result = new Map(); + for (const day of record?.days || []) { + for (const punch of day.punches || []) { + const key = `${day.date}:${punch.rowIndex}:${punch.slot}`; + if (result.has(key)) throw Object.assign(new Error('business_delta_invalid'), { code: 'business_delta_invalid' }); + result.set(key, punch); + } + } + return result; +} +function unresolved(record) { + const values = new Set(); + for (const day of record?.days || []) { + for (const slot of day.unresolvedSlots || []) values.add(`${day.date}:${slot}`); + } + return values; +} +function intersection(left, right) { return [...left].filter(value => right.has(value)); } +function difference(left, right) { return [...left].filter(value => !right.has(value)); } + +const ROSTER_KEYS = Object.freeze([ + 'addedCount', 'profileChangedCount', 'summaryChangedCount', 'recordChangedCount', + 'becameUnknownCount', 'returnedFromUnknownCount', +]); +const TIMECARD_KEYS = Object.freeze(['addedCount', 'changedCount', 'unchangedCount', 'removedCount']); +const DAY_KEYS = Object.freeze([ + 'addedCount', 'changedCount', 'removedCount', 'missingPunchAddedCount', 'missingPunchResolvedCount', + 'unresolvedSlotAddedCount', 'unresolvedSlotResolvedCount', 'commentSectionsChangedCount', + 'totalSectionsChangedCount', +]); +const PUNCH_KEYS = Object.freeze(['addedCount', 'editedCount', 'removedCount', 'kindChangedCount', 'addedByKind', 'removedByKind']); +const DETAIL_KEYS = Object.freeze([ + 'additionalRowSectionsChangedCount', 'approvalSectionsChangedCount', + 'attestationSectionsChangedCount', 'mealWaiverSectionsChangedCount', +]); +const DELTA_KEYS = Object.freeze(['roster', 'timecards', 'days', 'punches', 'details']); + +function validateKindCounts(value) { + if (!exact(value, KIND_KEYS) || KIND_KEYS.some(key => !nonnegative(value[key]))) throw Object.assign(new Error('business_delta_invalid'), { code: 'business_delta_invalid' }); + return value; +} +function validateBusinessDelta(value) { + if (!exact(value, DELTA_KEYS) + || !exact(value.roster, ROSTER_KEYS) || ROSTER_KEYS.some(key => !nonnegative(value.roster[key])) + || !exact(value.timecards, TIMECARD_KEYS) || TIMECARD_KEYS.some(key => !nonnegative(value.timecards[key])) + || !exact(value.days, DAY_KEYS) || DAY_KEYS.some(key => !nonnegative(value.days[key])) + || !exact(value.punches, PUNCH_KEYS) + || ['addedCount', 'editedCount', 'removedCount', 'kindChangedCount'].some(key => !nonnegative(value.punches[key])) + || !exact(value.details, DETAIL_KEYS) || DETAIL_KEYS.some(key => !nonnegative(value.details[key]))) { + throw Object.assign(new Error('business_delta_invalid'), { code: 'business_delta_invalid' }); + } + validateKindCounts(value.punches.addedByKind); + validateKindCounts(value.punches.removedByKind); + if (KIND_KEYS.reduce((sum, key) => sum + value.punches.addedByKind[key], 0) !== value.punches.addedCount + || KIND_KEYS.reduce((sum, key) => sum + value.punches.removedByKind[key], 0) !== value.punches.removedCount) { + throw Object.assign(new Error('business_delta_invalid'), { code: 'business_delta_invalid' }); + } + return value; +} + +function computeBusinessDelta(previousRows, nextRows, mirrorCounts) { + if (!Array.isArray(previousRows) || !Array.isArray(nextRows) || !plain(mirrorCounts)) { + throw Object.assign(new Error('business_delta_invalid'), { code: 'business_delta_invalid' }); + } + const previous = keyed(previousRows, row => row.employeeCode, 'business_delta_invalid'); + const next = keyed(nextRows, row => row.employeeCode, 'business_delta_invalid'); + const previousCodes = new Set(previous.keys()); + const nextCodes = new Set(next.keys()); + const addedCodes = difference(nextCodes, previousCodes); + const removedCodes = difference(previousCodes, nextCodes); + const commonCodes = intersection(nextCodes, previousCodes); + const changedCodes = commonCodes.filter(code => recordHash(previous.get(code)) !== recordHash(next.get(code))); + const unchangedCodes = commonCodes.filter(code => recordHash(previous.get(code)) === recordHash(next.get(code))); + const delta = { + roster: { + addedCount: mirrorCounts.rosterAddedCount, + profileChangedCount: mirrorCounts.rosterProfileChangedCount, + summaryChangedCount: mirrorCounts.rosterSummaryChangedCount, + recordChangedCount: mirrorCounts.rosterRecordChangedCount, + becameUnknownCount: mirrorCounts.becameUnknownCount, + returnedFromUnknownCount: mirrorCounts.returnedFromUnknownCount, + }, + timecards: { + addedCount: addedCodes.length, + changedCount: changedCodes.length, + unchangedCount: unchangedCodes.length, + removedCount: removedCodes.length, + }, + days: { + addedCount: 0, changedCount: 0, removedCount: 0, + missingPunchAddedCount: 0, missingPunchResolvedCount: 0, + unresolvedSlotAddedCount: 0, unresolvedSlotResolvedCount: 0, + commentSectionsChangedCount: 0, totalSectionsChangedCount: 0, + }, + punches: { + addedCount: 0, editedCount: 0, removedCount: 0, kindChangedCount: 0, + addedByKind: counts(), removedByKind: counts(), + }, + details: { + additionalRowSectionsChangedCount: 0, + approvalSectionsChangedCount: 0, + attestationSectionsChangedCount: 0, + mealWaiverSectionsChangedCount: 0, + }, + }; + + for (const code of commonCodes) { + const before = previous.get(code).record; + const after = next.get(code).record; + const beforeDays = keyed(before.days, day => day.date, 'business_delta_invalid'); + const afterDays = keyed(after.days, day => day.date, 'business_delta_invalid'); + const beforeDates = new Set(beforeDays.keys()); + const afterDates = new Set(afterDays.keys()); + delta.days.addedCount += difference(afterDates, beforeDates).length; + delta.days.removedCount += difference(beforeDates, afterDates).length; + for (const date of intersection(afterDates, beforeDates)) { + const left = beforeDays.get(date); + const right = afterDays.get(date); + if (!equal(left, right)) delta.days.changedCount += 1; + if (!left.missingPunch && right.missingPunch) delta.days.missingPunchAddedCount += 1; + if (left.missingPunch && !right.missingPunch) delta.days.missingPunchResolvedCount += 1; + if (!equal(left.comments, right.comments)) delta.days.commentSectionsChangedCount += 1; + if (!equal( + { hours: left.hours, totalHours: left.totalHours, dollars: left.dollars }, + { hours: right.hours, totalHours: right.totalHours, dollars: right.dollars }, + )) delta.days.totalSectionsChangedCount += 1; + } + if (!equal(before.weeklyTotals, after.weeklyTotals) || before.periodTotalHours !== after.periodTotalHours) { + delta.days.totalSectionsChangedCount += 1; + } + const beforeUnresolved = unresolved(before); + const afterUnresolved = unresolved(after); + delta.days.unresolvedSlotAddedCount += difference(afterUnresolved, beforeUnresolved).length; + delta.days.unresolvedSlotResolvedCount += difference(beforeUnresolved, afterUnresolved).length; + + const beforePunches = punchMap(before); + const afterPunches = punchMap(after); + const beforePunchKeys = new Set(beforePunches.keys()); + const afterPunchKeys = new Set(afterPunches.keys()); + for (const key of difference(afterPunchKeys, beforePunchKeys)) { + delta.punches.addedCount += 1; + delta.punches.addedByKind[kindKey(afterPunches.get(key))] += 1; + } + for (const key of difference(beforePunchKeys, afterPunchKeys)) { + delta.punches.removedCount += 1; + delta.punches.removedByKind[kindKey(beforePunches.get(key))] += 1; + } + for (const key of intersection(afterPunchKeys, beforePunchKeys)) { + const left = beforePunches.get(key); + const right = afterPunches.get(key); + if (!equal(left, right)) { + delta.punches.editedCount += 1; + if (kindKey(left) !== kindKey(right)) delta.punches.kindChangedCount += 1; + } + } + if (!equal(before.additionalRows, after.additionalRows)) delta.details.additionalRowSectionsChangedCount += 1; + if (!equal(before.approvals, after.approvals)) delta.details.approvalSectionsChangedCount += 1; + if (!equal(before.attestations, after.attestations)) delta.details.attestationSectionsChangedCount += 1; + if (!equal(before.mealWaivers, after.mealWaivers)) delta.details.mealWaiverSectionsChangedCount += 1; + } + + for (const code of addedCodes) { + const after = next.get(code).record; + delta.days.addedCount += after.days.length; + for (const day of after.days) { + delta.days.missingPunchAddedCount += Number(day.missingPunch); + delta.days.unresolvedSlotAddedCount += day.unresolvedSlots.length; + for (const punch of day.punches) { + delta.punches.addedCount += 1; + delta.punches.addedByKind[kindKey(punch)] += 1; + } + } + } + for (const code of removedCodes) { + const before = previous.get(code).record; + delta.days.removedCount += before.days.length; + for (const day of before.days) { + delta.days.missingPunchResolvedCount += Number(day.missingPunch); + delta.days.unresolvedSlotResolvedCount += day.unresolvedSlots.length; + for (const punch of day.punches) { + delta.punches.removedCount += 1; + delta.punches.removedByKind[kindKey(punch)] += 1; + } + } + } + return validateBusinessDelta(delta); +} + +module.exports = { + KIND_KEYS, ROSTER_KEYS, TIMECARD_KEYS, DAY_KEYS, PUNCH_KEYS, DETAIL_KEYS, + validateBusinessDelta, computeBusinessDelta, +}; diff --git a/plugins/paycom/backend/src/sync-publication.js b/plugins/paycom/backend/src/sync-publication.js new file mode 100644 index 0000000..ab9c81b --- /dev/null +++ b/plugins/paycom/backend/src/sync-publication.js @@ -0,0 +1,162 @@ +'use strict'; + +const { + canonicalStringify, rosterProfileSha256, rosterSummarySha256, timecardBusinessSha256, +} = require('./fingerprints'); +const { TIMECARD_SUMMARY, linkRows } = require('./resource-links'); + +function fail(code) { + const error = new Error(code); + error.code = code; + throw error; +} + +const LIFECYCLE_STATUSES = new Set(['active', 'inactive', 'unknown']); + +function employeeMap(rows, { source = false } = {}) { + if (!Array.isArray(rows) || rows.length < 1 || rows.length > 5000) fail('candidate_invalid'); + const result = new Map(); + for (const row of rows) { + if (typeof row?.employeeCode !== 'string' || !/^[A-Za-z0-9]{4}$/.test(row.employeeCode) + || typeof row.employeeName !== 'string' || !row.employeeName.trim() + || typeof row.isActive !== 'boolean' + || (row.lifecycleStatus !== undefined && !LIFECYCLE_STATUSES.has(row.lifecycleStatus))) fail('candidate_invalid'); + const code = row.employeeCode.toUpperCase(); + if (result.has(code)) fail('candidate_invalid'); + const lifecycleStatus = source + ? (row.isActive ? 'active' : 'inactive') + : (row.lifecycleStatus || (row.isActive ? 'active' : 'inactive')); + if (source && row.lifecycleStatus === 'unknown') fail('candidate_invalid'); + result.set(code, { ...row, employeeCode: code, lifecycleStatus }); + } + return result; +} + +function timecardMap(rows) { + if (!Array.isArray(rows) || rows.length > 5000) fail('candidate_invalid'); + const result = new Map(); + for (const row of rows) { + if (typeof row?.employeeCode !== 'string' || !/^[A-Za-z0-9]{4}$/.test(row.employeeCode) + || typeof row.employeeName !== 'string' || !row.employeeName.trim() + || !row.record || row.record.employeeCode !== row.employeeCode) fail('candidate_invalid'); + const code = row.employeeCode.toUpperCase(); + if (result.has(code)) fail('candidate_invalid'); + result.set(code, { ...row, employeeCode: code }); + } + return result; +} + +function rowBusinessSha256(row) { + return row.businessSha256 || timecardBusinessSha256(row.record); +} + +function planWorkforceMirror({ + period, + priorRosterRows, + priorTimecardRows, + sourceEmployees, + collectedTimecardRows, +}) { + if (!period || typeof period.key !== 'string') fail('candidate_invalid'); + const priorRoster = Array.isArray(priorRosterRows) && priorRosterRows.length === 0 + ? new Map() : employeeMap(priorRosterRows); + const source = employeeMap(sourceEmployees, { source: true }); + const priorTimecards = timecardMap(priorTimecardRows); + const collected = timecardMap(collectedTimecardRows); + + const merged = new Map(source); + let retainedMissingCount = 0; + let becameUnknownCount = 0; + for (const [code, employee] of priorRoster) { + if (!merged.has(code)) { + const retained = employee.lifecycleStatus === 'inactive' + ? employee + : { ...employee, lifecycleStatus: 'unknown' }; + merged.set(code, retained); + retainedMissingCount += 1; + becameUnknownCount += Number(employee.lifecycleStatus === 'active'); + } + } + + let rosterAddedCount = 0; + let rosterProfileChangedCount = 0; + let rosterSummaryChangedCount = 0; + let rosterRecordChangedCount = 0; + let deactivatedCount = 0; + let reactivatedCount = 0; + let returnedFromUnknownCount = 0; + for (const [code, employee] of source) { + const previous = priorRoster.get(code); + if (!previous) { + rosterAddedCount += 1; + rosterRecordChangedCount += 1; + continue; + } + rosterProfileChangedCount += Number(rosterProfileSha256(previous) !== rosterProfileSha256(employee)); + rosterSummaryChangedCount += Number(rosterSummarySha256(previous) !== rosterSummarySha256(employee)); + rosterRecordChangedCount += Number(canonicalStringify(previous) !== canonicalStringify(employee)); + deactivatedCount += Number(previous.lifecycleStatus === 'active' && employee.lifecycleStatus === 'inactive'); + reactivatedCount += Number(previous.lifecycleStatus === 'inactive' && employee.lifecycleStatus === 'active'); + returnedFromUnknownCount += Number(previous.lifecycleStatus === 'unknown' && employee.lifecycleStatus === 'active'); + } + + rosterRecordChangedCount += becameUnknownCount; + + const rosterRows = [...merged.values()].sort((left, right) => left.employeeCode.localeCompare(right.employeeCode)); + const activeEmployees = rosterRows.filter(row => row.isActive === true); + if (!activeEmployees.length) fail('roster_invalid'); + + const completeTimecards = new Map(); + for (const employee of activeEmployees) { + const replacement = collected.get(employee.employeeCode); + const retained = priorTimecards.get(employee.employeeCode); + const row = replacement || retained; + if (!row || row.employeeName !== employee.employeeName + || row.record.employeeCode !== employee.employeeCode || row.record.periodEnd !== period.end) { + fail('timecard_refresh_required'); + } + completeTimecards.set(employee.employeeCode, row); + } + for (const code of collected.keys()) { + if (!completeTimecards.has(code)) fail('membership_mismatch'); + } + + let timecardAddedCount = 0; + let timecardChangedCount = 0; + for (const [code, row] of collected) { + const previous = priorTimecards.get(code); + if (!previous) timecardAddedCount += 1; + else timecardChangedCount += Number(rowBusinessSha256(previous) !== rowBusinessSha256(row)); + } + + const timecardRows = [...completeTimecards.values()] + .sort((left, right) => left.employeeCode.localeCompare(right.employeeCode)); + const resourceLinkRows = linkRows(TIMECARD_SUMMARY, activeEmployees, period); + const unknownEmployeeCount = rosterRows.filter(row => row.lifecycleStatus === 'unknown').length; + const hasChanges = rosterRecordChangedCount > 0 || timecardAddedCount > 0 || timecardChangedCount > 0; + + return { + hasChanges, + absencePolicy: 'retain', + rosterRows, + timecardRows, + resourceLinkRows, + counts: { + rosterAddedCount, + rosterProfileChangedCount, + rosterSummaryChangedCount, + rosterRecordChangedCount, + timecardAddedCount, + timecardChangedCount, + retainedMissingCount, + becameUnknownCount, + returnedFromUnknownCount, + unknownEmployeeCount, + deactivatedCount, + reactivatedCount, + activeEmployeeCount: activeEmployees.length, + }, + }; +} + +module.exports = { planWorkforceMirror }; diff --git a/plugins/paycom/backend/src/timecard-dom.js b/plugins/paycom/backend/src/timecard-dom.js new file mode 100644 index 0000000..eb279b4 --- /dev/null +++ b/plugins/paycom/backend/src/timecard-dom.js @@ -0,0 +1,407 @@ +'use strict'; + +const {parsePeriodKey, validateCode} = require('./timecard-period'); + +const HEADERS = Object.freeze(['date', 'paycode', 'i1', 'allocation1', 'o1', 'i2', 'allocation2', 'o2', 'hours', 'total_hours', 'amount', 'exception-points', 'waiver', 'comment', 'missing-punch', 'delete']); +const HEADERS_NO_WAIVER = Object.freeze(HEADERS.filter(value => value !== 'waiver')); +const SLOTS = Object.freeze(['i1', 'o1', 'i2', 'o2']); +const KINDS = Object.freeze(['IN DAY', 'OUT LUNCH', 'IN LUNCH', 'OUT DAY']); +const LABELS = Object.freeze(['SUN', 'MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT']); +const TIME = /^(0[1-9]|1[0-2]):[0-5][0-9] [AP]M$/; +const CHANGE_OPERATIONS = Object.freeze(['add', 'edit', 'delete', 'type_change']); +const CHANGE_DETAIL_STATES = Object.freeze(['not_applicable', 'unavailable', 'complete']); +const RECORD_KEYS = Object.freeze(['additionalRows', 'approvals', 'attestations', 'days', 'employeeCode', 'headers', 'mealWaivers', 'pageTitle', 'periodEnd', 'periodKey', 'periodStart', 'periodTotalHours', 'sourceFormat', 'sourceUrl', 'version', 'weeklyTotals']); +const DAY_KEYS = Object.freeze(['allocation1', 'allocation2', 'comments', 'date', 'dollars', 'exceptionText', 'hours', 'label', 'missingPunch', 'payCode', 'punches', 'totalHours', 'unresolvedSlots', 'waiverChecked']); +const ADDITIONAL_ROW_KEYS = Object.freeze(['allocation1', 'allocation2', 'comments', 'date', 'dollars', 'exceptionText', 'hours', 'payCode', 'punchOrdinals', 'rowClass', 'rowIndex', 'totalHours', 'unresolvedSlots', 'waiverChecked']); +const PUNCH_KEYS = Object.freeze(['actualTime', 'approved', 'changeDetailState', 'changeNote', 'changeOperation', 'changeRequestStatus', 'clockCode', 'clockName', 'comment', 'currentKind', 'currentTime', 'displayTime', 'kind', 'ordinal', 'provenanceAvailable', 'requestedKind', 'requestedTime', 'roundedTime', 'rowIndex', 'slot']); + +// These are the only validation diagnostics that may cross the collector/CLI boundary. +const TIME_CARD_VALIDATION_CODES = Object.freeze([ + 'timecard_identity_invalid', + 'timecard_header_invalid', + 'timecard_day_count_invalid', + 'timecard_date_sequence_invalid', + 'timecard_day_label_invalid', + 'timecard_pay_code_invalid', + 'timecard_allocation_invalid', + 'timecard_exception_invalid', + 'timecard_day_number_invalid', + 'timecard_day_comments_invalid', + 'timecard_missing_punch_invalid', + 'timecard_punch_invalid', + 'timecard_provenance_invalid', + 'timecard_pcr_marker_invalid', + 'pending_change_detail_unavailable', + 'timecard_additional_row_invalid', + 'timecard_weekly_total_invalid', + 'timecard_period_total_invalid', + 'timecard_approval_invalid', + 'timecard_attestation_invalid', + 'timecard_waiver_invalid' +]); +const TIME_CARD_VALIDATION_CODE_SET = new Set(TIME_CARD_VALIDATION_CODES); + +function invalid(code) { + const safeCode = TIME_CARD_VALIDATION_CODE_SET.has(code) ? code : 'timecard_identity_invalid'; + const error = new Error(safeCode); + error.code = safeCode; + throw error; +} + +function bounded(value, max) { + return typeof value === 'string' && value.length <= max && !/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/.test(value); +} + +function numberOrNull(value) { + return value === null || (typeof value === 'number' && Number.isFinite(value) && value >= 0); +} + +function exactKeys(value, keys) { + return value && typeof value === 'object' && !Array.isArray(value) && Object.getPrototypeOf(value) === Object.prototype && Object.keys(value).length === keys.length && keys.every(key => Object.prototype.hasOwnProperty.call(value, key)); +} + +function validateGenericRows(value, code) { + if (!Array.isArray(value) || value.length > 500) invalid(code); + for (const row of value) { + if (!Array.isArray(row) || row.length > 20 || row.some(cell => !bounded(cell, 1000))) invalid(code); + } +} + +function validatePunch(punch, index, seen) { + const key = punch ? `${punch.rowIndex}:${punch.slot}` : ''; + if (!exactKeys(punch, PUNCH_KEYS) || punch.ordinal !== index + 1 || !Number.isInteger(punch.rowIndex) || punch.rowIndex < 0 || punch.rowIndex > 16 || !SLOTS.includes(punch.slot) || seen.has(key) || !TIME.test(punch.displayTime)) { + invalid('timecard_punch_invalid'); + } + if (!bounded(punch.clockName, 200) || !bounded(punch.clockCode, 50) || !bounded(punch.comment, 2000) || typeof punch.provenanceAvailable !== 'boolean' || ![null, 'approved', 'pending', 'rejected'].includes(punch.changeRequestStatus) || typeof punch.approved !== 'boolean' || punch.approved !== (punch.changeRequestStatus === 'approved')) { + invalid('timecard_provenance_invalid'); + } + if (punch.provenanceAvailable) { + if (!KINDS.includes(punch.kind) || !TIME.test(punch.actualTime) || !TIME.test(punch.roundedTime)) invalid('timecard_provenance_invalid'); + } else if (punch.kind !== '' || punch.actualTime !== '' || punch.roundedTime !== '' || punch.clockName !== '' || punch.clockCode !== '' || punch.comment !== '') { + invalid('timecard_provenance_invalid'); + } + const nullableKind = value => value === null || KINDS.includes(value); + const nullableTime = value => value === null || TIME.test(value); + if (!CHANGE_DETAIL_STATES.includes(punch.changeDetailState) || !nullableKind(punch.currentKind) || !nullableKind(punch.requestedKind) || !nullableTime(punch.currentTime) || !nullableTime(punch.requestedTime) || (punch.changeNote !== null && !bounded(punch.changeNote, 2000)) || (punch.changeOperation !== null && !CHANGE_OPERATIONS.includes(punch.changeOperation))) invalid('timecard_provenance_invalid'); + const emptyDirection = punch.changeOperation === null && punch.currentKind === null && punch.currentTime === null && punch.requestedKind === null && punch.requestedTime === null && punch.changeNote === null; + if (punch.changeRequestStatus === null) { + if (punch.changeDetailState !== 'not_applicable' || !emptyDirection) invalid('timecard_provenance_invalid'); + } else if (punch.changeDetailState === 'unavailable') { + if (!emptyDirection) invalid('timecard_provenance_invalid'); + } else if (punch.changeDetailState !== 'complete') { + invalid('timecard_provenance_invalid'); + } else { + const currentPresent = punch.currentKind !== null && punch.currentTime !== null; + const requestedPresent = punch.requestedKind !== null && punch.requestedTime !== null; + if (punch.changeOperation === 'add') { + if (punch.currentKind !== null || punch.currentTime !== null || !requestedPresent) invalid('timecard_provenance_invalid'); + } else if (punch.changeOperation === 'delete') { + if (!currentPresent || punch.requestedKind !== null || punch.requestedTime !== null) invalid('timecard_provenance_invalid'); + } else if (punch.changeOperation === 'edit') { + if (!currentPresent || !requestedPresent || punch.currentKind !== punch.requestedKind) invalid('timecard_provenance_invalid'); + } else if (punch.changeOperation === 'type_change') { + if (!currentPresent || !requestedPresent || punch.currentKind === punch.requestedKind) invalid('timecard_provenance_invalid'); + } else invalid('timecard_provenance_invalid'); + } + seen.add(key); +} + +function validateTimecardRecord(value, {employeeCode, period, sourceUrl}) { + validateCode(employeeCode); + const expected = parsePeriodKey(period.key); + const headerValid = Array.isArray(value?.headers) && [HEADERS, HEADERS_NO_WAIVER].some(schema => value.headers.length === schema.length && value.headers.every((item, index) => item === schema[index])); + + if (!exactKeys(value, RECORD_KEYS) || value.version !== 2 || value.sourceFormat !== 'paycom-timecard-dom.v2' || value.employeeCode !== employeeCode || value.periodStart !== expected.start || value.periodEnd !== expected.end || value.periodKey !== expected.key || value.sourceUrl !== sourceUrl || value.pageTitle !== 'Timecard Editor') { + invalid('timecard_identity_invalid'); + } + if (!headerValid) invalid('timecard_header_invalid'); + if (!Array.isArray(value.days) || value.days.length !== 14) invalid('timecard_day_count_invalid'); + if (!Array.isArray(value.additionalRows) || value.additionalRows.length > 200) invalid('timecard_additional_row_invalid'); + + const hasWaiverColumn = value.headers.length === HEADERS.length; + for (let index = 0; index < 14; index++) { + const day = value.days[index]; + if (!exactKeys(day, DAY_KEYS) || day.date !== expected.dates[index]) invalid('timecard_date_sequence_invalid'); + if (day.label !== LABELS[index % 7]) invalid('timecard_day_label_invalid'); + if (!bounded(day.payCode, 100)) invalid('timecard_pay_code_invalid'); + if (!bounded(day.allocation1, 300) || !bounded(day.allocation2, 300)) invalid('timecard_allocation_invalid'); + if (!bounded(day.exceptionText, 2000)) invalid('timecard_exception_invalid'); + if (!numberOrNull(day.hours) || !numberOrNull(day.totalHours) || !numberOrNull(day.dollars)) invalid('timecard_day_number_invalid'); + if (!Array.isArray(day.comments) || day.comments.length > 20 || day.comments.some(item => !bounded(item, 2000))) invalid('timecard_day_comments_invalid'); + if (typeof day.missingPunch !== 'boolean' || !Array.isArray(day.unresolvedSlots) || day.unresolvedSlots.length > 32 || day.unresolvedSlots.some(item => !/^(?:[1-9]|1[0-6]):(?:i1|o1|i2|o2)$|^(?:i1|o1|i2|o2)$/.test(item)) || new Set(day.unresolvedSlots).size !== day.unresolvedSlots.length || day.missingPunch !== (day.unresolvedSlots.length > 0)) invalid('timecard_missing_punch_invalid'); + if (day.waiverChecked !== null && typeof day.waiverChecked !== 'boolean') invalid('timecard_waiver_invalid'); + if (!hasWaiverColumn && day.waiverChecked !== null) invalid('timecard_waiver_invalid'); + if (!Array.isArray(day.punches) || day.punches.length > 32) invalid('timecard_punch_invalid'); + const seen = new Set(); + day.punches.forEach((punch, punchIndex) => validatePunch(punch, punchIndex, seen)); + } + + const rowIds = new Set(); + for (const row of value.additionalRows) { + if (!exactKeys(row, ADDITIONAL_ROW_KEYS) || !expected.dates.includes(row.date) || !Number.isInteger(row.rowIndex) || row.rowIndex < 1 || row.rowIndex > 16 || rowIds.has(`${row.date}:${row.rowIndex}`) || !bounded(row.rowClass, 300) || !bounded(row.payCode, 2000) || !bounded(row.allocation1, 300) || !bounded(row.allocation2, 300) || !numberOrNull(row.hours) || !numberOrNull(row.totalHours) || !numberOrNull(row.dollars) || !bounded(row.exceptionText, 2000) || !Array.isArray(row.comments) || row.comments.length > 20 || row.comments.some(item => !bounded(item, 2000)) || !Array.isArray(row.unresolvedSlots) || row.unresolvedSlots.length > 32 || row.unresolvedSlots.some(item => !SLOTS.includes(item)) || new Set(row.unresolvedSlots).size !== row.unresolvedSlots.length || !Array.isArray(row.punchOrdinals) || row.punchOrdinals.length > 32 || row.punchOrdinals.some(item => !Number.isInteger(item) || item < 1 || item > 32) || new Set(row.punchOrdinals).size !== row.punchOrdinals.length) { + invalid('timecard_additional_row_invalid'); + } + if (row.waiverChecked !== null && typeof row.waiverChecked !== 'boolean') invalid('timecard_waiver_invalid'); + if (!hasWaiverColumn && row.waiverChecked !== null) invalid('timecard_waiver_invalid'); + rowIds.add(`${row.date}:${row.rowIndex}`); + const day = value.days[expected.dates.indexOf(row.date)]; + for (const ordinal of row.punchOrdinals) { + const punch = day.punches[ordinal - 1]; + if (!punch || punch.rowIndex !== row.rowIndex) invalid('timecard_additional_row_invalid'); + } + for (const slot of row.unresolvedSlots) { + if (!day.unresolvedSlots.includes(`${row.rowIndex}:${slot}`)) invalid('timecard_additional_row_invalid'); + } + } + + if (!Array.isArray(value.weeklyTotals) || value.weeklyTotals.length !== 2 || value.weeklyTotals.some(item => typeof item !== 'number' || !Number.isFinite(item) || item < 0)) invalid('timecard_weekly_total_invalid'); + if (typeof value.periodTotalHours !== 'number' || !Number.isFinite(value.periodTotalHours) || value.periodTotalHours < 0) invalid('timecard_period_total_invalid'); + if (Math.abs(value.periodTotalHours - value.weeklyTotals.reduce((sum, item) => sum + item, 0)) > 0.011) invalid('timecard_period_total_invalid'); + + validateGenericRows(value.approvals, 'timecard_approval_invalid'); + validateGenericRows(value.attestations, 'timecard_attestation_invalid'); + validateGenericRows(value.mealWaivers, 'timecard_waiver_invalid'); + return value; +} + +function renderedDayHeader(value) { + const text = String(value ?? '').replace(/\s+/g, ' ').trim(); + const match = text.match(/^([A-Z]{3})\s*\(([0-9]{2}\/[0-9]{2})\)$/); + return match ? {label: match[1], dateText: match[2]} : null; +} + +function bindRenderedDayDate(value, expectedDate) { + const header = renderedDayHeader(value); + return header && typeof expectedDate === 'string' && header.dateText === expectedDate.slice(5).replace('-', '/') ? expectedDate : ''; +} + +function resolveObservedEmployeeCode(values) { + if (!Array.isArray(values) || values.length < 1 + || values.some(value => typeof value !== 'string' || !/^[A-Za-z0-9]{4}$/.test(value))) return ''; + const codes = [...new Set(values.map(value => value.toUpperCase()))]; + return codes.length === 1 ? codes[0] : ''; +} + +function normalizeRequestDetail(observed) { + const empty = state => ({changeOperation: null, currentKind: null, currentTime: null, requestedKind: null, requestedTime: null, changeNote: null, changeDetailState: state}); + const unavailable = () => empty('unavailable'); + const keys = ['operation', 'currentKind', 'currentTime', 'requestedKind', 'requestedTime', 'note']; + if (!observed || typeof observed !== 'object' || Array.isArray(observed) || keys.some(key => !Array.isArray(observed[key]))) throw new Error('timecard_provenance_invalid'); + // No directly exposed direction is the safe permission-limited state. Once + // any direction field is exposed, the whole observation must be singular + // and complete; partial or malformed detail must not be collapsed into the + // permission-limited state. + if (keys.every(key => observed[key].length === 0)) return unavailable(); + // Direction fields are required even when the directly observed value is an + // empty string. An explicit empty value is absence evidence; a missing, + // duplicate, or contradictory observation is not. + if (keys.slice(0, 5).some(key => observed[key].length !== 1) || observed.note.length > 1) throw new Error('timecard_provenance_invalid'); + const raw = Object.fromEntries(keys.slice(0, 5).map(key => [key, observed[key][0]])); + if (Object.values(raw).some(value => typeof value !== 'string' || value.length > 200)) throw new Error('timecard_provenance_invalid'); + const operationAliases = new Map([['add', 'add'], ['edit', 'edit'], ['delete', 'delete'], ['type_change', 'type_change'], ['type change', 'type_change']]); + const kindAliases = new Map([['IN DAY', 'IN DAY'], ['OUT LUNCH', 'OUT LUNCH'], ['IN LUNCH', 'IN LUNCH'], ['OUT DAY', 'OUT DAY'], ['OUT BREAK', 'OUT LUNCH'], ['IN BREAK', 'IN LUNCH']]); + const operation = operationAliases.get(raw.operation.toLowerCase()); + const kind = value => value === '' ? null : kindAliases.get(value.toUpperCase()); + const time = value => value === '' ? null : /^(0[1-9]|1[0-2]):[0-5][0-9] [AP]M$/.test(value.toUpperCase()) ? value.toUpperCase() : undefined; + const currentKind = kind(raw.currentKind); + const requestedKind = kind(raw.requestedKind); + const currentTime = time(raw.currentTime); + const requestedTime = time(raw.requestedTime); + const note = observed.note.length === 0 ? null : observed.note[0]; + if (!operation || currentKind === undefined || requestedKind === undefined || currentTime === undefined || requestedTime === undefined || typeof note !== 'string' && note !== null || note !== null && (note.length > 2000 || /[\u0000-\u001f\u007f]/.test(note))) throw new Error('timecard_provenance_invalid'); + const complete = operation === 'add' ? currentKind === null && currentTime === null && requestedKind !== null && requestedTime !== null : operation === 'delete' ? currentKind !== null && currentTime !== null && requestedKind === null && requestedTime === null : operation === 'edit' ? currentKind !== null && currentKind === requestedKind && currentTime !== null && requestedTime !== null : currentKind !== null && requestedKind !== null && currentKind !== requestedKind && currentTime !== null && requestedTime !== null; + if (!complete) throw new Error('timecard_provenance_invalid'); + return {changeOperation: operation, currentKind, currentTime, requestedKind, requestedTime, changeNote: note, changeDetailState: 'complete'}; +} + +function runtimeExtract(config, normalizeRequestDetail, resolveObservedEmployeeCode) { + const clean = value => String(value ?? '').replace(/\s+/g, ' ').trim(); + const decode = value => { + let out = String(value ?? ''); + for (let index = 0; index < 2; index++) { + const div = document.createElement('div'); + div.innerHTML = out.replace(//gi, '\n'); + out = div.textContent || ''; + } + return out.replace(/\u00a0/g, ' ').replace(/[ \t]+/g, ' ').trim(); + }; + const numeric = value => { + const text = clean(value); + if (!text) return null; + const number = Number(text.replace(/[$,]/g, '')); + return Number.isFinite(number) && number >= 0 ? number : null; + }; + const visible = element => { + if (!element) return ''; + const clone = element.cloneNode(true); + clone.querySelectorAll('script,style').forEach(node => node.remove()); + return clean(clone.textContent); + }; + const controlValue = element => { + if (!element) return ''; + const select = element.querySelector('select'); + if (select) return clean(select.value || select.selectedOptions?.[0]?.textContent); + const input = element.querySelector('input,textarea'); + if (input) return clean(input.value); + const triggers = Array.from(element.querySelectorAll('a.popoverTrigger.popoverTrigger--text')).filter(e => e.offsetParent !== null); + return triggers.length === 1 ? clean(triggers[0].textContent) : visible(element); + }; + const renderedPunchTime = element => { + if (!element) return ''; + const pattern = /^(0?[1-9]|1[0-2]):([0-5][0-9]) ([AP])M$/; + const values = Array.from(element.children) + .filter(child => child.offsetParent !== null && child.getClientRects().length > 0) + .map(child => clean(child.textContent)) + .filter(value => pattern.test(value)); + if (values.length !== 1) return controlValue(element); + const match = values[0].match(pattern); + return `${match[1].padStart(2, '0')}:${match[2]} ${match[3]}M`; + }; + const dayHeader = value => { + const match = clean(value).match(/^([A-Z]{3})\s*\(([0-9]{2}\/[0-9]{2})\)$/); + return match ? {label: match[1], dateText: match[2]} : null; + }; + const boundDate = (value, expectedDate) => { + const header = dayHeader(value); + return header && header.dateText === expectedDate.slice(5).replace('-', '/') ? expectedDate : ''; + }; + const table = document.querySelector('#tbltimesheet'); + if (!table) return null; + const identityElements = Array.from(document.querySelectorAll('input[name="firstrefno"],input#firstrefno,[data-firstrefno],[data-employee-code]')); + const identityValues = identityElements.map(element => clean(element.value || element.getAttribute('data-firstrefno') || element.getAttribute('data-employee-code'))).filter(Boolean); + const observedEmployeeCode = resolveObservedEmployeeCode(identityValues); + const headers = Array.from(table.querySelectorAll('thead [data-column]')).map(element => element.getAttribute('data-column')); + const column = name => headers.indexOf(name); + const cell = (cells, name) => cells[column(name)]; + const allRows = Array.from(table.querySelectorAll(':scope > tbody > tr')); + const isDay = row => Boolean(dayHeader(row.children[0]?.textContent)); + const isWeekly = row => /^Weekly Totals$/i.test(clean(row.children[0]?.textContent)); + const slots = [['i1', column('i1')], ['o1', column('o1')], ['i2', column('i2')], ['o2', column('o2')]]; + const parsePunch = (element, slot, rowIndex) => { + const displayTime = renderedPunchTime(element); + if (!displayTime || displayTime === '??') return null; + const nodes = [element, ...Array.from(element.querySelectorAll('*'))]; + if (nodes.length > 128) throw new Error('timecard_pcr_marker_invalid'); + const markerTokens = []; + for (const candidate of nodes) { + for (const token of Array.from(candidate.classList || [])) { + if (/^pcr/i.test(token)) { + if (token.length > 100) throw new Error('timecard_pcr_marker_invalid'); + markerTokens.push(token); + } + if (markerTokens.length > 256) throw new Error('timecard_pcr_marker_invalid'); + } + } + const markerStates = {pcrApproved: 'approved', pcrPending: 'pending', pcrRejected: 'rejected'}; + if (markerTokens.some(token => !Object.prototype.hasOwnProperty.call(markerStates, token)) || new Set(markerTokens).size !== markerTokens.length || markerTokens.length > 1) throw new Error('timecard_pcr_marker_invalid'); + const changeRequestStatus = markerTokens.length ? markerStates[markerTokens[0]] : null; + const approved = changeRequestStatus === 'approved'; + const directChange = () => { + if (changeRequestStatus === null) return {changeOperation: null, currentKind: null, currentTime: null, requestedKind: null, requestedTime: null, changeNote: null, changeDetailState: 'not_applicable'}; + const attributes = { + operation: ['data-pcr-operation', 'data-change-operation'], + currentKind: ['data-pcr-current-kind', 'data-pcr-current-type'], + currentTime: ['data-pcr-current-time'], + requestedKind: ['data-pcr-requested-kind', 'data-pcr-requested-type'], + requestedTime: ['data-pcr-requested-time'], + note: ['data-pcr-note', 'data-change-note'] + }; + const labels = { + operation: ['Operation', 'Request Operation'], + currentKind: ['Current Kind', 'Current Type'], + currentTime: ['Current Time'], + requestedKind: ['Requested Kind', 'Requested Type'], + requestedTime: ['Requested Time'], + note: ['Request Note', 'Change Note'] + }; + const observed = Object.fromEntries(Object.keys(attributes).map(key => [key, []])); + for (const candidate of nodes) { + for (const [key, aliases] of Object.entries(attributes)) for (const alias of aliases) if (candidate.hasAttribute?.(alias)) observed[key].push(clean(candidate.getAttribute(alias))); + for (const attribute of ['title', 'data-content', 'data-original-title']) { + if (!candidate.hasAttribute?.(attribute)) continue; + const rawContent = candidate.getAttribute(attribute); + if (typeof rawContent !== 'string' || rawContent.length > 4000) throw new Error('timecard_provenance_invalid'); + const content = decode(rawContent); + if (content.length > 4000) throw new Error('timecard_provenance_invalid'); + for (const line of content.split(/\n+/).map(clean).filter(Boolean)) { + for (const [key, aliases] of Object.entries(labels)) for (const alias of aliases) if (line.startsWith(`${alias}:`)) observed[key].push(clean(line.slice(alias.length + 1))); + } + } + } + return normalizeRequestDetail(observed); + }; + const change = directChange(); + const node = element.querySelector('[title*="Actual:"]'); + const raw = node?.getAttribute('title'); + if (!raw) return {ordinal: 0, rowIndex, slot, kind: '', displayTime, actualTime: '', roundedTime: '', clockName: '', clockCode: '', comment: '', provenanceAvailable: false, changeRequestStatus, approved, ...change}; + const text = decode(raw); + const lines = text.split(/\n+/).map(clean).filter(Boolean); + const first = lines[0] || ''; + const sourceKind = (first.match(/^(IN DAY|OUT LUNCH|IN LUNCH|OUT DAY|OUT BREAK|IN BREAK)\b/i) || [])[1]?.toUpperCase() || ''; + const kind = sourceKind === 'OUT BREAK' ? 'OUT LUNCH' : sourceKind === 'IN BREAK' ? 'IN LUNCH' : sourceKind; + const field = name => { + const line = lines.find(item => item.toLowerCase().startsWith(name.toLowerCase() + ':')); + return line ? clean(line.slice(name.length + 1)) : ''; + }; + const clock = field('Clock'); + const match = clock.match(/^(.*?)\s*\(([^()]*)\)\s*$/); + return {ordinal: 0, rowIndex, slot, kind, displayTime, actualTime: field('Actual'), roundedTime: field('Rounded'), clockName: match ? clean(match[1]) : clock, clockCode: match ? clean(match[2]) : '', comment: field('Comment'), provenanceAvailable: true, changeRequestStatus, approved, ...change}; + }; + const projectRow = (row, rowIndex) => { + const cells = Array.from(row.children); + const unresolvedSlots = slots.filter(([, index]) => clean(cells[index]?.textContent) === '??').map(([slot]) => slot); + const punches = slots.map(([slot, index]) => parsePunch(cells[index], slot, rowIndex)).filter(Boolean); + const waiver = cell(cells, 'waiver')?.querySelector('input[type=checkbox]'); + const comments = Array.from(cell(cells, 'comment')?.querySelectorAll('[title]') || []).map(element => decode(element.getAttribute('title')).replace(/^Comment:\s*/i, '').trim()).filter(Boolean); + return {payCode: controlValue(cell(cells, 'paycode')), allocation1: visible(cell(cells, 'allocation1')), allocation2: visible(cell(cells, 'allocation2')), hours: numeric(visible(cell(cells, 'hours'))), totalHours: numeric(visible(cell(cells, 'total_hours'))), dollars: numeric(visible(cell(cells, 'amount'))), exceptionText: visible(cell(cells, 'exception-points')), waiverChecked: waiver ? Boolean(waiver.checked) : null, comments, unresolvedSlots, punches}; + }; + const dayRows = allRows.filter(isDay); + const days = dayRows.map((row, index) => { + const heading = dayHeader(row.children[0]?.textContent); + const projected = projectRow(row, 0); + projected.punches.forEach((punch, punchIndex) => { punch.ordinal = punchIndex + 1; }); + return {date: boundDate(row.children[0]?.textContent, config.period.dates[index] || ''), label: heading?.label || '', payCode: projected.payCode, allocation1: projected.allocation1, allocation2: projected.allocation2, hours: projected.hours, totalHours: projected.totalHours, dollars: projected.dollars, exceptionText: projected.exceptionText, waiverChecked: projected.waiverChecked, comments: projected.comments, missingPunch: projected.unresolvedSlots.length > 0, unresolvedSlots: [...projected.unresolvedSlots], punches: projected.punches}; + }); + const additionalRows = []; + const nextRowIndex = Array(14).fill(1); + let dayIndex = -1; + for (const row of allRows) { + if (isDay(row)) { dayIndex++; continue; } + if (isWeekly(row) || dayIndex < 0 || row.children.length !== headers.length || clean(row.children[0]?.textContent) !== '') continue; + const rowIndex = nextRowIndex[dayIndex]++; + const projected = projectRow(row, rowIndex); + const day = days[dayIndex]; + const start = day.punches.length; + projected.punches.forEach((punch, punchIndex) => { punch.ordinal = start + punchIndex + 1; day.punches.push(punch); }); + const unresolved = projected.unresolvedSlots.map(slot => `${rowIndex}:${slot}`); + day.unresolvedSlots.push(...unresolved); + day.missingPunch = day.unresolvedSlots.length > 0; + additionalRows.push({date: day.date, rowIndex, rowClass: clean(row.className), payCode: projected.payCode, allocation1: projected.allocation1, allocation2: projected.allocation2, hours: projected.hours, totalHours: projected.totalHours, dollars: projected.dollars, exceptionText: projected.exceptionText, waiverChecked: projected.waiverChecked, comments: projected.comments, unresolvedSlots: projected.unresolvedSlots, punchOrdinals: projected.punches.map(punch => punch.ordinal)}); + } + const weeklyTotals = allRows.filter(isWeekly).map(row => numeric(row.children[1]?.textContent)); + const generic = id => { + const found = document.querySelector(id); + if (!found) return []; + return Array.from(found.querySelectorAll(':scope > tbody > tr')).map(row => Array.from(row.children).map(visible)).filter(row => row.some(Boolean) && !/^No Records Found$/i.test(row.join(' '))); + }; + const periodTotalHours = weeklyTotals.every(item => typeof item === 'number') ? Number(weeklyTotals.reduce((a, b) => a + b, 0).toFixed(2)) : null; + return {version: 2, sourceFormat: 'paycom-timecard-dom.v2', employeeCode: observedEmployeeCode, periodStart: config.period.start, periodEnd: config.period.end, periodKey: config.period.key, sourceUrl: location.href, pageTitle: document.title, headers, days, additionalRows, weeklyTotals, periodTotalHours, approvals: generic('#approvals-table'), attestations: generic('#timecard-attestation-table'), mealWaivers: generic('#meal-waivers-table')}; +} + +function buildExtractionExpression({employeeCode, period, sourceUrl}) { + validateCode(employeeCode); + const config = JSON.stringify({employeeCode, period: parsePeriodKey(period.key), sourceUrl}); + return `(${runtimeExtract.toString()})(${config},${normalizeRequestDetail.toString()},${resolveObservedEmployeeCode.toString()})`; +} + +module.exports = { + HEADERS, + HEADERS_NO_WAIVER, + TIME_CARD_VALIDATION_CODES, + renderedDayHeader, + bindRenderedDayDate, + resolveObservedEmployeeCode, + normalizeRequestDetail, + buildExtractionExpression, + validateTimecardRecord +}; diff --git a/plugins/paycom/backend/src/timecard-period.js b/plugins/paycom/backend/src/timecard-period.js new file mode 100644 index 0000000..993e7f2 --- /dev/null +++ b/plugins/paycom/backend/src/timecard-period.js @@ -0,0 +1,18 @@ +'use strict'; +const DAY=86400000,ANCHOR_START='2026-07-26',BASE='https://www.paycomonline.net/v4/cl/web.php/timecard/index'; +function isoDate(value){if(typeof value!=='string'||!/^\d{4}-\d{2}-\d{2}$/.test(value))throw new Error('invalid_period');const date=new Date(`${value}T00:00:00Z`);if(Number.isNaN(date.valueOf())||date.toISOString().slice(0,10)!==value)throw new Error('invalid_period');return date;} +function iso(date){return date.toISOString().slice(0,10);} +function fromBounds(start,end){const a=isoDate(start),b=isoDate(end);if(a.getUTCDay()!==0||b.getUTCDay()!==6||(b-a)!==13*DAY)throw new Error('invalid_period');const dates=Array.from({length:14},(_,i)=>iso(new Date(a.valueOf()+i*DAY)));return Object.freeze({start,end,key:`${start}_${end}`,dates:Object.freeze(dates)});} +function periodFromEnd(end){const b=isoDate(end);if(b.getUTCDay()!==6)throw new Error('invalid_period');return fromBounds(iso(new Date(b.valueOf()-13*DAY)),end);} +function periodContaining(value){const target=isoDate(value),anchor=isoDate(ANCHOR_START),index=Math.floor((target-anchor)/(14*DAY)),start=new Date(anchor.valueOf()+index*14*DAY);return fromBounds(iso(start),iso(new Date(start.valueOf()+13*DAY)));} +function parsePeriodKey(key){if(typeof key!=='string'||!/^[0-9]{4}-[0-9]{2}-[0-9]{2}_[0-9]{4}-[0-9]{2}-[0-9]{2}$/.test(key))throw new Error('invalid_period');const [start,end]=key.split('_');return fromBounds(start,end);} +function shift(period,days){const p=parsePeriodKey(period.key);return fromBounds(iso(new Date(isoDate(p.start).valueOf()+days*DAY)),iso(new Date(isoDate(p.end).valueOf()+days*DAY)));} +function nextPeriod(period){return shift(period,14);} +function previousPeriod(period){return shift(period,-14);} +function validateCode(code){if(typeof code!=='string'||!/^[A-Za-z0-9]{4}$/.test(code))throw new Error('invalid_employee_code');return code;} +function canonicalTimecardUrl(code,period){validateCode(code);const p=parsePeriodKey(period.key),url=new URL(BASE);url.searchParams.set('firstrefno',code);url.searchParams.set('perioddates',p.key);url.searchParams.set('formtype','SUMMARY');return url.href;} +function buildTimecardUrl(code,period,variant){if(variant!==1&&variant!==2)throw new Error('navigation_policy_violation');const url=new URL(canonicalTimecardUrl(code,period));url.searchParams.set('dispatch_timecards',String(variant));return url.href;} +function sameTimecardIdentity(url,expected){return url.protocol===expected.protocol&&url.hostname===expected.hostname&&url.port===''&&url.username.length===0&&url.password.length===0&&url.pathname===expected.pathname&&url.hash===''&&url.searchParams.getAll('firstrefno').length===1&&url.searchParams.get('firstrefno')===expected.searchParams.get('firstrefno')&&url.searchParams.getAll('perioddates').length===1&&url.searchParams.get('perioddates')===expected.searchParams.get('perioddates')&&url.searchParams.getAll('formtype').length===1&&url.searchParams.get('formtype')==='SUMMARY';} +function isCapturedTimecardUrl(value,{employeeCode,period}){try{const url=new URL(value),expected=new URL(canonicalTimecardUrl(employeeCode,period)),keys=[...url.searchParams.keys()].sort();return sameTimecardIdentity(url,expected)&&keys.join(',')==='dispatch_timecards,firstrefno,formtype,perioddates'&&url.searchParams.getAll('dispatch_timecards').length===1&&['1','2'].includes(url.searchParams.get('dispatch_timecards'));}catch{return false;}} +function isCanonicalTimecardUrl(value,{employeeCode,period}){try{const url=new URL(value),expected=new URL(canonicalTimecardUrl(employeeCode,period)),keys=[...url.searchParams.keys()].sort();return sameTimecardIdentity(url,expected)&&keys.join(',')==='firstrefno,formtype,perioddates';}catch{return false;}} +module.exports={BASE,ANCHOR_START,periodFromEnd,periodContaining,parsePeriodKey,nextPeriod,previousPeriod,validateCode,canonicalTimecardUrl,buildTimecardUrl,isCapturedTimecardUrl,isCanonicalTimecardUrl}; diff --git a/plugins/paycom/backend/tests/collector.test.js b/plugins/paycom/backend/tests/collector.test.js new file mode 100644 index 0000000..63f2219 --- /dev/null +++ b/plugins/paycom/backend/tests/collector.test.js @@ -0,0 +1,351 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const test = require('node:test'); +const { validateRequest, periodsFor, resolvedTargets, safeFailure } = require('../src/collector'); +const { ROSTER_REQUEST_FIELDS, rosterRequest, rosterAuthorityAssessment, fetchRosterRequest, + rosterMembershipAssessment, rosterMembership, collectTimecards } = require('../src/browser'); +const { periodContaining } = require('../src/timecard-period'); +const { resolveObservedEmployeeCode } = require('../src/timecard-dom'); +const { boundedJson } = require('dispatch-runtime-kit/collection-manager/src/validation'); + +function request(method = 'collector.health', input = {}) { + return { + protocolVersion: 1, + runId: 'run-1', + plan: 'paycom-health', + source: { id: 'paycom-main', collector: 'paycom', authProfile: 'paycom-main', config: { timezone: 'America/Los_Angeles', maxConcurrency: 3 } }, + method, + input, + attempt: 1, + deadline: new Date(Date.now() + 60_000).toISOString(), + }; +} + +test('collector request contract is closed and supports all registered methods', () => { + assert.equal(validateRequest(request()).method, 'collector.health'); + assert.equal(validateRequest(request('timecards.period', { periodEnd: '2026-09-05' })).input.periodEnd, '2026-09-05'); + assert.equal(validateRequest(request('timecards.from-published-roster', { periodEnd: '2026-09-05' })).input.periodEnd, '2026-09-05'); + assert.equal(validateRequest(request('timecards.audit', { periodEnd: '2026-09-05' })).input.periodEnd, '2026-09-05'); + assert.equal(validateRequest(request('roster.period', { periodEnd: '2026-09-05' })).input.periodEnd, '2026-09-05'); + assert.equal(validateRequest(request('resource-links.current-period', { resourceType: 'paycom.timecard.summary' })).input.resourceType, 'paycom.timecard.summary'); + assert.equal(validateRequest(request('resource-links.audit', { resourceType: 'paycom.timecard.summary', periodEnd: '2026-09-05' })).input.periodEnd, '2026-09-05'); + const syncInput = { + reconcileBatchSize: 10, fullReconcileMinutes: 1440, + lookbackPeriods: 1, publishMode: 'additions_edits', + }; + assert.equal(validateRequest(request('sync.current-workforce', syncInput)).input.publishMode, 'additions_edits'); + assert.throws(() => validateRequest(request('sync.current-workforce', { ...syncInput, lookbackPeriods: 2 })), /invalid_request/); + assert.equal(validateRequest(request('sync.current-workforce', { + ...syncInput, publishMode: 'additions_edits_preview', + })).input.publishMode, 'additions_edits_preview'); + assert.throws(() => validateRequest(request('sync.current-workforce', { ...syncInput, publishMode: 'deletions' })), /invalid_request/); + assert.throws(() => validateRequest({ ...request(), password: 'forbidden' }), /invalid_request/); + assert.throws(() => validateRequest(request('timecards.period', {})), /invalid_request/); + assert.throws(() => validateRequest(request('timecards.incremental', { periodEnd: '2026-09-05' })), /invalid_request/); + assert.throws(() => validateRequest(request('resource-links.current-period', { resourceType: 'paycom.arbitrary' })), /invalid_request/); + assert.throws(() => validateRequest({ ...request(), source: { ...request().source, config: { timezone: 'UTC', maxConcurrency: 7 } } }), /invalid_request/); +}); + +test('period discovery returns one previous, current, and next period', () => { + const rows = periodsFor('2026-08-25'); + assert.deepEqual(rows.map(row => row.relation), ['previous', 'current', 'next']); + assert.equal(rows[1].key, periodContaining('2026-08-25').key); +}); + +test('standard selectors resolve to unique exact Paycom pay periods without collecting', () => { + const date = resolvedTargets({ selectorKind: 'date', date: '2026-08-18' }); + assert.equal(date.targetType, 'pay-period'); + assert.deepEqual(date.targets.map(target => target.key), ['2026-08-22']); + assert.deepEqual(resolvedTargets({ selectorKind: 'latest-complete', date: '2026-08-18' }).targets.map(target => target.key), ['2026-08-08']); + assert.deepEqual(resolvedTargets({ selectorKind: 'date-range', start: '2026-08-01', end: '2026-08-31' }).targets.map(target => target.key), + ['2026-08-08', '2026-08-22', '2026-09-05']); + assert.deepEqual(resolvedTargets({ selectorKind: 'exact-target', key: '2026-09-05' }).targets[0].values, { periodEnd: '2026-09-05' }); + assert.equal(validateRequest(request('collection.resolve-targets', { selectorKind: 'date', date: '2026-08-18' })).method, 'collection.resolve-targets'); + assert.throws(() => validateRequest(request('collection.resolve-targets', { selectorKind: 'date-range', start: '2026-08-01' })), /invalid_request/); +}); + +test('roster interception accepts only exact API membership and period bounds', () => { + const period = periodContaining('2026-08-25'); + const event = { + requestId: 'request-1', + request: { + method: 'POST', + url: 'https://time-and-attendance.paycomonline.net/api/cl/timecard-search/employees', + postData: JSON.stringify({ startDate: period.start, endDate: period.end, eeCodes: ['A001', 'A002'] }), + }, + }; + assert.deepEqual(rosterRequest(event, period), { requestId: 'request-1', codes: ['A001', 'A002'] }); + assert.equal(rosterRequest({ ...event, request: { ...event.request, url: `${event.request.url}/evil` } }, period), null); + assert.equal(rosterRequest({ ...event, request: { ...event.request, url: `${event.request.url}?unexpected=1` } }, period), null); + assert.equal(rosterMembership(Buffer.from(JSON.stringify({ eeCodes: ['A002', 'A001'] })), ['A001', 'A002']), true); + assert.equal(rosterMembership(Buffer.from(JSON.stringify({ eeCodes: ['A001'] })), ['A001', 'A002']), false); + assert.deepEqual(rosterMembershipAssessment( + Buffer.from(JSON.stringify({ eeCodes: ['A001'] })), ['A001', 'A002'], + ), { exact: false, returnedCount: 1 }); + assert.equal(rosterMembershipAssessment( + Buffer.from(JSON.stringify({ eeCodes: ['A003'] })), ['A001', 'A002'], + ), null); + const fetchBody = Object.fromEntries(ROSTER_REQUEST_FIELDS.map(key => [key, null])); + Object.assign(fetchBody, { startDate: '2026-08-09', endDate: '2026-08-22', eeCodes: ['A001', 'A002'] }); + const fetchRequest = fetchRosterRequest({ + requestId: 'fetch-1', request: { method: 'POST', url: event.request.url, postData: JSON.stringify(fetchBody) }, + }, period); + assert.deepEqual(fetchRequest.codes, ['A001', 'A002']); + assert.deepEqual(fetchRequest.uiPeriod, { start: '2026-08-09', end: '2026-08-22' }); + const rewritten = JSON.parse(Buffer.from(fetchRequest.postData, 'base64').toString('utf8')); + assert.equal(rewritten.startDate, period.start); + assert.equal(rewritten.endDate, period.end); + assert.equal(fetchRosterRequest({ + requestId: 'fetch-1', request: { method: 'POST', url: event.request.url, postData: JSON.stringify({ ...fetchBody, unexpected: true }) }, + }, period), null); +}); + +test('bounded timecard workers reuse browser targets and emit identifier-free performance metrics', async () => { + const employees = Array.from({ length: 8 }, (_, index) => ({ employeeCode: `A00${index}`, employeeName: `Employee ${index}` })); + let opened = 0; + let closed = 0; + let active = 0; + let peak = 0; + const uses = []; + const result = await collectTimecards('http://127.0.0.1:1', employees, periodContaining('2026-08-25'), 3, () => {}, { + openSession: async workerIndex => { + opened += 1; + uses[workerIndex] = 0; + return { + collect: async (employee, collectedPeriod, variant, onTiming) => { + uses[workerIndex] += 1; + active += 1; + peak = Math.max(peak, active); + await new Promise(resolve => setTimeout(resolve, 5)); + active -= 1; + onTiming({ navigateMs: 10, responseMs: 20, loadMs: 30, bodyMs: 40, readyMs: 50, extractMs: 60, validateMs: 70 }); + return { employeeCode: employee.employeeCode }; + }, + close: async () => { closed += 1; }, + }; + }, + }); + assert.equal(result.rows.length, employees.length); + assert.equal(opened, 3); + assert.equal(closed, 3); + assert.equal(peak, 3); + assert.equal(uses.some(count => count > 1), true); + const phases = ['targetOpen', 'navigate', 'response', 'load', 'body', 'ready', 'extract', 'validate']; + assert.deepEqual(Object.keys(result.performance).sort(), [ + 'itemCount', 'itemMaxMs', 'itemP50Ms', 'itemP95Ms', 'openedTargets', 'retryCount', 'totalMs', 'workerCount', + ...phases.flatMap(phase => [`${phase}MaxMs`, `${phase}P50Ms`, `${phase}P95Ms`]), + ].sort()); + assert.equal(result.performance.navigateP50Ms, 10); + assert.equal(result.performance.validateP95Ms, 70); + assert.equal(JSON.stringify(result.performance).includes('Employee'), false); + assert.doesNotThrow(() => boundedJson({ performance: result.performance })); +}); + +test('timecard workers enforce one absolute item deadline and recycle only the timed-out session', async () => { + const employees = [{ employeeCode: 'A001', employeeName: 'Employee 1' }]; + let opened = 0; + let closed = 0; + const result = await collectTimecards('http://127.0.0.1:1', employees, periodContaining('2026-08-25'), 1, () => {}, { + itemTimeoutMs: 20, + openSession: async (workerIndex, signal) => { + opened += 1; + const attempt = opened; + return { + collect: async (employee, period, variant, onTiming) => { + if (attempt === 1) { + await new Promise((resolve, reject) => signal.addEventListener('abort', () => reject(Object.assign(new Error('cancelled'), { code: 'acquisition_cancelled' })), { once: true })); + } + onTiming({ navigateMs: 1, responseMs: 1, loadMs: 1, bodyMs: 1, readyMs: 1, extractMs: 1, validateMs: 1 }); + return { employeeCode: employee.employeeCode }; + }, + close: async () => { closed += 1; }, + }; + }, + }); + assert.equal(result.rows.length, 1); + assert.equal(result.performance.retryCount, 1); + assert.equal(opened, 2); + assert.equal(closed, 2); +}); + +test('rendered timecard identity must be present, singular, and internally consistent', () => { + assert.equal(resolveObservedEmployeeCode(['a001', 'A001']), 'A001'); + assert.equal(resolveObservedEmployeeCode([]), ''); + assert.equal(resolveObservedEmployeeCode(['A001', 'A002']), ''); + assert.equal(resolveObservedEmployeeCode(['not-an-employee']), ''); +}); + +test('collector failures expose only stable sanitized error codes', () => { + for (const code of ['invalid_credentials', 'primary_credentials_rejected', 'security_answers_rejected', 'browser_profile_busy']) { + assert.deepEqual(safeFailure(Object.assign(new Error('details'), { code })), { ok: false, status: 'failed', data: null, error: { code } }); + } + assert.equal(safeFailure(new Error('password=secret')).error.code, 'collection_failed'); +}); + +test('worker rejects duplicate JSON keys with one sanitized receipt', () => { + const worker = path.resolve(__dirname, "../bin/dispatch-paycom-collector"); + const result = spawnSync(worker, [], { input: '{"protocolVersion":1,"protocolVersion":1}\n', encoding: 'utf8' }); + assert.equal(result.status, 0); + assert.deepEqual(JSON.parse(result.stdout), { ok: false, status: 'failed', data: null, error: { code: 'invalid_request' } }); + assert.equal(result.stderr, ''); +}); + +test('sync roster request clears saved soft filters while preserving closed request and full-membership guards', () => { + const period = periodContaining('2026-09-08'); + const body = { ...Object.fromEntries(ROSTER_REQUEST_FIELDS.map(key => [key, null])), + startDate: period.start, endDate: period.end, eeCodes: ['A001', 'A002'], + q: '', isAdvancedFilterApplied: true, onlyBorrowedEmployees: false, + payClassCodes: 'selected-class', selectedEarnings: [], approvalMode: 0, + }; + const event = selected => ({ requestId: 'unfiltered-roster', request: { + method: 'POST', url: 'https://time-and-attendance.paycomonline.net/api/cl/timecard-search/employees', + postData: JSON.stringify(selected), + } }); + assert.equal(fetchRosterRequest(event(body), period).authoritative, false); + const request = fetchRosterRequest(event(body), period, { unfiltered: true }); + assert.equal(request.authoritative, true); + assert.deepEqual(request.codes, ['A001', 'A002']); + const sent = JSON.parse(Buffer.from(request.postData, 'base64').toString()); + assert.equal(sent.isAdvancedFilterApplied, false); + assert.equal(sent.payClassCodes, 'selected-class'); + assert.equal(sent.approvalMode, null); + assert.equal(body.isAdvancedFilterApplied, true, 'Never change saved UI preferences'); + for (const patch of [{ q: 'employee' }, { skip: 1 }, { onlyBorrowedEmployees: true }, { isAdvancedFilterApplied: null }]) { + const refused = fetchRosterRequest(event({ ...body, ...patch }), period, { unfiltered: true }); + assert.equal(refused.observable, false); + assert.equal(refused.authoritative, false); + } + assert.equal(fetchRosterRequest(event({ ...body, unexpected: true }), period, { unfiltered: true }), null); + assert.equal(rosterMembership(Buffer.from(JSON.stringify({ eeCodes: ['A001'] })), request.codes), false); +}); + +for (const readFails of [false, true]) test(`roster capture owns a fresh page and closes it after ${readFails ? 'failure' : 'success'}`, async t => { + const http = require('node:http'); + const { CdpConnection } = require('../../../../runtime/auth-broker/src/cdp'); + const { collectRoster, ROSTER_API, TIMECARD_SEARCH_URL } = require('../src/browser'); + const period = periodContaining('2026-09-08'); + const body = { ...Object.fromEntries(ROSTER_REQUEST_FIELDS.map(key => [key, null])), + startDate: period.start, endDate: period.end, eeCodes: ['A001', 'A002'], + q: '', isAdvancedFilterApplied: true, onlyBorrowedEmployees: false, payClassCodes: 'selected', approvalMode: 0 }; + const requestEvent = { requestId: 'request', networkId: 'network-original', request: { + method: 'POST', url: ROSTER_API, headers: { Authorization: 'Bearer fixture-session', 'Content-Type': 'application/json' }, postData: JSON.stringify(body) } }; + const calls = []; + let navigated = false, resolveRequest; + const socket = new EventTarget(); + const connection = { + socket, close() {}, + evaluate: async expression => { + return require('node:vm').runInNewContext(expression, { AbortSignal, TextDecoder, fetch: async (url, options) => { + assert.equal(url, ROSTER_API); + assert.equal(options.redirect, 'error'); + assert.equal(options.cache, 'no-store'); + assert.equal(options.headers.Authorization, 'Bearer fixture-session'); + const sent = JSON.parse(options.body); + assert.equal(sent.startDate, period.start); + assert.equal(sent.endDate, period.end); + assert.equal(sent.isAdvancedFilterApplied, false); + assert.equal(sent.payClassCodes, 'selected'); + if (readFails) return new Response('unavailable', { status: 503 }); + return new Response(JSON.stringify({ eeCodes: ['A001', 'A002'], employees: [] }), { headers: { 'Content-Type': 'application/json' } }); + } }); + }, + waitFor: async (method, accepts) => { + assert.equal(method, 'Fetch.requestPaused'); + assert.equal(accepts(requestEvent), true); + return new Promise(resolve => { resolveRequest = resolve; }); + }, + command: async (name, params = {}) => { + calls.push([name, params]); + if (name === 'Fetch.enable') assert.equal(navigated, false, 'Install capture while the new page is still blank'); + if (name === 'Page.navigate') { + assert.equal(params.url, TIMECARD_SEARCH_URL); + navigated = true; + resolveRequest(requestEvent); + for (const params of [requestEvent, + { requestId: 'preflight', request: { method: 'OPTIONS', url: ROSTER_API } }, + { ...requestEvent, requestId: 'dependent-request', networkId: 'dependent-network' }]) { + socket.dispatchEvent(new MessageEvent('message', { data: JSON.stringify({ method: 'Fetch.requestPaused', params }) })); + } + } + if (name === 'Fetch.continueRequest' && params.requestId === 'request') { + assert.equal(params.postData, undefined, 'Preserve the original page request'); + } + return {}; + }, + }; + t.mock.method(CdpConnection, 'connect', async url => { + assert.ok(url.endsWith('/devtools/page/capture'), 'Never reuse the handoff page and its pending requests'); + return connection; + }); + const browserCalls = []; + const server = http.createServer((request, response) => { + browserCalls.push([request.method, request.url]); + if (request.method === 'PUT' && request.url === '/json/new?about%3Ablank') return response.end(JSON.stringify({ + type: 'page', id: 'capture', url: 'about:blank', + webSocketDebuggerUrl: `ws://127.0.0.1:${server.address().port}/devtools/page/capture`, + })); + if (request.url === '/json/close/capture') return response.end(JSON.stringify({ success: true })); + response.writeHead(400); response.end('{}'); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => server.close(resolve))); + const collection = collectRoster(`http://127.0.0.1:${server.address().port}`, period, { unfiltered: true }); + if (readFails) await assert.rejects(collection, { code: 'roster_response_invalid' }); + else assert.equal((await collection).completeness.authoritative, true); + assert.deepEqual(browserCalls, [['PUT', '/json/new?about%3Ablank'], ['GET', '/json/close/capture']]); + assert.ok(calls.some(([method, params]) => method === 'Fetch.continueRequest' && params.requestId === 'preflight')); + assert.ok(calls.some(([method, params]) => method === 'Fetch.continueRequest' && params.requestId === 'dependent-request')); + assert.equal(calls.filter(([method]) => method === 'Fetch.enable').length, 1, 'Do not change interception while the request is paused'); + assert.ok(calls.some(([method]) => method === 'Fetch.disable')); +}); + +test('dedicated roster reads bound response size and reject non-JSON, redirects, and timeouts', async () => { + const { rosterReadExpression } = require('../src/browser'); + const expression = rosterReadExpression({ headers: { 'Content-Type': 'application/json' }, body: '{}' }); + const run = fetch => require('node:vm').runInNewContext(expression, { fetch, AbortSignal, TextDecoder }); + assert.equal((await run(async () => new Response('login page', { headers: { 'Content-Type': 'text/html' } }))).error, 'invalid_response'); + assert.equal((await run(async () => new Response(new Uint8Array(2_097_153), { headers: { 'Content-Type': 'application/json' } }))).error, 'too_large'); + assert.equal((await run(async (_url, options) => { + assert.equal(options.redirect, 'error'); + throw new TypeError('redirect rejected'); + })).error, 'request_failed'); + assert.equal((await run(async () => { throw Object.assign(new Error(), { name: 'TimeoutError' }); })).error, 'timeout'); +}); + + +test('timecard sessions subscribe to network events before navigating and clean up on invalid HTML', async t => { + const http = require('node:http'); + const { CdpConnection } = require('../../../../runtime/auth-broker/src/cdp'); + const { collectOneTimecard } = require('../src/browser'); + let networkEnabled = false, navigated = false, closed = false; + const connection = { + close() { closed = true; }, + command: async (method) => { + if (method === 'Network.enable') networkEnabled = true; + if (method === 'Network.disable') networkEnabled = false; + if (method === 'Page.navigate') { + assert.equal(networkEnabled, true, 'Response events require a subscription on this session'); + navigated = true; + return { loaderId: 'loader' }; + } + if (method === 'Network.getResponseBody') return { body: 'Incomplete timecard' }; + return {}; + }, + waitFor: async () => ({ requestId: 'response' }), + }; + t.mock.method(CdpConnection, 'connect', async () => connection); + const server = http.createServer((_request, response) => response.end(JSON.stringify({ + type: 'page', id: 'fixture', url: 'about:blank', + webSocketDebuggerUrl: `ws://127.0.0.1:${server.address().port}/devtools/page/fixture`, + }))); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => server.close(resolve))); + await assert.rejects(collectOneTimecard(`http://127.0.0.1:${server.address().port}`, + { employeeCode: 'A001', employeeName: 'Fixture' }, periodContaining('2026-09-08')), /timecard_html_invalid/); + assert.equal(navigated, true); + assert.equal(networkEnabled, false); + assert.equal(closed, true); +}); diff --git a/plugins/paycom/backend/tests/helpers.js b/plugins/paycom/backend/tests/helpers.js new file mode 100644 index 0000000..739838f --- /dev/null +++ b/plugins/paycom/backend/tests/helpers.js @@ -0,0 +1,54 @@ +'use strict'; + +const { periodFromEnd, buildTimecardUrl } = require('../src/timecard-period'); + +const LABELS = ['SUN', 'MON', 'TUE', 'WED', 'THU', 'FRI', 'SAT']; + +function timecardRecord(code = 'A001', end = '2026-09-05', variant = 1) { + const period = periodFromEnd(end); + return { + version: 2, + sourceFormat: 'paycom-timecard-dom.v2', + employeeCode: code, + periodStart: period.start, + periodEnd: period.end, + periodKey: period.key, + sourceUrl: buildTimecardUrl(code, period, variant), + pageTitle: 'Timecard Editor', + headers: ['date', 'paycode', 'i1', 'allocation1', 'o1', 'i2', 'allocation2', 'o2', 'hours', 'total_hours', 'amount', 'exception-points', 'waiver', 'comment', 'missing-punch', 'delete'], + additionalRows: [], + periodTotalHours: 8, + weeklyTotals: [8, 0], + approvals: [], + attestations: [], + mealWaivers: [], + days: period.dates.map((date, index) => ({ + date, + label: LABELS[index % 7], + payCode: '', + allocation1: '', + allocation2: '', + hours: index === 0 ? 8 : null, + totalHours: index === 0 ? 8 : null, + dollars: null, + exceptionText: '', + waiverChecked: null, + comments: [], + missingPunch: false, + unresolvedSlots: [], + punches: index === 0 ? [{ + ordinal: 1, rowIndex: 0, slot: 'i1', kind: 'IN DAY', displayTime: '09:00 AM', + actualTime: '09:00 AM', roundedTime: '09:00 AM', clockName: 'Clock', clockCode: 'WEB00', + comment: '', provenanceAvailable: true, changeRequestStatus: null, approved: false, + changeOperation: null, currentKind: null, currentTime: null, requestedKind: null, + requestedTime: null, changeNote: null, changeDetailState: 'not_applicable', + }] : [], + })), + }; +} + +function rosterRow(code = 'A001', name = 'Employee One') { + return { employeeCode: code, employeeName: name, isActive: true, isActiveDriver: true }; +} + +module.exports = { timecardRecord, rosterRow }; diff --git a/plugins/paycom/backend/tests/oci-lifecycle-seed.js b/plugins/paycom/backend/tests/oci-lifecycle-seed.js new file mode 100644 index 0000000..666029a --- /dev/null +++ b/plugins/paycom/backend/tests/oci-lifecycle-seed.js @@ -0,0 +1,56 @@ +#!/usr/local/bin/node +'use strict'; +// Installed only in the disposable acceptance image, never the runtime image. +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const { PaycomStore, stageCandidate, cleanupStage } = require('/opt/dispatch/plugins/paycom/backend/src/store'); +const { DATABASE, STAGING_ROOT } = require('/opt/dispatch/plugins/paycom/backend/src/paths'); +const { periodFromEnd } = require('/opt/dispatch/plugins/paycom/backend/src/timecard-period'); +const { TIMECARD_SUMMARY, ROUTE_VERSION, linkRows } = require('/opt/dispatch/plugins/paycom/backend/src/resource-links'); +const { timecardRecord, rosterRow } = require('/opt/dispatch/plugins/paycom/backend/tests/helpers'); +const PAYCOM_SYNC_ID = 'paycom-main-workforce'; +process.umask(0o077); +const store = new PaycomStore(DATABASE); +const changed = process.argv[2] === '--changed'; +const period = periodFromEnd(changed ? '2026-09-19' : '2026-09-05'); +const collectedAt = new Date().toISOString(); +function publish(candidate) { + const stage = stageCandidate(STAGING_ROOT, { attempt: 1, collectedAt, ...candidate, runId: candidate.runId + (changed ? '_changed' : '') }); + try { return store.publish(stage); } finally { cleanupStage(stage, STAGING_ROOT); } +} +const payPeriods = publish({ kind: 'pay_periods', target: period.end, runId: 'run_periods_fixture', metadata: {}, + rows: [{ start: period.start, end: period.end, key: period.key, relation: 'current' }] }); +const rows = [rosterRow('Z999', changed ? 'Synthetic Updated Employee' : 'Synthetic Fixture Employee')]; +const roster = publish({ kind: 'roster', target: period.end, runId: 'run_fixture_roster', metadata: {}, rows }); +const timecards = publish({ kind: 'timecards', target: period.end, periodKey: period.key, + runId: 'run_fixture_timecards', metadata: { periodStart: period.start, periodEnd: period.end, mode: 'full', + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256 }, + rows: [{ employeeCode: rows[0].employeeCode, employeeName: rows[0].employeeName, + record: timecardRecord(rows[0].employeeCode, period.end), sourceSha256: 'b'.repeat(64) }] }); +const links = publish({ kind: 'resource_links', target: period.end, periodKey: period.key, + runId: 'run_fixture_links', metadata: { resourceType: TIMECARD_SUMMARY, periodStart: period.start, + periodEnd: period.end, rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256, + routeVersion: ROUTE_VERSION }, rows: linkRows(TIMECARD_SUMMARY, rows, period) }); +store.close(); +if (!changed) { +const schema = { type: 'object', properties: { behavior: { type: 'string', enum: ['no_change'] } }, + required: ['behavior'], additionalProperties: false }; +const spec = { schemaVersion: 1, + collectors: [{ id: 'fixture', version: '1.0.0', description: 'Synthetic acceptance collector', + command: '/opt/dispatch/fixture-collector', sourceSchema: { type: 'object', properties: {}, required: [], additionalProperties: false }, + methods: { 'fixture.sync': { description: 'No provider access', inputSchema: schema, timeoutSeconds: 5, + maxAttempts: 1, backoffSeconds: [], concurrencyKeys: ['collector:{collector}'] } } }], + sources: [{ id: 'fixture-main', collector: 'fixture', authProfile: null, config: {}, enabled: true }], + plans: [{ id: 'fixture-plan', source: 'fixture-main', method: 'fixture.sync', schedule: { type: 'manual' }, + input: { behavior: 'no_change' }, dependsOn: [], enabled: true }], + syncs: [{ id: PAYCOM_SYNC_ID, plan: 'fixture-plan', intervalSeconds: 3600, jitterSeconds: 0, + overlap: 'coalesce', settingsSchema: schema, settings: { behavior: 'no_change' }, desiredState: 'stopped' }] }; +const file = path.join(process.env.DISPATCH_RUNTIME_ROOT, 'synthetic-spec.json'); +fs.writeFileSync(file, JSON.stringify(spec), { mode: 0o600 }); +const applied = spawnSync('/usr/local/bin/node', ['/opt/dispatch/runtime/collection-manager/bin/dispatch-collectionctl', 'apply', file], + { encoding: 'utf8', env: process.env, timeout: 30_000 }); +fs.unlinkSync(file); +if (applied.status !== 0 || JSON.parse(applied.stdout).status !== 'applied') throw new Error(`fixture_apply_failed_${JSON.parse(applied.stdout || '{}').status || applied.status}`); +} +process.stdout.write(`${JSON.stringify({ payPeriods, roster, timecards, links })}\n`); diff --git a/plugins/paycom/backend/tests/publication-continuity.test.js b/plugins/paycom/backend/tests/publication-continuity.test.js new file mode 100644 index 0000000..59df834 --- /dev/null +++ b/plugins/paycom/backend/tests/publication-continuity.test.js @@ -0,0 +1,76 @@ +'use strict'; +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { PaycomStore, stageCandidate, cleanupStage } = require('../src/store'); +const { periodFromEnd } = require('../src/timecard-period'); +const { TIMECARD_SUMMARY, ROUTE_VERSION, linkRows } = require('../src/resource-links'); +const { rosterRow, timecardRecord } = require('./helpers'); +const { createPublicationBaseline } = require('dispatch-protocol/contracts/src/publication-baseline'); +const { verifyPublicationContinuity } = require('../src/publication-continuity'); + +test('continuity audits active publication content, target and original producer identities', t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'legacy-sourceation-proof-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const stageRoot = path.join(root, 'stage'); fs.mkdirSync(stageRoot, { mode: 0o700 }); + const database = path.join(root, 'paycom.sqlite3'); + let store = new PaycomStore(database); + const period = periodFromEnd('2026-09-05'); + const rows = [rosterRow('Z999', 'Synthetic Fixture')]; + const proof = { target: period.end, publications: {} }; + function publish(name, candidate) { + const runId = `run_fixture_${name}`; + const stage = stageCandidate(stageRoot, { target: period.end, attempt: 1, + collectedAt: '2026-09-04T12:00:00.000Z', runId, ...candidate }); + try { + const value = store.publish(stage); + proof.publications[name] = { id: value.publicationId, originRunId: runId, contentSha256: value.contentSha256 }; + return value; + } finally { cleanupStage(stage, stageRoot); } + } + try { + publish('payPeriods', { kind: 'pay_periods', metadata: {}, + rows: [{ start: period.start, end: period.end, key: period.key, relation: 'current' }] }); + const roster = publish('roster', { kind: 'roster', metadata: {}, rows }); + publish('timecards', { kind: 'timecards', periodKey: period.key, + metadata: { periodStart: period.start, periodEnd: period.end, mode: 'full', + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256 }, + rows: [{ employeeCode: 'Z999', employeeName: 'Synthetic Fixture', record: timecardRecord('Z999'), sourceSha256: 'b'.repeat(64) }] }); + publish('resourceLinks', { kind: 'resource_links', periodKey: period.key, + metadata: { resourceType: TIMECARD_SUMMARY, periodStart: period.start, periodEnd: period.end, + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256, routeVersion: ROUTE_VERSION }, + rows: linkRows(TIMECARD_SUMMARY, rows, period) }); + } finally { store.close(); } + const baseline = createPublicationBaseline(proof.target, proof.publications); + assert.deepEqual(verifyPublicationContinuity(database, { mode: 'capture' }), { status: 'verified', publicationBaseline: baseline }); + assert.deepEqual(verifyPublicationContinuity(database, { mode: 'verify', baseline }), { status: 'verified', publicationBaselineDigest: baseline.digest }); + for (const alter of [ + value => { value.publications.timecards.contentSha256 = '0'.repeat(64); }, + value => { value.publications.roster.originRunId = 'run_substituted'; }, + value => { value.publications.resourceLinks.id = 'pub_substituted'; }, + value => { value.target = '2026-09-19'; }, + ]) { + const value = structuredClone(proof); alter(value); + assert.throws(() => verifyPublicationContinuity(database, { mode: 'verify', baseline: createPublicationBaseline(value.target, value.publications) }), /first_publication_failed/); + } + store = new PaycomStore(database); + try { + const changedRows = [rosterRow('Z999', 'Synthetic Updated')]; + const roster = publish('roster', { kind: 'roster', metadata: {}, rows: changedRows }); + publish('timecards', { kind: 'timecards', periodKey: period.key, + metadata: { periodStart: period.start, periodEnd: period.end, mode: 'full', + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256 }, + rows: [{ employeeCode: 'Z999', employeeName: 'Synthetic Updated', record: timecardRecord('Z999'), sourceSha256: 'c'.repeat(64) }] }); + publish('resourceLinks', { kind: 'resource_links', periodKey: period.key, + metadata: { resourceType: TIMECARD_SUMMARY, periodStart: period.start, periodEnd: period.end, + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256, routeVersion: ROUTE_VERSION }, + rows: linkRows(TIMECARD_SUMMARY, changedRows, period) }); + } finally { store.close(); } + const updated = verifyPublicationContinuity(database, { mode: 'capture' }).publicationBaseline; + assert.notEqual(updated.digest, baseline.digest); + assert.throws(() => verifyPublicationContinuity(database, { mode: 'verify', baseline }), /first_publication_failed/); + assert.equal(verifyPublicationContinuity(database, { mode: 'verify', baseline: updated }).publicationBaselineDigest, updated.digest); + +}); diff --git a/plugins/paycom/backend/tests/published-fixture.js b/plugins/paycom/backend/tests/published-fixture.js new file mode 100644 index 0000000..79b6541 --- /dev/null +++ b/plugins/paycom/backend/tests/published-fixture.js @@ -0,0 +1,15 @@ +'use strict'; +const { rosterRow, timecardRecord } = require('./helpers'); +const { periodFromEnd } = require('../src/timecard-period'); +const { linkRows, TIMECARD_SUMMARY } = require('../src/resource-links'); +function workforceFixture({ name = 'Fixture', count = 3, target = '2026-09-19' } = {}) { + const period = periodFromEnd(target), collected = '2026-09-11T12:00:00.000Z'; + const employees = Array.from({ length: count }, (_, index) => rosterRow(index.toString(36).toUpperCase().padStart(4, '0'), `${name} Employee ${index + 1}`)); + const publication = { id: `roster-${name}-${target}`, target, collected_at: collected, content_sha256: 'a'.repeat(64) }; + return { roster: { publication, employees }, + timecards: { publication: { id: `timecards-${name}-${target}`, target, collected_at: collected }, + rows: employees.map(row => ({ ...row, observedAt: collected, record: timecardRecord(row.employeeCode, target) })) }, + resourceLinks: { publication: { id: `links-${name}-${target}`, target, collected_at: collected, + period_key: period.key, resource_type: TIMECARD_SUMMARY }, rows: linkRows(TIMECARD_SUMMARY, employees, period) } }; +} +module.exports = { workforceFixture }; diff --git a/plugins/paycom/backend/tests/published.test.js b/plugins/paycom/backend/tests/published.test.js new file mode 100644 index 0000000..93d929a --- /dev/null +++ b/plugins/paycom/backend/tests/published.test.js @@ -0,0 +1,95 @@ +'use strict'; +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { PaycomStore, stageCandidate, cleanupStage } = require('../src/store'); +const { periodFromEnd } = require('../src/timecard-period'); +const { TIMECARD_SUMMARY, ROUTE_VERSION, linkRows } = require('../src/resource-links'); +const { rosterRow, timecardRecord } = require('./helpers'); +const { publishWorkforce } = require('../adapters/published'); +const { LocalPaycomWorkforcePort } = require('../adapters/workforce'); +const { createPublishedClient, SORTS } = require('../../dashboard/published'); +const { WorkforceClient } = require('../../../../runtime/sdk/src/workforce-client'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'published-workforce-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const stageRoot = path.join(root, 'stage'); fs.mkdirSync(stageRoot, { mode: 0o700 }); + const database = path.join(root, 'paycom.sqlite3'), directory = path.join(root, 'published'); + function seed(end = '2026-09-05', name = 'Employee One') { + const store = new PaycomStore(database), period = periodFromEnd(end); + const rows = [rosterRow('Z999', name), rosterRow('A001', 'Employee 10'), rosterRow('B002', 'Employee 2')]; + const publish = candidate => { + const stage = stageCandidate(stageRoot, { target: end, attempt: 1, collectedAt: '2026-09-04T12:00:00.000Z', + runId: `run_${candidate.kind}_${end}_${name.replaceAll(' ', '_')}`, ...candidate }); + try { return store.publish(stage); } finally { cleanupStage(stage, stageRoot); } + }; + try { + const roster = publish({ kind: 'roster', metadata: {}, rows }); + publish({ kind: 'timecards', periodKey: period.key, metadata: { periodStart: period.start, periodEnd: end, mode: 'full', + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256 }, + rows: rows.map(row => ({ employeeCode: row.employeeCode, employeeName: row.employeeName, record: timecardRecord(row.employeeCode, end), sourceSha256: 'b'.repeat(64) })) }); + publish({ kind: 'resource_links', periodKey: period.key, metadata: { resourceType: TIMECARD_SUMMARY, periodStart: period.start, periodEnd: end, + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256, routeVersion: ROUTE_VERSION }, rows: linkRows(TIMECARD_SUMMARY, rows, period) }); + } finally { store.close(); } + } + const timezone = 'America/Los_Angeles'; + return { root, database, directory, seed, + publish: () => publishWorkforce({ database, publishedDatabase: path.join(directory, 'paycom.sqlite3'), timezone }), + published: createPublishedClient({ directory }).workforce, + original: new WorkforceClient({ port: new LocalPaycomWorkforcePort({ database, timezone }) }) }; +} + +test('published reads match the existing workforce contract, including historical dates and sorting', async t => { + const c = fixture(t); c.seed('2026-08-22'); c.seed(); + assert.equal((await c.publish()).changed, 2); + assert.equal((await c.publish()).changed, 0); + for (const method of ['snapshot', 'employees', 'employee', 'timecards', 'resourceLinks', 'punches', 'day']) { + const inputs = method === 'employee' ? ['Z999'] : method === 'punches' || method === 'day' ? [{ date: '2026-08-23', limit: 2, offset: 1 }] : method === 'snapshot' ? [] : [{ limit: 2, offset: 1 }]; + const expected = await c.original[method](...inputs); + assert.equal(expected.ok, true, JSON.stringify(expected)); + assert.deepEqual(await c.published[method](...inputs), expected, method); + } + for (const date of ['2026-08-09', '2026-08-24', '2026-10-01']) { + for (const sort of SORTS) for (const direction of ['asc', 'desc']) { + const query = { date, sort, direction, limit: 2, search: 'Employee' }; + assert.deepEqual(await c.published.day(query), await c.original.day(query)); + } + } + fs.renameSync(c.database, c.database + '.offline'); + assert.equal((await c.published.employees()).data.total, 3, 'reading has no source database or DSP dependency'); +}); + +test('incomplete publication preserves the last complete period; changes replace only that period', async t => { + const c = fixture(t); c.seed('2026-08-22'); c.seed(); await c.publish(); + const before = await c.published.day({ date: '2026-08-09' }); + c.seed('2026-09-05', 'Changed Employee'); await c.publish(); + assert.equal((await c.published.employee('Z999')).data.employee.employeeName, 'Changed Employee'); + assert.deepEqual(await c.published.day({ date: '2026-08-09' }), before); + const store = new PaycomStore(c.database); + store.db.prepare("DELETE FROM active_resource_link_publications WHERE target='2026-09-05'").run(); store.close(); + assert.equal((await c.publish()).changed, 0); + assert.equal((await c.published.employee('Z999')).data.employee.employeeName, 'Changed Employee'); +}); + +test('missing data stays unavailable and symlinked publications fail closed', async t => { + const c = fixture(t); + assert.equal((await c.published.employees()).status, 'not_initialized'); + c.seed(); await c.publish(); + const file = path.join(c.directory, 'paycom.sqlite3'); + fs.renameSync(file, file + '.actual'); fs.symlinkSync(file + '.actual', file); + assert.equal((await c.published.employees()).status, 'unsafe_storage'); +}); + +test('temporary publisher writes an atomic read model and exits, then unchanged periods need no worker', async t => { + const c = fixture(t); c.seed(); + const job = require('../adapters/published-job'); + const options = { database: c.database, publishedDatabase: path.join(c.directory, 'paycom.sqlite3'), timezone: 'America/Los_Angeles' }; + assert.equal(job.needsPublication(options), true); + assert.equal((await job.publish(options)).changed, 1); + assert.equal(job.needsPublication(options), false); + assert.equal((await job.publish(options)).changed, 0); + assert.deepEqual(await c.published.day({ date: '2026-09-01' }), await c.original.day({ date: '2026-09-01' })); +}); diff --git a/plugins/paycom/backend/tests/settings.test.js b/plugins/paycom/backend/tests/settings.test.js new file mode 100644 index 0000000..d31f9b9 --- /dev/null +++ b/plugins/paycom/backend/tests/settings.test.js @@ -0,0 +1,242 @@ +"use strict"; +const test = require("node:test"), + assert = require("node:assert/strict"), + fs = require("node:fs"), + os = require("node:os"), + path = require("node:path"); +const { openDatabase } = require("dispatch-protocol/published/database"); +const { schema, publishPeriod } = require("../adapters/published"); +const { PublishedWorkforcePort } = require("../../dashboard/published"); +const { + WorkforceClient, +} = require("dispatch-protocol/contracts/src/workforce-client"); +const { workforceFixture } = require("./published-fixture"); +test("selected DSP driver departments filter Timecards before pagination and summaries while retaining the full directory", async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "paycom-settings-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const file = path.join(root, "published/paycom.sqlite3"), + db = openDatabase(file, { write: true, journalMode: "DELETE" }); + schema(db); + const raw = workforceFixture({ count: 103 }); + raw.roster.employees.forEach((row, index) => { + row.departmentCode = index < 101 ? "D1" : "D2"; + row.departmentDesc = index < 101 ? "Dispatch" : "Driver"; + row.deliveryStationCode = "S1"; + row.deliveryStationDesc = "North"; + }); + publishPeriod(db, raw, "America/Chicago"); + db.close(); + const a = new WorkforceClient({ + port: new PublishedWorkforcePort(file, { driver_departments: ["D2"] }), + }); + const b = new WorkforceClient({ + port: new PublishedWorkforcePort(file, { driver_departments: null }), + }); + const query = { + date: "2026-09-11", + limit: 1, + offset: 1, + sort: "employeeName", + direction: "desc", + }; + const selected = await a.day(query), + all = await b.day(query); + assert.equal(selected.ok, true); + assert.equal(selected.data.total, 2); + assert.equal(selected.data.items.length, 1); + assert.equal(selected.data.hasMore, false); + assert.equal(selected.data.summary.employees, 2); + assert.equal(all.data.total, 103); + assert.equal((await a.employees({})).data.total, 103); + const none = new WorkforceClient({ + port: new PublishedWorkforcePort(file, { driver_departments: [] }), + }); + const empty = await none.day({ date: "2026-09-11" }); + assert.equal(empty.data.total, 0); + assert.equal(empty.data.available, true); + assert.equal(empty.data.summary.employees, 0); + const options = new PublishedWorkforcePort(file).settingsOptions(); + assert.equal( + options.departments.find((item) => item.value === "D2").count, + 2, + ); + assert.equal((await a.day({ ...query, department: "D1" })).data.total, 0); +}); + +test("name order preserves Paycom spelling, compound names and suffixes without guessing ambiguous names", () => { + const { displayEmployeeName } = require("../../dashboard/published"); + for (const [source, first] of [ + ["DOE, JANE", "JANE DOE"], + ["DE LA CRUZ, MARÍA ELENA", "MARÍA ELENA DE LA CRUZ"], + ["SMITH JR, JOHN", "JOHN SMITH JR"], + ["O’NEILL-SMITH, ANNE-MARIE", "ANNE-MARIE O’NEILL-SMITH"], + ["PRINCE", "PRINCE"], + ["DOE, JR., JOHN", "DOE, JR., JOHN"], + ["DOE, ", "DOE, "], + ]) { + assert.equal(displayEmployeeName(source, "first_last"), first); + assert.equal(displayEmployeeName(source, "last_first"), source); + assert.equal(displayEmployeeName(source), source); + } +}); + +test("DSP name order changes displayed records and sorts before pagination while preserving source data and sibling views", async (t) => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "paycom-name-order-")); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const file = path.join(root, "published/paycom.sqlite3"); + const db = openDatabase(file, { write: true, journalMode: "DELETE" }); + schema(db); + const raw = workforceFixture({ count: 103 }); + raw.roster.employees.forEach((employee, i) => { + employee.employeeName = `SURNAME ${String(i).padStart(3, "0")}, GIVEN ${String(102 - i).padStart(3, "0")}`; + raw.timecards.rows[i].employeeName = employee.employeeName; + }); + publishPeriod(db, raw, "America/Chicago"); + db.close(); + const bytes = fs.readFileSync(file); + const first = new WorkforceClient({ + port: new PublishedWorkforcePort(file, { name_order: "first_last" }), + }); + const last = new WorkforceClient({ + port: new PublishedWorkforcePort(file, { name_order: "last_first" }), + }); + const query = { + date: "2026-09-11", + sort: "employeeName", + direction: "asc", + limit: 2, + offset: 100, + }; + const a = await first.day(query), + b = await last.day(query); + assert(a.ok); + assert(b.ok); + assert.equal(a.data.total, 103); + assert.equal(a.data.summary.employees, 103); + assert.deepEqual( + a.data.items.map((row) => row.employeeName), + ["GIVEN 100 SURNAME 002", "GIVEN 101 SURNAME 001"], + ); + assert.deepEqual( + b.data.items.map((row) => row.employeeName), + ["SURNAME 100, GIVEN 002", "SURNAME 101, GIVEN 001"], + ); + assert.equal( + (await first.day({ ...query, direction: "desc", offset: 0, limit: 1 })).data + .items[0].employeeName, + "GIVEN 102 SURNAME 000", + ); + const search = await first.day({ + ...query, + offset: 0, + search: "given 100 surname 002", + }); + assert.equal(search.data.total, 1); + assert.equal( + (await first.day({ ...query, offset: 0, search: "surname 002, given 100" })) + .data.total, + 1, + ); + const employees = await first.employees({ limit: 100 }); + assert(employees.ok); + assert.equal(employees.data.total, 103); + const record = employees.data.items.find( + (row) => row.employeeCode === "0000", + ); + assert.equal(record.employeeName, "GIVEN 102 SURNAME 000"); + const detail = await first.employee("0000"); + assert(detail.ok); + assert.equal(detail.data.employee.employeeName, record.employeeName); + assert.equal(detail.data.timecard.employeeName, record.employeeName); + const portDetail = new PublishedWorkforcePort(file, { + name_order: "first_last", + }).employee("0000"); + assert( + portDetail.days.every((row) => row.employeeName === record.employeeName), + ); + assert.equal( + (await last.employee("0000")).data.employee.employeeName, + "SURNAME 000, GIVEN 102", + ); + assert.deepEqual(fs.readFileSync(file), bytes); +}); + +for (const oldVersion of [1, 2, 3]) + test(`SDK migration sets First Last once from definition ${oldVersion}, preserving other DSP preferences`, (t) => { + const { + settingsStore, + } = require("dispatch-core/core/plugins/settings-store.js"); + const definition = require("../../dispatch-plugin.json").settings; + const { migrations, previews, ...base } = definition; + const old = { + ...base, + version: oldVersion, + fields: definition.fields + .filter((field) => oldVersion !== 1 || field.id !== "name_order") + .map((field) => + field.id === "name_order" + ? { + ...field, + default: oldVersion === 3 ? "first_last" : "last_first", + } + : field, + ), + }; + const root = fs.mkdtempSync( + path.join(os.tmpdir(), "paycom-name-migration-"), + ); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const storage = settingsStore(root, "paycom"); + const before = storage.initialize(old, { + automatic_sync: false, + sync_interval_seconds: 7200, + driver_departments: ["D2"], + rows_per_page: 25, + columns: ["totalHours", "inDay"], + }); + const migrated = storage.initialize(definition); + assert.equal(migrated.definitionVersion, 4); + assert.equal(migrated.revision, before.revision + 1); + assert.deepEqual(migrated.values, { + ...before.values, + name_order: "first_last", + }); + const saved = storage.update( + definition, + { + values: { ...migrated.values, name_order: "last_first" }, + expectedRevision: migrated.revision, + definitionVersion: 4, + idempotencyKey: "test:name-order", + }, + "owner:test", + ); + const restarted = settingsStore(root, "paycom").initialize(definition); + assert.deepEqual(restarted, saved); + assert.equal(saved.values.automatic_sync, false); + assert.deepEqual(saved.values.driver_departments, ["D2"]); + assert.throws( + () => + storage.update( + definition, + { + values: { ...saved.values, name_order: "invalid" }, + expectedRevision: saved.revision, + definitionVersion: 4, + idempotencyKey: "test:invalid-order", + }, + "owner:test", + ), + { code: "settings_invalid" }, + ); + const sibling = settingsStore( + path.join(root, "sibling"), + "paycom", + ).initialize(definition); + assert.equal(sibling.values.name_order, "first_last"); + const { settingsValues } = require("dispatch-sdk/settings"); + assert.equal( + settingsValues(definition, {}, { defaults: true }).name_order, + "first_last", + ); + }); diff --git a/plugins/paycom/backend/tests/staging-cleanup.test.js b/plugins/paycom/backend/tests/staging-cleanup.test.js new file mode 100644 index 0000000..18d4304 --- /dev/null +++ b/plugins/paycom/backend/tests/staging-cleanup.test.js @@ -0,0 +1,88 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { execute, safeFailure } = require('../src/collector'); +const { PaycomStore, stageCandidate, cleanupRunStages } = require('../src/store'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'paycom-cleanup-')); + fs.chmodSync(root, 0o700); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + return { root, stagingRoot: path.join(root, 'staging'), database: path.join(root, 'db/paycom.sqlite3'), + businessClock: () => new Date('2026-08-30T18:00:00.000Z') }; +} +function request() { + return { protocolVersion: 1, runId: 'cleanup-run', attempt: 2, plan: 'paycom-periods', + source: { id: 'paycom-main', collector: 'paycom', authProfile: 'paycom-main', + config: { timezone: 'UTC', maxConcurrency: 1 } }, + method: 'pay-periods.discover', input: {}, deadline: new Date(Date.now() + 60000).toISOString() }; +} +function leavePartial(directory) { + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + fs.writeFileSync(path.join(directory, '.candidate.tmp'), 'partial fixture', { mode: 0o600 }); +} + +test('partial staging writes and rename failures remove their owned files', t => { + const { stagingRoot } = fixture(t); + const candidate = { kind: 'roster', target: '2026-09-05', runId: 'partial-run', attempt: 1, + collectedAt: '2026-08-30T18:00:00.000Z', metadata: {}, + rows: [{ employeeCode: 'A001', employeeName: 'Fixture', isActive: true, isActiveDriver: true }] }; + for (const method of ['writeFileSync', 'renameSync']) { + const original = fs[method]; + const mocked = t.mock.method(fs, method, (...args) => { + if (method === 'writeFileSync') original(...args); + throw Object.assign(new Error('fixture_io_failure'), { code: 'ENOSPC' }); + }); + try { assert.throws(() => stageCandidate(stagingRoot, candidate), /fixture_io_failure/); } + finally { mocked.mock.restore(); } + assert.deepEqual(fs.readdirSync(stagingRoot), []); + } +}); + +test('successful publication removes all of its attempts and preserves other runs', async t => { + const paths = fixture(t); + leavePartial(path.join(paths.stagingRoot, 'cleanup-run.attempt-1')); + const other = path.join(paths.stagingRoot, 'other-run.attempt-1'); + leavePartial(other); + const result = await execute(request(), paths); + assert.equal(result.ok, true); + assert.deepEqual(fs.readdirSync(paths.stagingRoot), ['other-run.attempt-1']); + assert.equal(fs.readFileSync(path.join(other, '.candidate.tmp'), 'utf8'), 'partial fixture'); + const store = new PaycomStore(paths.database, { readOnly: true }); + try { assert.equal(store.audit('pay_periods').verified, true); } + finally { store.close(); } +}); + +test('leftover files prevent a success acknowledgement; retry keeps committed data and finishes cleanup', async t => { + const paths = fixture(t); + const owned = path.join(paths.stagingRoot, 'cleanup-run.attempt-2'); + const remove = fs.rmSync; + // Also proves cleanup verifies absence rather than trusting a return value. + const mocked = t.mock.method(fs, 'rmSync', (directory, options) => directory === owned ? undefined : remove(directory, options)); + try { + await assert.rejects(execute(request(), paths), error => { + assert.equal(safeFailure(error).error.code, 'stage_cleanup_failed'); + return true; + }); + } finally { mocked.mock.restore(); } + assert.equal(fs.existsSync(owned), true); + const store = new PaycomStore(paths.database, { readOnly: true }); + try { assert.equal(store.audit('pay_periods').verified, true); } + finally { store.close(); } + assert.equal((await execute(request(), paths)).status, 'no_change'); + assert.deepEqual(fs.readdirSync(paths.stagingRoot), []); +}); + +test('cleanup refuses links outside the private staging root', t => { + const paths = fixture(t); + const outside = path.join(paths.root, 'retained'); + leavePartial(outside); + fs.mkdirSync(paths.stagingRoot, { mode: 0o700 }); + fs.symlinkSync(outside, path.join(paths.stagingRoot, 'cleanup-run.attempt-1')); + assert.throws(() => cleanupRunStages(paths.stagingRoot, 'cleanup-run'), /stage_cleanup_failed/); + assert.equal(fs.readFileSync(path.join(outside, '.candidate.tmp'), 'utf8'), 'partial fixture'); +}); diff --git a/plugins/paycom/backend/tests/store.test.js b/plugins/paycom/backend/tests/store.test.js new file mode 100644 index 0000000..3a1b91f --- /dev/null +++ b/plugins/paycom/backend/tests/store.test.js @@ -0,0 +1,269 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { PaycomStore, stageCandidate, cleanupStage } = require('../src/store'); +const { periodFromEnd } = require('../src/timecard-period'); +const { TIMECARD_SUMMARY, ROUTE_VERSION, linkRows } = require('../src/resource-links'); +const { timecardRecord, rosterRow } = require('./helpers'); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-paycom-store-')); + fs.chmodSync(root, 0o700); + return { root, staging: path.join(root, 'staging'), database: path.join(root, 'db', 'paycom.sqlite3') }; +} + +function publish(store, staging, candidate) { + const stage = stageCandidate(staging, candidate); + try { return store.publish(stage); } + finally { cleanupStage(stage, staging); } +} + +test('staged roster publication is atomic, private, versioned, and idempotent', () => { + const { root, staging, database } = fixture(); + const store = new PaycomStore(database); + try { + const candidate = { + kind: 'roster', target: '2026-08-22', runId: 'run-roster-1', attempt: 1, collectedAt: '2026-08-25T20:00:00.000Z', + metadata: { sourceSha256: 'a'.repeat(64) }, rows: [rosterRow()], + }; + const first = publish(store, staging, candidate); + assert.equal(first.disposition, 'published'); + assert.equal(store.audit('roster').verified, true); + const second = publish(store, staging, { ...candidate, runId: 'run-roster-2', collectedAt: '2026-08-25T20:01:00.000Z' }); + assert.equal(second.disposition, 'no_change'); + assert.equal(second.publicationId, first.publicationId); + assert.equal(fs.statSync(database).mode & 0o777, 0o600); + assert.deepEqual(fs.readdirSync(staging), []); + assert.throws(() => publish(store, staging, { ...candidate, runId: 'run-bad', rows: [] }), /candidate_invalid/); + assert.equal(store.audit('roster').publicationId, first.publicationId); + const third = publish(store, staging, { ...candidate, runId: 'run-roster-3', collectedAt: '2026-08-25T20:02:00.000Z', metadata: { sourceSha256: 'b'.repeat(64) }, rows: [rosterRow('A001', 'Beta')] }); + const fourth = publish(store, staging, { ...candidate, runId: 'run-roster-4', collectedAt: '2026-08-25T20:03:00.000Z', metadata: { sourceSha256: 'c'.repeat(64) }, rows: [rosterRow('A001', 'Gamma')] }); + assert.equal(store.audit('roster').publicationId, fourth.publicationId); + assert.equal(store.db.prepare('SELECT COUNT(*) count FROM publications WHERE kind=? AND target=?').get('roster', candidate.target).count, 2); + assert.equal(store.db.prepare('SELECT COUNT(*) count FROM publications WHERE id=?').get(third.publicationId).count, 1); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('complete timecard publication reconciles exactly to the active roster', () => { + const { root, staging, database } = fixture(); + const store = new PaycomStore(database); + const period = periodFromEnd('2026-09-05'); + try { + const rosterResult = publish(store, staging, { + kind: 'roster', target: period.end, runId: 'run-roster', attempt: 1, collectedAt: '2026-08-25T20:00:00.000Z', + metadata: {}, rows: [rosterRow('A001', 'One'), rosterRow('A002', 'Two')], + }); + const result = publish(store, staging, { + kind: 'timecards', target: period.end, periodKey: period.key, runId: 'run-timecards', attempt: 1, collectedAt: '2026-08-25T20:02:00.000Z', + metadata: { + periodStart: period.start, periodEnd: period.end, mode: 'full', + rosterPublicationId: rosterResult.publicationId, + rosterContentSha256: rosterResult.contentSha256, + }, + rows: [ + { employeeCode: 'A001', employeeName: 'One', record: timecardRecord('A001'), sourceSha256: 'b'.repeat(64) }, + { employeeCode: 'A002', employeeName: 'Two', record: timecardRecord('A002'), sourceSha256: 'c'.repeat(64) }, + ], + }); + assert.equal(result.disposition, 'published'); + assert.deepEqual(store.reconcileCurrent(period.end), { + verified: true, + code: 'verified', + periodEnd: period.end, + rosterPublicationId: store.active('roster').id, + timecardPublicationId: result.publicationId, + activeEmployees: 2, + timecards: 2, + missing: [], + unexpected: [], + omitted: 0, + }); + assert.equal(store.audit('timecards', period.end).verified, true); + const exactAudit = store.auditTimecards(period.end); + assert.equal(exactAudit.verified, true); + assert.equal(exactAudit.rosterBindingValid, true); + assert.deepEqual({ + activeEmployees: exactAudit.activeEmployees, timecards: exactAudit.timecards, + missingCount: exactAudit.missingCount, unexpectedCount: exactAudit.unexpectedCount, + duplicateCount: exactAudit.duplicateCount, identityMismatchCount: exactAudit.identityMismatchCount, + }, { activeEmployees: 2, timecards: 2, missingCount: 0, unexpectedCount: 0, duplicateCount: 0, identityMismatchCount: 0 }); + store.db.prepare('UPDATE timecards SET period_total_hours=999 WHERE publication_id=? AND employee_code=?').run(result.publicationId, 'A001'); + const tampered = store.audit('timecards', period.end); + assert.equal(tampered.verified, false); + assert.equal(tampered.projectionValid, false); + assert.equal(store.auditTimecards(period.end).verified, false); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('partial membership and mismatched source URLs cannot replace a good timecard publication', () => { + const { root, staging, database } = fixture(); + const store = new PaycomStore(database); + const period = periodFromEnd('2026-09-05'); + try { + const roster = publish(store, staging, { + kind: 'roster', target: period.end, runId: 'run-roster-safe', attempt: 1, + collectedAt: '2026-08-25T20:00:00.000Z', metadata: {}, + rows: [rosterRow('A001', 'One'), rosterRow('A002', 'Two')], + }); + const metadata = { + periodStart: period.start, periodEnd: period.end, mode: 'full', + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256, + }; + const good = publish(store, staging, { + kind: 'timecards', target: period.end, periodKey: period.key, runId: 'run-good', attempt: 1, + collectedAt: '2026-08-25T20:01:00.000Z', metadata, + rows: [ + { employeeCode: 'A001', employeeName: 'One', record: timecardRecord('A001'), sourceSha256: 'a'.repeat(64) }, + { employeeCode: 'A002', employeeName: 'Two', record: timecardRecord('A002'), sourceSha256: 'b'.repeat(64) }, + ], + }); + assert.throws(() => publish(store, staging, { + kind: 'timecards', target: period.end, periodKey: period.key, runId: 'run-partial', attempt: 1, + collectedAt: '2026-08-25T20:02:00.000Z', metadata, + rows: [{ employeeCode: 'A001', employeeName: 'One', record: timecardRecord('A001'), sourceSha256: 'c'.repeat(64) }], + }), /membership_mismatch/); + const wrongUrl = { ...timecardRecord('A001'), sourceUrl: timecardRecord('A002').sourceUrl }; + assert.throws(() => stageCandidate(staging, { + kind: 'timecards', target: period.end, periodKey: period.key, runId: 'run-wrong-url', attempt: 1, + collectedAt: '2026-08-25T20:03:00.000Z', metadata, + rows: [{ employeeCode: 'A001', employeeName: 'One', record: wrongUrl, sourceSha256: 'd'.repeat(64) }], + }), /candidate_invalid/); + publish(store, staging, { + kind: 'roster', target: period.end, runId: 'run-roster-revised', attempt: 1, + collectedAt: '2026-08-25T20:04:00.000Z', metadata: { revision: 2 }, + rows: [rosterRow('A001', 'One'), rosterRow('A002', 'Two')], + }); + const staleAudit = store.auditTimecards(period.end); + assert.equal(staleAudit.verified, false); + assert.equal(staleAudit.rosterBindingValid, false); + assert.throws(() => publish(store, staging, { + kind: 'timecards', target: period.end, periodKey: period.key, runId: 'run-stale-roster', attempt: 1, + collectedAt: '2026-08-25T20:05:00.000Z', metadata, + rows: [ + { employeeCode: 'A001', employeeName: 'One', record: timecardRecord('A001'), sourceSha256: 'e'.repeat(64) }, + { employeeCode: 'A002', employeeName: 'Two', record: timecardRecord('A002'), sourceSha256: 'f'.repeat(64) }, + ], + }), /membership_mismatch/); + assert.equal(store.active('timecards', period.end).id, good.publicationId); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('verification failure rolls back activation and stale attempts are fenced', () => { + const { root, staging, database } = fixture(); + const store = new PaycomStore(database); + const period = periodFromEnd('2026-09-05'); + try { + const roster = publish(store, staging, { + kind: 'roster', target: period.end, runId: 'run-roster-fence', attempt: 1, + collectedAt: '2026-08-25T20:00:00.000Z', metadata: {}, rows: [rosterRow('A001', 'One')], + }); + const metadata = { + periodStart: period.start, periodEnd: period.end, mode: 'full', + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256, + }; + const good = publish(store, staging, { + kind: 'timecards', target: period.end, periodKey: period.key, runId: 'run-verified', attempt: 1, + collectedAt: '2026-08-25T20:01:00.000Z', metadata, + rows: [{ employeeCode: 'A001', employeeName: 'One', record: timecardRecord('A001'), sourceSha256: 'a'.repeat(64) }], + }); + store.db.exec(`CREATE TRIGGER corrupt_new_timecard AFTER INSERT ON timecards BEGIN + UPDATE timecards SET employee_name='Corrupt' WHERE publication_id=NEW.publication_id AND employee_code=NEW.employee_code; + END;`); + assert.throws(() => publish(store, staging, { + kind: 'timecards', target: period.end, periodKey: period.key, runId: 'run-corrupt', attempt: 1, + collectedAt: '2026-08-25T20:02:00.000Z', metadata: { ...metadata, mode: 'incremental' }, + rows: [{ employeeCode: 'A001', employeeName: 'One', record: timecardRecord('A001'), sourceSha256: 'b'.repeat(64) }], + }), /publication_verification_failed/); + assert.equal(store.active('timecards', period.end).id, good.publicationId); + store.db.exec('DROP TRIGGER corrupt_new_timecard'); + + const newer = publish(store, staging, { + kind: 'roster', target: period.end, runId: 'run-recovered', attempt: 2, + collectedAt: '2026-08-25T20:04:00.000Z', metadata: { revision: 2 }, rows: [rosterRow('A001', 'Newer')], + }); + assert.throws(() => publish(store, staging, { + kind: 'roster', target: period.end, runId: 'run-recovered', attempt: 1, + collectedAt: '2026-08-25T20:03:00.000Z', metadata: { revision: 1 }, rows: [rosterRow('A001', 'Older')], + }), /stale_collection_attempt/); + assert.equal(store.active('roster', period.end).id, newer.publicationId); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('resource-link manifests cover the exact active period roster and invalidate on roster change', () => { + const { root, staging, database } = fixture(); + const store = new PaycomStore(database); + const period = periodFromEnd('2026-09-05'); + try { + assert.equal(store.db.prepare('SELECT version FROM schema_meta').get().version, 5); + const rosterRows = [rosterRow('A001', 'One'), rosterRow('A002', 'Two'), { ...rosterRow('A003', 'Three'), isActive: false }]; + const roster = publish(store, staging, { + kind: 'roster', target: period.end, runId: 'run-links-roster', attempt: 1, + collectedAt: '2026-08-25T20:00:00.000Z', metadata: {}, rows: rosterRows, + }); + const rows = linkRows(TIMECARD_SUMMARY, rosterRows.filter(row => row.isActive), period); + const candidate = { + kind: 'resource_links', target: period.end, periodKey: period.key, + runId: 'run-links', attempt: 1, collectedAt: '2026-08-25T20:01:00.000Z', + metadata: { + resourceType: TIMECARD_SUMMARY, periodStart: period.start, periodEnd: period.end, + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256, + routeVersion: ROUTE_VERSION, + }, + rows, + }; + const first = publish(store, staging, candidate); + assert.equal(first.disposition, 'published'); + assert.deepEqual(first.membership, { rosterPublicationId: roster.publicationId, activeEmployees: 2, links: 2 }); + assert.equal(store.auditResourceLinks(TIMECARD_SUMMARY, period.end).verified, true); + assert.equal(publish(store, staging, { ...candidate, runId: 'run-links-repeat' }).disposition, 'no_change'); + assert.throws(() => publish(store, staging, { ...candidate, runId: 'run-links-partial', rows: rows.slice(0, 1) }), /membership_mismatch/); + assert.equal(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).publication.id, first.publicationId); + + publish(store, staging, { + kind: 'roster', target: period.end, runId: 'run-links-roster-new', attempt: 1, + collectedAt: '2026-08-25T20:02:00.000Z', metadata: { revision: 2 }, rows: rosterRows, + }); + assert.equal(store.activeResourceLinks(TIMECARD_SUMMARY, period.end), null); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('aggregate change-history compaction preserves published runs and one daily no-change row', () => { + const { root, database } = fixture(); + const store = new PaycomStore(database); + try { + const insert = store.db.prepare(`INSERT INTO paycom_sync_change_history( + run_id,source_id,target,business_date,business_timezone,observed_at,disposition,delta_json,persistence_json + ) VALUES(?,?,?,?,?,?,?,?,?)`); + const common = ['paycom-main', '2026-09-05', '2026-08-30', 'America/Los_Angeles']; + insert.run('history-1', ...common, '2026-08-30T01:00:00.000Z', 'no_change', '{}', '{}'); + insert.run('history-2', ...common, '2026-08-30T02:00:00.000Z', 'no_change', '{}', '{}'); + insert.run('history-3', ...common, '2026-08-30T03:00:00.000Z', 'no_change', '{}', '{}'); + insert.run('history-published', ...common, '2026-08-30T04:00:00.000Z', 'published', '{}', '{}'); + const compacted = store.compactSyncChangeHistory('2026-08-31T00:00:00.000Z'); + assert.equal(compacted.deleted, 2); + assert.deepEqual(store.db.prepare(`SELECT run_id FROM paycom_sync_change_history ORDER BY run_id`).all().map(row => row.run_id), + ['history-3', 'history-published']); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/plugins/paycom/backend/tests/sync.test.js b/plugins/paycom/backend/tests/sync.test.js new file mode 100644 index 0000000..9025db5 --- /dev/null +++ b/plugins/paycom/backend/tests/sync.test.js @@ -0,0 +1,946 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const test = require('node:test'); +const { execute } = require('../src/collector'); +const { authoritativeRosterBody, rosterAuthorityAssessment } = require('../src/browser'); +const { + canonicalBusinessTimecard, timecardBusinessSha256, +} = require('../src/fingerprints'); +const { PaycomStore, validateCandidate, stageCandidate, cleanupStage } = require('../src/store'); +const { TIMECARD_SUMMARY, ROUTE_VERSION, linkRows } = require('../src/resource-links'); +const { planWorkforceMirror } = require('../src/sync-publication'); +const { computeBusinessDelta } = require('../src/sync-delta'); +const { timecardRecord } = require('./helpers'); +const { boundedJson } = require('dispatch-runtime-kit/collection-manager/src/validation'); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-paycom-sync-')); + fs.chmodSync(root, 0o700); + return { + root, + database: path.join(root, 'db', 'paycom.sqlite3'), + staging: path.join(root, 'staging'), + }; +} + +function publishCandidate(store, staging, candidate) { + const stage = stageCandidate(staging, candidate); + try { return store.publish(stage); } + finally { cleanupStage(stage, staging); } +} + +function activeWorkforceBase(store, target) { + const roster = store.active('roster', target); + const timecards = store.active('timecards', target); + const links = store.activeResourceLinks(TIMECARD_SUMMARY, target)?.publication || null; + return { + rosterPublicationId: roster.id, + rosterContentSha256: roster.content_sha256, + timecardPublicationId: timecards.id, + timecardContentSha256: timecards.content_sha256, + resourceLinkPublicationId: links?.id || null, + resourceLinkContentSha256: links?.content_sha256 || null, + }; +} + +function employee(code, overrides = {}) { + return { + employeeCode: code, + employeeName: `Employee ${code}`, + status: 'A', + lifecycleStatus: 'active', + departmentCode: 'D1', + departmentDesc: 'Driver', + deliveryStationCode: 'S1', + deliveryStationDesc: 'Station', + positionTitle: 'Driver', + payClass: 'PC', + terminalGroup: 'TG', + payType: 'Hourly', + primarySupervisor: 'Supervisor', + missingPunches: '0', + totalHours: '8', + totalOvertimeHours: '0', + employeeApprovalPercentage: '0', + supervisorApprovalPercentage: '0', + isActive: true, + isDriverDepartment: true, + isDriverPosition: true, + isActiveDriver: true, + ...overrides, + }; +} + +function rawEmployee(code, overrides = {}) { + return { + employeeCode: code, + fullName: `Employee ${code}`, + eestatus: 'A', + allocation: { selections: [ + { categoryName: 'Department', isDepartment: true, code: 'D1', description: 'Driver' }, + { categoryName: 'Delivery Station Code', isDepartment: false, code: 'S1', description: 'Station' }, + ] }, + position: 'Driver', + payClassCode: 'PC', + terminalCode: 'TG', + payType: 'Hourly', + primarySupervisor: 'Supervisor', + missingPunches: 0, + totals: { totalHours: 8, otHours: 0 }, + approvalPercentages: { employee: 0, supervisor: 0 }, + ...overrides, + }; +} + +function rosterBytes(rows) { + return Buffer.from(JSON.stringify({ eeCodes: rows.map(row => row.employeeCode), employees: rows })); +} + +function shadowTimecardRow(code, end = '2026-09-05', overrides = {}, observedAt = '2026-08-27T06:00:00.000Z') { + const record = canonicalBusinessTimecard({ ...timecardRecord(code, end, 1), ...overrides }); + return { + employeeCode: code, + employeeName: `Employee ${code}`, + record, + sourceSha256: 'e'.repeat(64), + businessSha256: timecardBusinessSha256(record), + observedAt, + }; +} + +function fixturePerformance(itemCount) { + return { totalMs: 1, itemCount, workerCount: Math.min(3, itemCount), openedTargets: Math.min(3, itemCount), retryCount: 0 }; +} + +function syncRequest(runId = 'shadow-1') { + return { + protocolVersion: 1, + runId, + plan: 'paycom-current-workforce-sync', + source: { + id: 'paycom-main', collector: 'paycom', authProfile: 'paycom-main', + config: { timezone: 'America/Los_Angeles', maxConcurrency: 3 }, + }, + method: 'sync.current-workforce', + input: { + reconcileBatchSize: 10, + fullReconcileMinutes: 1440, + lookbackPeriods: 1, + }, + attempt: 1, + deadline: new Date(Date.now() + 60_000).toISOString(), + }; +} + +test('timecard business identity ignores cache-buster evidence but detects business edits', () => { + const first = timecardRecord('A001', '2026-09-05', 1); + const second = timecardRecord('A001', '2026-09-05', 2); + assert.equal(timecardBusinessSha256(first), timecardBusinessSha256(second)); + const canonical = canonicalBusinessTimecard(first); + assert.equal(canonical.sourceUrl.includes('dispatch_timecards'), false); + assert.notEqual(timecardBusinessSha256(canonical), timecardBusinessSha256({ + ...canonical, + approvals: [['approved']], + })); + const businessSha256 = timecardBusinessSha256(canonical); + assert.doesNotThrow(() => validateCandidate({ + kind: 'timecards', target: canonical.periodEnd, periodKey: canonical.periodKey, + runId: 'semantic-1', attempt: 1, collectedAt: '2026-08-27T06:00:00.000Z', + metadata: { periodStart: canonical.periodStart, periodEnd: canonical.periodEnd }, + rows: [{ + employeeCode: 'A001', employeeName: 'Employee A001', record: canonical, + sourceSha256: 'a'.repeat(64), businessSha256, observedAt: '2026-08-27T06:00:00.000Z', + }], + })); +}); + +test('business delta classifies punch additions, edits, and nested approval changes without identities', () => { + const before = shadowTimecardRow('A001'); + const after = structuredClone(before); + after.record.days[0].punches[0].comment = 'changed'; + after.record.days[0].punches.push({ + ...structuredClone(after.record.days[0].punches[0]), + ordinal: 2, slot: 'o2', kind: 'OUT DAY', displayTime: '05:00 PM', + actualTime: '05:00 PM', roundedTime: '05:00 PM', comment: '', + }); + after.record.approvals = [['approved']]; + after.businessSha256 = timecardBusinessSha256(after.record); + const mirror = { + rosterAddedCount: 0, rosterProfileChangedCount: 0, rosterSummaryChangedCount: 0, + rosterRecordChangedCount: 0, timecardAddedCount: 0, timecardChangedCount: 1, + retainedMissingCount: 0, becameUnknownCount: 0, returnedFromUnknownCount: 0, + unknownEmployeeCount: 0, deactivatedCount: 0, reactivatedCount: 0, activeEmployeeCount: 1, + }; + const delta = computeBusinessDelta([before], [after], mirror); + assert.equal(delta.timecards.changedCount, 1); + assert.equal(delta.days.changedCount, 1); + assert.equal(delta.punches.addedCount, 1); + assert.equal(delta.punches.addedByKind.outDayCount, 1); + assert.equal(delta.punches.editedCount, 1); + assert.equal(delta.details.approvalSectionsChangedCount, 1); + assert.equal(JSON.stringify(delta).includes('A001'), false); + assert.equal(JSON.stringify(delta).includes('05:00 PM'), false); +}); + +test('authoritative roster policy rejects filtered and incomplete request shapes', () => { + const body = { + eeCodes: ['A001', 'A002'], q: '', isAdvancedFilterApplied: false, + onlyBorrowedEmployees: false, payClassCodes: [], selectedEarnings: [], approvalMode: null, + skip: 0, take: 2, getCount: true, + }; + assert.equal(authoritativeRosterBody(body), true); + assert.equal(authoritativeRosterBody({ ...body, q: 'employee' }), false); + assert.deepEqual(rosterAuthorityAssessment({ ...body, q: 'employee' }), { + observable: false, authoritative: false, code: 'roster_filter_search', + }); + assert.equal(rosterAuthorityAssessment({ ...body, take: 1 }).code, 'roster_filter_page_size'); + assert.equal(authoritativeRosterBody({ ...body, isAdvancedFilterApplied: true }), false); + assert.deepEqual(rosterAuthorityAssessment({ ...body, isAdvancedFilterApplied: true }), { + observable: true, authoritative: false, code: 'roster_filters_present', + }); + assert.equal(authoritativeRosterBody({ ...body, onlyBorrowedEmployees: true }), false); + assert.deepEqual(rosterAuthorityAssessment({ ...body, payClassCodes: ['selected'] }), { + observable: true, authoritative: false, code: 'roster_filters_present', + }); + assert.equal(authoritativeRosterBody({ ...body, take: 1 }), false); + assert.equal(authoritativeRosterBody({ ...body, skip: 1 }), false); +}); + +test('shadow observations retain missing employees and are retry-idempotent', () => { + const { root, database } = fixture(); + const store = new PaycomStore(database); + const base = { + sourceId: 'paycom-main', target: '2026-09-05', observedAt: '2026-08-27T06:00:00.000Z', + sourceSha256: 'a'.repeat(64), + }; + try { + const baseline = store.observeWorkforceShadow({ + ...base, runId: 'shadow-1', employees: [employee('A001'), employee('A002')], + }); + assert.deepEqual(baseline, { + baseline: true, absencePolicy: 'retain', sourceCompleteness: 'observation_only', + observedCount: 2, addedCount: 0, profileChangedCount: 0, + summaryChangedCount: 0, missingCount: 0, candidateCount: 0, + }); + const changed = store.observeWorkforceShadow({ + ...base, runId: 'shadow-2', observedAt: '2026-08-27T06:15:00.000Z', sourceSha256: 'b'.repeat(64), + employees: [employee('A001', { positionTitle: 'Lead', totalHours: '10' }), employee('A003')], + }); + assert.equal(changed.addedCount, 1); + assert.equal(changed.profileChangedCount, 1); + assert.equal(changed.summaryChangedCount, 1); + assert.equal(changed.missingCount, 1); + assert.equal(changed.absencePolicy, 'retain'); + assert.equal(changed.candidateCount, 2); + const repeated = store.observeWorkforceShadow({ + ...base, runId: 'shadow-3', observedAt: '2026-08-27T06:30:00.000Z', sourceSha256: 'c'.repeat(64), + employees: [employee('A001', { positionTitle: 'Lead', totalHours: '10' }), employee('A003')], + }); + assert.equal(repeated.absencePolicy, 'retain'); + assert.equal(repeated.missingCount, 1); + assert.deepEqual(store.observeWorkforceShadow({ + ...base, runId: 'shadow-3', observedAt: '2026-08-27T06:31:00.000Z', sourceSha256: 'd'.repeat(64), + employees: [employee('A001'), employee('A003')], + }), repeated); + assert.equal(JSON.stringify(repeated).includes('A002'), false); + assert.equal(store.syncState('paycom-main', '2026-09-05').pendingRemovalCount, 0); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('shadow timecard selection rotates, prioritizes obvious changes, and performs due full reconciliation', () => { + const { root, database } = fixture(); + const store = new PaycomStore(database); + const target = '2026-09-05'; + const firstEmployees = ['A001', 'A002', 'A003', 'A004'].map(code => employee(code)); + const common = { + sourceId: 'paycom-main', target, sourceSha256: 'a'.repeat(64), + reconcileBatchSize: 2, fullReconcileMinutes: 60, + }; + try { + const firstPlan = store.planWorkforceShadow({ + sourceId: common.sourceId, target, observedAt: '2026-08-27T06:00:00.000Z', + employees: firstEmployees, reconcileBatchSize: 2, fullReconcileMinutes: 60, + }); + assert.deepEqual(firstPlan.selectedEmployees.map(row => row.employeeCode), ['A001', 'A002']); + assert.equal(firstPlan.rotationCount, 2); + assert.equal(firstPlan.fullReconciliation, false); + const first = store.observeWorkforceShadow({ + ...common, runId: 'rotation-1', observedAt: '2026-08-27T06:00:00.000Z', employees: firstEmployees, + timecardRows: firstPlan.selectedEmployees.map(row => shadowTimecardRow(row.employeeCode)), + }); + assert.equal(first.timecardBaselineCount, 2); + + const changedEmployees = firstEmployees.map(row => { + if (row.employeeCode === 'A001') return employee('A001', { totalHours: '10' }); + if (row.employeeCode === 'A002') return employee('A002', { + status: 'I', isActive: false, isActiveDriver: false, + }); + return row; + }); + const secondPlan = store.planWorkforceShadow({ + sourceId: common.sourceId, target, observedAt: '2026-08-27T06:15:00.000Z', + employees: changedEmployees, reconcileBatchSize: 2, fullReconcileMinutes: 60, + }); + assert.deepEqual(secondPlan.selectedEmployees.map(row => row.employeeCode), ['A001', 'A003', 'A004']); + assert.equal(secondPlan.obviousCandidateCount, 1); + assert.equal(secondPlan.rotationCount, 2); + const changedRecord = { approvals: [['approved']] }; + const second = store.observeWorkforceShadow({ + ...common, runId: 'rotation-2', observedAt: '2026-08-27T06:15:00.000Z', + sourceSha256: 'b'.repeat(64), employees: changedEmployees, + timecardRows: secondPlan.selectedEmployees.map(row => shadowTimecardRow( + row.employeeCode, target, row.employeeCode === 'A001' ? changedRecord : {}, '2026-08-27T06:15:00.000Z', + )), + }); + assert.equal(second.timecardChangedCount, 1); + assert.equal(second.timecardBaselineCount, 2); + + const notYetFull = store.planWorkforceShadow({ + sourceId: common.sourceId, target, observedAt: '2026-08-27T06:59:00.000Z', + employees: changedEmployees, reconcileBatchSize: 2, fullReconcileMinutes: 60, + }); + assert.equal(notYetFull.fullReconciliation, false); + + const fullPlan = store.planWorkforceShadow({ + sourceId: common.sourceId, target, observedAt: '2026-08-27T07:15:00.000Z', + employees: changedEmployees, reconcileBatchSize: 2, fullReconcileMinutes: 60, + }); + assert.equal(fullPlan.fullReconciliation, true); + assert.equal(fullPlan.selectedEmployees.length, 3); + const full = store.observeWorkforceShadow({ + ...common, runId: 'rotation-3', observedAt: '2026-08-27T07:15:00.000Z', + sourceSha256: 'c'.repeat(64), employees: changedEmployees, + timecardRows: fullPlan.selectedEmployees.map(row => shadowTimecardRow( + row.employeeCode, target, row.employeeCode === 'A001' ? changedRecord : {}, '2026-08-27T07:15:00.000Z', + )), + }); + assert.equal(full.fullReconciliation, true); + assert.equal(full.selectedTimecardCount, 3); + assert.equal(full.timecardUnchangedCount, 3); + assert.equal(store.syncState(common.sourceId, target).lastFullReconciledAt, '2026-08-27T07:15:00.000Z'); + assert.equal(store.planWorkforceShadow({ + sourceId: common.sourceId, target, observedAt: '2026-08-27T07:30:00.000Z', + employees: changedEmployees, reconcileBatchSize: 2, fullReconcileMinutes: 60, + }).fullReconciliation, false); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('workforce mirror planner applies additions and edits while retaining untrusted absences', () => { + const period = { start: '2026-08-23', end: '2026-09-05', key: '2026-08-23_2026-09-05' }; + const priorRosterRows = [employee('A001'), employee('A002')]; + const priorTimecardRows = [shadowTimecardRow('A001'), shadowTimecardRow('A002')]; + const sourceEmployees = [ + employee('A001', { positionTitle: 'Lead', totalHours: '10' }), + employee('A003'), + ]; + const changedA001 = shadowTimecardRow('A001', period.end, { approvals: [['approved']] }); + const addedA003 = shadowTimecardRow('A003'); + const plan = planWorkforceMirror({ + period, priorRosterRows, priorTimecardRows, sourceEmployees, + collectedTimecardRows: [changedA001, addedA003], + }); + assert.equal(plan.hasChanges, true); + assert.deepEqual(plan.rosterRows.map(row => row.employeeCode), ['A001', 'A002', 'A003']); + assert.deepEqual(plan.timecardRows.map(row => row.employeeCode), ['A001', 'A002', 'A003']); + assert.equal(plan.resourceLinkRows.length, 3); + assert.deepEqual(plan.counts, { + rosterAddedCount: 1, + rosterProfileChangedCount: 1, + rosterSummaryChangedCount: 1, + rosterRecordChangedCount: 3, + timecardAddedCount: 1, + timecardChangedCount: 1, + retainedMissingCount: 1, + becameUnknownCount: 1, + returnedFromUnknownCount: 0, + unknownEmployeeCount: 1, + deactivatedCount: 0, + reactivatedCount: 0, + activeEmployeeCount: 3, + }); + assert.throws(() => planWorkforceMirror({ + period, priorRosterRows, priorTimecardRows, sourceEmployees, + collectedTimecardRows: [changedA001], + }), /timecard_refresh_required/); +}); + +test('atomic workforce publication activates additions and edits together and rolls back every pointer on failure', () => { + const { root, database, staging } = fixture(); + const store = new PaycomStore(database); + const period = { start: '2026-08-23', end: '2026-09-05', key: '2026-08-23_2026-09-05' }; + const priorRosterRows = [employee('A001'), employee('A002')]; + const priorTimecardRows = [shadowTimecardRow('A001'), shadowTimecardRow('A002')]; + try { + const roster = publishCandidate(store, staging, { + kind: 'roster', target: period.end, runId: 'atomic-prior-roster', attempt: 1, + collectedAt: '2026-08-27T05:00:00.000Z', metadata: { sourceSha256: '1'.repeat(64) }, + rows: priorRosterRows, + }); + publishCandidate(store, staging, { + kind: 'timecards', target: period.end, periodKey: period.key, + runId: 'atomic-prior-timecards', attempt: 1, collectedAt: '2026-08-27T05:01:00.000Z', + metadata: { + periodStart: period.start, periodEnd: period.end, + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256, + mode: 'published_roster', + }, + rows: priorTimecardRows, + }); + publishCandidate(store, staging, { + kind: 'resource_links', target: period.end, periodKey: period.key, + runId: 'atomic-prior-links', attempt: 1, collectedAt: '2026-08-27T05:02:00.000Z', + metadata: { + resourceType: TIMECARD_SUMMARY, periodStart: period.start, periodEnd: period.end, + rosterPublicationId: roster.publicationId, rosterContentSha256: roster.contentSha256, + routeVersion: ROUTE_VERSION, + }, + rows: linkRows(TIMECARD_SUMMARY, priorRosterRows, period), + }); + store.observeWorkforceShadow({ + sourceId: 'paycom-main', target: period.end, runId: 'atomic-baseline', + observedAt: '2026-08-27T05:03:00.000Z', sourceSha256: '2'.repeat(64), + employees: priorRosterRows, + timecardRows: priorTimecardRows, + reconcileBatchSize: 2, fullReconcileMinutes: 1440, + }); + + const sourceEmployees = [employee('A001', { positionTitle: 'Lead', totalHours: '10' }), employee('A003')]; + const collected = [ + shadowTimecardRow('A001', period.end, { approvals: [['approved']] }, '2026-08-27T06:00:00.000Z'), + shadowTimecardRow('A003', period.end, {}, '2026-08-27T06:00:00.000Z'), + ]; + const mirrorPlan = planWorkforceMirror({ + period, priorRosterRows, priorTimecardRows, sourceEmployees, + collectedTimecardRows: collected, + }); + const publishInput = { + runId: 'atomic-sync-1', attempt: 1, collectedAt: '2026-08-27T06:00:00.000Z', + businessTimezone: 'America/Los_Angeles', period, sourceSha256: '3'.repeat(64), sourceFormat: 'paycom-employees-json.v1', + sourceEmployees, mirrorPlan, stagingRoot: staging, + base: activeWorkforceBase(store, period.end), + observation: { + sourceId: 'paycom-main', target: period.end, runId: 'atomic-sync-1', + observedAt: '2026-08-27T06:00:00.000Z', sourceSha256: '3'.repeat(64), + employees: sourceEmployees, + timecardRows: collected, + reconcileBatchSize: 2, fullReconcileMinutes: 1440, + }, + }; + const activeIdsBeforePreview = ['roster', 'timecards'].map(kind => store.active(kind, period.end).id) + .concat(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).publication.id); + const previewInput = { + ...publishInput, + runId: 'atomic-preview-1', + observation: { ...publishInput.observation, runId: 'atomic-preview-1' }, + }; + const preview = store.previewWorkforceSync(previewInput); + assert.equal(preview.disposition, 'no_change'); + assert.equal(preview.wouldPublish, true); + assert.equal(preview.counts.rosterAddedCount, 1); + assert.equal(preview.rosterPublicationId, undefined); + assert.deepEqual(['roster', 'timecards'].map(kind => store.active(kind, period.end).id) + .concat(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).publication.id), activeIdsBeforePreview); + assert.deepEqual(fs.readdirSync(staging), []); + const replayedPreview = store.previewWorkforceSync(previewInput); + assert.equal(replayedPreview.wouldPublish, true); + assert.deepEqual(replayedPreview.counts, preview.counts); + assert.deepEqual(['roster', 'timecards'].map(kind => store.active(kind, period.end).id) + .concat(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).publication.id), activeIdsBeforePreview); + const result = store.publishWorkforceSync(publishInput); + assert.equal(result.disposition, 'published'); + assert.equal(result.counts.rosterAddedCount, 1); + assert.equal(result.counts.retainedMissingCount, 1); + assert.equal(result.counts.becameUnknownCount, 1); + assert.equal(result.businessDate, '2026-08-27'); + assert.equal(result.businessTimezone, 'America/Los_Angeles'); + assert.deepEqual(result.delta.timecards, { + addedCount: 1, changedCount: 1, unchangedCount: 1, removedCount: 0, + }); + assert.equal(result.delta.punches.addedCount, 1); + assert.equal(result.delta.punches.addedByKind.inDayCount, 1); + assert.equal(result.delta.punches.editedCount, 0); + assert.equal(result.delta.details.approvalSectionsChangedCount, 1); + assert.equal(result.persistence.verified, true); + assert.equal(result.persistence.timecardCount, 3); + assert.equal(result.persistence.dateRowCount, 3); + assert.equal(result.persistence.selectedTimecardCount, 2); + assert.equal(result.persistence.persistedSelectedTimecardCount, 2); + assert.equal(result.persistence.selectedMismatchCount, 0); + const persistedPunches = store.auditTimecardPersistence(period.end, period.start, collected); + assert.equal(persistedPunches.verified, true); + assert.equal(persistedPunches.punchCount, 3); + assert.equal(persistedPunches.inDayPunchCount, 3); + assert.equal(persistedPunches.inDayTimecardCount, 3); + const mismatchedPersistence = store.auditTimecardPersistence(period.end, period.start, [ + shadowTimecardRow('A001', period.end), + ]); + assert.equal(mismatchedPersistence.verified, false); + assert.equal(mismatchedPersistence.selectedMismatchCount, 1); + const boundedReceipt = { + ok: true, status: 'published', data: { + method: 'sync.current-workforce', mode: 'additions_edits', + businessDate: result.businessDate, + businessTimezone: result.businessTimezone, + mirror: result.counts, + delta: result.delta, + persistence: result.persistence, + publications: { + rosterPublicationId: result.rosterPublicationId, + timecardPublicationId: result.timecardPublicationId, + resourceLinkPublicationId: result.resourceLinkPublicationId, + }, + }, + }; + assert.doesNotThrow(() => boundedJson(boundedReceipt, 262144)); + assert.equal(JSON.stringify(boundedReceipt).includes('A001'), false); + assert.deepEqual(store.activeRoster(period.end).employees.map(row => row.employeeCode), ['A001', 'A002', 'A003']); + assert.equal(store.activeRoster(period.end).employees.find(row => row.employeeCode === 'A002').lifecycleStatus, 'unknown'); + assert.deepEqual(store.activeTimecards(period.end).rows.map(row => row.employeeCode), ['A001', 'A002', 'A003']); + assert.equal(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).rows.length, 3); + assert.equal(store.auditTimecards(period.end).verified, true); + assert.equal(store.auditResourceLinks(TIMECARD_SUMMARY, period.end).verified, true); + assert.equal(store.db.prepare('SELECT COUNT(*) count FROM publications WHERE kind=? AND target=?').get('roster', period.end).count, 2); + assert.equal(store.db.prepare('SELECT COUNT(*) count FROM publications WHERE kind=? AND target=?').get('timecards', period.end).count, 2); + assert.equal(store.db.prepare('SELECT COUNT(*) count FROM resource_link_publications WHERE resource_type=? AND target=?') + .get(TIMECARD_SUMMARY, period.end).count, 2); + const replayed = store.publishWorkforceSync(publishInput); + assert.equal(replayed.disposition, 'published'); + assert.deepEqual(replayed.counts, result.counts); + assert.deepEqual(replayed.delta, result.delta); + assert.deepEqual(replayed.persistence, result.persistence); + const changeHistory = store.syncChangeHistory('paycom-main', period.end); + assert.equal(changeHistory.total, 1); + assert.equal(changeHistory.items[0].runId, 'atomic-sync-1'); + assert.equal(changeHistory.items[0].businessDate, '2026-08-27'); + assert.equal(changeHistory.items[0].businessTimezone, 'America/Los_Angeles'); + assert.deepEqual(changeHistory.items[0].delta, result.delta); + assert.equal(replayed.rosterPublicationId, result.rosterPublicationId); + assert.equal(replayed.timecardPublicationId, result.timecardPublicationId); + assert.equal(replayed.resourceLinkPublicationId, result.resourceLinkPublicationId); + assert.equal(store.db.prepare('SELECT COUNT(*) count FROM publications WHERE kind=? AND target=?').get('roster', period.end).count, 2); + const activeIdsBeforeMissing = ['roster', 'timecards'].map(kind => store.active(kind, period.end).id) + .concat(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).publication.id); + const activeRosterBeforeMissing = store.activeRoster(period.end).employees; + const activeTimecardsBeforeMissing = store.activeTimecards(period.end).rows; + const missingOnlySource = activeRosterBeforeMissing.filter(row => row.employeeCode !== 'A002'); + const missingOnlyCollected = activeTimecardsBeforeMissing.filter(row => row.employeeCode !== 'A002'); + const missingOnlyPlan = planWorkforceMirror({ + period, + priorRosterRows: activeRosterBeforeMissing, + priorTimecardRows: activeTimecardsBeforeMissing, + sourceEmployees: missingOnlySource, + collectedTimecardRows: missingOnlyCollected, + }); + assert.equal(missingOnlyPlan.hasChanges, false); + const missingOperationInput = { + runId: 'atomic-missing-preview', attempt: 1, collectedAt: '2026-08-27T06:10:00.000Z', + businessTimezone: 'America/Los_Angeles', period, sourceSha256: '5'.repeat(64), sourceFormat: 'paycom-employees-json.v1', + sourceEmployees: missingOnlySource, mirrorPlan: missingOnlyPlan, stagingRoot: staging, + base: activeWorkforceBase(store, period.end), + observation: { + sourceId: 'paycom-main', target: period.end, runId: 'atomic-missing-preview', + observedAt: '2026-08-27T06:10:00.000Z', sourceSha256: '5'.repeat(64), + employees: missingOnlySource, + timecardRows: missingOnlyCollected, + reconcileBatchSize: 2, fullReconcileMinutes: 1440, + }, + }; + const missingPreview = store.previewWorkforceSync(missingOperationInput); + assert.equal(missingPreview.disposition, 'no_change'); + assert.equal(missingPreview.wouldPublish, false); + assert.equal(missingPreview.counts.retainedMissingCount, 1); + assert.deepEqual(['roster', 'timecards'].map(kind => store.active(kind, period.end).id) + .concat(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).publication.id), activeIdsBeforeMissing); + const missingOnly = store.publishWorkforceSync({ + ...missingOperationInput, + runId: 'atomic-missing-only', + observation: { ...missingOperationInput.observation, runId: 'atomic-missing-only' }, + }); + assert.equal(missingOnly.disposition, 'no_change'); + assert.equal(missingOnly.counts.retainedMissingCount, 1); + assert.equal(missingOnly.persistence.verified, true); + assert.equal(missingOnly.persistence.selectedTimecardCount, missingOnlyCollected.length); + assert.equal(missingOnly.persistence.persistedSelectedTimecardCount, missingOnlyCollected.length); + assert.equal(missingOnly.persistence.selectedMismatchCount, 0); + assert.equal(missingOnly.delta.timecards.changedCount, 0); + assert.equal(missingOnly.delta.timecards.unchangedCount, 3); + assert.equal(missingOnly.delta.punches.addedCount, 0); + assert.equal(store.syncChangeHistory('paycom-main', period.end).total, 2); + assert.deepEqual(['roster', 'timecards'].map(kind => store.active(kind, period.end).id) + .concat(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).publication.id), activeIdsBeforeMissing); + + const beforeReturnRoster = store.activeRoster(period.end).employees; + const beforeReturnTimecards = store.activeTimecards(period.end).rows; + const returnedSource = beforeReturnRoster.map(row => row.employeeCode === 'A002' + ? { ...row, status: 'A', lifecycleStatus: 'active', isActive: true, isActiveDriver: true } + : row); + const returnedSelection = store.planWorkforceShadow({ + sourceId: 'paycom-main', target: period.end, observedAt: '2026-08-27T06:10:30.000Z', + employees: returnedSource, reconcileBatchSize: 2, fullReconcileMinutes: 1440, + }); + const returnedCards = beforeReturnTimecards.filter(row => returnedSelection.selectedEmployees + .some(employeeRow => employeeRow.employeeCode === row.employeeCode)); + const returnPlan = planWorkforceMirror({ + period, + priorRosterRows: beforeReturnRoster, + priorTimecardRows: beforeReturnTimecards, + sourceEmployees: returnedSource, + collectedTimecardRows: returnedCards, + }); + assert.equal(returnPlan.counts.returnedFromUnknownCount, 1); + assert.equal(returnPlan.counts.unknownEmployeeCount, 0); + const returned = store.publishWorkforceSync({ + runId: 'atomic-return-unknown', attempt: 1, collectedAt: '2026-08-27T06:10:30.000Z', + businessTimezone: 'America/Los_Angeles', period, sourceSha256: '8'.repeat(64), sourceFormat: 'paycom-employees-json.v1', + sourceEmployees: returnedSource, mirrorPlan: returnPlan, stagingRoot: staging, + base: activeWorkforceBase(store, period.end), + observation: { + sourceId: 'paycom-main', target: period.end, runId: 'atomic-return-unknown', + observedAt: '2026-08-27T06:10:30.000Z', sourceSha256: '8'.repeat(64), + employees: returnedSource, + timecardRows: returnedCards, + reconcileBatchSize: 2, fullReconcileMinutes: 1440, + }, + }); + assert.equal(returned.disposition, 'published'); + assert.equal(store.activeRoster(period.end).employees.find(row => row.employeeCode === 'A002').lifecycleStatus, 'active'); + + const beforeDeactivationRoster = store.activeRoster(period.end).employees; + const beforeDeactivationTimecards = store.activeTimecards(period.end).rows; + const deactivatedSource = beforeDeactivationRoster.map(row => row.employeeCode === 'A002' + ? { ...row, status: 'I', lifecycleStatus: 'inactive', isActive: false, isActiveDriver: false } + : row); + const deactivationCards = beforeDeactivationTimecards.filter(row => row.employeeCode !== 'A002'); + const deactivationPlan = planWorkforceMirror({ + period, + priorRosterRows: beforeDeactivationRoster, + priorTimecardRows: beforeDeactivationTimecards, + sourceEmployees: deactivatedSource, + collectedTimecardRows: deactivationCards, + }); + assert.equal(deactivationPlan.counts.deactivatedCount, 1); + assert.equal(deactivationPlan.counts.reactivatedCount, 0); + assert.equal(deactivationPlan.counts.activeEmployeeCount, 2); + const deactivated = store.publishWorkforceSync({ + runId: 'atomic-deactivate', attempt: 1, collectedAt: '2026-08-27T06:11:00.000Z', + businessTimezone: 'America/Los_Angeles', period, sourceSha256: '6'.repeat(64), sourceFormat: 'paycom-employees-json.v1', + sourceEmployees: deactivatedSource, mirrorPlan: deactivationPlan, stagingRoot: staging, + base: activeWorkforceBase(store, period.end), + observation: { + sourceId: 'paycom-main', target: period.end, runId: 'atomic-deactivate', + observedAt: '2026-08-27T06:11:00.000Z', sourceSha256: '6'.repeat(64), + employees: deactivatedSource, + timecardRows: deactivationCards, + reconcileBatchSize: 2, fullReconcileMinutes: 1440, + }, + }); + assert.equal(deactivated.disposition, 'published'); + assert.equal(store.activeRoster(period.end).employees.find(row => row.employeeCode === 'A002').isActive, false); + assert.deepEqual(store.activeTimecards(period.end).rows.map(row => row.employeeCode), ['A001', 'A003']); + assert.deepEqual(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).rows.map(row => row.employeeCode), ['A001', 'A003']); + assert.equal(store.auditTimecards(period.end).activeEmployees, 2); + + const beforeReactivationRoster = store.activeRoster(period.end).employees; + const beforeReactivationTimecards = store.activeTimecards(period.end).rows; + const reactivatedSource = beforeReactivationRoster.map(row => row.employeeCode === 'A002' + ? { ...row, status: 'A', lifecycleStatus: 'active', isActive: true, isActiveDriver: true } + : row); + const reactivationCards = [...beforeReactivationTimecards, shadowTimecardRow( + 'A002', period.end, {}, '2026-08-27T06:12:00.000Z', + )].sort((left, right) => left.employeeCode.localeCompare(right.employeeCode)); + const reactivationPlan = planWorkforceMirror({ + period, + priorRosterRows: beforeReactivationRoster, + priorTimecardRows: beforeReactivationTimecards, + sourceEmployees: reactivatedSource, + collectedTimecardRows: reactivationCards, + }); + assert.equal(reactivationPlan.counts.deactivatedCount, 0); + assert.equal(reactivationPlan.counts.reactivatedCount, 1); + assert.equal(reactivationPlan.counts.activeEmployeeCount, 3); + const reactivated = store.publishWorkforceSync({ + runId: 'atomic-reactivate', attempt: 1, collectedAt: '2026-08-27T06:12:00.000Z', + businessTimezone: 'America/Los_Angeles', period, sourceSha256: '7'.repeat(64), sourceFormat: 'paycom-employees-json.v1', + sourceEmployees: reactivatedSource, mirrorPlan: reactivationPlan, stagingRoot: staging, + base: activeWorkforceBase(store, period.end), + observation: { + sourceId: 'paycom-main', target: period.end, runId: 'atomic-reactivate', + observedAt: '2026-08-27T06:12:00.000Z', sourceSha256: '7'.repeat(64), + employees: reactivatedSource, + timecardRows: reactivationCards, + reconcileBatchSize: 2, fullReconcileMinutes: 1440, + }, + }); + assert.equal(reactivated.disposition, 'published'); + assert.equal(store.activeRoster(period.end).employees.find(row => row.employeeCode === 'A002').isActive, true); + assert.deepEqual(store.activeTimecards(period.end).rows.map(row => row.employeeCode), ['A001', 'A002', 'A003']); + assert.equal(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).rows.length, 3); + assert.equal(store.auditTimecards(period.end).activeEmployees, 3); + + const capturedBase = activeWorkforceBase(store, period.end); + const priorRosterPublication = store.db.prepare(`SELECT id FROM publications + WHERE kind='roster' AND target=? AND id<>?`).get(period.end, capturedBase.rosterPublicationId); + store.db.prepare(`UPDATE active_publications SET publication_id=? + WHERE kind='roster' AND target=?`).run(priorRosterPublication.id, period.end); + try { + assert.throws(() => store.previewWorkforceSync({ + ...publishInput, + runId: 'atomic-preview-stale-base', + base: capturedBase, + observation: { ...publishInput.observation, runId: 'atomic-preview-stale-base' }, + }), /publication_base_changed/); + assert.throws(() => store.publishWorkforceSync({ + ...publishInput, + runId: 'atomic-stale-base', + base: capturedBase, + observation: { ...publishInput.observation, runId: 'atomic-stale-base' }, + }), /publication_base_changed/); + } finally { + store.db.prepare(`UPDATE active_publications SET publication_id=? + WHERE kind='roster' AND target=?`).run(capturedBase.rosterPublicationId, period.end); + } + assert.equal(store.shadowReceiptForRun('paycom-main', period.end, 'atomic-preview-stale-base'), null); + assert.equal(store.shadowReceiptForRun('paycom-main', period.end, 'atomic-stale-base'), null); + const activeBeforeFailure = ['roster', 'timecards'].map(kind => store.active(kind, period.end).id) + .concat(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).publication.id); + + const badSource = [employee('A001', { positionTitle: 'Manager', totalHours: '10' }), employee('A003')]; + const badCollected = [shadowTimecardRow('A001', period.end, { attestations: [['changed']] }, '2026-08-27T06:15:00.000Z')]; + const badPlan = planWorkforceMirror({ + period, + priorRosterRows: store.activeRoster(period.end).employees, + priorTimecardRows: store.activeTimecards(period.end).rows, + sourceEmployees: badSource, + collectedTimecardRows: badCollected, + }); + badPlan.resourceLinkRows[0] = { ...badPlan.resourceLinkRows[0], canonicalUrl: 'https://invalid.example/' }; + assert.throws(() => store.publishWorkforceSync({ + runId: 'atomic-sync-fail', attempt: 1, collectedAt: '2026-08-27T06:15:00.000Z', + businessTimezone: 'America/Los_Angeles', period, sourceSha256: '4'.repeat(64), sourceFormat: 'paycom-employees-json.v1', + sourceEmployees: badSource, mirrorPlan: badPlan, stagingRoot: staging, + base: activeWorkforceBase(store, period.end), + observation: { + sourceId: 'paycom-main', target: period.end, runId: 'atomic-sync-fail', + observedAt: '2026-08-27T06:15:00.000Z', sourceSha256: '4'.repeat(64), + employees: badSource, + timecardRows: badCollected, + reconcileBatchSize: 2, fullReconcileMinutes: 1440, + }, + }), /candidate_invalid/); + const activeAfterFailure = ['roster', 'timecards'].map(kind => store.active(kind, period.end).id) + .concat(store.activeResourceLinks(TIMECARD_SUMMARY, period.end).publication.id); + assert.deepEqual(activeAfterFailure, activeBeforeFailure); + assert.equal(store.shadowReceiptForRun('paycom-main', period.end, 'atomic-sync-fail'), null); + assert.deepEqual(fs.readdirSync(staging), []); + } finally { + store.close(); + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('sync.current-workforce records a private shadow baseline without publishing Paycom data', async () => { + const { root, database } = fixture(); + const bytes = rosterBytes([rawEmployee('A001'), rawEmployee('A002')]); + const captured = { + bytes, + sourceSha256: crypto.createHash('sha256').update(bytes).digest('hex'), + completeness: { observable: true, authoritative: true, requestedCount: 2, returnedCount: 2 }, + }; + let browserRuns = 0; + const browserRunner = async (request, callback) => { + browserRuns += 1; + return callback({ endpoint: 'fixture' }); + }; + const timecardCollector = async (endpoint, employees, period) => ({ + rows: employees.map(item => shadowTimecardRow(item.employeeCode, period.end)), + performance: fixturePerformance(employees.length), + }); + try { + const receipt = await execute(syncRequest(), { + database, + browserRunner, + rosterCollector: async () => captured, + timecardCollector, + }); + assert.equal(receipt.ok, true); + assert.equal(receipt.status, 'no_change'); + assert.equal(receipt.data.mode, 'shadow'); + assert.equal(receipt.data.baseline, true); + assert.equal(receipt.data.observedCount, 2); + assert.equal(receipt.data.selectedTimecardCount, 2); + assert.equal(receipt.data.timecardBaselineCount, 2); + assert.equal(receipt.data.fullReconciliation, false); + assert.equal(browserRuns, 1); + assert.equal(JSON.stringify(receipt).includes('A001'), false); + assert.doesNotThrow(() => boundedJson(receipt)); + const store = new PaycomStore(database); + try { + assert.equal(store.db.prepare('SELECT COUNT(*) count FROM publications').get().count, 0); + assert.equal(store.syncState('paycom-main', receipt.data.target).employeeCount, 2); + } finally { store.close(); } + const replay = await execute(syncRequest(), { + database, browserRunner, rosterCollector: async () => captured, timecardCollector, + }); + assert.equal(replay.data.replayed, true); + assert.equal(browserRuns, 1); + await assert.rejects(() => execute(syncRequest('shadow-2'), { + database, + browserRunner, + rosterCollector: async () => ({ + ...captured, + completeness: { + ...captured.completeness, observable: false, authoritative: false, + authorityCode: 'roster_filter_search', + }, + }), + }), /roster_filter_search/); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('sync.current-workforce rejects a source business-date rollover before persistence', async () => { + const { root, database } = fixture(); + const bytes = rosterBytes([rawEmployee('A001')]); + const captured = { + bytes, + sourceSha256: crypto.createHash('sha256').update(bytes).digest('hex'), + completeness: { observable: true, authoritative: true, requestedCount: 1, returnedCount: 1 }, + }; + const times = [new Date('2026-08-31T06:59:00.000Z'), new Date('2026-08-31T07:01:00.000Z')]; + try { + await assert.rejects(() => execute(syncRequest('date-rollover'), { + database, + businessClock: () => times.shift(), + browserRunner: async (request, callback) => callback({ endpoint: 'fixture' }), + rosterCollector: async () => captured, + timecardCollector: async (endpoint, employees, period) => ({ + rows: employees.map(item => shadowTimecardRow(item.employeeCode, period.end)), + performance: fixturePerformance(employees.length), + }), + }), /business_date_changed/); + const store = new PaycomStore(database); + try { assert.equal(store.syncState('paycom-main', '2026-09-05'), null); } + finally { store.close(); } + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('sync.current-workforce enforces the requested period even when the UI defaults to another period', async () => { + const { root, database } = fixture(); + const bytes = rosterBytes([rawEmployee('A001')]); + const captured = { + bytes, + sourceSha256: crypto.createHash('sha256').update(bytes).digest('hex'), + completeness: { observable: true, authoritative: true, requestedCount: 1, returnedCount: 1 }, + uiPeriod: { start: '2026-08-09', end: '2026-08-22' }, + }; + try { + const receipt = await execute(syncRequest('period-rewrite'), { + database, + businessClock: () => new Date('2026-08-30T18:00:00.000Z'), + browserRunner: async (request, callback) => callback({ endpoint: 'fixture' }), + rosterCollector: async () => captured, + timecardCollector: async (endpoint, employees, period) => ({ + rows: employees.map(item => shadowTimecardRow(item.employeeCode, period.end)), + performance: fixturePerformance(employees.length), + }), + }); + assert.equal(receipt.status, 'no_change'); + assert.equal(receipt.data.target, '2026-09-05'); + assert.equal(receipt.data.performance.requestedPeriodEnforced, true); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +test('first publishing sync rejects an incomplete source without recording success', async () => { + const { root, database } = fixture(); + const bytes = rosterBytes([rawEmployee('A001')]); + const request = syncRequest('incomplete-baseline'); + request.input.publishMode = 'additions_edits'; + try { + await assert.rejects(execute(request, { + database, + browserRunner: async (value, callback) => callback({ endpoint: 'fixture' }), + rosterCollector: async () => ({ bytes, sourceSha256: crypto.createHash('sha256').update(bytes).digest('hex'), + completeness: { observable: true, authoritative: false, returnedCount: 1 } }), + timecardCollector: async () => assert.fail('Do not collect an incomplete baseline'), + }), /roster_source_not_authoritative/); + const store = new PaycomStore(database); + try { assert.equal(store.db.prepare('SELECT COUNT(*) count FROM publications').get().count, 0); } + finally { store.close(); } + } finally { fs.rmSync(root, { recursive: true, force: true }); } +}); + +test('first sync publishes a complete baseline atomically, replays, and starts a new pay period', async () => { + const { root, database, staging } = fixture(); + const bytes = rosterBytes([rawEmployee('A001'), rawEmployee('A002')]); + let clock = new Date('2026-08-30T18:00:00.000Z'); + let failCollection = true; + let corruptRow = false; + let browserRuns = 0; + const dependencies = { + database, stagingRoot: staging, businessClock: () => clock, + browserRunner: async (value, callback) => { browserRuns++; return callback({ endpoint: 'fixture' }); }, + rosterCollector: async () => ({ bytes, sourceSha256: crypto.createHash('sha256').update(bytes).digest('hex'), + completeness: { observable: true, authoritative: true, returnedCount: 2 } }), + timecardCollector: async (endpoint, employees, period) => { + assert.equal(employees.length, 2, 'Initial import exceeds the rotating batch of one'); + return { rows: employees.slice(0, failCollection ? 1 : 2).map(item => ({ ...shadowTimecardRow(item.employeeCode, period.end, {}, clock.toISOString()), + ...(corruptRow ? { businessSha256: 'b'.repeat(64) } : {}) })), + performance: fixturePerformance(employees.length) }; + }, + }; + const request = syncRequest('complete-baseline'); + request.input.publishMode = 'additions_edits'; + request.input.reconcileBatchSize = 1; + try { + await assert.rejects(execute(request, dependencies), /membership_mismatch/); + let store = new PaycomStore(database); + assert.equal(Boolean(store.active('roster', '2026-09-05')), false); + store.close(); + failCollection = false; + corruptRow = true; + await assert.rejects(execute(request, dependencies), /timecards_invalid|candidate_invalid/); + store = new PaycomStore(database); + assert.equal(store.db.prepare('SELECT COUNT(*) count FROM publications').get().count, 0, 'Roll back periods and roster when timecard validation fails'); + store.close(); + corruptRow = false; + const first = await execute(request, dependencies); + assert.equal(first.data.publicationStatus, 'ready'); + assert.equal(first.data.persistence.verified, true); + assert.deepEqual(fs.readdirSync(staging), []); + const interrupted = path.join(staging, `${request.runId}.attempt-1`); + fs.mkdirSync(interrupted, { mode: 0o700 }); + fs.writeFileSync(path.join(interrupted, '.candidate.tmp'), 'interrupted fixture', { mode: 0o600 }); + const replay = await execute(request, dependencies); + assert.equal(replay.data.replayed, true); + assert.deepEqual(fs.readdirSync(staging), [], 'Committed replay still removes interrupted staging'); + assert.equal(browserRuns, 3); + clock = new Date('2026-09-08T18:00:00.000Z'); + const next = await execute({ ...request, runId: 'period-rollover' }, dependencies); + assert.equal(next.data.target, '2026-09-19'); + assert.equal(next.data.publicationStatus, 'ready'); + store = new PaycomStore(database); + try { + for (const target of ['2026-09-05', '2026-09-19']) { + assert.equal(store.activeRoster(target).employees.length, 2); + assert.equal(store.auditTimecards(target).verified, true); + assert.equal(store.auditResourceLinks(TIMECARD_SUMMARY, target).verified, true); + } + } finally { store.close(); } + } finally { fs.rmSync(root, { recursive: true, force: true }); } +}); diff --git a/plugins/paycom/dashboard/published.js b/plugins/paycom/dashboard/published.js new file mode 100644 index 0000000..2538992 --- /dev/null +++ b/plugins/paycom/dashboard/published.js @@ -0,0 +1,128 @@ +'use strict'; + +const path = require('node:path'); +const { openDatabase } = require('dispatch-protocol/published/database'); +const { WorkforceClient } = require('dispatch-protocol/contracts/src/workforce-client'); +const { DAY_SORT_KEYS: SORTS } = require('dispatch-protocol/contracts/src/workforce'); +const emptySummary = () => Object.fromEntries(['employees', 'activeEmployees', 'inDayPunches', 'completeTimecards', + 'needsReview', 'noActivity', 'missingOutDay', 'incompleteLunch', 'unclassifiedPunches'].map(key => [key, 0])); + +function displayEmployeeName(name, order) { + if (order !== 'first_last' || typeof name !== 'string') return name; + // Paycom supplies Last, First. Keep ambiguous or undelimited names intact; + // never guess where a multiword surname, given name or suffix belongs. + const parts = name.split(','); + if (parts.length !== 2 || parts.some(part => !part.trim())) return name; + return `${parts[1].trim()} ${parts[0].trim()}`; +} + +class PublishedWorkforcePort { + constructor(database, settings = {}) { this.database = database; this.settings = settings; } + named(row) { + return row && typeof row.employeeName === 'string' + ? { ...row, employeeName: displayEmployeeName(row.employeeName, this.settings.name_order) } : row; + } + read(action) { + const db = openDatabase(this.database); + if (!db) return null; + try { + if (db.prepare('PRAGMA user_version').get().user_version !== 1) throw Object.assign(new Error('schema_invalid'), { code: 'schema_invalid' }); + db.function('paycom_name', { deterministic: true }, name => displayEmployeeName(name, this.settings.name_order)); + db.function('paycom_name_search', { deterministic: true }, name => displayEmployeeName(name, this.settings.name_order)?.toLocaleLowerCase('en-US') ?? ''); + db.exec('BEGIN'); + const latest = db.prepare('SELECT * FROM periods ORDER BY target DESC LIMIT 1').get(); + const result = latest ? action(db, latest) : null; + db.exec('COMMIT'); + return result; + } finally { db.close(); } + } + snapshot() { return this.read((db, period) => JSON.parse(period.snapshot_json)); } + page(db, table, where, parameters, order, query) { + const total = db.prepare(`SELECT count(*) n FROM ${table} WHERE ${where}`).get(...parameters).n; + const items = db.prepare(`SELECT body FROM ${table} WHERE ${where} ORDER BY ${order} LIMIT ? OFFSET ?`) + .all(...parameters, query.limit, query.offset).map(row => this.named(JSON.parse(row.body))); + return { items, total, limit: query.limit, offset: query.offset, hasMore: query.offset + items.length < total }; + } + employees(query) { + return this.read((db, period) => ({ target: period.target, collectedAt: JSON.parse(period.snapshot_json).collectedAt.roster, + ...this.page(db, 'employees', 'target=?' + (query.lifecycleStatus ? ' AND lifecycle=?' : ''), + [period.target, ...(query.lifecycleStatus ? [query.lifecycleStatus] : [])], 'ordinal', query) })); + } + employee(code) { + return this.read((db, period) => { + const record = db.prepare('SELECT detail FROM employees WHERE target=? AND code=?').get(period.target, code); + if (!record) return { target: period.target, + collectedAt: JSON.parse(period.snapshot_json).collectedAt.roster, employee: null, timecard: null }; + const detail = JSON.parse(record.detail); + return { ...detail, employee: this.named(detail.employee), timecard: this.named(detail.timecard), + days: detail.days?.map(row => this.named(row)) }; + }); + } + day(query) { + return this.read((db, latest) => { + const period = db.prepare('SELECT * FROM periods WHERE start<=? AND target>=? ORDER BY target DESC LIMIT 1').get(query.date, query.date) || latest; + const metadata = db.prepare('SELECT summary_json FROM dates WHERE target=? AND date=?').get(period.target, query.date); + let where = 'target=? AND date=?'; + const parameters = [period.target, query.date]; + const departments = this.settings.driver_departments; + if (Array.isArray(departments)) { + if (!departments.length) where += ' AND 0'; + else { where += ` AND code IN (SELECT code FROM employees WHERE target=? AND json_extract(body,'$.department.code') IN (${departments.map(()=>'?').join(',')}))`; + parameters.push(period.target,...departments); } + } + for (const [field, jsonPath] of [['department','$.department.code'],['station','$.deliveryStation.code']]) if (query[field]) { + where += ` AND code IN (SELECT code FROM employees WHERE target=? AND json_extract(body,'${jsonPath}')=?)`; + parameters.push(period.target,query[field]); + } + const summary = Array.isArray(departments) || query.department || query.station + ? require('dispatch-protocol/contracts/src/workforce-summary').dailySummary(db.prepare(`SELECT body FROM days WHERE ${where}`).all(...parameters).map(row=>JSON.parse(row.body))) + : metadata ? JSON.parse(metadata.summary_json) : emptySummary(); + if (query.lifecycleStatus) { where += ' AND lifecycle=?'; parameters.push(query.lifecycleStatus); } + if (query.search) { + where += " AND (instr(search,?)>0 OR instr(paycom_name_search(json_extract(body,'$.employeeName')),?)>0)"; + const search = query.search.toLocaleLowerCase('en-US'); parameters.push(search, search); + } + if (query.attention === 'incomplete') where += " AND condition IN ('incomplete','needs_review')"; + else if (query.attention) { where += ' AND condition=?'; parameters.push(query.attention); } + const direction = query.direction === 'desc' ? 'desc' : 'asc'; + const order = query.sort === 'employeeName' && this.settings.name_order === 'first_last' + ? `paycom_name(json_extract(body,'$.employeeName')) COLLATE NOCASE ${direction},code` + : query.sort && SORTS.includes(query.sort) ? `rank_${query.sort}_${direction}` : 'ordinal'; + return { target: period.target, businessDate: query.date, businessTimezone: period.timezone, + periodStart: period.start, periodEnd: period.target, available: Boolean(metadata), + collectedAt: JSON.parse(period.snapshot_json).collectedAt.timecards, + summary, + ...this.page(db, 'days', where, parameters, order, query) }; + }); + } + items(kind, query) { + return this.read((db, period) => { + let where = 'target=? AND kind=?'; const parameters = [period.target, kind]; + if (query.lifecycleStatus) { where += ' AND lifecycle=?'; parameters.push(query.lifecycleStatus); } + if (kind === 'punches') { + where += ' AND date=?'; parameters.push(query.date); + for (const [field, operator, value] of [['punch_kind', '=', query.kind], ['time', '>=', query.fromTime], ['time', '<=', query.throughTime]]) { + if (value !== null) { where += ` AND ${field}${operator}?`; parameters.push(value); } + } + } + return { target: period.target, collectedAt: JSON.parse(period.snapshot_json).collectedAt[kind === 'resourceLinks' ? 'resourceLinks' : 'timecards'], + ...(kind === 'punches' ? { businessDate: query.date, businessTimezone: period.timezone } : {}), + ...this.page(db, 'items', where, parameters, 'ordinal', query) }; + }); + } + timecards(query) { return this.items('timecards', query); } + resourceLinks(query) { return this.items('resourceLinks', query); } + punches(query) { return this.items('punches', query); } + settingsOptions() { + return this.read((db,period)=>Object.fromEntries([['departments','department'],['stations','deliveryStation']].map(([key,field])=>{ + const rows=db.prepare(`SELECT json_extract(body,'$.${field}.code') value,json_extract(body,'$.${field}.name') label,count(*) count + FROM employees WHERE target=? GROUP BY value,label ORDER BY label,value`).all(period.target); + return [key,rows.filter(row=>typeof row.value==='string'&&typeof row.label==='string').map(row=>({...row}))]; + }))) || {departments:[],stations:[]}; + } +} +function createPublishedClient({ directory, settings = {} }) { + const port = new PublishedWorkforcePort(path.join(directory,'paycom.sqlite3'),settings); + return { workforce: new WorkforceClient({port}), settingsOptions:()=>port.settingsOptions() }; +} +module.exports = { createPublishedClient, PublishedWorkforcePort, displayEmployeeName, SORTS }; diff --git a/plugins/paycom/dispatch-plugin.json b/plugins/paycom/dispatch-plugin.json new file mode 100644 index 0000000..d3f1005 --- /dev/null +++ b/plugins/paycom/dispatch-plugin.json @@ -0,0 +1,501 @@ +{ + "schemaVersion": 1, + "id": "paycom", + "name": "Paycom", + "version": "0.18.9", + "description": "Connect Paycom to collect your workforce, employees and timecards.", + "frontend": "frontend/index.tsx", + "dashboard": null, + "published": "dashboard/published.js", + "runtime": "backend/plugin.js", + "pages": [ + { + "id": "paycom", + "label": "Paycom", + "icon": "calendar", + "permission": "workforce.read" + } + ], + "httpPrefixes": [ + "/api/paycom" + ], + "gatewayActions": [ + "paycom.setup", + "workforce.day", + "workforce.employees", + "workforce.employee" + ], + "services": [ + "paycom" + ], + "collectors": [ + "paycom" + ], + "syncs": [ + "paycom-main-workforce" + ], + "legacyProfile": "paycom-main", + "actions": [ + { + "id": "workforce.day", + "permission": "workforce.read", + "input": { + "type": "object", + "properties": { + "query": { + "type": "object", + "properties": { + "date": { + "type": "string", + "maxLength": 256 + }, + "search": { + "type": "string", + "maxLength": 256 + }, + "attention": { + "type": "string", + "maxLength": 256 + }, + "lifecycleStatus": { + "type": "string", + "maxLength": 256 + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + "offset": { + "type": "integer", + "minimum": 0 + }, + "sort": { + "type": "string", + "maxLength": 256 + }, + "direction": { + "type": "string", + "maxLength": 256 + }, + "department": { + "type": "string", + "maxLength": 256 + }, + "station": { + "type": "string", + "maxLength": 256 + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [ + "query" + ], + "additionalProperties": false + } + }, + { + "id": "workforce.employees", + "permission": "workforce.read", + "input": { + "type": "object", + "properties": { + "query": { + "type": "object", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + "offset": { + "type": "integer", + "minimum": 0 + }, + "lifecycleStatus": { + "type": "string", + "maxLength": 256 + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [ + "query" + ], + "additionalProperties": false + } + }, + { + "id": "workforce.employee", + "permission": "workforce.read", + "input": { + "type": "object", + "properties": { + "code": { + "type": "string", + "maxLength": 256 + } + }, + "required": [ + "code" + ], + "additionalProperties": false + } + }, + { + "id": "sync.status", + "permission": "workforce.read", + "input": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "paycom-main-workforce" + ] + } + }, + "required": [ + "id" + ], + "additionalProperties": false + } + }, + { + "id": "sync.run_now", + "permission": "sync.run", + "input": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "paycom-main-workforce" + ] + }, + "options": { + "type": "object", + "properties": { + "idempotencyKey": { + "type": "string", + "minLength": 16, + "maxLength": 128 + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [ + "id" + ], + "additionalProperties": false + } + } + ], + "jobs": [ + "paycom-health", + "paycom-periods", + "paycom-roster", + "paycom-period-roster", + "paycom-current-resource-links", + "paycom-period-resource-links", + "paycom-period-resource-links-audit", + "paycom-resource-links-audit", + "paycom-current-timecards", + "paycom-period-timecards", + "paycom-period-timecards-from-roster", + "paycom-period-timecards-audit", + "paycom-incremental-timecards", + "paycom-reconcile", + "paycom-current-workforce-sync" + ], + "settings": { + "version": 4, + "sections": [ + { + "id": "sync", + "label": "Sync schedule" + }, + { + "id": "view", + "label": "Workspace view" + }, + { + "id": "drivers", + "label": "Driver departments" + } + ], + "optionsView": "settings-options", + "schedule": { + "id": "paycom-main-workforce", + "enabled": "automatic_sync", + "interval": "sync_interval_seconds" + }, + "fields": [ + { + "id": "automatic_sync", + "section": "sync", + "label": "Automatic sync", + "description": "Keep employees and timecards up to date. Pausing lets the current collection finish.", + "type": "boolean", + "default": true, + "applies": "schedule" + }, + { + "id": "sync_interval_seconds", + "section": "sync", + "label": "Sync every", + "type": "integer", + "default": 3600, + "minimum": 1800, + "maximum": 14400, + "options": [ + { + "label": "30 minutes", + "value": 1800 + }, + { + "label": "1 hour", + "value": 3600 + }, + { + "label": "2 hours", + "value": 7200 + }, + { + "label": "4 hours", + "value": 14400 + } + ], + "applies": "schedule", + "enabledWhen": { + "field": "automatic_sync", + "equals": true + }, + "disabledReason": "Turn on Automatic sync to change its interval. Your saved interval is remembered." + }, + { + "id": "opening_page", + "section": "view", + "label": "Opening page", + "type": "string", + "default": "timecards", + "options": [ + { + "label": "Timecards", + "value": "timecards" + }, + { + "label": "Employees", + "value": "employees" + } + ], + "applies": "immediate" + }, + { + "id": "rows_per_page", + "section": "view", + "label": "Rows per page", + "type": "integer", + "default": 100, + "minimum": 25, + "maximum": 100, + "options": [ + { + "label": "25", + "value": 25 + }, + { + "label": "50", + "value": 50 + }, + { + "label": "100", + "value": 100 + } + ], + "applies": "immediate" + }, + { + "id": "name_order", + "section": "view", + "label": "Name order", + "description": "Choose how employee names appear and sort in Paycom.", + "type": "string", + "default": "first_last", + "options": [ + { + "label": "First Last", + "value": "first_last" + }, + { + "label": "Last, First", + "value": "last_first" + } + ], + "applies": "immediate" + }, + { + "id": "default_sort", + "section": "view", + "label": "Default sort", + "type": "string", + "default": "employeeName", + "options": [ + { + "label": "Employee name, A\u2013Z", + "value": "employeeName" + }, + { + "label": "Punch status", + "value": "condition" + }, + { + "label": "Clock-in time, earliest first", + "value": "inDay" + } + ], + "applies": "immediate" + }, + { + "id": "department", + "section": "view", + "label": "Default department", + "type": "string", + "nullable": true, + "default": null, + "optionsSource": "departments", + "applies": "immediate" + }, + { + "id": "station", + "section": "view", + "label": "Default delivery station", + "type": "string", + "nullable": true, + "default": null, + "optionsSource": "stations", + "applies": "immediate" + }, + { + "id": "columns", + "section": "view", + "label": "Timecard columns", + "description": "Employee names always appear. Select other columns and use the arrows to order them.", + "type": "strings", + "ordered": true, + "default": [ + "inDay", + "outLunch", + "inLunch", + "outDay", + "totalHours", + "condition" + ], + "options": [ + { + "label": "Clock in", + "value": "inDay" + }, + { + "label": "Lunch out", + "value": "outLunch" + }, + { + "label": "Lunch in", + "value": "inLunch" + }, + { + "label": "Clock out", + "value": "outDay" + }, + { + "label": "Hours", + "value": "totalHours" + }, + { + "label": "Punch status", + "value": "condition" + } + ], + "applies": "immediate" + }, + { + "id": "driver_departments", + "section": "drivers", + "label": "Departments shown on Timecards", + "description": "Only employees from the selected departments appear on your DSP\u2019s Timecard page. Selecting none shows no employees.", + "type": "strings", + "nullable": true, + "default": null, + "optionsSource": "departments", + "applies": "immediate" + } + ], + "migrations": [ + { + "fromVersion": 2, + "remove": [ + "name_order" + ] + } + ], + "rules": [ + { + "id": "timecard_department_excluded", + "kind": "included", + "field": "department", + "selection": "driver_departments", + "severity": "warning", + "message": "Your default department is excluded from Timecards. Choose an included department or update Driver departments." + } + ], + "previews": [ + { + "id": "name_example", + "section": "view", + "field": "name_order", + "label": "Name preview", + "kind": "choice", + "examples": [ + { + "value": "first_last", + "text": "JANE DOE" + }, + { + "value": "last_first", + "text": "DOE, JANE" + } + ] + }, + { + "id": "column_example", + "section": "view", + "field": "columns", + "label": "Timecard column preview", + "kind": "columns", + "leading": "Employee" + }, + { + "id": "department_coverage", + "section": "drivers", + "field": "driver_departments", + "label": "Timecard coverage", + "kind": "selection_count", + "unit": "employees", + "groupLabel": "departments" + } + ] + }, + "package": { + "runtime": "backend/installed.js", + "authentication": "backend/auth/adapter.js", + "collections": "backend/config/collection-manager.json" + } +} diff --git a/plugins/paycom/frontend/Paycom.tsx b/plugins/paycom/frontend/Paycom.tsx new file mode 100644 index 0000000..393749e --- /dev/null +++ b/plugins/paycom/frontend/Paycom.tsx @@ -0,0 +1,322 @@ +import { useEffect, useState } from "react"; +import { + ArrowRight, + CircleCheck, + CircleHelp, + Info, + LoaderCircle, + Plug, + RotateCw, + TriangleAlert, +} from "lucide-react"; +import { useQuery } from "@tanstack/react-query"; +import { + activeMembership, + isDspOwner, + mutation, + request, +} from "dispatch-sdk/ui"; +import { useSession } from "dispatch-sdk/ui"; +import { useTimezone } from "dispatch-sdk/ui"; +import { dateTime } from "dispatch-sdk/ui"; +import { Button } from "dispatch-sdk/ui"; +import { Badge } from "dispatch-sdk/ui"; +import { PaycomWorkforce } from "./PaycomWorkforce.tsx"; +import { PaycomSettings } from "./PaycomSettings.tsx"; +import { ErrorNotice, Loading, Notice, PageHeading } from "dispatch-sdk/ui"; + +type Connection = { + status: + "not_started" | "enrolling" | "queued" | "running" | "succeeded" | "failed"; + failureCode: string | null; + canSubmit: boolean; + canRetry: boolean; + retryState?: string | null; + retryAt?: string | null; + workforceAvailable: boolean; +}; +const pending = (status?: string) => + ["enrolling", "queued", "running"].includes(status || ""); +const connectionCopy = { + not_started: { + label: "Not connected", + title: "Connect your Paycom account", + description: + "Paycom is not connected. You can keep using your DSP and connect it later.", + icon: CircleHelp, + }, + enrolling: { + label: "Connecting", + title: "Your connection is in progress", + description: + "Verifying your Paycom login. You can leave this page while it finishes.", + icon: LoaderCircle, + }, + queued: { + label: "Connecting", + title: "Your connection is in progress", + description: + "Verifying your Paycom login. You can leave this page while it finishes.", + icon: LoaderCircle, + }, + running: { + label: "Connecting", + title: "Your connection is in progress", + description: + "Verifying your Paycom login. You can leave this page while it finishes.", + icon: LoaderCircle, + }, + succeeded: { + label: "Connected", + title: "Your Paycom account is connected", + description: + "Your Paycom login is verified. Workforce data has not been imported.", + icon: CircleCheck, + }, + failed: { + label: "Needs attention", + title: "Let’s get Paycom connected", + description: + "Paycom could not finish connecting. Your DSP is still ready to use.", + icon: TriangleAlert, + }, +} as const; +export function Paycom() { + const { session } = useSession(); + const [settings, setSettings] = useState(() => + new URLSearchParams(location.hash.split("?")[1] || "").has("settings"), + ); + useEffect(() => { + const changed = () => + setSettings( + new URLSearchParams(location.hash.split("?")[1] || "").has("settings"), + ); + window.addEventListener("hashchange", changed); + return () => window.removeEventListener("hashchange", changed); + }, []); + if (settings) + return isDspOwner(session) ? ( + + ) : ( + Your DSP owner can manage Paycom settings. + ); + return ; +} +function PaycomContent() { + const { timeZone } = useTimezone(); + const { session } = useSession(); + const membership = activeMembership(session); + const owner = isDspOwner(session); + const connection = useQuery({ + queryKey: ["paycom-connection", membership?.organizationId], + queryFn: ({ signal }) => + request("/api/organization/paycom-setup", { signal }), + enabled: owner, + refetchInterval: (q) => { + const value = q.state.data; + if (pending(value?.status)) return 2000; + // Recovery happens outside this page; observe it without submitting a login. + return value?.status === "failed" ? 5000 : false; + }, + }); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const state = connection.data; + const needsSetup = + state?.status !== "succeeded" && !state?.workforceAvailable; + // Share the Settings connection cache and read the broker's current result. + // The onboarding job also prepares collections; it is not a login status. + const credentials = useQuery({ + queryKey: ["connections", membership?.organizationId], + queryFn: ({ signal }) => + request<{ items: { service: string; state: string }[] }>( + "/api/organization/connections", + { signal }, + ), + enabled: owner && needsSetup, + refetchInterval: pending(state?.status) ? 2000 : 5000, + }); + const verified = credentials.data?.items.some( + (item) => item.service === "paycom" && item.state === "connected", + ); + async function retry() { + setBusy(true); + setError(null); + try { + await mutation("/api/organization/paycom-setup/retry", "POST", {}); + await connection.refetch(); + } catch (err) { + setError(err); + } finally { + setBusy(false); + } + } + const copy = state ? connectionCopy[state.status] : null; + const StatusIcon = copy?.icon || CircleHelp; + if (!owner || state?.status === "succeeded" || state?.workforceAvailable) + return ; + if (verified && state) + return ( + + +

+ {pending(state.status) + ? "Paycom is connected. Preparing workforce sync. Timecards and employees will appear after the first collection." + : "Paycom is connected, but workforce setup has not finished."} +

+ {state.canRetry ? ( + + ) : !pending(state.status) ? ( + + ) : null} +
+ + + } + /> + ); + return ( +
+ + {!owner ? ( + Your DSP owner can manage the Paycom connection. + ) : ( + <> + + {connection.isPending ? ( + + ) : ( + state && + copy && ( +
+
+
+ +
+

Workforce connection

+

Confirm that your Paycom login works.

+
+
+ + {state.status === "not_started" ? ( + +
+
+
+

{copy.title}

+

{copy.description}

+
+ {state.status === "failed" && + (state.failureCode === "provider_auth_required" ? ( + + Paycom could not verify your login. Retry the connection + or update your Paycom credentials. + + ) : ( + + ))} + {state.status === "failed" && + state.retryState === "cooldown" && + state.retryAt && ( + + Another attempt is available after{" "} + {dateTime(state.retryAt, timeZone)}. + + )} + {state.status === "failed" && + state.retryState === "unavailable" && ( + + Connection status is temporarily unavailable. Checking + again shortly. + + )} + {state.status === "failed" && state.retryState === "busy" && ( + + Paycom is still finishing an operation. Checking again + shortly. + + )} + {(state.canRetry || state.canSubmit) && ( +
+ {state.canRetry && ( + + )} + {state.canSubmit && ( + + )} +
+ )} + {state.canSubmit && state.status === "not_started" && ( +

+ Have your client code, login, and five numbered security + PINs ready. +

+ )} +
+
+
+
+ ) + )} + + )} +
+ ); +} diff --git a/plugins/paycom/frontend/PaycomSettings.tsx b/plugins/paycom/frontend/PaycomSettings.tsx new file mode 100644 index 0000000..b6b8c34 --- /dev/null +++ b/plugins/paycom/frontend/PaycomSettings.tsx @@ -0,0 +1,77 @@ +import { usePaycomSync } from "./usePaycomSync.ts"; +import { + Button, + ErrorNotice, + PluginSettingsForm, + activeMembership, + dateTime, + useSession, + useTimezone, +} from "dispatch-sdk/ui"; + +export type PaycomPreferences = { + automatic_sync: boolean; + sync_interval_seconds: number; + opening_page: "timecards" | "employees"; + rows_per_page: number; + name_order: "last_first" | "first_last"; + default_sort: "employeeName" | "condition" | "inDay"; + department: string | null; + station: string | null; + columns: string[]; + driver_departments: string[] | null; +}; +function ScheduleStatus() { + const { timeZone } = useTimezone(); + const { session } = useSession(), + scope = `${activeMembership(session)?.organizationId}:${session.dspView?.viewRef || "member"}`; + const { query: status, run, message } = usePaycomSync(scope); + return ( +
+ +

+ Last successful sync{" "} + + {status.data?.lastSucceededAt + ? dateTime(status.data.lastSucceededAt, timeZone) + : "Not yet synced"} + +

+

+ Next scheduled sync{" "} + + {status.data?.nextDueAt + ? dateTime(status.data.nextDueAt, timeZone) + : "Not scheduled"} + +

+
+ + +
+ {message &&

{message}

} +
+ ); +} +export function PaycomSettings() { + return ( + + section === "sync" ? : null + } + /> + ); +} diff --git a/plugins/paycom/frontend/PaycomWorkforce.tsx b/plugins/paycom/frontend/PaycomWorkforce.tsx new file mode 100644 index 0000000..d5506da --- /dev/null +++ b/plugins/paycom/frontend/PaycomWorkforce.tsx @@ -0,0 +1,902 @@ +import { usePaycomSync } from "./usePaycomSync.ts"; +import { useState, type ReactNode } from "react"; +import { useQuery } from "@tanstack/react-query"; +import { + ArrowDown, + ArrowUp, + ArrowUpDown, + ArrowLeft, + ChevronLeft, + ChevronRight, + ExternalLink, +} from "lucide-react"; +import { + activeMembership, + ApiError, + has, + request, + isDspOwner, + usePluginSettings, +} from "dispatch-sdk/ui"; +import type { PaycomPreferences } from "./PaycomSettings.tsx"; +import { useSession } from "dispatch-sdk/ui"; +import { useTimezone, useBusinessToday } from "dispatch-sdk/ui"; +import { + calendarDateLabel as dateLabel, + moveCalendarDate as moveDate, + dateTime as timestamp, +} from "dispatch-sdk/ui"; +import { Button } from "dispatch-sdk/ui"; +import { Badge } from "dispatch-sdk/ui"; +import { Tabs, TabsList, TabsTrigger, TabsContent } from "dispatch-sdk/ui"; +import { + Table, + TableHeader, + TableHead, + TableBody, + TableRow, + TableCell, +} from "dispatch-sdk/ui"; +import { + EmptyState, + ErrorNotice, + Loading, + Notice, + PageHeading, + TextField, +} from "dispatch-sdk/ui"; +import "./paycom-workforce.css"; + +type PunchKey = "inDay" | "outLunch" | "inLunch" | "outDay"; +type SortKey = "employeeName" | PunchKey | "totalHours" | "condition"; +type Direction = "asc" | "desc"; +type Punch = { time: string; timeBasis: "actual" | "displayed" }; +type DayRow = { + employeeCode: string; + employeeName: string; + businessDate: string; + condition: "complete" | "incomplete" | "needs_review" | "no_activity"; + missingPunch: boolean; + totalHours: string | null; + observedAt: string; + punches: Record; +}; +type Day = { + available: boolean; + items: DayRow[]; + total: number; + offset: number; + hasMore: boolean; + businessDate: string; + businessTimezone: string; + collectedAt: string; + periodStart: string; + periodEnd: string; +}; +type Employee = { + employeeCode: string; + employeeName: string; + lifecycleStatus: string; + department: { code: string; name: string }; + deliveryStation: { code: string; name: string }; + positionTitle: string; + payClass: string; + payType: string; + primarySupervisor: string; +}; +type Directory = { + items: Employee[]; + target: string; + collectedAt: string; + total: number; + hasMore: boolean; +}; +type EmployeeDetail = { + employee: Employee; + days?: DayRow[]; + collectedAt: string; + timecard: null | { + periodStart: string; + periodEnd: string; + periodTotalHours: string; + missingDays: number; + canonicalUrl: string; + observedAt: string; + }; +}; +const columns: { key: SortKey; label: string }[] = [ + { key: "employeeName", label: "Employee" }, + { key: "inDay", label: "Clock in" }, + { key: "outLunch", label: "Lunch out" }, + { key: "inLunch", label: "Lunch in" }, + { key: "outDay", label: "Clock out" }, + { key: "totalHours", label: "Hours" }, + { key: "condition", label: "Punch status" }, +]; +const punchKeys: PunchKey[] = ["inDay", "outLunch", "inLunch", "outDay"]; +function awaitingCollection(error: unknown) { + return error instanceof ApiError && error.code === "not_initialized"; +} +function AwaitingCollection() { + return ( + + ); +} +const nameOrder = new Intl.Collator("en", { + sensitivity: "base", + numeric: true, +}); +function status(row: DayRow) { + if (row.missingPunch) return "Missing punch"; + if (row.condition === "needs_review") return "Needs review"; + if (row.condition === "no_activity") return "No punches"; + if (row.condition === "complete") return "Clocked out"; + return row.punches.outLunch.length > row.punches.inLunch.length + ? "On lunch" + : "Clocked in"; +} +function punchTime(value: string) { + const match = /^([01]\d|2[0-3]):([0-5]\d)$/.exec(value); + if (!match) return value; + const hour = Number(match[1]); + return `${hour % 12 || 12}:${match[2]} ${hour < 12 ? "AM" : "PM"}`; +} +function PunchCells({ + row, + visible = [...punchKeys, "totalHours", "condition"], +}: { + row: DayRow; + visible?: string[]; +}) { + return ( + <> + {visible.map((column) => + punchKeys.includes(column as PunchKey) ? ( + (() => { + const key = column as PunchKey; + return ( + +
+ {row.punches[key].length + ? row.punches[key].map((punch, index) => ( + + {punchTime(punch.time)} + + {" "} + {punch.timeBasis} time + + + )) + : "—"} +
+
+ ); + })() + ) : column === "totalHours" ? ( + + {row.totalHours === null ? "—" : Number(row.totalHours).toFixed(2)} + + ) : column === "condition" ? ( + + + {status(row)} + + {row.punches.unclassified.length > 0 && ( +
+ Unclassified:{" "} + {row.punches.unclassified + .map((p) => punchTime(p.time)) + .join(", ")} +
+ )} +
+ ) : null, + )} + + ); +} +function SortHeader({ + label, + active, + direction, + onClick, +}: { + label: string; + active: boolean; + direction: Direction; + onClick: () => void; +}) { + const Icon = active + ? direction === "asc" + ? ArrowUp + : ArrowDown + : ArrowUpDown; + return ( + + + + ); +} +async function allEmployees(signal: AbortSignal) { + const items: Employee[] = []; + let first: Directory | undefined; + // Bound the complete directory to the collector's 5,000-employee limit. + for (let offset = 0; offset < 5000; offset += 100) { + const page = await request( + `/api/paycom/employees?limit=100&offset=${offset}`, + { signal }, + ); + if ( + first && + (page.target !== first.target || + page.collectedAt !== first.collectedAt || + page.total !== first.total) + ) + throw new ApiError("workforce_changed"); + first ||= page; + items.push(...page.items); + if (!page.hasMore) return items; + if (page.items.length !== 100) throw new ApiError("workforce_unavailable"); + } + throw new ApiError("workforce_unavailable"); +} +function DailyTimecards({ + scope, + timezone, + displayTimezone, + preferences, +}: { + scope: string; + timezone: string; + displayTimezone: string; + preferences: PaycomPreferences; +}) { + const today = useBusinessToday(timezone); + const [selectedDate, setSelectedDate] = useState(null); + const [sort, setSort] = useState(preferences.default_sort); + const [direction, setDirection] = useState("asc"); + const [offset, setOffset] = useState(0); + const date = selectedDate || today; + const size = preferences.rows_per_page; + const visibleColumns = [ + columns[0], + ...preferences.columns + .map((key) => columns.find((column) => column.key === key)) + .filter((column): column is (typeof columns)[number] => !!column), + ]; + const filters = new URLSearchParams(); + if (preferences.department) filters.set("department", preferences.department); + if (preferences.station) filters.set("station", preferences.station); + const dayQuery = useQuery({ + queryKey: [ + "paycom-day", + scope, + date, + sort, + direction, + offset, + size, + preferences.driver_departments, + preferences.name_order, + filters.toString(), + ], + queryFn: ({ signal }) => + request<{ day: Day }>( + `/api/paycom/daily?date=${date}&sort=${sort}&direction=${direction}&limit=${size}&offset=${offset}&${filters}`, + { signal }, + ), + refetchInterval: date === today ? 30_000 : false, + }); + const day = dayQuery.data?.day; + const waiting = awaitingCollection(dayQuery.error); + const chooseDate = (value: string) => { + if (/^\d{4}-\d{2}-\d{2}$/.test(value) && value <= today) { + setSelectedDate(value === today ? null : value); + setOffset(0); + } + }; + const changeSort = (key: SortKey) => { + setDirection(sort === key && direction === "asc" ? "desc" : "asc"); + setSort(key); + setOffset(0); + }; + return ( +
+
+
+

{date === today ? "Today’s timecards" : "Daily timecards"}

+

+ {dateLabel(date)} · {timezone.replaceAll("_", " ")} +

+
+
+ + chooseDate(event.target.value)} + /> + + +
+
+ + {dayQuery.isError && !waiting && ( + + )} + {waiting ? ( + + ) : dayQuery.isPending ? ( + + ) : ( + day && ( + <> + {!day.available ? ( + + ) : ( +
+
+

Employee timecards

+ {day.total} employees +
+ + + + {visibleColumns.map((column) => ( + changeSort(column.key)} + /> + ))} + + + + {day.items.map((row) => ( + + {row.employeeName} + + + ))} + +
+ {day.items.length === 0 && ( + + )} + {(offset > 0 || day.hasMore) && ( +
+ + {offset + 1}–{offset + day.items.length} of {day.total} + + + +
+ )} +
+ )} +

+ Last collected {timestamp(day.collectedAt, displayTimezone)}. + Times reflect the last collection, and hours may change after + corrections. +

+

+ An open shift or empty punch does not automatically mean a missing + punch. Multiple punches are shown in source order; sorting uses + the first punch. +

+ + ) + )} +
+ ); +} +function EmployeeTimecard({ + code, + scope, + timezone, + back, + nameOrder, +}: { + code: string; + scope: string; + timezone: string; + back: () => void; + nameOrder: PaycomPreferences["name_order"]; +}) { + const detail = useQuery({ + queryKey: ["paycom-employee", scope, code, nameOrder], + queryFn: ({ signal }) => + request( + `/api/paycom/employees/${encodeURIComponent(code)}`, + { signal }, + ), + }); + const data = detail.data; + return ( +
+ + + {detail.isError && ( + + )} + {detail.isPending ? ( + + ) : ( + data && ( + <> +
+
+

{data.employee.employeeName}

+

+ {data.employee.positionTitle} +

+
+ {data.employee.lifecycleStatus} +
+
+ {[ + ["Department", data.employee.department.name], + ["Delivery station", data.employee.deliveryStation.code], + ["Supervisor", data.employee.primarySupervisor], + ["Pay class", data.employee.payClass], + ["Pay type", data.employee.payType], + ].map(([label, value]) => ( +
+
{label}
+
{value || "—"}
+
+ ))} +
+ {data.employee.lifecycleStatus === "unknown" && ( + + Not present in the latest active roster. Last verified + information is retained. + + )} + {data.timecard ? ( + <> +
+
+

Employee timecard

+

+ {dateLabel(data.timecard.periodStart)} –{" "} + {dateLabel(data.timecard.periodEnd)} +

+
+ +
+ {data.days?.length ? ( +
+ + + + {[ + "Date", + ...columns.slice(1).map((c) => c.label), + ].map((label) => ( + + {label} + + ))} + + + + {data.days.map((row) => ( + + {dateLabel(row.businessDate)} + + + ))} + +
+
+ ) : ( + + Detailed daily rows are unavailable. Open the collected + period in Paycom. + + )} +

+ Period hours:{" "} + + {Number(data.timecard.periodTotalHours).toFixed(2)} + {" "} + · Days with missing punches: {data.timecard.missingDays} +

+

+ Last observed {timestamp(data.timecard.observedAt, timezone)}. + Blank days indicate no recorded activity, not an absence. +

+ + ) : ( + + )} + + ) + )} +
+ ); +} +function Employees({ + scope, + timezone, + preferences, +}: { + scope: string; + timezone: string; + preferences: PaycomPreferences; +}) { + const [search, setSearch] = useState(""); + const [page, setPage] = useState(0); + const [selected, setSelected] = useState(null); + const [direction, setDirection] = useState("asc"); + const employees = useQuery({ + queryKey: ["paycom-employees", scope, preferences.name_order], + queryFn: ({ signal }) => allEmployees(signal), + refetchInterval: (query) => + awaitingCollection(query.state.error) ? 30_000 : false, + }); + const waiting = awaitingCollection(employees.error); + const size = preferences.rows_per_page; + const rows = (employees.data || []) + .filter( + (row) => + (!preferences.department || + row.department.code === preferences.department) && + (!preferences.station || + row.deliveryStation.code === preferences.station), + ) + .filter((row) => + row.employeeName + .toLocaleLowerCase() + .includes(search.trim().toLocaleLowerCase()), + ) + .sort( + (a, b) => + nameOrder.compare(a.employeeName, b.employeeName) * + (direction === "asc" ? 1 : -1) || + a.employeeCode.localeCompare(b.employeeCode), + ); + if (selected) + return ( + setSelected(null)} + nameOrder={preferences.name_order} + /> + ); + return ( +
+
+ { + setSearch(e.target.value); + setPage(0); + }} + /> +
+ + {employees.isError && !waiting && ( + + )} + {waiting ? ( + + ) : employees.isPending ? ( + + ) : ( + !employees.isError && ( +
+
+

Employees

+ + {rows.length} {rows.length === 1 ? "employee" : "employees"} + +
+ + + + { + setDirection(direction === "asc" ? "desc" : "asc"); + setPage(0); + }} + /> + + + + {rows.slice(page * size, (page + 1) * size).map((row) => ( + + + + + + ))} + +
+ {rows.length > size && ( +
+ + {page * size + 1}–{Math.min((page + 1) * size, rows.length)}{" "} + of {rows.length} + + + +
+ )} + {rows.length === 0 && ( + + )} +
+ ) + )} +
+ ); +} +const syncLabels: Record = { + idle: "Waiting for next sync", + queued: "Queued", + waiting_for_capacity: "Waiting for capacity", + syncing: "Collecting", + stopping: "Stopping", + backing_off: "Waiting to retry", + blocked: "Needs attention", +}; +function PaycomSyncStatus({ + scope, + timezone, + canSync, +}: { + scope: string; + timezone: string; + canSync: boolean; +}) { + const { query, run, busy, authentication, message } = usePaycomSync(scope); + const sync = query.data; + const label = query.isError + ? "Sync status unavailable" + : !sync + ? "Checking sync status" + : sync.queuedRequest?.status === "failed" + ? "Requested sync could not start" + : busy + ? syncLabels[sync.activity] + : authentication + ? "Needs authentication" + : sync.desiredState === "stopped" && sync.activity !== "stopping" + ? "Sync paused" + : sync.activity === "idle" && sync.lastError + ? "Last collection failed" + : sync.activity === "idle" && !sync.lastSucceededAt + ? "Waiting for first collection" + : syncLabels[sync.activity] || "Needs attention"; + return ( +
+ {label} + {sync?.lastSucceededAt ? ( + + Last successful sync {timestamp(sync.lastSucceededAt, timezone)} + + ) : null} + {sync?.desiredState === "running" && sync.nextDueAt && !authentication ? ( + + Next scheduled sync {timestamp(sync.nextDueAt, timezone)} + + ) : null} + {canSync ? ( + + ) : null} + {authentication ? ( + + Click Sync now to sign in to Paycom and sync your data. + {sync?.lastSucceededAt + ? " Previously synced data remains available below." + : " Workforce data will appear after verification and the first successful sync."} + + ) : null} + {message ? {message} : null} + {run.isError ? ( + Could not request a sync. Please try again. + ) : null} +
+ ); +} + +export function PaycomWorkforce({ + setupNotice, +}: { setupNotice?: ReactNode } = {}) { + const { session } = useSession(); + const { timeZone: displayTimezone } = useTimezone(); + const membership = activeMembership(session); + const settings = usePluginSettings("paycom"); + if (!has(membership, "workforce.read")) + return You do not have permission to view workforce data.; + const scope = `${membership!.organizationId}:${session?.dspView?.viewRef || "member"}`; + const timezone = membership!.organization.timezone; + if (settings.query.isPending) return ; + if (!settings.query.data) + return ( + <> + + + + ); + const preferences = settings.query.data.values; + return ( +
+ + {isDspOwner(session) && ( + + )} + {setupNotice || ( + + )} + + + Timecard + Employees + + + + + + + + +
+ ); +} diff --git a/plugins/paycom/frontend/index.tsx b/plugins/paycom/frontend/index.tsx new file mode 100644 index 0000000..f938e32 --- /dev/null +++ b/plugins/paycom/frontend/index.tsx @@ -0,0 +1,2 @@ +import { Paycom } from "./Paycom.tsx"; +export default { id: "paycom", pages: { paycom: Paycom } }; diff --git a/plugins/paycom/frontend/paycom-workforce.css b/plugins/paycom/frontend/paycom-workforce.css new file mode 100644 index 0000000..24fdc7a --- /dev/null +++ b/plugins/paycom/frontend/paycom-workforce.css @@ -0,0 +1,123 @@ +.paycom-workforce { + min-width: 0; +} +.paycom-data-view { + display: flex; + flex-direction: column; + align-items: stretch; + gap: 18px; + min-width: 0; + padding-top: 24px; +} +.paycom-data-view > button { + align-self: flex-start; +} +.paycom-day-toolbar { + display: flex; + justify-content: space-between; + align-items: center; + flex-wrap: wrap; + gap: 18px; +} +.paycom-date-controls { + display: flex; + align-items: flex-end; + gap: 8px; + flex-wrap: wrap; +} +.paycom-date-controls [data-slot="field"] { + width: 165px; +} +.paycom-source-note { + color: var(--muted-foreground); + font-size: 12px; + line-height: 1.6; +} +.paycom-data-table { + min-width: 0; + border: 1px solid var(--border); + border-radius: var(--radius); + overflow: hidden; +} +.paycom-table-heading { + display: flex; + justify-content: space-between; + align-items: center; + gap: 16px; + padding: 18px; +} +.paycom-table-heading > span { + color: var(--muted-foreground); + font-size: 12px; +} +.paycom-data-table [data-slot="table-cell"] { + padding: 14px 18px; + font-variant-numeric: tabular-nums; +} +.paycom-data-table [data-slot="table-head"] { + background: var(--muted); +} +.paycom-punches { + display: flex; + flex-direction: column; + gap: 4px; +} +.paycom-pagination { + display: flex; + justify-content: flex-end; + align-items: center; + gap: 12px; + flex-wrap: wrap; + padding: 16px; +} +.paycom-pagination > span { + margin-right: auto; + color: var(--muted-foreground); + font-size: 12px; +} +.paycom-employee-search { + max-width: 360px; +} +.paycom-employee-details { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); + gap: 18px; + padding: 20px; + background: var(--muted); + border-radius: var(--radius); +} +.paycom-employee-details dt { + color: var(--muted-foreground); + font-size: 12px; +} +.paycom-employee-details dd { + margin: 5px 0 0; + overflow-wrap: anywhere; +} +@media (max-width: 600px) { + .paycom-date-controls { + width: 100%; + } + .paycom-date-controls [data-slot="field"] { + width: 145px; + } + .paycom-data-table [data-slot="table-cell"] { + padding: 12px; + } +} + +.paycom-sync-status { + display: flex; + flex-wrap: wrap; + gap: 6px 20px; + margin-bottom: 20px; + font-size: 13px; +} +.paycom-settings-status{margin-top:2rem;padding-top:1.5rem;border-top:1px solid var(--border);display:flex;flex-direction:column;gap:1rem;font-size:.875rem} +.paycom-settings-status p{display:flex;justify-content:space-between;gap:1rem;flex-wrap:wrap;color:var(--muted-foreground)} +.paycom-settings-status strong{font-weight:400;color:var(--foreground)} +.paycom-settings-status>div{display:flex;gap:.5rem;flex-wrap:wrap} +.paycom-settings-note{margin-top:1.5rem;font-size:.875rem;color:var(--muted-foreground);line-height:1.7} +.paycom-settings-preview{border:1px solid var(--border);border-radius:.5rem;margin-top:1.5rem;overflow:hidden;font-size:.8125rem} +.paycom-settings-preview>span{display:block;padding:.7rem 1rem;background:var(--muted);color:var(--muted-foreground)} +.paycom-settings-preview>div{display:flex;flex-wrap:wrap;gap:1.25rem;padding:1rem} diff --git a/plugins/paycom/frontend/usePaycomSync.ts b/plugins/paycom/frontend/usePaycomSync.ts new file mode 100644 index 0000000..46a659f --- /dev/null +++ b/plugins/paycom/frontend/usePaycomSync.ts @@ -0,0 +1,86 @@ +import { useEffect, useRef } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { idempotent, request } from "dispatch-sdk/ui"; + +export type SyncSummary = { + queuedRequest?: { id: string; status: string; error: string | null } | null; + activeRun?: { id: string; status: string } | null; + activity: string; + desiredState: string; + lastSucceededAt: string | number | null; + nextDueAt: string | number | null; + lastError: string | null; + alerts: Array<{ code: string }>; +}; +type SyncRequest = { + sync?: SyncSummary; + run?: { id: string; status: string } | null; +}; +const active = new Set([ + "queued", + "waiting_for_capacity", + "syncing", + "stopping", +]); + +export function usePaycomSync(scope: string) { + const cache = useQueryClient(); + const pending = useRef | null>(null); + const lastSuccess = useRef(null); + const requestedAfter = useRef(null); + const query = useQuery({ + queryKey: ["paycom-sync", scope], + queryFn: ({ signal }) => + request("/api/paycom/sync", { signal }), + refetchInterval: 5000, + }); + const run = useMutation({ + mutationFn: () => { + if (!pending.current) { + requestedAfter.current = query.data?.lastSucceededAt ?? null; + pending.current = idempotent( + `paycom-sync:${scope}`, + "/api/paycom/sync", + {}, + ).finally(() => { + pending.current = null; + }); + } + return pending.current; + }, + onSuccess: () => + cache.invalidateQueries({ queryKey: ["paycom-sync", scope] }), + }); + const sync = query.data; + useEffect(() => { + if (!sync?.lastSucceededAt || sync.lastSucceededAt === lastSuccess.current) + return; + lastSuccess.current = sync.lastSucceededAt; + for (const key of ["paycom-day", "paycom-employees", "paycom-employee"]) { + void cache.invalidateQueries({ queryKey: [key, scope] }); + } + }, [sync?.lastSucceededAt, cache, scope]); + const busy = Boolean(sync && active.has(sync.activity)); + const authentication = + !busy && + sync?.alerts?.some((alert) => alert.code === "authentication_blocked"); + const completed = Boolean( + run.isSuccess && + sync?.lastSucceededAt && + sync.lastSucceededAt !== requestedAfter.current, + ); + const message = run.isPending + ? "Requesting sync…" + : run.isSuccess && busy + ? "Sync is in progress. Paycom will sign in automatically if needed." + : completed + ? "Sync completed." + : run.isSuccess && authentication + ? "Paycom could not finish signing in. You can retry with Sync now or check your connection settings." + : run.isSuccess && sync?.lastError + ? "Sync could not finish. Click Sync now to retry." + : run.isSuccess + ? "Sync requested. Paycom will sign in automatically if needed." + : null; + return { query, run, busy, authentication, message }; +} diff --git a/plugins/paycom/generated/client.d.ts b/plugins/paycom/generated/client.d.ts new file mode 100644 index 0000000..2cd6dc1 --- /dev/null +++ b/plugins/paycom/generated/client.d.ts @@ -0,0 +1,10 @@ +// Generated from dispatch-plugin.json. Do not edit. +import type { Json, RequestOptions } from "dispatch-sdk"; +export interface Client { + "workforce.day"(input: { "query": { "date"?: string; "search"?: string; "attention"?: string; "lifecycleStatus"?: string; "limit"?: number; "offset"?: number; "sort"?: string; "direction"?: string; "department"?: string; "station"?: string } }, options?: RequestOptions): Promise; + "workforce.employees"(input: { "query": { "limit"?: number; "offset"?: number; "lifecycleStatus"?: string } }, options?: RequestOptions): Promise; + "workforce.employee"(input: { "code": string }, options?: RequestOptions): Promise; + "sync.status"(input: { "id": "paycom-main-workforce" }, options?: RequestOptions): Promise; + "sync.run_now"(input: { "id": "paycom-main-workforce"; "options"?: { "idempotencyKey"?: string } }, options?: RequestOptions): Promise; +} +export function createClient(invoke: (action: string, input: Json, options?: RequestOptions) => Promise): Client; diff --git a/plugins/paycom/generated/client.js b/plugins/paycom/generated/client.js new file mode 100644 index 0000000..404fbbd --- /dev/null +++ b/plugins/paycom/generated/client.js @@ -0,0 +1,168 @@ +'use strict'; +// Generated from dispatch-plugin.json. Do not edit. +const { createOperationClient } = require('dispatch-sdk/operations'); +const actions = [ + { + "id": "workforce.day", + "permission": "workforce.read", + "input": { + "type": "object", + "properties": { + "query": { + "type": "object", + "properties": { + "date": { + "type": "string", + "maxLength": 256 + }, + "search": { + "type": "string", + "maxLength": 256 + }, + "attention": { + "type": "string", + "maxLength": 256 + }, + "lifecycleStatus": { + "type": "string", + "maxLength": 256 + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + "offset": { + "type": "integer", + "minimum": 0 + }, + "sort": { + "type": "string", + "maxLength": 256 + }, + "direction": { + "type": "string", + "maxLength": 256 + }, + "department": { + "type": "string", + "maxLength": 256 + }, + "station": { + "type": "string", + "maxLength": 256 + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [ + "query" + ], + "additionalProperties": false + } + }, + { + "id": "workforce.employees", + "permission": "workforce.read", + "input": { + "type": "object", + "properties": { + "query": { + "type": "object", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + "offset": { + "type": "integer", + "minimum": 0 + }, + "lifecycleStatus": { + "type": "string", + "maxLength": 256 + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [ + "query" + ], + "additionalProperties": false + } + }, + { + "id": "workforce.employee", + "permission": "workforce.read", + "input": { + "type": "object", + "properties": { + "code": { + "type": "string", + "maxLength": 256 + } + }, + "required": [ + "code" + ], + "additionalProperties": false + } + }, + { + "id": "sync.status", + "permission": "workforce.read", + "input": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "paycom-main-workforce" + ] + } + }, + "required": [ + "id" + ], + "additionalProperties": false + } + }, + { + "id": "sync.run_now", + "permission": "sync.run", + "input": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "paycom-main-workforce" + ] + }, + "options": { + "type": "object", + "properties": { + "idempotencyKey": { + "type": "string", + "minLength": 16, + "maxLength": 128 + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [ + "id" + ], + "additionalProperties": false + } + } +]; +function createClient(invoke) { return createOperationClient({ actions, invoke }); } +module.exports = { createClient }; diff --git a/plugins/paycom/generated/openapi.json b/plugins/paycom/generated/openapi.json new file mode 100644 index 0000000..36cddf6 --- /dev/null +++ b/plugins/paycom/generated/openapi.json @@ -0,0 +1,499 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Paycom operations", + "version": "0.18.9" + }, + "paths": { + "/api/plugins/paycom/workforce.day": { + "post": { + "operationId": "paycom.workforce.day", + "summary": "workforce.day", + "x-dispatch-permission": "workforce.read", + "description": "Requires an enabled installation and current DSP authority. DSP scope comes from the authenticated session or signed support view.", + "security": [ + { + "session": [], + "csrf": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "query": { + "type": "object", + "properties": { + "date": { + "type": "string", + "maxLength": 256 + }, + "search": { + "type": "string", + "maxLength": 256 + }, + "attention": { + "type": "string", + "maxLength": 256 + }, + "lifecycleStatus": { + "type": "string", + "maxLength": 256 + }, + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + "offset": { + "type": "integer", + "minimum": 0 + }, + "sort": { + "type": "string", + "maxLength": 256 + }, + "direction": { + "type": "string", + "maxLength": 256 + }, + "department": { + "type": "string", + "maxLength": 256 + }, + "station": { + "type": "string", + "maxLength": 256 + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [ + "query" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Operation completed", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ok", + "data", + "status", + "contractVersion" + ], + "properties": { + "ok": { + "const": true + }, + "contractVersion": { + "const": 1 + }, + "status": { + "type": "string" + }, + "data": {} + }, + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Invalid input" + }, + "401": { + "description": "Authentication required" + }, + "403": { + "description": "Permission or request verification failed" + }, + "409": { + "description": "Operation rejected or installation changed" + }, + "503": { + "description": "Service unavailable" + } + } + } + }, + "/api/plugins/paycom/workforce.employees": { + "post": { + "operationId": "paycom.workforce.employees", + "summary": "workforce.employees", + "x-dispatch-permission": "workforce.read", + "description": "Requires an enabled installation and current DSP authority. DSP scope comes from the authenticated session or signed support view.", + "security": [ + { + "session": [], + "csrf": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "query": { + "type": "object", + "properties": { + "limit": { + "type": "integer", + "minimum": 1, + "maximum": 100 + }, + "offset": { + "type": "integer", + "minimum": 0 + }, + "lifecycleStatus": { + "type": "string", + "maxLength": 256 + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [ + "query" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Operation completed", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ok", + "data", + "status", + "contractVersion" + ], + "properties": { + "ok": { + "const": true + }, + "contractVersion": { + "const": 1 + }, + "status": { + "type": "string" + }, + "data": {} + }, + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Invalid input" + }, + "401": { + "description": "Authentication required" + }, + "403": { + "description": "Permission or request verification failed" + }, + "409": { + "description": "Operation rejected or installation changed" + }, + "503": { + "description": "Service unavailable" + } + } + } + }, + "/api/plugins/paycom/workforce.employee": { + "post": { + "operationId": "paycom.workforce.employee", + "summary": "workforce.employee", + "x-dispatch-permission": "workforce.read", + "description": "Requires an enabled installation and current DSP authority. DSP scope comes from the authenticated session or signed support view.", + "security": [ + { + "session": [], + "csrf": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "code": { + "type": "string", + "maxLength": 256 + } + }, + "required": [ + "code" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Operation completed", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ok", + "data", + "status", + "contractVersion" + ], + "properties": { + "ok": { + "const": true + }, + "contractVersion": { + "const": 1 + }, + "status": { + "type": "string" + }, + "data": {} + }, + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Invalid input" + }, + "401": { + "description": "Authentication required" + }, + "403": { + "description": "Permission or request verification failed" + }, + "409": { + "description": "Operation rejected or installation changed" + }, + "503": { + "description": "Service unavailable" + } + } + } + }, + "/api/plugins/paycom/sync.status": { + "post": { + "operationId": "paycom.sync.status", + "summary": "sync.status", + "x-dispatch-permission": "workforce.read", + "description": "Requires an enabled installation and current DSP authority. DSP scope comes from the authenticated session or signed support view.", + "security": [ + { + "session": [], + "csrf": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "paycom-main-workforce" + ] + } + }, + "required": [ + "id" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Operation completed", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ok", + "data", + "status", + "contractVersion" + ], + "properties": { + "ok": { + "const": true + }, + "contractVersion": { + "const": 1 + }, + "status": { + "type": "string" + }, + "data": {} + }, + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Invalid input" + }, + "401": { + "description": "Authentication required" + }, + "403": { + "description": "Permission or request verification failed" + }, + "409": { + "description": "Operation rejected or installation changed" + }, + "503": { + "description": "Service unavailable" + } + } + } + }, + "/api/plugins/paycom/sync.run_now": { + "post": { + "operationId": "paycom.sync.run_now", + "summary": "sync.run_now", + "x-dispatch-permission": "sync.run", + "description": "Requires an enabled installation and current DSP authority. DSP scope comes from the authenticated session or signed support view.", + "security": [ + { + "session": [], + "csrf": [] + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": [ + "paycom-main-workforce" + ] + }, + "options": { + "type": "object", + "properties": { + "idempotencyKey": { + "type": "string", + "minLength": 16, + "maxLength": 128 + } + }, + "required": [], + "additionalProperties": false + } + }, + "required": [ + "id" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Operation completed", + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "ok", + "data", + "status", + "contractVersion" + ], + "properties": { + "ok": { + "const": true + }, + "contractVersion": { + "const": 1 + }, + "status": { + "type": "string" + }, + "data": {} + }, + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Invalid input" + }, + "401": { + "description": "Authentication required" + }, + "403": { + "description": "Permission or request verification failed" + }, + "409": { + "description": "Operation rejected or installation changed" + }, + "503": { + "description": "Service unavailable" + } + } + } + } + }, + "components": { + "securitySchemes": { + "session": { + "type": "apiKey", + "in": "cookie", + "name": "dispatch_session" + }, + "csrf": { + "type": "apiKey", + "in": "header", + "name": "X-Dispatch-CSRF" + } + } + } +} diff --git a/shared/dashboard/public/assets/inter.woff2 b/shared/dashboard/public/assets/inter.woff2 new file mode 100644 index 0000000..d15208d Binary files /dev/null and b/shared/dashboard/public/assets/inter.woff2 differ diff --git a/shared/dashboard/src/App.tsx b/shared/dashboard/src/App.tsx new file mode 100644 index 0000000..477b7a9 --- /dev/null +++ b/shared/dashboard/src/App.tsx @@ -0,0 +1,418 @@ +import { workspaceRoutes, Workspace } from "./Workspace"; +import { PasswordRecovery } from "@/pages/PasswordRecovery"; +import { DefaultShellLayout } from "@/themes/defaults/ShellLayout"; +import { PluginBoundary } from "@/plugins/loader"; +import { useCallback, useEffect, useState } from "react"; +import { + Building2, + FlaskConical, + ArrowUpFromLine, + Database, + Puzzle, + Settings as SettingsIcon, + House, + CalendarDays, + Users, + Menu, + LogOut, + ChevronDown, + Eye, + type LucideIcon, +} from "lucide-react"; +import { + activeMembership, + has, + isPlatform, + isDspOwner, + mutation, + queryClient, + request, + setSession, + setDspView, + ApiError, +} from "@/lib/api"; +import type { Session } from "@/lib/types"; +import { SessionContext } from "@/lib/session"; +import { ThemeProvider, useTheme } from "@/lib/theme"; +import { TimezoneProvider } from "@/lib/timezone"; +import { ReleasePopup } from "@/components/ReleasePopup"; +import { Brand } from "@/components/Brand"; +import { Button } from "@/components/ui/button"; +import { + Sheet, + SheetContent, + SheetTitle, + SheetDescription, +} from "@/components/ui/sheet"; +import { + DropdownMenu, + DropdownMenuTrigger, + DropdownMenuContent, + DropdownMenuGroup, + DropdownMenuItem, +} from "@/components/ui/dropdown-menu"; +import { PageHeading, Loading, ErrorNotice, Notice } from "@/components/shared"; +import { Settings, DspOnboarding } from "@/pages/Settings"; +import { Plugins } from "@/pages/Plugins"; +import { + pluginPage, + pluginPages, + usePlugins, + type PluginView, +} from "@/plugins/registry"; +import { Auth } from "@/pages/Auth"; +export type NavItem = { id: string; label: string; icon: LucideIcon }; +function Shell({ + session, + refresh, + hash, + viewEnded, +}: { + session: Session; + refresh: (session?: Session) => Promise; + hash: string; + viewEnded: boolean; +}) { + const Layout = + useTheme().themePack.components?.ShellLayout || DefaultShellLayout; + const platform = isPlatform(session); + const membership = activeMembership(session); + const plugins = usePlugins(session); + const installed = plugins.data?.items || []; + const navigation = workspaceRoutes(session, installed); + const requested = hash.replace(/^#\//, "").split(/[/?]/)[0]; + const route = navigation.find((n) => n.id === requested) || navigation[0]; + const PluginPage = pluginPage(installed, route.id); + const [mobile, setMobile] = useState(false); + const [error, setError] = useState(null); + const [busy, setBusy] = useState(false); + async function exitView() { + setBusy(true); + setError(null); + try { + setDspView(null); + await refresh(); + location.hash = "#/platform"; + } catch (e) { + setError(e); + } finally { + setBusy(false); + } + } + useEffect(() => { + if (requested !== route.id) { + history.replaceState( + {}, + "", + `${location.pathname}${location.search}#/${route.id}`, + ); + window.dispatchEvent(new HashChangeEvent("hashchange")); + } + document.title = `${route.label} · Dispatch`; + setMobile(false); + }, [requested, route.id, route.label]); + const nav = ( + <> +
+ +

+ {platform ? "Platform" : membership?.organization.name || "Workspace"} +

+
+ +
+ + + + + + + + + Account settings + + + { + setBusy(true); + try { + await mutation("/api/auth/logout", "POST", {}); + setDspView(null); + setSession(null); + queryClient.clear(); + location.hash = ""; + await refresh(); + } catch (e) { + setError(e); + } finally { + setBusy(false); + } + }} + > + + Sign out + + + + +
+ + ); + return ( + + { + e.preventDefault(); + document.getElementById("main-content")?.focus(); + }} + > + Skip to content + + + + Navigation + + Your Dispatch workspace pages. + + {nav} + + + } + banner={ + session.dspView && ( +
+
+ ) + } + header={ + <> + +
+ + {platform + ? "Platform" + : membership?.organization.name || "Workspace"} + + + {route.label} +
+ + } + > + {viewEnded && !session.dspView && ( + + The DSP view expired or is no longer available. You’re back in the + platform console. + + )} + + +
+ +
+ ); +} +export function App() { + const [session, saveSession] = useState(null); + const [loaded, setLoaded] = useState(false); + const [dashboardChanged,setDashboardChanged]=useState(false); + const [error, setError] = useState(null); + const [hash, setHash] = useState(location.hash); + const [navigation, setNavigation] = useState(0); + const [viewEnded, setViewEnded] = useState(false); + const refresh = useCallback(async (next?: Session) => { + let value: Session; + try { + value = next || (await request("/api/auth/session")); + } catch (e) { + if (!(e instanceof ApiError) || e.code !== "dsp_view_unavailable") + throw e; + setViewEnded(true); + value = await request("/api/auth/session"); + } + if (value.dspView || !value.authenticated) setViewEnded(false); + if (!value.authenticated) setDspView(null); + const target = value.authenticated && !isPlatform(value) ? "dsp" : "core"; + if (window.__dispatchDashboard && window.__dispatchDashboard.product !== target) { + window.location.reload(); return; + } + setSession(value); + saveSession(value); + setLoaded(true); + setError(null); + }, []); + useEffect(() => { + void refresh().catch((e) => { + setError(e); + setLoaded(true); + }); + const route = () => { + setHash(location.hash); + setNavigation((value) => value + 1); + }; + const expired = () => { + setSession(null); + saveSession(null); + }; + const viewEnded = () => { + setViewEnded(true); + void refresh().catch(setError); + }; + window.addEventListener("hashchange", route); + window.addEventListener("dispatch-session-expired", expired); + window.addEventListener("dispatch-dsp-view-ended", viewEnded); + return () => { + window.removeEventListener("hashchange", route); + window.removeEventListener("dispatch-session-expired", expired); + window.removeEventListener("dispatch-dsp-view-ended", viewEnded); + }; + }, [refresh]); + useEffect(() => { + if (!session?.authenticated) return; + const reload = () => { + void refresh().catch(() => {}); + }; + window.addEventListener("focus", reload); + return () => window.removeEventListener("focus", reload); + }, [session?.authenticated, refresh]); + useEffect(() => { + if (!session?.dspView) return; + const timer = window.setTimeout( + () => void refresh().catch(setError), + Math.max(0, Date.parse(session.dspView.expiresAt) - Date.now()) + 100, + ); + return () => window.clearTimeout(timer); + }, [session?.dspView?.expiresAt, refresh]); + useEffect(() => { + if(!session?.authenticated || !window.__dispatchDashboard)return; + let cancelled=false; + const check=async()=>{try{const current=await request<{product:string;digest:string}>("/api/dashboard?identity=1"); + if(!cancelled)setDashboardChanged(current.digest!==window.__dispatchDashboard?.digest); + }catch{ /* Normal session/error handling owns authentication failures. */ }}; + void check(); const timer=window.setInterval(check,30000); + return ()=>{cancelled=true;window.clearInterval(timer);}; + },[session?.authenticated,session?.activeOrganizationId,session?.dspView?.viewRef]); + function renderContent() { + if (!loaded) + return ( +
+ + +
+ ); + if (error) + return ( +
+ + + +
+ ); + if ( + hash === "#/forgot-password" || + hash === "#/reset-password" || + hash.startsWith("#/reset-password/") + ) + return ( + + ); + const token = + /^#\/invitation\/([A-Za-z0-9_-]{43})$/.exec(hash)?.[1] || null; + if (!session?.authenticated || token) + return ; + if ( + !session.dspView && + hash === "#/onboarding" && + session.memberships.some( + (m) => + m.organizationId === session.activeOrganizationId && + m.organization.status !== "suspended" && + m.permissions.includes("organization.owner"), + ) + ) + return ( + + + + ); + return ( + + ); + } + const userId = session?.authenticated ? session.user.id : null; + return {dashboardChanged &&
An update is ready. Refresh before making further changes.
}{renderContent()}
; +} diff --git a/shared/dashboard/src/backups.css b/shared/dashboard/src/backups.css new file mode 100644 index 0000000..57061db --- /dev/null +++ b/shared/dashboard/src/backups.css @@ -0,0 +1,637 @@ +/* Approved F: open sections and a contextual rail, using the active theme. */ +.backup-workspace.backup-streamlined { + --backup-section-gap: 36px; + font-size: 15px; + min-width: 0; +} +.backup-streamlined .backup-header { + align-items: center; + margin-bottom: 46px; +} +.backup-workspace.backup-streamlined h1 { + font-size: 38px; + font-weight: 600; + letter-spacing: -1.2px; +} +.backup-streamlined .backup-tabs { + margin-bottom: 0; + gap: 36px; +} +.backup-streamlined .backup-tabs a { + font-size: 15px; + padding: 0 6px 20px; + font-weight: 450; +} +.backup-streamlined .backup-header .backup-actions { + gap: 28px; +} +.backup-streamlined .backup-button { + font-size: 15px; + min-height: 44px; + padding: 11px 18px; +} +.backup-streamlined .backup-link { + font-size: 15px; + font-weight: 450; +} +.backup-streamlined .backup-button-text { + border: 0; + background: transparent; + color: var(--primary); + justify-content: flex-start; + padding: 0; + min-height: 32px; + font-weight: 450; +} +.backup-streamlined .backup-button-text:hover { + text-decoration: underline; +} +.backup-streamlined .backup-split { + display: grid; + grid-template-columns: minmax(0, 2.2fr) minmax(250px, 1fr); + gap: 0; + min-height: 550px; +} +.backup-streamlined .backup-split:has(> aside[hidden]) { + grid-template-columns: minmax(0, 1fr); +} +.backup-streamlined .backup-main { + min-width: 0; + padding: 36px 42px 40px 0; +} +.backup-streamlined .backup-rail { + border-left: 1px solid var(--border); + padding: 36px 0 40px 34px; + min-width: 0; +} +.backup-streamlined .backup-rail[hidden] { + display: none; +} +.backup-streamlined .backup-section { + min-width: 0; +} +.backup-streamlined .backup-section > h2, +.backup-streamlined .backup-scope-heading h2 { + font-size: 19px; + font-weight: 600; + letter-spacing: -0.35px; + margin: 0 0 24px; +} +.backup-streamlined .backup-main > .backup-section + .backup-section { + margin-top: var(--backup-section-gap); + padding-top: var(--backup-section-gap); + border-top: 1px solid var(--border); +} +.backup-streamlined .backup-rail > .backup-section + .backup-section { + border-top: 1px solid var(--border); + margin-top: 32px; + padding-top: 32px; +} +.backup-streamlined .backup-section p { + margin: 0 0 16px; +} +.backup-streamlined .backup-rail p { + line-height: 1.6; + overflow-wrap: anywhere; +} +.backup-streamlined .backup-rail > p { + border-top: 1px solid var(--border); + padding-top: 28px; + margin-top: 28px; +} +.backup-streamlined .backup-latest { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 16px; + margin: 26px 0 14px; +} +.backup-streamlined .backup-latest > p { + font-size: 25px; + letter-spacing: -0.65px; + line-height: 1.4; + margin: 0; +} +.backup-streamlined .backup-latest + p { + margin-bottom: 26px; +} +.backup-streamlined .backup-latest-label { + margin-top: 32px; +} +.backup-streamlined .backup-latest-label + .backup-latest { + margin-top: 8px; +} +.backup-streamlined .backup-status { + font-size: 14px; + white-space: nowrap; +} +.backup-streamlined .backup-status::before { + flex-shrink: 0; + width: 7px; + height: 7px; +} +.backup-streamlined .backup-latest .backup-status::before, +.backup-streamlined .backup-scope-heading .backup-status::before { + display: none; +} +.backup-streamlined .backup-latest .backup-status svg, +.backup-streamlined .backup-scope-heading .backup-status svg { + display: block; +} +.backup-streamlined .backup-latest .backup-status.verified, +.backup-streamlined .backup-scope-heading .backup-status.verified { + color: var(--success) !important; +} +.backup-streamlined .backup-off { + color: var(--warning); + margin-left: 8px; +} +.backup-streamlined .backup-table-scroll { + border: 0; + border-radius: 0; + overflow-x: auto; +} +.backup-streamlined .backup-table { + display: table; + width: 100%; + min-width: 0; + table-layout: auto; +} +.backup-streamlined .backup-table thead { + display: table-header-group; +} +.backup-streamlined .backup-table tbody { + display: table-row-group; +} +.backup-streamlined .backup-table tr { + display: table-row; + border: 0; +} +.backup-streamlined .backup-table th, +.backup-streamlined .backup-table td { + display: table-cell; + border: 0; + border-bottom: 1px solid var(--border); + padding: 18px 12px; + font-size: 14px; + text-align: left; + white-space: normal; + vertical-align: middle; +} +.backup-streamlined .backup-table th { + font-weight: 450; + padding-top: 8px; + padding-bottom: 14px; +} +.backup-streamlined .backup-table td::before { + display: none; +} +.backup-streamlined .backup-table td:first-child, +.backup-streamlined .backup-table th:first-child { + padding-left: 2px; +} +.backup-streamlined .backup-table td:last-child, +.backup-streamlined .backup-table th:last-child { + padding-right: 2px; + text-align: right; +} +.backup-streamlined .backup-table td { + font-weight: 400; +} +.backup-streamlined .backup-table .backup-link { + font-size: 14px; + color: var(--primary); + white-space: nowrap; +} +.backup-streamlined .backup-table .backup-dsp-name .backup-link { + color: var(--foreground); + white-space: normal; +} +.backup-streamlined .backup-table td:last-child .backup-icon { + display: none; +} +.backup-streamlined .backup-dsp-picker { + max-width: 320px; + margin-bottom: 32px; +} +.backup-streamlined select, +.backup-streamlined input:not([type="checkbox"]) { + font-size: 14px; + min-height: 42px; +} +.backup-streamlined .backup-scope-heading { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 24px; + margin-bottom: 18px; +} +.backup-streamlined .backup-scope-heading h2 { + margin: 0; + font-size: 23px; +} +.backup-streamlined .backup-toolbar { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 16px; + margin: 20px 0 24px; +} +.backup-streamlined .backup-toolbar .backup-search-wrap { + flex: 1 1 220px; + min-width: 0; +} +.backup-streamlined .backup-toolbar .backup-field { + flex: 0 1 200px; +} +.backup-streamlined .backup-toolbar .backup-field > span { + position: absolute; + width: 1px; + height: 1px; + overflow: hidden; + clip-path: inset(50%); +} +.backup-streamlined .backup-toolbar select { + width: 100%; +} +.backup-streamlined .backup-rail > .backup-field { + margin-bottom: 26px; +} +.backup-streamlined .backup-rail > .backup-button-text { + border-top: 1px solid var(--border); + padding-top: 24px; + width: 100%; +} +.backup-streamlined .backup-filter-tabs { + gap: 20px; + margin: 0 0 24px; + padding: 0; + border-bottom: 1px solid var(--border); +} +.backup-streamlined .backup-filter-tabs button { + padding: 0 8px 12px; + min-height: 36px; + border: 0; + border-bottom: 2px solid transparent; + border-radius: 0; + background: none; + color: var(--foreground); + font-weight: 400; +} +.backup-streamlined .backup-filter-tabs button[aria-pressed="true"] { + border-color: var(--foreground); + font-weight: 600; + background: none; +} +.backup-streamlined .backup-event-type { + display: flex; + flex-direction: column; + gap: 4px; +} +.backup-streamlined .backup-event-type small { + font-size: 12px; +} +.backup-streamlined .backup-event-count { + margin-top: 20px; +} +.backup-streamlined .backup-storage-total { + font-size: 28px; + margin: 0 0 10px; +} +.backup-streamlined .backup-disclosure { + border-bottom: 1px solid var(--border); +} +.backup-streamlined .backup-disclosure > summary { + list-style: none; + cursor: pointer; + display: flex; + justify-content: space-between; + align-items: center; + gap: 16px; + padding: 20px 2px; + font-weight: 500; +} +.backup-streamlined .backup-disclosure > summary::-webkit-details-marker { + display: none; +} +.backup-streamlined .backup-disclosure[open] > summary > svg { + transform: rotate(90deg); +} +.backup-streamlined .backup-disclosure > p, +.backup-streamlined .backup-disclosure > .backup-pairs { + margin: 16px 0; +} +.backup-streamlined .backup-disclosure > .backup-button { + margin: 12px 12px 20px 0; +} +.backup-streamlined .backup-settings-scope { + max-width: 460px; + margin: 28px 0 34px; +} +.backup-streamlined .backup-form-grid { + border: 0; + padding: 0; + margin: 28px 0 36px; + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 22px; + min-width: 0; +} +.backup-streamlined .backup-form-grid input, +.backup-streamlined .backup-form-grid select { + width: 100%; +} +.backup-streamlined input:disabled, +.backup-streamlined select:disabled { + opacity: 0.55; + cursor: not-allowed; +} +.backup-streamlined .backup-toggle-label { + display: inline-flex; + align-items: center; + gap: 10px; +} +.backup-streamlined .backup-toggle-label .backup-switch { + width: 40px; + height: 24px; +} +.backup-streamlined .backup-toggle-label .backup-switch::after { + width: 18px; + height: 18px; + top: 3px; + left: 3px; +} +.backup-streamlined .backup-toggle-label .backup-switch:checked::after { + transform: translateX(16px); +} +.backup-streamlined .backup-form-section { + padding-top: 28px; +} +.backup-streamlined .backup-form-section .backup-field { + max-width: 460px; + margin: 22px 0 16px; +} +.backup-streamlined .backup-note { + margin: 24px 0; +} +.backup-streamlined .backup-empty { + text-align: left; + padding: 24px 0; +} +.backup-streamlined .backup-empty > svg { + display: none; +} +.backup-streamlined > .backup-card, +.backup-streamlined > .backup-page-title { + margin-top: 32px; +} +@media (max-width: 1100px) { + .backup-streamlined .backup-split { + grid-template-columns: minmax(0, 1.8fr) minmax(220px, 1fr); + } + .backup-streamlined .backup-main { + padding-right: 24px; + } + .backup-streamlined .backup-rail { + padding-left: 24px; + } + .backup-streamlined .backup-table { + min-width: 560px; + } + .backup-streamlined .backup-form-grid { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} +@media (max-width: 800px) { + .backup-streamlined .backup-split { + display: flex; + flex-direction: column; + min-height: 0; + } + .backup-streamlined .backup-main { + padding: 28px 0; + } + .backup-streamlined .backup-rail { + border-left: 0; + border-top: 1px solid var(--border); + padding: 28px 0; + } + .backup-streamlined .backup-header { + align-items: flex-start; + flex-direction: column; + gap: 20px; + margin-bottom: 30px; + } + .backup-streamlined .backup-header .backup-actions { + justify-content: flex-start; + gap: 20px; + } + .backup-workspace.backup-streamlined h1 { + font-size: 32px; + } + .backup-streamlined .backup-tabs { + gap: 24px; + } + .backup-streamlined .backup-tabs a { + padding-bottom: 16px; + font-size: 14px; + } + .backup-streamlined .backup-latest > p { + font-size: 22px; + } + .backup-streamlined .backup-form-grid { + grid-template-columns: minmax(0, 1fr); + } + .backup-streamlined .backup-settings-scope { + max-width: none; + } + .backup-streamlined .backup-table th, + .backup-streamlined .backup-table td { + padding: 16px 10px; + } + .backup-streamlined .backup-table-scroll { + max-width: 100%; + } +} +.backup-streamlined .backup-header a[aria-current="page"] { + color: var(--foreground); + font-weight: 600; +} +.backup-streamlined .backup-tabs a[aria-current="page"] { + border-bottom-color: var(--primary); +} +.backup-streamlined .backup-filter-tabs button[aria-pressed="true"] { + color: var(--foreground); +} +.backup-streamlined .backup-rail > .backup-button-text { + border-radius: 0; +} +.backup-streamlined .backup-field > span { + font-size: 14px; + font-weight: 450; +} +.backup-streamlined .backup-card-heading h3, +.backup-streamlined .backup-form-section h3 { + font-size: 18px; + font-weight: 600; +} +@media (max-width: 800px) { + .backup-streamlined .backup-storage-table .backup-table { + min-width: 0; + table-layout: fixed; + } + .backup-streamlined .backup-storage-table .backup-table th:first-child { + width: 50%; + } + .backup-streamlined .backup-storage-table .backup-table th:not(:first-child) { + width: auto; + } + .backup-streamlined .backup-storage-table th, + .backup-streamlined .backup-storage-table td { + padding: 14px 6px; + font-size: 13px; + overflow-wrap: anywhere; + } + .backup-streamlined .backup-storage-table .backup-link { + white-space: normal; + } + .backup-streamlined + .backup-table-scroll:not(.backup-storage-table) + .backup-table { + min-width: 0; + } + .backup-streamlined .backup-table-scroll:not(.backup-storage-table) thead { + position: absolute; + width: 1px; + height: 1px; + overflow: hidden; + clip-path: inset(50%); + } + .backup-streamlined .backup-table-scroll:not(.backup-storage-table) tbody { + display: block; + } + .backup-streamlined .backup-table-scroll:not(.backup-storage-table) tr { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 12px 20px; + padding: 18px 0; + border-bottom: 1px solid var(--border); + } + .backup-streamlined .backup-table-scroll:not(.backup-storage-table) td { + display: block; + border: 0; + padding: 0; + text-align: left; + overflow-wrap: anywhere; + } + .backup-streamlined + .backup-table-scroll:not(.backup-storage-table) + td:first-child { + grid-column: 1 / -1; + } + .backup-streamlined + .backup-table-scroll:not(.backup-storage-table) + td:not(:first-child)::before { + display: block; + content: attr(data-label); + font-size: 12px; + color: var(--muted-foreground); + margin-bottom: 5px; + } + .backup-streamlined + .backup-table-scroll:not(.backup-storage-table) + .backup-status { + white-space: normal; + } +} +/* The reference uses a 1536px content canvas. Scale its typography up when + that space is available, while retaining readable laptop/mobile controls. */ +.backup-workspace.backup-streamlined { + container: backups / inline-size; +} +.backup-streamlined .backup-muted { + font-size: inherit; +} +.backup-streamlined .backup-toolbar > select { + flex: 0 1 200px; + width: 200px; +} +.backup-streamlined[data-view="dsps"] + .backup-main + > .backup-section + + .backup-section, +.backup-streamlined[data-view="storage"] + .backup-main + > .backup-section + + .backup-section { + border-top: 0; + padding-top: 0; +} +@container backups (min-width: 1300px) { + .backup-streamlined .backup-header { + padding-top: 24px; + margin-bottom: 60px; + } + .backup-workspace.backup-streamlined h1 { + font-size: 46px; + } + .backup-streamlined .backup-tabs { + gap: 40px; + } + .backup-streamlined .backup-tabs a { + font-size: 18px; + padding-bottom: 24px; + } + .backup-streamlined .backup-main { + padding: 60px 56px 48px 0; + } + .backup-streamlined .backup-rail { + padding: 54px 0 48px 42px; + } + .backup-streamlined .backup-section > h2, + .backup-streamlined .backup-scope-heading h2 { + font-size: 22px; + } + .backup-streamlined .backup-section p, + .backup-streamlined .backup-rail p, + .backup-streamlined .backup-link, + .backup-streamlined .backup-button, + .backup-streamlined .backup-tabs a { + font-size: 18px; + } + .backup-streamlined .backup-button { + min-height: 50px; + } + .backup-streamlined .backup-button-text { + min-height: 36px; + } + .backup-streamlined .backup-latest { + margin-top: 36px; + } + .backup-streamlined .backup-latest > p { + font-size: 30px; + } + .backup-streamlined .backup-main > .backup-section + .backup-section { + margin-top: 48px; + padding-top: 48px; + } + .backup-streamlined .backup-rail > .backup-section + .backup-section { + margin-top: 40px; + padding-top: 40px; + } + .backup-streamlined .backup-table th, + .backup-streamlined .backup-table td, + .backup-streamlined .backup-table .backup-link, + .backup-streamlined .backup-status, + .backup-streamlined .backup-field > span, + .backup-streamlined input:not([type="checkbox"]), + .backup-streamlined select { + font-size: 16px; + } + .backup-streamlined .backup-section .backup-storage-total { + font-size: 32px; + } + .backup-streamlined .backup-scope-heading h2 { + font-size: 28px; + } +} diff --git a/shared/dashboard/src/components/AuditLog.tsx b/shared/dashboard/src/components/AuditLog.tsx new file mode 100644 index 0000000..fffd428 --- /dev/null +++ b/shared/dashboard/src/components/AuditLog.tsx @@ -0,0 +1,90 @@ +import { useQuery } from "@tanstack/react-query"; +import { activeMembership, has, request } from "@/lib/api"; +import { useSession } from "@/lib/session"; +import { useTimezone } from "@/lib/timezone"; +import type { AuditLogData } from "@/lib/types"; +import { + dateTime, + EmptyState, + ErrorNotice, + Loading, + Notice, + RefreshButton, + Status, +} from "@/components/shared"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; + +export function AuditLog() { + const { session } = useSession(); + const { timeZone } = useTimezone(); + const membership = activeMembership(session); + const suspended = membership?.organization.status === "suspended"; + const audit = useQuery({ + queryKey: ["organization-audit", membership?.organizationId], + queryFn: ({ signal }) => + request("/api/organization/audit", { signal }), + enabled: has(membership, "audit.read") && !suspended, + refetchInterval: 15000, + }); + + if (suspended) + return ( + This DSP is suspended. The audit log is unavailable. + ); + if (!has(membership, "audit.read")) return null; + + return ( + <> +
+

+ Recent changes to your DSP, team, and access. +

+ void audit.refetch()} + busy={audit.isFetching} + /> +
+ + {audit.isPending ? ( + + ) : audit.data && !audit.error ? ( + audit.data.audit.length ? ( + + + + Action + By + Date + Result + + + + {audit.data.audit.map((event, index) => ( + + {event.action.replaceAll(".", " ")} + {event.actor} + {dateTime(event.createdAt, timeZone)} + + {event.result} + + + ))} + +
+ ) : ( + + ) + ) : null} + + ); +} diff --git a/shared/dashboard/src/components/Brand.tsx b/shared/dashboard/src/components/Brand.tsx new file mode 100644 index 0000000..bace827 --- /dev/null +++ b/shared/dashboard/src/components/Brand.tsx @@ -0,0 +1,13 @@ +export function Brand() { + return ( + + + Dispatch + + ); +} diff --git a/shared/dashboard/src/components/DspAvatar.tsx b/shared/dashboard/src/components/DspAvatar.tsx new file mode 100644 index 0000000..c6a7f0f --- /dev/null +++ b/shared/dashboard/src/components/DspAvatar.tsx @@ -0,0 +1,16 @@ +import { useTheme } from "@/lib/theme"; +import { dspIdentity } from "@/lib/identity"; + +export function DefaultDspAvatar({ name }: { name: string }) { + const { initials, className } = dspIdentity(name); + return ( + + ); +} + +export function DspAvatar(props: { name: string }) { + const View = useTheme().themePack.components?.DspAvatar || DefaultDspAvatar; + return ; +} diff --git a/shared/dashboard/src/components/ReleasePopup.tsx b/shared/dashboard/src/components/ReleasePopup.tsx new file mode 100644 index 0000000..4af1caa --- /dev/null +++ b/shared/dashboard/src/components/ReleasePopup.tsx @@ -0,0 +1,184 @@ +import { useEffect, useRef, useState } from "react"; +import { X } from "lucide-react"; +import { mutation, request } from "@/lib/api"; +import { useSession } from "@/lib/session"; +import { useTimezone } from "@/lib/timezone"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogTitle, +} from "@/components/ui/dialog"; + +type Change = { + kind: "added" | "improved" | "changed" | "fixed" | "removed"; + title: string; + description: string; +}; +type Release = { + releaseId: string; + version: string; + publishedAt: string; + changelog: Change[]; + afterUpdating: { title: string; description: string }[]; +}; +const sections = [ + { title: "New", kinds: ["added"] }, + { title: "Improved", kinds: ["improved", "changed"] }, + { title: "Fixed", kinds: ["fixed"] }, + { title: "Removed", kinds: ["removed"] }, +]; + +export function ReleasePopup() { + const { session } = useSession(); + const { timeZone } = useTimezone(); + const [release, setRelease] = useState(null); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(false); + const inFlight = useRef(false); + const heading = useRef(null); + useEffect(() => { + let cancelled = false; + // Check on entry, not on every navigation or while the user is working. + void request<{ release: Release | null }>("/api/updates/popup") + .then((data) => { + if (!cancelled) setRelease(data.release); + }) + .catch(() => {}); + return () => { + cancelled = true; + }; + }, []); + async function dismiss() { + if (!release || inFlight.current) return; + inFlight.current = true; + setSaving(true); + setError(false); + try { + await mutation("/api/updates/popup", "POST", { + releaseId: release.releaseId, + }); + setRelease(null); + } catch { + setError(true); + } finally { + inFlight.current = false; + setSaving(false); + } + } + // Another open tab can acknowledge the release without interrupting this tab. + useEffect(() => { + if (!release) return; + const check = () => { + void request<{ release: Release | null }>("/api/updates/popup") + .then((data) => { + if (!data.release) setRelease(null); + }) + .catch(() => {}); + }; + window.addEventListener("focus", check); + return () => window.removeEventListener("focus", check); + }, [release]); + if (!release || session.dspView) return null; + return ( + { + if (!open) void dismiss(); + }} + > + event.preventDefault()} + onOpenAutoFocus={(event) => { + event.preventDefault(); + heading.current?.focus(); + }} + onCloseAutoFocus={(event) => { + event.preventDefault(); + document.getElementById("main-content")?.focus(); + }} + > +
+

What’s new

+ + Dispatch {release.version} + + + + + Here’s what changed in the latest release. + + + +
+
+ {sections.map((section) => { + const items = release.changelog.filter((item) => + section.kinds.includes(item.kind), + ); + if (!items.length) return null; + return ( +
+

{section.title}

+
    + {items.map((item, index) => ( +
  • + {item.title} + {item.description &&

    {item.description}

    } +
  • + ))} +
+
+ ); + })} + {release.afterUpdating.length > 0 && ( +
+

After updating

+
    + {release.afterUpdating.map((item, index) => ( +
  • + {item.title} +

    {item.description}

    +
  • + ))} +
+
+ )} +
+
+ {error && ( +

+ We couldn’t save your dismissal. Please try again. +

+ )} + +
+
+
+ ); +} diff --git a/shared/dashboard/src/components/ThemeSection.tsx b/shared/dashboard/src/components/ThemeSection.tsx new file mode 100644 index 0000000..9c2b4e1 --- /dev/null +++ b/shared/dashboard/src/components/ThemeSection.tsx @@ -0,0 +1,117 @@ +import { useTheme, type Theme } from "@/lib/theme"; + +import { themePacks } from "@/themes/registry"; + +const choices: { value: Theme; label: string; description: string }[] = [ + { value: "light", label: "Light", description: "A bright, clean workspace." }, + { value: "dark", label: "Dark", description: "A calm, low-light workspace." }, + { + value: "system", + label: "System", + description: "Match your device settings.", + }, +]; + +function ThemePreview({ mode }: { mode: "light" | "dark" }) { + const { themePack } = useTheme(); + return ( + + + + + + + + + + + + {[0, 1, 2].map((row) => ( + + + + + + ))} + + + + ); +} + +export function ThemeSection() { + const { + appearance, + setAppearance, + themePack, + setThemePack, + storageUnavailable, + } = useTheme(); + return ( +
+
+ + +

{themePack.description}

+
+
+ Appearance +

Choose how Dispatch looks for you.

+
+ {choices.map(({ value, label, description }) => ( + + ))} +
+

+ Saved for your account on this browser. Other users keep their own + theme. +

+ {storageUnavailable && ( +

+ Theme applied for this visit. Browser storage is unavailable, so it + could not be saved. +

+ )} +
+
+ ); +} diff --git a/shared/dashboard/src/components/TimezoneSection.tsx b/shared/dashboard/src/components/TimezoneSection.tsx new file mode 100644 index 0000000..be11bce --- /dev/null +++ b/shared/dashboard/src/components/TimezoneSection.tsx @@ -0,0 +1,26 @@ +import { useTimezone } from "@/lib/timezone"; +import { validTimeZone } from "@/lib/date-time"; + +const supported = typeof Intl.supportedValuesOf === "function" ? Intl.supportedValuesOf("timeZone") : [ + "America/Los_Angeles", "America/Phoenix", "America/Denver", "America/Chicago", "America/New_York", "Europe/London", +]; + +export function TimezoneSection() { + const { timeZone, deviceZone, preference, setPreference, storageUnavailable } = useTimezone(); + const options = [...new Set(["UTC", deviceZone, preference, ...supported].filter(validTimeZone))].sort(); + return
+

Date & time

Choose how event times appear for you.

+
+ + +

Sync and activity times use {timeZone.replaceAll("_", " ")}. Timecards keep the DSP’s business timezone.

+

Saved for your account on this browser.

+ {storageUnavailable &&

Applied for this visit. Browser storage is unavailable, so this preference could not be saved.

} +
+
; +} diff --git a/shared/dashboard/src/components/Turnstile.tsx b/shared/dashboard/src/components/Turnstile.tsx new file mode 100644 index 0000000..0ada331 --- /dev/null +++ b/shared/dashboard/src/components/Turnstile.tsx @@ -0,0 +1,141 @@ +import { useEffect, useRef, useState } from "react"; +import { Button } from "@/components/ui/button"; + +type TurnstileApi = { + render: ( + container: HTMLElement, + options: { + sitekey: string; + action: string; + size: "flexible"; + "response-field": false; + callback: (token: string) => void; + "error-callback": () => void; + "expired-callback": () => void; + "timeout-callback": () => void; + "unsupported-callback": () => void; + }, + ) => string; + remove: (id: string) => void; +}; +declare global { + interface Window { + turnstile?: TurnstileApi; + } +} + +let scriptPromise: Promise | null = null; +function loadTurnstile(): Promise { + if (window.turnstile) return Promise.resolve(window.turnstile); + if (scriptPromise) return scriptPromise; + scriptPromise = new Promise((resolve, reject) => { + const script = document.createElement("script"); + script.src = + "https://challenges.cloudflare.com/turnstile/v0/api.js?render=explicit"; + script.async = true; + const fail = () => { + clearTimeout(timer); + script.onload = null; + script.onerror = null; + script.remove(); + reject(new Error("turnstile_unavailable")); + }; + const timer = window.setTimeout(fail, 15000); + script.onerror = fail; + script.onload = () => { + if (!window.turnstile) return fail(); + clearTimeout(timer); + script.onload = null; + script.onerror = null; + resolve(window.turnstile); + }; + document.head.appendChild(script); + }).catch((error) => { + scriptPromise = null; + throw error; + }); + return scriptPromise; +} + +export function Turnstile({ + siteKey, + action, + onToken, + busy, +}: { + siteKey: string; + action: "login" | "register" | "forgot_password"; + onToken: (token: string) => void; + busy: boolean; +}) { + const container = useRef(null); + const [attempt, setAttempt] = useState(0); + const [status, setStatus] = useState< + "checking" | "ready" | "expired" | "error" + >("checking"); + useEffect(() => { + let disposed = false; + let api: TurnstileApi | undefined; + let widget: string | undefined; + onToken(""); + setStatus("checking"); + const invalidate = (next: "expired" | "error") => { + if (disposed) return; + onToken(""); + setStatus(next); + }; + void loadTurnstile() + .then((loaded) => { + if (disposed || !container.current) return; + api = loaded; + widget = api.render(container.current, { + sitekey: siteKey, + action, + size: "flexible", + "response-field": false, + callback: (token) => { + if (disposed) return; + onToken(token); + setStatus("ready"); + }, + "error-callback": () => invalidate("error"), + "expired-callback": () => invalidate("expired"), + "timeout-callback": () => invalidate("expired"), + "unsupported-callback": () => invalidate("error"), + }); + }) + .catch(() => invalidate("error")); + return () => { + disposed = true; + if (widget !== undefined) api?.remove(widget); + }; + }, [siteKey, action, attempt, onToken]); + return ( +
+
+

+ {status === "checking" + ? "Checking your browser…" + : status === "ready" + ? "Security check complete." + : status === "expired" + ? "Security check expired. Please verify again." + : "Security check could not load. Check your connection and try again."} +

+ {(status === "error" || status === "expired") && ( + + )} +
+ ); +} diff --git a/shared/dashboard/src/components/shared.tsx b/shared/dashboard/src/components/shared.tsx new file mode 100644 index 0000000..67ad5ca --- /dev/null +++ b/shared/dashboard/src/components/shared.tsx @@ -0,0 +1,382 @@ +import { useTheme } from "@/lib/theme"; +import type { PageHeadingProps } from "@/themes/types"; +import { + useId, + useRef, + useState, + type ComponentProps, + type ReactNode, +} from "react"; +import { RefreshCw, Search, LoaderCircle } from "lucide-react"; +import { Button } from "./ui/button.tsx"; +import { Field, FieldDescription, FieldLabel } from "./ui/field.tsx"; +import { Input } from "./ui/input.tsx"; +import { + Sheet, + SheetContent, + SheetHeader, + SheetTitle, + SheetDescription, +} from "./ui/sheet.tsx"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, + DialogFooter, +} from "./ui/dialog.tsx"; +import { Alert, AlertDescription } from "./ui/alert.tsx"; +import { Skeleton } from "./ui/skeleton.tsx"; +import { Empty, EmptyHeader, EmptyTitle, EmptyDescription } from "./ui/empty.tsx"; +import { cn } from "@/lib/utils"; +import { errorMessage } from "@/lib/errors"; +import type { InvitationResult } from "@/lib/types"; +export function PageHeading(props: PageHeadingProps) { + const View = + useTheme().themePack.components?.PageHeading || DefaultPageHeading; + return ; +} +export function DefaultPageHeading({ + title, + description, + children, +}: { + title: string; + description?: string; + children?: ReactNode; +}) { + return ( +
+
+

{title}

+ {description &&

{description}

} +
+ {children &&
{children}
} +
+ ); +} +export function TextField({ + label, + description, + ...props +}: ComponentProps & { label: string; description?: string }) { + const id = useId(); + return ( + + {label} + + {description && {description}} + + ); +} +export function Notice({ + children, + error = false, +}: { + children: ReactNode; + error?: boolean; +}) { + return ( + + {children} + + ); +} +export function ErrorNotice({ error }: { error: unknown }) { + return error ? {errorMessage(error)} : null; +} +export function Loading() { + return ( +
+ + {[0, 1, 2].map((i) => ( + + ))} +
+ ); +} +export function EmptyState({ + title, + description, +}: { + title: string; + description?: string; +}) { + return ( + + + {title} + {description && {description}} + + + ); +} +export function RefreshButton({ + onClick, + busy = false, +}: { + onClick: () => void; + busy?: boolean; +}) { + return ( + + ); +} +export function SearchInput({ + value, + onChange, + placeholder, +}: { + value: string; + onChange: (value: string) => void; + placeholder: string; +}) { + return ( + + ); +} +export function SubmitButton({ + busy, + children, + ...props +}: ComponentProps & { busy: boolean }) { + return ( + + ); +} +export function Status({ + value, + children, +}: { + value: string; + children?: ReactNode; +}) { + return ( + + + ); +} +export function Panel({ + open, + onClose, + title, + description, + children, + busy = false, +}: { + open: boolean; + onClose: () => void; + title: ReactNode; + description?: string; + children: ReactNode; + busy?: boolean; +}) { + const prior = useRef(null); + return ( + { + if (!v && !busy) onClose(); + }} + > + { + prior.current = document.activeElement as HTMLElement; + }} + onCloseAutoFocus={(e) => { + e.preventDefault(); + if (prior.current?.isConnected) prior.current.focus(); + else document.querySelector(".page-heading h1")?.focus(); + }} + > + + {title} + + {description || "Review details and manage access."} + + + {children} + + + ); +} +export function ConfirmAction({ + title, + description, + confirmation, + passwordRequired = false, + onConfirm, + onClose, +}: { + title: string; + description: string; + confirmation?: string; + passwordRequired?: boolean; + onConfirm: (password?: string) => Promise; + onClose: () => void; +}) { + const [text, setText] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + return ( + { + if (!v && !busy) onClose(); + }} + > + + + {title} + {description} + +
{ + e.preventDefault(); + if (confirmation && text !== confirmation) return; + setBusy(true); + setError(null); + try { + await onConfirm(passwordRequired ? text : undefined); + onClose(); + } catch (err) { + if (passwordRequired) setText(""); + setError(err); + } finally { + setBusy(false); + } + }} + className="flex flex-col gap-5" + > + {passwordRequired && ( + setText(e.target.value)} + required + disabled={busy} + /> + )} + {confirmation && ( + setText(e.target.value)} + autoComplete="off" + disabled={busy} + required + /> + )} + + + + + {title} + + + +
+
+ ); +} +export function InvitationNotice({ + result, +}: { + result: InvitationResult | null; +}) { + if (!result) return null; + const email = + result.ownerInvitation?.email || + result.invitation?.email || + "the recipient"; + if (result.delivery?.status === "accepted") + return Invitation sent to {email}.; + if (!result.invitationPath) + return ( + + The invitation request was already processed. No new invitation was + sent. + + ); + const url = new URL(result.invitationPath, window.location.origin).href; + return ( + +
+

+ {result.delivery?.status === "unknown" + ? "Email delivery could not be confirmed. Use this same link if a private handoff is needed." + : result.delivery?.status === "failed" + ? "The invitation email could not be sent. Share this one-time invitation through a private channel." + : "No email was sent because invitation email is not configured. Share this one-time invitation through a private channel."} +

+ e.target.select()} + /> + +
+
+ ); +} +export { dateTime } from "@/lib/date-time"; diff --git a/shared/dashboard/src/components/ui/alert.tsx b/shared/dashboard/src/components/ui/alert.tsx new file mode 100644 index 0000000..5b9859d --- /dev/null +++ b/shared/dashboard/src/components/ui/alert.tsx @@ -0,0 +1,65 @@ +import * as React from "react"; +import { cva, type VariantProps } from "class-variance-authority"; +import { cn } from "@/lib/utils"; + +const alertVariants = cva( + "relative grid w-full grid-cols-[0_1fr] items-start gap-y-0.5 rounded-lg border px-4 py-3 text-sm has-[>svg]:grid-cols-[calc(var(--spacing)*4)_1fr] has-[>svg]:gap-x-3 [&>svg]:size-4 [&>svg]:translate-y-0.5 [&>svg]:text-current", + { + variants: { + variant: { + default: "bg-card text-card-foreground", + destructive: + "bg-card text-destructive *:data-[slot=alert-description]:text-destructive/90 [&>svg]:text-current", + }, + }, + defaultVariants: { + variant: "default", + }, + }, +); + +function Alert({ + className, + variant, + ...props +}: React.ComponentProps<"div"> & VariantProps) { + return ( +
+ ); +} + +function AlertTitle({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +function AlertDescription({ + className, + ...props +}: React.ComponentProps<"div">) { + return ( +
+ ); +} + +export { Alert, AlertTitle, AlertDescription }; diff --git a/shared/dashboard/src/components/ui/badge.tsx b/shared/dashboard/src/components/ui/badge.tsx new file mode 100644 index 0000000..eee82f0 --- /dev/null +++ b/shared/dashboard/src/components/ui/badge.tsx @@ -0,0 +1,47 @@ +import * as React from "react"; +import { cva, type VariantProps } from "class-variance-authority"; +import { cn } from "@/lib/utils"; +import { Slot } from "radix-ui"; + +const badgeVariants = cva( + "inline-flex w-fit shrink-0 items-center justify-center gap-1 overflow-hidden rounded-full border border-transparent px-2 py-0.5 text-xs font-medium whitespace-nowrap transition-[color,box-shadow] focus-visible:border-ring focus-visible:ring-[3px] focus-visible:ring-ring/50 aria-invalid:border-destructive aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 [&>svg]:pointer-events-none [&>svg]:size-3", + { + variants: { + variant: { + default: "bg-primary text-primary-foreground [a&]:hover:bg-primary/90", + secondary: + "bg-secondary text-secondary-foreground [a&]:hover:bg-secondary/90", + destructive: + "bg-destructive text-white focus-visible:ring-destructive/20 dark:bg-destructive/60 dark:focus-visible:ring-destructive/40 [a&]:hover:bg-destructive/90", + outline: + "border-border text-foreground [a&]:hover:bg-accent [a&]:hover:text-accent-foreground", + ghost: "[a&]:hover:bg-accent [a&]:hover:text-accent-foreground", + link: "text-primary underline-offset-4 [a&]:hover:underline", + }, + }, + defaultVariants: { + variant: "default", + }, + }, +); + +function Badge({ + className, + variant = "default", + asChild = false, + ...props +}: React.ComponentProps<"span"> & + VariantProps & { asChild?: boolean }) { + const Comp = asChild ? Slot.Root : "span"; + + return ( + + ); +} + +export { Badge, badgeVariants }; diff --git a/shared/dashboard/src/components/ui/button.tsx b/shared/dashboard/src/components/ui/button.tsx new file mode 100644 index 0000000..ada3314 --- /dev/null +++ b/shared/dashboard/src/components/ui/button.tsx @@ -0,0 +1,63 @@ +import * as React from "react"; +import { cva, type VariantProps } from "class-variance-authority"; +import { cn } from "@/lib/utils"; +import { Slot } from "radix-ui"; + +const buttonVariants = cva( + "inline-flex shrink-0 items-center justify-center gap-2 rounded-md text-sm font-medium whitespace-nowrap transition-all outline-none focus-visible:border-ring focus-visible:ring-[3px] focus-visible:ring-ring/50 disabled:pointer-events-none disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4", + { + variants: { + variant: { + default: "bg-primary text-primary-foreground hover:bg-primary/90", + destructive: + "bg-destructive text-white hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:bg-destructive/60 dark:focus-visible:ring-destructive/40", + outline: + "border bg-background hover:bg-accent hover:text-accent-foreground dark:border-input dark:bg-input/30 dark:hover:bg-input/50", + secondary: + "bg-secondary text-secondary-foreground hover:bg-secondary/80", + ghost: + "hover:bg-accent hover:text-accent-foreground dark:hover:bg-accent/50", + link: "text-primary underline-offset-4 hover:underline", + }, + size: { + default: "h-9 px-4 py-2 has-[>svg]:px-3", + xs: "h-6 gap-1 rounded-md px-2 text-xs has-[>svg]:px-1.5 [&_svg:not([class*='size-'])]:size-3", + sm: "h-8 gap-1.5 rounded-md px-3 has-[>svg]:px-2.5", + lg: "h-10 rounded-md px-6 has-[>svg]:px-4", + icon: "size-9", + "icon-xs": "size-6 rounded-md [&_svg:not([class*='size-'])]:size-3", + "icon-sm": "size-8", + "icon-lg": "size-10", + }, + }, + defaultVariants: { + variant: "default", + size: "default", + }, + }, +); + +function Button({ + className, + variant = "default", + size = "default", + asChild = false, + ...props +}: React.ComponentProps<"button"> & + VariantProps & { + asChild?: boolean; + }) { + const Comp = asChild ? Slot.Root : "button"; + + return ( + + ); +} + +export { Button, buttonVariants }; diff --git a/shared/dashboard/src/components/ui/card.tsx b/shared/dashboard/src/components/ui/card.tsx new file mode 100644 index 0000000..ea2e699 --- /dev/null +++ b/shared/dashboard/src/components/ui/card.tsx @@ -0,0 +1,91 @@ +import * as React from "react"; +import { cn } from "@/lib/utils"; + +function Card({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +function CardHeader({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +function CardTitle({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +function CardDescription({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +function CardAction({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +function CardContent({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +function CardFooter({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +export { + Card, + CardHeader, + CardFooter, + CardTitle, + CardAction, + CardDescription, + CardContent, +}; diff --git a/shared/dashboard/src/components/ui/dialog.tsx b/shared/dashboard/src/components/ui/dialog.tsx new file mode 100644 index 0000000..009cc58 --- /dev/null +++ b/shared/dashboard/src/components/ui/dialog.tsx @@ -0,0 +1,158 @@ +"use client"; + +import * as React from "react"; +import { cn } from "@/lib/utils"; +import { XIcon } from "lucide-react"; +import { Dialog as DialogPrimitive } from "radix-ui"; + +import { Button } from "@/components/ui/button"; + +function Dialog({ + ...props +}: React.ComponentProps) { + return ; +} + +function DialogTrigger({ + ...props +}: React.ComponentProps) { + return ; +} + +function DialogPortal({ + ...props +}: React.ComponentProps) { + return ; +} + +function DialogClose({ + ...props +}: React.ComponentProps) { + return ; +} + +function DialogOverlay({ + className, + ...props +}: React.ComponentProps) { + return ( + + ); +} + +function DialogContent({ + className, + children, + showCloseButton = true, + ...props +}: React.ComponentProps & { + showCloseButton?: boolean; +}) { + return ( + + + + {children} + {showCloseButton && ( + + + Close + + )} + + + ); +} + +function DialogHeader({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +function DialogFooter({ + className, + showCloseButton = false, + children, + ...props +}: React.ComponentProps<"div"> & { + showCloseButton?: boolean; +}) { + return ( +
+ {children} + {showCloseButton && ( + + + + )} +
+ ); +} + +function DialogTitle({ + className, + ...props +}: React.ComponentProps) { + return ( + + ); +} + +function DialogDescription({ + className, + ...props +}: React.ComponentProps) { + return ( + + ); +} + +export { + Dialog, + DialogClose, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogOverlay, + DialogPortal, + DialogTitle, + DialogTrigger, +}; diff --git a/shared/dashboard/src/components/ui/dropdown-menu.tsx b/shared/dashboard/src/components/ui/dropdown-menu.tsx new file mode 100644 index 0000000..7849a88 --- /dev/null +++ b/shared/dashboard/src/components/ui/dropdown-menu.tsx @@ -0,0 +1,254 @@ +import * as React from "react"; +import { cn } from "@/lib/utils"; +import { CheckIcon, ChevronRightIcon, CircleIcon } from "lucide-react"; +import { DropdownMenu as DropdownMenuPrimitive } from "radix-ui"; + +function DropdownMenu({ + ...props +}: React.ComponentProps) { + return ; +} + +function DropdownMenuPortal({ + ...props +}: React.ComponentProps) { + return ( + + ); +} + +function DropdownMenuTrigger({ + ...props +}: React.ComponentProps) { + return ( + + ); +} + +function DropdownMenuContent({ + className, + sideOffset = 4, + ...props +}: React.ComponentProps) { + return ( + + + + ); +} + +function DropdownMenuGroup({ + ...props +}: React.ComponentProps) { + return ( + + ); +} + +function DropdownMenuItem({ + className, + inset, + variant = "default", + ...props +}: React.ComponentProps & { + inset?: boolean; + variant?: "default" | "destructive"; +}) { + return ( + + ); +} + +function DropdownMenuCheckboxItem({ + className, + children, + checked, + ...props +}: React.ComponentProps) { + return ( + + + + + + + {children} + + ); +} + +function DropdownMenuRadioGroup({ + ...props +}: React.ComponentProps) { + return ( + + ); +} + +function DropdownMenuRadioItem({ + className, + children, + ...props +}: React.ComponentProps) { + return ( + + + + + + + {children} + + ); +} + +function DropdownMenuLabel({ + className, + inset, + ...props +}: React.ComponentProps & { + inset?: boolean; +}) { + return ( + + ); +} + +function DropdownMenuSeparator({ + className, + ...props +}: React.ComponentProps) { + return ( + + ); +} + +function DropdownMenuShortcut({ + className, + ...props +}: React.ComponentProps<"span">) { + return ( + + ); +} + +function DropdownMenuSub({ + ...props +}: React.ComponentProps) { + return ; +} + +function DropdownMenuSubTrigger({ + className, + inset, + children, + ...props +}: React.ComponentProps & { + inset?: boolean; +}) { + return ( + + {children} + + + ); +} + +function DropdownMenuSubContent({ + className, + ...props +}: React.ComponentProps) { + return ( + + ); +} + +export { + DropdownMenu, + DropdownMenuPortal, + DropdownMenuTrigger, + DropdownMenuContent, + DropdownMenuGroup, + DropdownMenuLabel, + DropdownMenuItem, + DropdownMenuCheckboxItem, + DropdownMenuRadioGroup, + DropdownMenuRadioItem, + DropdownMenuSeparator, + DropdownMenuShortcut, + DropdownMenuSub, + DropdownMenuSubTrigger, + DropdownMenuSubContent, +}; diff --git a/shared/dashboard/src/components/ui/empty.tsx b/shared/dashboard/src/components/ui/empty.tsx new file mode 100644 index 0000000..0cf293e --- /dev/null +++ b/shared/dashboard/src/components/ui/empty.tsx @@ -0,0 +1,103 @@ +import { cva, type VariantProps } from "class-variance-authority"; +import { cn } from "@/lib/utils"; + +function Empty({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +function EmptyHeader({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +const emptyMediaVariants = cva( + "mb-2 flex shrink-0 items-center justify-center [&_svg]:pointer-events-none [&_svg]:shrink-0", + { + variants: { + variant: { + default: "bg-transparent", + icon: "flex size-10 shrink-0 items-center justify-center rounded-lg bg-muted text-foreground [&_svg:not([class*='size-'])]:size-6", + }, + }, + defaultVariants: { + variant: "default", + }, + }, +); + +function EmptyMedia({ + className, + variant = "default", + ...props +}: React.ComponentProps<"div"> & VariantProps) { + return ( +
+ ); +} + +function EmptyTitle({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +function EmptyDescription({ className, ...props }: React.ComponentProps<"p">) { + return ( +
a]:underline [&>a]:underline-offset-4 [&>a:hover]:text-primary", + className, + )} + {...props} + /> + ); +} + +function EmptyContent({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +export { + Empty, + EmptyHeader, + EmptyTitle, + EmptyDescription, + EmptyContent, + EmptyMedia, +}; diff --git a/shared/dashboard/src/components/ui/field.tsx b/shared/dashboard/src/components/ui/field.tsx new file mode 100644 index 0000000..51c608c --- /dev/null +++ b/shared/dashboard/src/components/ui/field.tsx @@ -0,0 +1,246 @@ +import { useMemo } from "react"; +import { cva, type VariantProps } from "class-variance-authority"; +import { cn } from "@/lib/utils"; + +import { Label } from "@/components/ui/label"; +import { Separator } from "@/components/ui/separator"; + +function FieldSet({ className, ...props }: React.ComponentProps<"fieldset">) { + return ( +
[data-slot=checkbox-group]]:gap-3 has-[>[data-slot=radio-group]]:gap-3", + className, + )} + {...props} + /> + ); +} + +function FieldLegend({ + className, + variant = "legend", + ...props +}: React.ComponentProps<"legend"> & { variant?: "legend" | "label" }) { + return ( + + ); +} + +function FieldGroup({ className, ...props }: React.ComponentProps<"div">) { + return ( +
[data-slot=field-group]]:gap-4", + className, + )} + {...props} + /> + ); +} + +const fieldVariants = cva( + "group/field flex w-full gap-3 data-[invalid=true]:text-destructive", + { + variants: { + orientation: { + vertical: ["flex-col [&>*]:w-full [&>.sr-only]:w-auto"], + horizontal: [ + "flex-row items-center", + "[&>[data-slot=field-label]]:flex-auto", + "has-[>[data-slot=field-content]]:items-start has-[>[data-slot=field-content]]:[&>[role=checkbox],[role=radio]]:mt-px", + ], + responsive: [ + "flex-col @md/field-group:flex-row @md/field-group:items-center [&>*]:w-full @md/field-group:[&>*]:w-auto [&>.sr-only]:w-auto", + "@md/field-group:[&>[data-slot=field-label]]:flex-auto", + "@md/field-group:has-[>[data-slot=field-content]]:items-start @md/field-group:has-[>[data-slot=field-content]]:[&>[role=checkbox],[role=radio]]:mt-px", + ], + }, + }, + defaultVariants: { + orientation: "vertical", + }, + }, +); + +function Field({ + className, + orientation = "vertical", + ...props +}: React.ComponentProps<"div"> & VariantProps) { + return ( +
+ ); +} + +function FieldContent({ className, ...props }: React.ComponentProps<"div">) { + return ( +
+ ); +} + +function FieldLabel({ + className, + ...props +}: React.ComponentProps) { + return ( +