diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index b80db9a..74d1b59 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -2,7 +2,7 @@ name: Platform checks on: pull_request: push: - branches: [main] + branches: [main, dev] workflow_dispatch: permissions: contents: read @@ -30,6 +30,18 @@ jobs: - run: npm run test:artifact - run: npx playwright install --with-deps chromium - run: npm run test:ui + - run: python3 -m unittest discover -s tests -p '*_test.py' + - name: Package verified Dev build + if: github.event_name == 'push' && github.ref == 'refs/heads/dev' + run: tar -czf dispatch-dev.tar.gz -C .build . + - name: Upload verified Dev build + if: github.event_name == 'push' && github.ref == 'refs/heads/dev' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: dispatch-dev-${{ github.sha }} + path: dispatch-dev.tar.gz + if-no-files-found: error + retention-days: 14 - name: Remove temporary verification data if: always() run: node tooling/clean-test-output.mjs diff --git a/.gitignore b/.gitignore index c008405..1b94d59 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ test-results/ playwright-report/ *.tsbuildinfo .DS_Store +__pycache__/ +*.pyc +dispatch-dev.tar.gz diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index e2bb8a4..05ee923 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -1,67 +1,79 @@ # Development -## Start and stop - -`npm run dev` seeds three synthetic DSPs, starts the API on loopback 5180, and -starts Vite on loopback 5173. The permanent Dev DSP uses Preview’s separate job -database and browser manager. In this convenient local mode, both environments -run the current source. Testing two different artifacts uses the separate -Preview process and gateway, exercised by the operations and artifact tests. - -No npm command installs a system service, changes a proxy, opens a public port, -reads archived credentials, or changes the future live directories. -`npm run build` writes only `.build/` inside this repository. It bundles API and -worker code, installs locked production dependencies into the artifact, and -writes a complete SHA-256 inventory. It does not activate the artifact. - -Use Ctrl+C to stop `npm run dev`. Remove its explicitly selected fixture state -directory only after the process exits. The browser verification scripts remove -their own temporary state automatically; `tooling/clean-test-output.mjs` removes -the known screenshots and reports. Do not remove unrelated `/tmp` entries. - -## Workflows to exercise - -1. Sign in as the platform owner. Search DSPs and open Northline Logistics. -2. Browse employees, search by name/code, open an employee, and inspect timecards. -3. Open Connections, save synthetic Paycom credentials, and collect data. -4. Use password `require-verification` to exercise the owner verification flow; - fixture code is `123456`. `invalid-password` exercises a failed connection. -5. Enable a daily schedule in the DSP’s timezone. Check Jobs for completion. -6. Create a DSP, generate an owner invitation, accept it, and test owner/manager/ - member boundaries. Development mail is written privately to - `local/platform/development-mail` inside the selected fixture state root. -7. Suspend and resume a DSP. The permanent Dev DSP cannot be suspended. -8. Import a build into a disposable state root with the CLI and inspect Releases. - Deployment controls remain disabled unless an operator explicitly enables them. - -Browser assistance is available for real native fixture/provider sessions that -need verification. The fast in-memory fixtures support code verification and do -not fabricate browser screenshots. - -## Checks - -`npm test` covers account/role/CSRF boundaries, DSP view tampering, provisioning, -invitations, reset revocation, encrypted credential binding, schedules, durable -jobs, failed-publication preservation, artifact integrity, private-state -preservation, backup checksums, and separate Preview routing. Native and compiled -supervisor tests are opt-in commands because they require a built artifact. - -`npm run test:ui` checks the built API and dashboard together: owner login, DSP -search, employee detail, punches, credentials, verification, collection results, -restricted member navigation, mobile layout, and JavaScript errors. - -CI runs typechecking, service tests, dependency audit, the build, the compiled -supervisor simulation, and browser checks. It has read-only repository permission. -There is no release-publishing or deployment workflow in this rebuild. - -## Changes and data compatibility - -Update shared contracts and both producer/consumer paths together. Validate -provider data before publication and preserve the last successful dataset on -failure. Never accept a DSP filesystem path from a request. Provider workers -must not receive the platform state root or vault path. - -Shared account schemas are controlled by the production process. Preview refuses -to migrate that shared database. Shared authentication, routing, or account-schema -changes require an isolated full-platform staging run before promotion. A Dev -DSP alone cannot validate a replacement for the gateway currently routing to it. +## Shared Dev platform + +The persistent repository is `/home/thepickle/dispatch-platform/dev/live`, tracking +`dev`. It runs the compiled `.build/` artifact. Configuration is in sibling +`config/`, platform state in `data/`, and private DSP state in `dsps/`. This is an +independent platform with its own login, owner dashboard, Dev DSP and test DSPs. +Nothing depends on a Production account registry or gateway. + +Use isolated feature worktrees from `dev`; target PRs at `dev`. Merge only when +the owner explicitly requests it. After a verified merge, remove the clean feature +worktree and local/remote feature branch, preserving any unmerged work. Never +delete `dev`, `main`, or the persistent environment checkout. + +Successful **push checks on dev** upload a compiled artifact identified by the +commit SHA. A user-systemd timer checks every minute. It installs only the artifact +for the current merged `dev` head, after verifying the GitHub download digest, +runtime inventory and source commit. PR artifacts and failed/pending checks cannot +update the environment. Unfinished edits in the running checkout block updates. + +The updater stops Dev, swaps `.build/`, fast-forwards the source checkout, starts +Dev and verifies its health/digest. A failed start restores the previous code and +checkout. Accounts, configuration, credentials, DSP data and browser profiles stay +in place. An interrupted activation is recovered on the next updater run. Changes +to database schemas must preserve compatibility with the previous build; code +rollback does not reverse data migrations. + +First setup and operating commands are in [Dev setup](docs/DEV-SETUP.md). + +## Feature development and fixtures + +Inside a feature worktree: + +```bash +npm ci --ignore-scripts +npm run dev +``` + +This optional local runner starts Vite at `http://127.0.0.1:5173` and a fixture API +on 5180. Use a free API port via `PORT` and adjust the Vite proxy when the hosted +Dev service occupies 5180. It seeds synthetic accounts/DSPs in a temporary state +root; never point the fixture runner at the persistent Dev state. Frontend edits +hot reload; restart this local API runner after backend changes. Feature work does +not change the shared Dev environment before merge. + +`npm run build` writes only `.build/`. It bundles the API and browser workers, +installs locked runtime dependencies, records the source commit, and produces the +SHA-256 `release.json` inventory. Building alone does not activate it. + +Stop temporary servers before cleanup. Verification scripts remove their own +temporary state; `node tooling/clean-test-output.mjs` removes known reports and +screenshots. Remove only your own `/tmp` artifacts. + +## Verification + +```bash +npm run check +npm run format:check +npm test +python3 -m unittest discover -s tests -p '*_test.py' +npm run build +npm run test:artifact +npm run test:ui +npm run test:native +``` + +Checks cover login/role/CSRF boundaries, independent Dev accounts, provisioning, +encrypted credentials, collection jobs, schedules, backups, archive verification, +dirty-checkout protection, activation and rollback. Browser checks exercise the +built dashboard/API together. Native verification uses local fixture pages and +isolated Chromium profiles; real provider acceptance requires user-supplied DSP +credentials. Legacy gateway/supervisor tests remain for compatibility and do not +install or start Production. + +Update contracts and their producers/consumers together. Validate provider results +before publication and preserve the last successful dataset on failure. Never +accept a filesystem path from a DSP request. Collection workers receive no platform +state root, vault path or other DSP's profile. diff --git a/README.md b/README.md index 5f11714..0e59c74 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ A shared platform for DSP operations. One login and dashboard serve every DSP. Application code and browser workers are installed centrally; each DSP owns only private configuration, credentials, browser state, and databases. -**Repository rebuild only. Nothing deploys on install, build, push, or merge.** +**Independent Dev environment.** After explicit host setup, successful merged +`dev` builds automatically update the full test platform. Production setup is deferred. There is no Plugins page. Paycom is configured on a DSP’s **Connections** page. ## Develop @@ -50,16 +51,18 @@ tests/ Service, security, browser, Preview and artifact integration docs/ Architecture, implementation record and design reference ``` -The repository is `/home/thepickle/dispatch-platform/dev` directly. The future -working platform will use `/home/thepickle/dispatch-platform` directly, with -`preview/`, `dsps/`, `local/`, and retained `archive/` alongside the centrally -installed code. There is no `live/` directory and no nested repository directory. +The persistent repository is `/home/thepickle/dispatch-platform/dev/live`, tracking +`dev`. Its sibling `config/`, `data/` and `dsps/` directories contain private Dev +state. Feature work uses separate worktrees. The future Production layout is +`public/live` with its own sibling state directories. `archive/` is retained. +See [Dev setup](docs/DEV-SETUP.md) for owner bootstrap, services and access. ## Verify ```bash npm run check npm test +python3 -m unittest discover -s tests -p '*_test.py' npm run build npm run test:artifact npm run test:ui @@ -74,11 +77,11 @@ an already-running development server. `test:artifact` exercises two temporary API processes on ports 5200/5201 and a supervisor entirely under `/tmp`. Native tests use local fixture pages, real Chromium, separate Linux namespaces, -separate profiles, and the private CDP bridge. The fixture browser disables its -inner Chromium sandbox because this host’s AppArmor policy prohibits nested user -namespaces; the outer filesystem, process, and network isolation remains enabled. -**Production never uses that exception.** Production Chromium sandbox acceptance -and real Paycom acceptance require the later authorized host/connection setup. +profiles and the private CDP bridge. Host verification with +`npm run test:browser-host` additionally checks Chromium's internal namespace and +seccomp sandboxes without using provider credentials. See the Dev setup guide for +this host's scoped AppArmor configuration. Real Paycom acceptance requires the +owner to configure a Dev DSP connection. See [DEVELOPMENT.md](DEVELOPMENT.md), [architecture](docs/ARCHITECTURE.md), [security](docs/SECURITY.md), and [RELEASES.md](RELEASES.md). diff --git a/RELEASES.md b/RELEASES.md index 6c59da6..2dd04ea 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,89 +1,62 @@ -# Builds, release review and future operation +# Releases and operation -**This rebuild does not authorize live setup.** The commands below document the -operator tooling for a later approved setup. Builds, imports and plans alone do -not start services. There is no publishing or installation on Git merge. +## Dev updates -## Build and inspect +Feature PRs target `dev`. The owner explicitly requests each feature merge. +Successful merged-dev checks upload a compiled GitHub Actions artifact. Because +the repository is public, artifacts contain **code only**, never state or +credentials. The configured Dev updater downloads and verifies that artifact and +updates the full test platform automatically. See [DEVELOPMENT.md](DEVELOPMENT.md). -```bash -npm run build -npm run dispatch -- release-import /absolute/path/to/artifact "Release notes" -npm run dispatch -- release-plan DIGEST preview -``` - -Set `DISPATCH_STATE_ROOT` to a disposable development root while reviewing these -commands. `.build/release.json` contains the version, compatibility schema, Node -major, every runtime file hash/size and the aggregate immutable digest. Verification -rejects missing, changed, extra, duplicate, traversing, symlink or hard-linked files. -An import retains its own verified copy so editing the build directory cannot -change a previously imported release. - -## Later first setup - -The future state root is `/home/thepickle/dispatch-platform`; built code is placed -directly there. It must have private permissions. Configure Node, a compatible -Chromium/bubblewrap host, canonical HTTPS origin and SMTP separately. +## Prepare a release -1. Set `DISPATCH_STATE_ROOT`, `NODE_ENV=production`, `DISPATCH_PROVIDER_MODE=native` - and `DISPATCH_ORIGIN=https://your-dashboard-host`. -2. Bootstrap the owner with `dispatch bootstrap EMAIL NAME`, supplying the password - on stdin, not a command-line argument. This creates the permanent Dev DSP. -3. After explicit setup approval, set `DISPATCH_ENABLE_DEPLOYMENT=1` and use - `dispatch initialize ARTIFACT OWNER_EMAIL`. It installs an initial baseline - into both runtime locations while the fleet is empty. It starts nothing. -4. Run the built `tooling/supervisor.js` under the approved process manager. The - supervisor starts the two API processes, gives the gateway a private Preview - signing key, and processes approved release requests. -5. Verify Dev authentication, workforce, native browser isolation and provider - collection before adding production DSPs. This host’s inner Chromium sandbox - compatibility remains an explicit acceptance item. +When the owner says **Prepare a release**: -No systemd unit, proxy configuration, SMTP credentials, DNS record or live state -is installed by repository scripts. The first-install command refuses an existing -deployment or a fleet containing production DSPs. +1. Summarize changes since the last published release and show known running versions. +2. Ask for the version if it was not already supplied. +3. Pin the accepted `dev` revision, apply versioning, and verify the final compiled + candidate in Dev. Unmerged PRs and unfinished edits stay out. Application changes + require revalidation. +4. Merge the tested source into `main` through a release PR. +5. Publish one immutable `vX.Y.Z` release containing the verified runtime artifact + and its `release.json` inventory/digest. Verify assets and source provenance. -## Subsequent release flow +The agreed future Production workflow automatically installs a published stable +release into `public/live/`, preserving sibling `config/`, `data/` and `dsps/`. +Drafts, prereleases and main merges do not trigger Production activation. The +release request includes the resulting automatic update, without a separate +Promote action. -1. Build once and finish repository checks. Import the verified artifact. -2. Click **Update Dev** on Releases. Only Preview restarts with the candidate. -3. Test the permanent Dev DSP, then click **Mark tested** on the current candidate. -4. Click **Promote**. Only that exact tested digest can become Production. +**Production provisioning, release publishing automation and Production deployment +are not installed by this Dev setup.** Implement and verify them when requested. +The older gateway/Update Dev/Promote operator commands remain for compatibility +tests; standalone Dev disables those activation controls. -The supervisor stops the target API, allows graceful worker shutdown, replaces -only its managed code directories, starts the selected artifact and checks its -reported digest. A failed health check restores the previous code and restarts it. -Production activation affects all production DSPs and causes a short maintenance -window. DSP data, credentials, profiles, central state, `dev/` and `archive/` are -outside the replacement list. An update never copies credentials from Dev. +## Build integrity and rollback -The managed list is `dashboard`, `api`, `services`, `integrations`, `shared`, -`tooling`, `node_modules`, `package.json`, `package-lock.json` and `release.json`. -Only entries present in an artifact are installed. Source modules may be bundled -into the API or worker entrypoints rather than copied as separate runtime trees. +`.build/release.json` records the version, Node major, compatibility schema, every +runtime file hash/size and aggregate digest. `tooling/build-info.json` records the +source commit. Symlinks, hardlinks, unexpected files and unsafe paths are rejected. +The Dev updater additionally verifies the GitHub artifact archive digest and that +its workflow succeeded for a push to the current `dev` head. -## Interrupted activation - -Activation writes its intent receipt before the first directory rename. If the -supervisor crashes during an activation, it refuses to restart an ambiguous -running request automatically. With supervisor and both APIs stopped, inspect -`local/platform/activation-backups/*/receipt.json` and the pending request. Recover -with `dispatch release-recover RECEIPT_DIRECTORY REQUEST_ID` and the explicit -deployment switch. It checks the request/receipt match, restores code from the -recorded moves, and marks that request failed. Then restart the supervisor. - -Code rollback does not undo data migrations. This baseline uses schema version 1; -future schema changes must retain rollback compatibility or provide an explicit -offline migration/recovery plan before promotion. +Updates serialize through a lock and write an activation receipt before replacing +code. Failed health checks restore the prior artifact and source revision. Keep +schema changes compatible with that artifact; code rollback does not undo data +migrations. Recovery refuses to overwrite unrelated edits to the checkout. ## Backup and restore +With the Dev service and updater timer stopped, load `config/platform.env` into the +operator process environment and use the built CLI: + ```text -dispatch backup /absolute/private/backup-destination -dispatch restore /absolute/private/backup /absolute/empty/restore-target +node live/.build/tooling/cli.js backup /absolute/private/backup-destination +node live/.build/tooling/cli.js restore /absolute/private/backup /absolute/empty/restore-target ``` -Stop services before backup. The command takes API locks so they cannot start -during the snapshot. Restore verifies checksums, clears old authentication tokens, -cancels pending jobs and clears release paths. Reimport an artifact and complete -the later host setup before restarting a restored platform. +Standalone backups include `data/` and `dsps/`. Keep a separate private backup of +`config/`; environment configuration is not included in the state archive. Restore +validates checksums, revokes old sessions/invitations/reset links, clears stale +release state and cancels pending jobs. Configure and verify a compatible artifact +before restarting a restored platform. diff --git a/api/app.ts b/api/app.ts index b57187d..4b1ddd0 100644 --- a/api/app.ts +++ b/api/app.ts @@ -38,7 +38,10 @@ export async function createApp( // The local fixture runner exercises separate queues and sessions for both // environments. Deployed Preview uses a separate API process and release. const preview = - options.fixturePreview && config.development && config.providerMode === 'fixture' + !config.standalone && + options.fixturePreview && + config.development && + config.providerMode === 'fixture' ? new Runtime({ ...config, environment: 'preview' }) : undefined; const forContext = (context: Context) => { @@ -146,6 +149,8 @@ export async function createApp( environment: config.environment, release: config.release, separatePreview: Boolean(config.previewOrigin), + standalone: config.standalone, + providerMode: config.providerMode, }; }); app.post('/api/session/dsp', (request) => { @@ -189,10 +194,10 @@ export async function createApp( }); app.post('/api/invitations/:token/accept', async (request) => { runtime.accounts.throttle(`invite:${request.ip}`, 20, 3600_000); - const input = parse(z.object({ name, password }).strict(), request); + const input = parse(z.object({ firstName: name, lastName: name, password }).strict(), request); await runtime.accounts.acceptInvitation( z.string().length(43).parse(params(request).token), - input.name, + { firstName: input.firstName, lastName: input.lastName }, input.password, ); return { ok: true }; @@ -260,10 +265,24 @@ export async function createApp( }); app.get('/api/platform/releases', (request) => { owner(request); - return { releases: releases.list(), deploymentEnabled: config.allowDeployment }; + const statusFile = path.join(runtime.storage.paths.platform, 'dev-update.json'); + let update: unknown = null; + if (config.standalone && fs.existsSync(statusFile)) { + const status = JSON.parse(fs.readFileSync(statusFile, 'utf8')); + update = { status: status.status, commit: status.commit, updatedAt: status.updatedAt }; + } + return { + releases: releases.list(), + deploymentEnabled: config.allowDeployment, + standalone: config.standalone, + environment: config.environment, + release: config.release, + update, + }; }); app.post('/api/platform/releases/:digest/tested', (request) => { const a = owner(request, true); + assert(!config.standalone, 'github_manages_updates', 409); releases.markTested(params(request).digest!, a.user.id); return { ok: true }; }); diff --git a/api/main.ts b/api/main.ts index 9296304..f2980bd 100644 --- a/api/main.ts +++ b/api/main.ts @@ -4,7 +4,7 @@ import { fileURLToPath } from 'node:url'; import { createApp } from './app.js'; import { configuration } from '../services/config.js'; import { acquireLock } from '../services/storage/lock.js'; -import { privateDirectory } from '../services/storage/paths.js'; +import { Paths } from '../services/storage/paths.js'; const here = path.dirname(fileURLToPath(import.meta.url)); const releaseFile = path.join(here, '../release.json'); const config = configuration({ @@ -19,13 +19,12 @@ const unlock = () => { }; let closing = false; try { - locks.push( - acquireLock(privateDirectory(path.join(config.stateRoot, 'local', config.environment)), 'api'), - ); + const paths = new Paths(config.stateRoot, config.standalone); + if (config.standalone && !fs.existsSync(path.join(paths.platform, 'accounts.sqlite'))) + throw new Error('Platform accounts are missing. Run explicit bootstrap before starting.'); + locks.push(acquireLock(paths.environment(config.environment), 'api')); if (config.environment === 'production' && process.env.DISPATCH_FIXTURE_PREVIEW === '1') - locks.push( - acquireLock(privateDirectory(path.join(config.stateRoot, 'local', 'preview')), 'api'), - ); + locks.push(acquireLock(paths.environment('preview'), 'api')); const { app } = await createApp(config, { dashboardRoot: path.resolve(here, '../dashboard'), startWorkers: true, diff --git a/api/preview.ts b/api/preview.ts index d49429f..97e879a 100644 --- a/api/preview.ts +++ b/api/preview.ts @@ -14,6 +14,7 @@ function proof( } export function previewRouting(app: FastifyInstance, runtime: Runtime) { const config = runtime.config; + if (config.standalone) return; app.addHook('preHandler', async (request, reply) => { if (request.url === '/api/health') return; if (config.environment === 'preview' && config.previewKey) { diff --git a/dashboard/src/auth.tsx b/dashboard/src/auth.tsx index d79132b..36791f0 100644 --- a/dashboard/src/auth.tsx +++ b/dashboard/src/auth.tsx @@ -41,7 +41,11 @@ export function AuthScreen({ onLogin }: { onLogin: () => Promise }) { setNotice('Password updated. Sign in with your new password.'); } if (mode === 'invite') { - await api(`/api/invitations/${token}/accept`, { name: String(form.get('name')), password }); + await api(`/api/invitations/${token}/accept`, { + firstName: String(form.get('firstName')), + lastName: String(form.get('lastName')), + password, + }); window.location.hash = 'signin'; setMode('login'); setNotice('Invitation accepted. Sign in with your invited email address.'); @@ -107,10 +111,16 @@ export function AuthScreen({ onLogin }: { onLogin: () => Promise }) { )} {mode === 'invite' && ( - + <> + + + )} {mode !== 'forgot' && (