Fix Paycom credential saves under DSP capacity contention

core/accounts/src/onboarding-store.js

@@ -57,6 +57,10 @@ function createOnboardingStore(store, clock = Date.now) {
     if (db.prepare("UPDATE installation_onboarding_requests SET status=?,failure_code=?,lease_expires_at=NULL,updated_at=? WHERE id=? AND status='running' AND worker_id=? AND fence=? AND lease_expires_at>?")
       .run(code ? 'failed' : 'succeeded', code, clock(), row.id, row.worker_id, row.fence, clock()).changes !== 1) fail();
   }
-  return { get, latest, prior, begin, enrolled, enrollmentFailed, candidates, claim, renew, finish, requeue };
+  function defer(row) {
+    if (db.prepare("UPDATE installation_onboarding_requests SET status='queued',attempt=attempt-1,worker_id=NULL,lease_expires_at=NULL,updated_at=? WHERE id=? AND status='running' AND worker_id=? AND fence=? AND lease_expires_at>?")
+      .run(clock(), row.id, row.worker_id, row.fence, clock()).changes !== 1) fail();
+  }
+  return { get, latest, prior, begin, enrolled, enrollmentFailed, candidates, claim, renew, finish, requeue, defer };
 }
 module.exports = { createOnboardingStore, LEASE_MS };

core/accounts/src/owner-paycom-setup.js

@@ -7,7 +7,8 @@ const { managedInstallationContext } = require('./installation-authority');
 const { createAccessInstallationActivationAuthority } = require('./installation-activation');
 const { createOnboardingStore } = require('./onboarding-store');
 function fail(code, status = 409) { throw new AccessError(code, status); }
-function createOwnerPaycomSetup({ store, access, invoke, clock = Date.now }) {
+function createOwnerPaycomSetup({ store, access, invoke, clock = Date.now,
+  enroll = (key, input) => invoke(key, 'paycom.setup', input) }) {
   const requests = createOnboardingStore(store, clock);
   function context(session) {
     require('./plugins').requirePlugin(access, session, 'paycom');
@@ -89,7 +90,7 @@ function createOwnerPaycomSetup({ store, access, invoke, clock = Date.now }) {
           guard();
           row = requests.begin(selected.organization.id, session.user.id, input.idempotencyKey, input.intent, selected.manifest.revision);
         });
-        const result = await invoke(selected.manifest.runtime.key, 'paycom.setup', {
+        const result = await enroll(selected.manifest.runtime.key, {
           command: 'enroll', requestId: row.id, expiresAt: clock() + 30_000, credentials, intent: input.intent,
         });
         if (!result?.ok || result.status !== 'succeeded' || result.data?.configured !== true) fail(setupFailure(result?.status));
@@ -111,7 +112,7 @@ function createOwnerPaycomSetup({ store, access, invoke, clock = Date.now }) {
     try {
       row = requests.begin(selected.organization.id, session.user.id, input.idempotencyKey, input.intent, selected.manifest.revision);
       authority.guard(() => true);
-      const result = await invoke(selected.manifest.runtime.key, 'paycom.setup', {
+      const result = await enroll(selected.manifest.runtime.key, {
         command: 'enroll', requestId: row.id, expiresAt: clock() + 30_000, credentials, intent: input.intent,
       });
       authority.guard(() => true);

core/api/directory-platform.js

@@ -80,7 +80,9 @@ async function startDirectoryApi({ paths, installation, host, port = 4310, addre
       ? require('../../host/plugins/directory-lifecycle').createDirectoryInstallation({ paths, manager: runtime.manager, execution, store }) : null;
     const plugins = require('../accounts/src/plugins').createPluginService({ store, access, invoke, installationCoordinator,
       settingsPort: runtime.manager.pluginBackend ? (id,pluginId,request) => runtime.manager.pluginBackend.request(id,'plugin.settings',{pluginId,request}) : null });
-    const paycomSetup = createOwnerPaycomSetup({ store, access, invoke });
+    const paycomSetup = createOwnerPaycomSetup({ store, access, invoke,
+      ...(runtime.manager.pluginBackend ? { enroll: require('../../host/controller/paycom-enrollment')
+        .createPaycomEnrollment({ backend: runtime.manager.pluginBackend }) } : {}) });
     const connections = createOwnerConnections({ store, access, invoke, paycomSetup });
     const onboarding = createOwnerOnboardingWorker({ store, invoke, backends: [BACKEND] });
     const backups = new ManualBackups({ paths, store, access });

core/auth-broker/README.md

@@ -19,6 +19,12 @@ pin the DSP's acknowledged package digest and revision; declaring a service in a
 manifest does not grant it automatically. Core may relay owner credentials in
 memory to that DSP's worker, but does not persist them or log them.
 
+When another DSP is queued, idle authentication workers yield their slots even
+if status polling keeps them warm. Active requests, provider verification and
+plugin browser leases retain their workers. Directory DSPs enroll Paycom through
+the vault worker without waking the full collection runtime; subsequent setup
+waits in its existing onboarding queue when runtime capacity is occupied.
+
 The existing provider session, attempt-guard, native Chrome and assistance code
 is reused. Paycom's adapter comes from the DSP's installed package. Cortex stays
 a built-in automatic sign-in and owner-entered email-code connection. A plugin

core/auth-broker/coordinator.js

@@ -154,14 +154,22 @@ class AuthenticationCoordinator {
   poll() {
     if (this.polling) return this.polling;
     this.polling = (async () => {
-      for (const [dspId, entry] of this.dsps) {
-        if (!entry.row || entry.requests || entry.closing) continue;
+      for (const [dspId, entry] of [...this.dsps]) {
+        if (!entry.row || entry.requests || entry.waiters || entry.closing) continue;
         try {
           const response = await this.workers.request(entry.row, { action: 'activity' });
           if (!response.ok) throw new Error();
-          if (entry.requests) continue;
-          if (!response.busy && this.clock() - entry.lastUsed >= this.idleMs) await this.closeEntry(dspId, entry);
-          else await this.manager.renew(entry.context, entry.lease.leaseId);
+          if (entry.requests || entry.waiters) continue;
+          const leased = [...this.sessions.values()].some(session => session.entry === entry);
+          // Status polling can keep an otherwise idle worker warm forever.
+          // Give queued DSPs its slot after the worker confirms it is idle;
+          // in-progress sign-ins, verification and plugin leases stay intact.
+          const waiting = this.manager.status().queued > 0;
+          if (!response.busy && !leased && (waiting || this.clock() - entry.lastUsed >= this.idleMs)) {
+            await this.closeEntry(dspId, entry);
+            // Admit the queued DSP before considering another warm worker.
+            await this.manager.pump();
+          } else await this.manager.renew(entry.context, entry.lease.leaseId);
         } catch { await this.closeEntry(dspId, entry); }
       }
     })().finally(() => { this.polling = null; });

core/auth-broker/tests/coordinator.test.js

@@ -69,3 +69,78 @@ test('one admitted auth worker serves a DSP; SDK leases remain bound to the orig
   assert.equal(stopped.length, 2); assert.equal(manager.status().sessions, 0);
 });
 
+for (const signingIn of [false, true]) test(`a third DSP can save while status polling keeps idle authentication workers warm (sign-in: ${signingIn})`, async t => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-fairness-'));
+  const store = new BrowserStore(path.join(root, 'state/browser.sqlite3'));
+  const ids = ['a', 'b', 'c'].map(letter => 'dsp_' + letter.repeat(32));
+  const busy = new Set(signingIn ? [ids[0]] : []), stopped = [], saved = [];
+  const workers = {
+    start: async row => ({ protocol: 'worker', endpoint: 'worker://' + row.id, access: row.id }),
+    close: async row => { stopped.push(row.dsp_id); return true; },
+    request: async (row, request) => {
+      if (request.action === 'activity') return { ok: true, busy: busy.has(row.dsp_id) };
+      if (request.action === 'connections') return { ok: true, items: [] };
+      assert.equal(request.action, 'enroll-paycom');
+      saved.push(row.dsp_id); return { ok: true, status: 'configured' };
+    },
+  };
+  const manager = new BrowserManager({ store, workers, authorize: () => true, limits: { sessions: 2, tabs: 12 } });
+  await manager.start();
+  const coordinator = new AuthenticationCoordinator({ manager, workers, idleMs: 60000,
+    contextFor: dspId => ({ dspId, pluginId: 'core-auth', installationRevision: 1, jobId: 'auth' }),
+    authorizeRequest: () => true, authorizePlugin: () => true, relay: () => { throw new Error('no browser needed'); },
+  });
+  t.after(async () => { await coordinator.close(); await manager.close(); store.close(); fs.rmSync(root, { recursive: true, force: true }); });
+  for (const id of ids.slice(0, 2)) await coordinator.request(id, { action: 'connections', input: { command: 'list' } });
+  const pending = coordinator.request(ids[2], { action: 'enroll-paycom', intent: 'create', credentials: { password: 'synthetic-fair-save' } },
+    { signal: AbortSignal.timeout(3000) });
+  // Attach immediately so a regression's timeout is an ordinary test failure.
+  const outcome = pending.then(value => ({ value }), error => ({ error }));
+  await new Promise(resolve => setImmediate(resolve));
+  assert.equal(manager.status().queued, 1);
+  await coordinator.poll();
+  assert.deepEqual(stopped, [ids[signingIn ? 1 : 0]], 'yield only the one idle worker needed by the queue');
+  await manager.pump();
+  const result = await outcome;
+  assert.equal(result.error, undefined);
+  assert.equal(result.value.status, 'configured');
+  assert.deepEqual(saved, [ids[2]]);
+  assert.equal(manager.status().sessions, 2);
+  assert.equal(fs.readFileSync(path.join(root, 'state/browser.sqlite3')).includes('synthetic-fair-save'), false);
+});
+
+test('queued DSPs do not evict an in-flight request or an outstanding plugin browser lease', async t => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-auth-contention-'));
+  const store = new BrowserStore(path.join(root, 'state/browser.sqlite3'));
+  const ids = ['a', 'b'].map(letter => 'dsp_' + letter.repeat(32)), stopped = [];
+  let unblock, hold = false;
+  const workers = {
+    start: async row => ({ protocol: 'worker', endpoint: 'worker://' + row.id, access: row.id }),
+    close: async row => { stopped.push(row.dsp_id); return true; },
+    request: async (_row, request) => {
+      if (request.action === 'activity') return { ok: true, busy: false };
+      if (hold) await new Promise(resolve => { unblock = resolve; });
+      return { ok: true, items: [] };
+    },
+  };
+  const manager = new BrowserManager({ store, workers, authorize: () => true, limits: { sessions: 1, tabs: 6 } });
+  await manager.start();
+  const coordinator = new AuthenticationCoordinator({ manager, workers, idleMs: 60000,
+    contextFor: dspId => ({ dspId, pluginId: 'core-auth', installationRevision: 1, jobId: 'auth' }),
+    authorizeRequest: () => true, authorizePlugin: () => true, relay: () => {},
+  });
+  t.after(async () => { unblock?.(); coordinator.sessions.clear(); await coordinator.close(); await manager.close(); store.close(); fs.rmSync(root, { recursive: true, force: true }); });
+  const list = id => coordinator.request(id, { action: 'connections', input: { command: 'list' } });
+  await list(ids[0]); hold = true;
+  const reading = list(ids[0]);
+  await new Promise(resolve => setImmediate(resolve));
+  const cancel = new AbortController();
+  const queued = coordinator.request(ids[1], { action: 'connections', input: { command: 'list' } }, { signal: cancel.signal });
+  const cancelled = assert.rejects(queued, { code: 'cancelled' });
+  await new Promise(resolve => setImmediate(resolve));
+  await coordinator.poll(); assert.deepEqual(stopped, []);
+  hold = false; unblock(); await reading;
+  coordinator.sessions.set('retained', { entry: coordinator.dsps.get(ids[0]) });
+  await coordinator.poll(); assert.deepEqual(stopped, []);
+  coordinator.sessions.clear(); cancel.abort(); await cancelled;
+});

core/auth-broker/tests/enrollment.test.js

@@ -0,0 +1,43 @@
+'use strict';
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { createPaycomEnrollment } = require('../../../host/controller/paycom-enrollment');
+const dsp = 'dsp_' + 'a'.repeat(32);
+const input = () => ({ command: 'enroll', requestId: 'setup_' + 'b'.repeat(32), expiresAt: Date.now() + 30000,
+  intent: 'create', credentials: { clientCode: 'synthetic', username: 'synthetic', password: 'synthetic-secret',
+    pin1: 'one', pin2: 'two', pin3: 'three', pin4: 'four', pin5: 'five' } });
+
+test('direct enrollment forwards only to the selected DSP and confirms the vault acknowledgement', async () => {
+  const calls = [];
+  const enroll = createPaycomEnrollment({ backend: { request: async (...args) => {
+    calls.push(args); return { ok: true, status: 'configured' };
+  } } });
+  assert.deepEqual(await enroll(dsp, input()), { contractVersion: 1, ok: true, status: 'succeeded', data: { configured: true } });
+  assert.equal(calls[0][0], dsp); assert.equal(calls[0][1], 'auth.request');
+  assert.equal(calls[0][2].action, 'enroll-paycom'); assert.equal(calls[0][2].intent, 'create');
+  assert.ok(calls[0][3].signal instanceof AbortSignal);
+  assert.equal((await enroll(dsp, { ...input(), expiresAt: Date.now() - 1 })).status, 'invalid_input');
+  assert.equal(calls.length, 1);
+});
+
+test('only an explicit missing profile permits create fallback for replacement', async () => {
+  let status = 'profile_not_configured'; const intents = [];
+  const enroll = createPaycomEnrollment({ backend: { request: async (_id, _op, request) => {
+    intents.push(request.intent);
+    return request.intent === 'create' ? { ok: true, status: 'configured' } : { ok: false, status };
+  } } });
+  assert.equal((await enroll(dsp, { ...input(), intent: 'replace' })).ok, true);
+  assert.deepEqual(intents, ['replace', 'create']);
+  status = 'profile_locked'; intents.length = 0;
+  assert.equal((await enroll(dsp, { ...input(), intent: 'replace' })).status, 'profile_locked');
+  assert.deepEqual(intents, ['replace']);
+});
+
+test('lost or invalid enrollment acknowledgements report an unconfirmed save without replay', async () => {
+  for (const respond of [() => { throw new Error('lost response'); }, () => ({ ok: true, status: 'unexpected' })]) {
+    let calls = 0;
+    const enroll = createPaycomEnrollment({ backend: { request: async () => { calls++; return respond(); } } });
+    await assert.rejects(enroll(dsp, input()), { code: 'auth_unavailable', statusCode: 503 });
+    assert.equal(calls, 1);
+  }
+});

core/installations/src/owner-onboarding.js

@@ -43,6 +43,10 @@ function createOwnerOnboardingWorker({ store, invoke, backends = ['oci_container
           manifestAuthority: selected.manifestAuthority, parameters: {},
         });
         guard();
+        if (!result?.ok && result?.status === 'execution_capacity_wait' && selected.backend === 'directory_service_v1') {
+          requests.defer(row);
+          return { status: 'queued' };
+        }
         if (!result?.ok) fail(setupFailure(result?.status));
         if (result.status === 'succeeded') {
           if (step === 'sync') {
@@ -75,7 +79,7 @@ function createOwnerOnboardingWorker({ store, invoke, backends = ['oci_container
     for (const [index, row] of candidates.entries()) {
       try {
         const result = await run(row.id, `${workerId}_${index}`);
-        if (result.status === 'succeeded') completed += 1; else failed += 1;
+        if (result.status === 'succeeded') completed += 1; else if (result.status === 'failed') failed += 1;
       } catch { failed += 1; }
     }
     return { processed: candidates.length, completed, failed };

dashboard/frontend/src/pages/Connections.tsx

@@ -269,7 +269,9 @@ export function Connections() {
           caught.status >= 500)
       ) {
         setSaveUnconfirmed(true);
-        await query.refetch();
+        // Reconcile in the background so a slow status check cannot trap the
+        // owner in a disabled credential dialog after the save already failed.
+        void query.refetch().catch(() => {});
       }
     } finally {
       setBusy(null);
@@ -490,7 +492,10 @@ export function Connections() {
                   >
                     Cancel
                   </Button>
-                  <SubmitButton busy={busy !== null} disabled={busy !== null}>
+                  <SubmitButton
+                    busy={busy !== null}
+                    disabled={busy !== null || saveUnconfirmed}
+                  >
                     Save and connect
                   </SubmitButton>
                 </DialogFooter>