Update sync for credentials

integration-tests/cli/CHANGELOG.md

@@ -1,5 +1,12 @@
 # @openfn/integration-tests-cli
 
+## 1.0.14
+
+### Patch Changes
+
+- Updated dependencies [1bbc8c4]
+  - @openfn/project@0.14.0
+
 ## 1.0.13
 
 ### Patch Changes

integration-tests/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-cli",
   "private": true,
-  "version": "1.0.13",
+  "version": "1.0.14",
   "description": "CLI integration tests",
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",

packages/cli/CHANGELOG.md

@@ -1,5 +1,18 @@
 # @openfn/cli
 
+## 1.28.0
+
+### Minor Changes
+
+- 1bbc8c4: Update credentials to use credential id, not UUID. This enables credentials to sync better with app projects.
+
+  WARNING: existing credential maps will break after pulling after this change. Update your credential maps to index on the new id values rather than the UUIDs.
+
+### Patch Changes
+
+- Updated dependencies [1bbc8c4]
+  - @openfn/project@0.14.0
+
 ## 1.27.0
 
 ### Minor Changes
@@ -8,7 +21,7 @@
 
 ### Patch Changes
 
-- 5d6237d: DEfault endpoint to app.openfn.org and improve error message
+- 5d6237d: Default endpoint to app.openfn.org and improve error message
 - Updated dependencies [92c0b49]
 - Updated dependencies [dd88099]
   - @openfn/project@0.13.1

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.27.0",
+  "version": "1.28.0",
   "description": "CLI devtools for the OpenFn toolchain",
   "engines": {
     "node": ">=18",

packages/cli/src/projects/deploy.ts

@@ -245,7 +245,7 @@ export async function handler(options: DeployOptions, logger: Logger) {
 
   const active = ws.getActiveProject();
   const alias = options.alias ?? active?.alias;
-  // TODO this doesn't have an alias
+
   const localProject = await Project.from('fs', {
     root: options.workspace || '.',
     alias,
@@ -258,6 +258,8 @@ export async function handler(options: DeployOptions, logger: Logger) {
       endpoint: config.endpoint,
     };
   }
+  // generate a credential map
+  localProject.credentials = localProject.buildCredentialMap();
 
   logger.success(`Loaded local project ${printProjectName(localProject)}`);
 

packages/cli/test/execute/apply-credential-map.test.ts

@@ -12,11 +12,8 @@ const createWorkflow = (steps?: any[]) => ({
     steps: steps ?? [
       {
         id: 'a',
-        expression: `${fn}fn(() => ({ data: { count: 42 } }));`,
-        // project_credential_id must map here
-        // what about keychain_credential_id ?
-        // Should we map to credential, rather than configuration? I don't think so
-        configuration: 'A',
+        expression: `${fn}fn(() => ({ dat'admin@openfn.org-cred': { count: 42 } }));`,
+        configuration: 'admin@openfn.org-cred',
         next: { b: true },
       },
     ],
@@ -44,21 +41,41 @@ test('do nothing if map is empty', (t) => {
 test('apply a credential to a single step', (t) => {
   const wf = createWorkflow();
   const map = {
-    A: { user: 'Anne Arnold' },
+    'admin@openfn.org-cred': { user: 'Anne Arnold' },
   };
 
-  t.is(wf.workflow.steps[0].configuration, 'A');
+  t.is(wf.workflow.steps[0].configuration, 'admin@openfn.org-cred');
 
   applyCredentialMap(wf, map);
 
-  t.deepEqual(wf.workflow.steps[0].configuration, map.A);
+  t.deepEqual(wf.workflow.steps[0].configuration, map['admin@openfn.org-cred']);
+});
+
+test('apply a credential to a single step if UUIDs are used', (t) => {
+  const uuid = '4227ea57-6df9-4b6c-877f-04f00a6892b5';
+  const wf = createWorkflow([
+    {
+      id: 'a',
+      expression: `fn(s => s)`,
+      configuration: uuid,
+    },
+  ]);
+  const map = {
+    [uuid]: { user: 'Anne Arnold' },
+  };
+
+  t.is(wf.workflow.steps[0].configuration, uuid);
+
+  applyCredentialMap(wf, map);
+
+  t.deepEqual(wf.workflow.steps[0].configuration, map[uuid]);
 });
 
 test('apply a credential to a single step which already has config', (t) => {
   const wf = createWorkflow();
   wf.workflow.steps[0].configuration = { x: 1, [CREDENTIALS_KEY]: 'A' };
   const map = {
-    A: { user: 'Anne Arnold' },
+    'admin@openfn.org-cred': { user: 'Anne Arnold' },
   };
 
   applyCredentialMap(wf, map);
@@ -68,21 +85,21 @@ test('apply a credential to a single step which already has config', (t) => {
 
 test('apply a credential to several steps', (t) => {
   const wf = createWorkflow([
-    { id: 'a', configuration: 'A' },
-    { id: 'b', configuration: 'B' },
+    { id: 'a', configuration: 'admin@openfn.org-A' },
+    { id: 'b', configuration: 'admin@openfn.org-B' },
   ]);
   const map = {
-    A: { user: 'Anne Arnold' },
-    B: { user: 'Belle Bellvue' },
+    'admin@openfn.org-A': { user: 'Anne Arnold' },
+    'admin@openfn.org-B': { user: 'Belle Bellvue' },
   };
 
-  t.is(wf.workflow.steps[0].configuration, 'A');
-  t.is(wf.workflow.steps[1].configuration, 'B');
+  t.is(wf.workflow.steps[0].configuration, 'admin@openfn.org-A');
+  t.is(wf.workflow.steps[1].configuration, 'admin@openfn.org-B');
 
   applyCredentialMap(wf, map);
 
-  t.deepEqual(wf.workflow.steps[0].configuration, map.A);
-  t.deepEqual(wf.workflow.steps[1].configuration, map.B);
+  t.deepEqual(wf.workflow.steps[0].configuration, map['admin@openfn.org-A']);
+  t.deepEqual(wf.workflow.steps[1].configuration, map['admin@openfn.org-B']);
 });
 
 test('wipe string credential if unmapped', (t) => {

packages/lexicon/core.d.ts

@@ -193,7 +193,6 @@ export interface Step {
   // TODO a Step must ALWAYS have an id (util functions can default it)
   id?: StepId;
   name?: string; // user-friendly name used in logging
-
   next?: string | Record<StepId, StepEdge>;
   previous?: StepId;
 }

packages/lexicon/lightning.d.ts

@@ -231,9 +231,7 @@ export namespace Provisioner {
     workflows: Record<string, Workflow>;
     concurrency?: any; // TODO
 
-    // TODO typing isn't quite right here either
-    //project_credentials: Record<string | symbol, Credential>;
-    project_credentials: any[];
+    project_credentials: Array<{ id: string; name: string; owner: string }>;
 
     // this is clearly wrong?
     //collections: Record<string | symbol, Collection>;

packages/project/CHANGELOG.md

@@ -1,5 +1,13 @@
 # @openfn/project
 
+## 0.14.0
+
+### Minor Changes
+
+- 1bbc8c4: Update credentials to use credential id, not UUID. This enables credentials to sync better with app projects.
+
+  WARNING: existing credential maps will break after pulling after this change. Update your credential maps to index on the new id values rather than the UUIDs.
+
 ## 0.13.1
 
 ### Patch Changes

packages/project/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/project",
-  "version": "0.13.1",
+  "version": "0.14.0",
   "description": "Read, serialize, replicate and sync OpenFn projects",
   "scripts": {
     "test": "pnpm ava",

packages/project/src/Project.ts

@@ -15,6 +15,7 @@ import { Workspace } from './Workspace';
 import { buildConfig, extractConfig } from './util/config';
 import { Provisioner } from '@openfn/lexicon/lightning';
 import { SandboxMeta, UUID, WorkspaceConfig } from '@openfn/lexicon';
+import { parse } from './util/get-credential-name';
 
 const maybeCreateWorkflow = (wf: any) =>
   wf instanceof Workflow ? wf : new Workflow(wf);
@@ -34,6 +35,14 @@ type CLIMeta = {
   forked_from?: Record<string, string>;
 };
 
+export type Credential = {
+  uuid: string;
+  name: string;
+
+  // TODO having the owner in the credential is controvertial and we may need to rethink this later
+  owner: string;
+};
+
 export class Project {
   // what schema version is this?
   // And how are we tracking this?
@@ -70,7 +79,7 @@ export class Project {
 
   collections: any;
 
-  credentials: string[];
+  credentials: Credential[];
 
   sandbox?: SandboxMeta;
 
@@ -241,6 +250,26 @@ export class Project {
     return result;
   }
 
+  /**
+   * Find all project credentials referenced in all
+   * workflows and return it
+   */
+  buildCredentialMap(): Credential[] {
+    const creds: any = {};
+    for (const wf of this.workflows) {
+      for (const step of wf.steps) {
+        if (
+          typeof step.configuration === 'string' &&
+          !creds[step.configuration]
+        ) {
+          const { name, owner } = parse(step.configuration);
+          creds[step.configuration] = { name, owner };
+        }
+      }
+    }
+    return Object.values(creds);
+  }
+
   // Compare this project with another and return a list of workflow changes
   diff(project: Project, workflows: string[] = []) {
     return projectDiff(this, project, workflows);

packages/project/src/gen/generator.ts

@@ -35,7 +35,7 @@ const expectedNodeProps = [
   // TODO need to clarify adaptor/adaptors confusion
   'adaptor',
   'adaptors',
-
+  'configuration',
   'expression',
   'condition',
   'label',

packages/project/src/merge/merge-project.ts

@@ -1,12 +1,13 @@
 import { defaultsDeep, isEmpty } from 'lodash-es';
 
-import { Project } from '../Project';
+import { Credential, Project } from '../Project';
 import { mergeWorkflows } from './merge-workflow';
 import mapUuids from './map-uuids';
 import baseMerge from '../util/base-merge';
 import getDuplicates from '../util/get-duplicates';
 import Workflow from '../Workflow';
 import findChangedWorkflows from '../util/find-changed-workflows';
+import getCredentialName from '../util/get-credential-name';
 
 export const SANDBOX_MERGE = 'sandbox';
 
@@ -157,7 +158,12 @@ export function merge(
           name: source.name ?? target.name,
           alias: source.alias ?? target.alias,
           description: source.description ?? target.description,
-          credentials: source.credentials ?? target.credentials,
+
+          // when mapping credentials, we prefer the UUIDs on the target
+          credentials: replaceCredentials(
+            source.credentials,
+            target.credentials
+          ),
           collections: source.collections ?? target.collections,
         };
 
@@ -166,3 +172,29 @@ export function merge(
     baseMerge(target, source, ['collections'], assigns as any)
   );
 }
+
+export const replaceCredentials = (
+  sourceCreds: Credential[] = [],
+  targetCreds: Credential[] = []
+): Credential[] => {
+  const result = [...targetCreds];
+
+  // Build an object of existing target credential names for quick lookup
+  const targetCredNames = targetCreds.reduce((acc, cred) => {
+    acc[getCredentialName(cred)] = true;
+    return acc;
+  }, {} as Record<string, boolean>);
+
+  // Find credentials in source that don't exist in target
+  for (const sourceCred of sourceCreds) {
+    const credName = getCredentialName(sourceCred);
+    if (!targetCredNames[credName]) {
+      // This is a new credential - add it without the source uuid
+      // (a new UUID will be generated elsewhere)
+      const { uuid, ...credWithoutUuid } = sourceCred;
+      result.push(credWithoutUuid as Credential);
+    }
+  }
+
+  return result;
+};