CLI: Deploy straight from yaml

.changeset/fast-boats-grin.md

@@ -0,0 +1,5 @@
+---
+'@openfn/project': patch
+---
+
+Fix an issue where serializing a project without credential uuids to app state causes credentials on job bodies to be lost

.changeset/fast-cobras-throw.md

@@ -0,0 +1,5 @@
+---
+'@openfn/lexicon': patch
+---
+
+Project credentials can optionally have a UUID

.changeset/young-brooms-drum.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': patch
+---
+
+Enable a v2 project yaml file to be deployed directly, eg, `openfn project deploy .projects/main@app.openfn.org.yaml

packages/cli/src/projects/deploy.ts

@@ -76,7 +76,7 @@ const printProjectName = (project: Project) =>
 export const command: yargs.CommandModule<DeployOptions> = {
   command: 'deploy [project]',
   aliases: 'push',
-  describe: `Deploy the checked out project to a Lightning Instance`,
+  describe: `Deploy the passed or checked-out project to a Lightning Instance`,
   builder: (yargs: yargs.Argv<DeployOptions>) =>
     build(options, yargs)
       .positional('project', {
@@ -275,25 +275,52 @@ export async function handler(options: DeployOptions, logger: Logger) {
   );
   const config = loadAppAuthConfig(options, logger);
 
-  // TODO this is the hard way to load the local alias
-  // We need track alias in openfn.yaml to make this easier (and tracked in from fs)
-  const ws = new Workspace(options.workspace || '.');
+  // The local project that we want to actually deploy
+  let localProject: Project;
+  let ws;
+  let alias = options.alias ?? null;
 
-  const active = ws.getTrackedProject();
-  const alias = options.alias ?? active?.alias;
+  if (options.project) {
+    logger.debug('Reading project from path ', options.project);
+    localProject = await Project.from(
+      'path',
+      path.resolve(options.workspace ?? process.cwd(), options.project)
+    );
+
+    // If the local project doesn't have stateful stuff,
+    // flag this as a new upload
+    if (!localProject.uuid) {
+      logger.debug(
+        'Local project does not have a UUID: assuming this is a new project deployment'
+      );
+      options.new = true;
+
+      // TODO ensure the alias is unique if we're posting a new project
+    }
+  } else {
+    logger.debug('Reading checked-out project from workspace');
+    // TODO this is the hard way to load the local alias
+    // We need track alias in openfn.yaml to make this easier (and tracked in from fs)
+    ws = new Workspace(options.workspace || '.');
 
-  const localProject = await Project.from('fs', {
-    root: options.workspace || '.',
-    alias,
-    name: options.name,
-  });
+    const active = ws.getTrackedProject();
+
+    // messy
+    alias ??= active?.alias ?? null;
+
+    localProject = await Project.from('fs', {
+      root: options.workspace || '.',
+      alias,
+      name: options.name,
+    });
+  }
 
   // Track the remote we want to target
   // If the user passed a project alias, we need to use that
   // Otherwise just sync with the local project
   let tracker;
   if (!options.new) {
-    tracker = ws.get(options.project ?? localProject.uuid!);
+    tracker = ws?.get(options.project ?? localProject.uuid!);
     if (!tracker) {
       // Is this really an error? Unlikely to happen I thuink
       console.log(
@@ -344,7 +371,7 @@ export async function handler(options: DeployOptions, logger: Logger) {
     const syncResult = await syncProjects(
       options,
       config,
-      ws,
+      ws!,
       localProject,
       tracker!,
       logger
@@ -355,6 +382,10 @@ export async function handler(options: DeployOptions, logger: Logger) {
     ({ merged, remoteProject, locallyChangedWorkflows } = syncResult);
   }
 
+  /**
+   * Questions:
+   * 1. When this serializses, should project_credentials have uuids?
+   */
   const state = merged.serialize('state', {
     format: 'json',
   }) as Provisioner.Project_v1;
@@ -433,11 +464,16 @@ export async function handler(options: DeployOptions, logger: Logger) {
 
     updateForkedFrom(finalProject);
     const configData = finalProject.generateConfig();
+
+    // Write the updated openfn.yaml
+    // TODO: allow us to suppress writing this stuff
+    // (useful if posting from spec)
     await writeFile(
-      path.resolve(options.workspace!, configData.path),
+      path.resolve(options.workspace ?? process.cwd(), configData.path),
       configData.content
     );
 
+    // TODO if this was marked as new, we probably need to ensure a unique alias here
     const finalOutputPath = getSerializePath(localProject, options.workspace!);
     const fullFinalPath = await serialize(finalProject, finalOutputPath);
     logger.debug('Updated local project at ', fullFinalPath);

packages/cli/src/projects/util.ts

@@ -179,12 +179,13 @@ export async function deployProject(
       if (contentType.match('application/json')) {
         const body = await response.json();
         logger?.error(JSON.stringify(body, null, 2));
+        console.log(JSON.stringify(body, null, 2));
       } else {
         const content = await response.text();
         // TODO html errors are too long to be useful... figure this out later
         logger?.error(content);
+        console.log(JSON.stringify(content, null, 2));
       }
-
       throw new CLIError(
         `Failed to deploy project ${state.name}: ${response.status}`
       );

packages/cli/test/projects/create-credentials.test.ts

@@ -51,7 +51,7 @@ test('sync-credentials: ignores duplicate references', (t) => {
   t.deepEqual(findCredentialIds(project), ['same']);
 });
 
-test.only('sync-credentials: creates credential yaml file', (t) => {
+test('sync-credentials: creates credential yaml file', (t) => {
   mock({ '/ws': {} });
 
   const project = new Project(

packages/cli/test/projects/deploy.test.ts

@@ -18,6 +18,7 @@ import {
   UUID,
   two_workflows_yaml as twowfs,
   TWO_WORKFLOWS_UUID,
+  myProject_spec,
 } from './fixtures';
 import { checkout } from '../../src/projects';
 
@@ -77,25 +78,60 @@ test.beforeEach(() => {
   mock.restore();
 });
 
-test.serial('deploy a new project', async (t) => {
+test.serial(
+  'deploy a project as new from the checked out project',
+  async (t) => {
+    // the server should have 1 registered project by default - that's fine
+    t.is(Object.keys(server.state.projects).length, 1);
+
+    await setup();
+
+    await deploy(
+      {
+        endpoint: ENDPOINT,
+        apiKey: 'test-api-key',
+        workspace: '/ws',
+        new: true,
+      } as any,
+      logger
+    );
+
+    // We should now have a new project with a new UUID
+    t.is(Object.keys(server.state.projects).length, 2);
+
+    const success = logger._find('success', /Created new project at/);
+    t.truthy(success);
+  }
+);
+
+test.serial('deploy a project as new from a v2 spec yaml', async (t) => {
   // the server should have 1 registered project by default - that's fine
   t.is(Object.keys(server.state.projects).length, 1);
 
-  await setup();
+  // skip the usual setup and just set up the filesystem
+  mockFs({
+    '/ws/.projects/main@localhost.yaml': myProject_spec,
+    '/ws/openfn.yaml': '', // TODO this shouldn't be needed
+  });
 
   await deploy(
     {
       endpoint: ENDPOINT,
       apiKey: 'test-api-key',
-      workspace: '/ws',
-      new: true,
+      project: '/ws/.projects/main@localhost.yaml',
     } as any,
     logger
   );
 
+  console.log(logger._history);
+
   // We should now have a new project with a new UUID
   t.is(Object.keys(server.state.projects).length, 2);
 
+  // TODO more assertions
+  // project structure looks right (edges in particular)
+  // - credential is OK
+
   const success = logger._find('success', /Created new project at/);
   t.truthy(success);
 });

packages/cli/test/projects/fixtures.ts

@@ -104,6 +104,32 @@ workflows:
     id: my-workflow
     start: webhook`;
 
+export const myProject_spec = `id: my-project
+name: My Project
+schema_version: '4.0'
+description: my lovely project
+collections: []
+credentials:
+  - name: http1
+    owner: super@openfn.org
+workflows:
+  - name: My Workflow
+    steps:
+      - id: transform-data
+        name: Transform data
+        expression: fn()
+        adaptor: '@openfn/language-common@latest'
+        configuration: super@openfn.org|http1
+      - id: webhook
+        type: webhook
+        enabled: true
+        next:
+          transform-data:
+            disabled: false
+            condition: always
+    id: my-workflow
+    start: webhook`;
+
 export const TWO_WORKFLOWS_UUID = '4b09ddf1-35f4-4e40-9aa9-0d80c086dd9e';
 
 export const two_workflows_yaml = `id: my-project

packages/lexicon/portability.d.ts

@@ -92,6 +92,7 @@ export interface Trigger extends Step {
 export interface Credential {
   name: string;
   owner: string;
+  uuid?: string | number;
 }
 
 export interface Job extends Step {

packages/lightning-mock/test/rest.test.ts

@@ -125,7 +125,43 @@ test('validateProvisionPayload: returns null for a valid edge with source_job_id
   t.is(validateProvisionPayload(payload), null);
 });
 
-test('validateProvisionPayload: returns errors when edge has no source', (t) => {
+test('validateProvisionPayload: returns errors when edge has no source job or trigger id', (t) => {
+  const payload = {
+    id: 'proj-1',
+    workflows: [
+      {
+        name: 'wf1',
+        edges: [
+          {
+            id: 'edge-1',
+            source_trigger_id: null,
+            target_job_id: '',
+            enabled: true,
+          },
+        ],
+      },
+    ],
+  };
+  const result = validateProvisionPayload(payload);
+  t.truthy(result);
+  t.deepEqual(result, {
+    errors: {
+      workflows: {
+        wf1: {
+          edges: {
+            'edge-1': {
+              source_job_id: [
+                'source_job_id or source_trigger_id must be present',
+              ],
+            },
+          },
+        },
+      },
+    },
+  });
+});
+
+test('validateProvisionPayload: for new edges allow source_job', (t) => {
   const payload = {
     id: 'proj-1',
     workflows: [

packages/project/src/serialize/to-app-state.ts

@@ -49,6 +49,13 @@ export default function (
 
   state.id = (uuid as string) ?? randomUUID();
 
+  // Ensure each credential has a UUID
+  project.credentials =
+    project.credentials?.map((c) => ({
+      ...c,
+      uuid: c.uuid ?? randomUUID(),
+    })) ?? [];
+
   Object.assign(state, rest, project.options);
   if (options.asSpec) {
     for (const c of project.credentials) {
@@ -62,14 +69,10 @@ export default function (
       };
     }
   } else {
-    const credentialsWithUuids =
-      project.credentials?.map((c) => ({
-        ...c,
-        uuid: (c as CredentialState).uuid ?? randomUUID(),
-      })) ?? [];
-
-    state.project_credentials = credentialsWithUuids.map((c) => ({
-      // note the subtle conversion here
+    state.project_credentials = project.credentials.map((c) => ({
+      // note the subtle conversion here: uuid -> id
+      // That's because the local Project uses the uuid key to track UUIDs
+      // but the provisioner spec uses id
       id: c.uuid as string,
       name: c.name,
       owner: c.owner,
@@ -172,7 +175,9 @@ export const mapWorkflow = (
             return name === projectCredentialId;
           });
           if (mappedCredential && useUuids) {
-            projectCredentialId = mappedCredential.uuid;
+            // the mapped credential might have a uuid or an id depending on how it was fed in to us
+            // but this is bullshit right?
+            projectCredentialId = mappedCredential.uuid ?? mappedCredential.id;
           }
 
           if (useUuids) {