Release: CLI@1.43.0

integration-tests/cli/.gitignore

@@ -1 +1,2 @@
-repo
+repo
+.state.json

integration-tests/cli/CHANGELOG.md

@@ -1,5 +1,15 @@
 # @openfn/integration-tests-cli
 
+## 1.0.35
+
+### Patch Changes
+
+- Updated dependencies [d3f1a1a]
+- Updated dependencies [e90bfde]
+- Updated dependencies [d3f1a1a]
+  - @openfn/project@0.22.0
+  - @openfn/lightning-mock@2.5.5
+
 ## 1.0.34
 
 ### Patch Changes

integration-tests/cli/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@openfn/integration-tests-cli",
   "private": true,
-  "version": "1.0.34",
+  "version": "1.0.35",
   "description": "CLI integration tests",
   "author": "Open Function Group <admin@openfn.org>",
   "license": "ISC",

integration-tests/cli/src/run.ts

@@ -7,6 +7,8 @@ const getOptions = () => ({
   env: {
     ...process.env,
     OPENFN_REPO_DIR: path.resolve('repo'),
+    // workspaces live inside the kit git repo: don't pick up its branch
+    OPENFN_BRANCH: 'false',
   },
 });
 

integration-tests/cli/test/deploy.test.ts

@@ -273,7 +273,10 @@ test.serial('redirect to v2 protocol if openfn.yaml is present', async (t) => {
   t.falsy(bootstrap.stderr);
   assertLog(t, extractLogs(bootstrap.stdout), /Detected openfn.yaml file/i);
 
-  const yaml = await fs.readFile(path.join(tmpDir, 'openfn.yaml'), 'utf8');
+  const yaml = await fs.readFile(
+    path.join(tmpDir, '.openfn/checkout.yaml'),
+    'utf8'
+  );
   t.regex(yaml, new RegExp(`uuid\\: ${projectId}`));
 
   const workflowYaml = await fs.readFile(

integration-tests/cli/test/deploy.v2.test.ts

@@ -40,7 +40,10 @@ test.serial('pull a project', async (t) => {
 
   assertLog(t, extractLogs(stdout), /Checked out project locally/i);
 
-  const yaml = await fs.readFile(path.resolve(tmpDir, 'openfn.yaml'), 'utf8');
+  const yaml = await fs.readFile(
+    path.resolve(tmpDir, '.openfn/checkout.yaml'),
+    'utf8'
+  );
   t.regex(yaml, /uuid\: a/);
   t.regex(yaml, /id\: test-project/);
 });
@@ -56,7 +59,10 @@ test.serial('pull, change and re-deploy', async (t) => {
   t.falsy(pullResult.stderr);
   assertLog(t, extractLogs(pullResult.stdout), /Checked out project locally/i);
 
-  const yaml = await fs.readFile(path.resolve(tmpDir, 'openfn.yaml'), 'utf8');
+  const yaml = await fs.readFile(
+    path.resolve(tmpDir, '.openfn/checkout.yaml'),
+    'utf8'
+  );
   t.regex(yaml, /id\: test-project/);
 
   // modify expression to trigger a change
@@ -520,7 +526,7 @@ test.serial(
 );
 
 test.serial(
-  'deploy collections: add a collection via openfn.yaml',
+  'deploy collections: add a collection via resources.yaml',
   async (t) => {
     const projectId = 'gggggggg';
     server.addProject(makeProject(projectId) as any);
@@ -538,13 +544,13 @@ test.serial(
       /Checked out project locally/i
     );
 
-    const openfnPath = path.resolve(tmpDir, 'openfn.yaml');
-    const before: any = yamlToJson(await fs.readFile(openfnPath, 'utf8'));
-    t.falsy(before.project.collections);
+    const resourcesPath = path.resolve(tmpDir, 'resources.yaml');
+    const before: any = yamlToJson(await fs.readFile(resourcesPath, 'utf8'));
+    t.deepEqual(before.collections, {});
 
     // add a collection by hand - no workflow files are touched
-    before.project.collections = ['my-collection'];
-    await fs.writeFile(openfnPath, jsonToYaml(before));
+    before.collections = { 'my-collection': {} };
+    await fs.writeFile(resourcesPath, jsonToYaml(before));
 
     const { stdout, stderr } = await run(
       `openfn project deploy --no-confirm --log-json -l debug`
@@ -623,7 +629,7 @@ workflows:
 });
 
 test.serial(
-  'deploy collections: remove a collection via openfn.yaml',
+  'deploy collections: remove a collection via resources.yaml',
   async (t) => {
     const projectId = 'hhhhhhhh';
     server.addProject({
@@ -633,7 +639,7 @@ test.serial(
 
     t.is(server.state.projects[projectId].collections.length, 1);
 
-    // pull the project - openfn.yaml should list the fetched collection
+    // pull the project - resources.yaml should list the fetched collection
     const pullResult = await run(
       `openfn project pull ${projectId} --log-json -l debug`
     );
@@ -644,13 +650,13 @@ test.serial(
       /Checked out project locally/i
     );
 
-    const openfnPath = path.resolve(tmpDir, 'openfn.yaml');
-    const before: any = yamlToJson(await fs.readFile(openfnPath, 'utf8'));
-    t.deepEqual(before.project.collections, ['my-collection']);
+    const resourcesPath = path.resolve(tmpDir, 'resources.yaml');
+    const before: any = yamlToJson(await fs.readFile(resourcesPath, 'utf8'));
+    t.deepEqual(before.collections, { 'my-collection': {} });
 
     // remove the collection by hand - no workflow files are touched
-    before.project.collections = [];
-    await fs.writeFile(openfnPath, jsonToYaml(before));
+    before.collections = {};
+    await fs.writeFile(resourcesPath, jsonToYaml(before));
 
     const { stdout, stderr } = await run(
       `openfn project deploy --no-confirm --log-json -l debug`

integration-tests/cli/test/project-v2.test.ts

@@ -282,13 +282,12 @@ steps:
     );
 
     // add the credential map to the yaml
+    await writeFile(`${TMP_DIR}/openfn.yaml`, `credentials: creds.yaml`);
+
+    // point the checked out project at the mock server
     await writeFile(
-      `${TMP_DIR}/openfn.yaml`,
-      `
-project:
-  endpoint: http://localhost:1234
-workspace:
-  credentials: creds.yaml`
+      `${TMP_DIR}/.openfn/checkout.yaml`,
+      `endpoint: http://localhost:1234`
     );
 
     // write the credential map

integration-tests/cli/test/sync.test.ts

@@ -160,9 +160,9 @@ test('pull a new project', async (t) => {
   t.regex(proj_yaml, /id: patients/);
   t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
 
-  const openfn_yaml = read('openfn.yaml');
-  t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
-  t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`));
+  const checkout_yaml = read('.openfn/checkout.yaml');
+  t.regex(checkout_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
+  t.regex(checkout_yaml, new RegExp(`endpoint: ${endpoint}`));
 
   const job = read('workflows/workflow/job.js');
   t.is(job, 'fn()');
@@ -188,9 +188,9 @@ test('pull a new project with an alias', async (t) => {
   t.regex(proj_yaml, /id: patients/);
   t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
 
-  const openfn_yaml = read('openfn.yaml');
-  t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
-  t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`));
+  const checkout_yaml = read('.openfn/checkout.yaml');
+  t.regex(checkout_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
+  t.regex(checkout_yaml, new RegExp(`endpoint: ${endpoint}`));
 
   const job = read('workflows/workflow/job.js');
   t.is(job, 'fn()');
@@ -231,9 +231,9 @@ test('pull an update to project', async (t) => {
   t.regex(proj_yaml, /fn\(x\)/);
   t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
 
-  const openfn_yaml = read('openfn.yaml');
-  t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
-  t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`));
+  const checkout_yaml = read('.openfn/checkout.yaml');
+  t.regex(checkout_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
+  t.regex(checkout_yaml, new RegExp(`endpoint: ${endpoint}`));
 
   const job_updated = read('workflows/workflow/job.js');
   t.is(job_updated, 'fn()');

packages/cli/CHANGELOG.md

@@ -1,5 +1,27 @@
 # @openfn/cli
 
+## 1.43.0
+
+### Minor Changes
+
+- d3f1a1a: Restructure `openfn.yaml` to make it safer for merging on github.
+
+  Any checkout state (basically anything under the project key) have been moved into a new `.openfn` folder, which should not be tracked.
+
+  `openfn.yaml` now just contains workspace-wide configuration.
+
+### Patch Changes
+
+- d3f1a1a: Fix an issue where `project clean` can try to delete stuff even if nothing is checked out
+- d3f1a1a: Include collections and channels in sync diffs
+- e90bfde: Handle multiple local copies of the same project without reporting id conflicts
+- d3f1a1a: When running commands with v2 sync, if no checkout metadata is detected, prompt the user
+- d3f1a1a: Track collections in resources.yaml
+- Updated dependencies [d3f1a1a]
+- Updated dependencies [e90bfde]
+- Updated dependencies [d3f1a1a]
+  - @openfn/project@0.22.0
+
 ## 1.42.0
 
 ### Minor Changes

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@openfn/cli",
-  "version": "1.42.0",
+  "version": "1.43.0",
   "description": "CLI devtools for the OpenFn toolchain",
   "repository": {
     "url": "https://github.com/OpenFn/kit"

packages/cli/src/projects/checkout.ts

@@ -1,5 +1,6 @@
 import yargs from 'yargs';
 import Project, { Workspace } from '@openfn/project';
+import type l from '@openfn/lexicon';
 import path from 'path';
 import fs from 'fs';
 import { rimraf } from 'rimraf';
@@ -21,11 +22,23 @@ export type CheckoutOptions = Pick<
   | 'workspace'
   | 'log'
   | 'clean'
+  | 'confirm'
   | 'force'
   | 'createCredentials'
+  | 'branch'
+  | 'track'
 >;
 
-const options = [o.log, po.workspace, po.clean, o.force, po.creds];
+const options = [
+  o.log,
+  po.workspace,
+  po.branch,
+  po.track,
+  po.clean,
+  o.force,
+  o.confirm,
+  po.creds,
+];
 
 const command: yargs.CommandModule = {
   command: 'checkout <project>',
@@ -43,7 +56,9 @@ export default command;
 export const handler = async (options: CheckoutOptions, logger?: Logger) => {
   const projectIdentifier = options.project!;
   const workspacePath = options.workspace ?? process.cwd();
-  const workspace = new Workspace(workspacePath, logger);
+  const workspace = new Workspace(workspacePath, logger, true, {
+    branch: options.branch,
+  });
 
   // get the config
   // TODO: try to retain the endpoint for the projects
@@ -109,18 +124,81 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => {
   }
   // Check whether the checked out project has diverged from its forked from versions
 
+  // On a git branch, the checkout file binds the branch to a project.
+  // Checking out some other project is an ad-hoc checkout: expand its files,
+  // but keep the branch tracking the original project
+  const tracked = workspace.activeProject;
+  const isAdHoc = Boolean(
+    options.branch &&
+      !options.track &&
+      tracked &&
+      !isSameProject(tracked, switchProject)
+  );
+  if (isAdHoc) {
+    const trackedName = tracked!.alias ?? tracked!.id ?? tracked!.uuid;
+
+    logger?.warn('Watch out! Ad-hoc checkout detected!');
+    logger?.warn(
+      `You are on git branch ${options.branch}, which tracks project ${trackedName}`
+    );
+    logger?.warn(
+      `But you've asked to checkout project ${switchProject.alias}.`
+    );
+    logger?.warn(
+      `This will leave your branch (${trackedName}) inconsistent with your checked out workflows (${switchProject.alias}).`
+    );
+    logger?.warn(
+      `Pass --track to update your local tracker so that ${options.branch} tracks ${switchProject.alias} instead.`
+    );
+
+    // -f and -y both skip the prompt
+    const skip = options.force || options.confirm === false;
+
+    // Without a terminal there's nobody to ask, and the prompt would hang
+    if (!skip && !process.stdin.isTTY) {
+      abort(logger!, 'Ad-hoc checkout needs to be confirmed', {
+        details: 'There is no terminal to ask for confirmation',
+        fix: `Pass --force (-f) or --confirm (-y) to continue anyway, or --track to make ${options.branch} track ${switchProject.alias}`,
+      });
+    }
+    const doIt = logger
+      ? await logger.confirm(
+          `Continue and checkout ${switchProject.alias} anyway?`,
+          skip
+        )
+      : true;
+
+    if (!doIt) {
+      logger?.info('Checkout cancelled: nothing has been changed');
+      return;
+    }
+  }
+
   // delete workflow dir before expanding project
   if (options.clean) {
     await rimraf(workspace.workflowsPath);
   } else {
-    await tidyWorkflowDir(localProject, switchProject, false, workspacePath);
+    await tidyWorkflowDir(
+      localProject,
+      switchProject,
+      false,
+      workspacePath,
+      options.branch
+    );
   }
 
   // write the forked from map
   updateForkedFrom(switchProject);
 
   // expand project into directory
-  const files: any = switchProject.serialize('fs');
+  const files: any = switchProject.serialize('fs', {
+    branch: options.branch,
+  });
+  if (isAdHoc) {
+    // Don't touch the workspace or checkout metadata
+    delete files[switchProject.generateConfig().path];
+    delete files[switchProject.generateCheckout(options.branch).path];
+  }
   for (const f in files) {
     if (files[f]) {
       fs.mkdirSync(path.join(workspacePath, path.dirname(f)), {
@@ -143,6 +221,13 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => {
   logger?.success(`Expanded project to ${workspacePath}`);
 };
 
+const isSameProject = (tracked: l.ProjectMeta, project: Project) => {
+  if (tracked.uuid && project.openfn?.uuid) {
+    return tracked.uuid === project.openfn.uuid;
+  }
+  return tracked.id === project.id;
+};
+
 // This function will tell us if the active/checked out project
 // has any changes compared to the tracked state file
 // It implies that changes will be lost on checkout