Sync: split up openfn.yaml

.changeset/eager-houses-call.md

@@ -0,0 +1,10 @@
+---
+'@openfn/project': minor
+'@openfn/cli': minor
+---
+
+Restructure `openfn.yaml` to make it safer for merging on github.
+
+Any checkout state (basically anything under the project key) have been moved into a new `.openfn` folder, which should not be tracked.
+
+`openfn.yaml` now just contains workspace-wide configuration.

.changeset/hip-actors-switch.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': patch
+---
+
+Fix an issue where `project clean` can try to delete stuff even if nothing is checked out

.changeset/orange-lions-talk.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': patch
+---
+
+Include collections and channels in sync diffs

.changeset/small-cases-buy.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': patch
+---
+
+When running commands with v2 sync, if no checkout metadata is detected, prompt the user

.changeset/true-dolls-stay.md

@@ -0,0 +1,6 @@
+---
+'@openfn/project': patch
+'@openfn/cli': patch
+---
+
+Track collections in resources.yaml

integration-tests/cli/.gitignore

@@ -1 +1,2 @@
-repo
+repo
+.state.json

integration-tests/cli/src/run.ts

@@ -7,6 +7,8 @@ const getOptions = () => ({
   env: {
     ...process.env,
     OPENFN_REPO_DIR: path.resolve('repo'),
+    // workspaces live inside the kit git repo: don't pick up its branch
+    OPENFN_BRANCH: 'false',
   },
 });
 

integration-tests/cli/test/deploy.test.ts

@@ -273,7 +273,10 @@ test.serial('redirect to v2 protocol if openfn.yaml is present', async (t) => {
   t.falsy(bootstrap.stderr);
   assertLog(t, extractLogs(bootstrap.stdout), /Detected openfn.yaml file/i);
 
-  const yaml = await fs.readFile(path.join(tmpDir, 'openfn.yaml'), 'utf8');
+  const yaml = await fs.readFile(
+    path.join(tmpDir, '.openfn/checkout.yaml'),
+    'utf8'
+  );
   t.regex(yaml, new RegExp(`uuid\\: ${projectId}`));
 
   const workflowYaml = await fs.readFile(

integration-tests/cli/test/deploy.v2.test.ts

@@ -40,7 +40,10 @@ test.serial('pull a project', async (t) => {
 
   assertLog(t, extractLogs(stdout), /Checked out project locally/i);
 
-  const yaml = await fs.readFile(path.resolve(tmpDir, 'openfn.yaml'), 'utf8');
+  const yaml = await fs.readFile(
+    path.resolve(tmpDir, '.openfn/checkout.yaml'),
+    'utf8'
+  );
   t.regex(yaml, /uuid\: a/);
   t.regex(yaml, /id\: test-project/);
 });
@@ -56,7 +59,10 @@ test.serial('pull, change and re-deploy', async (t) => {
   t.falsy(pullResult.stderr);
   assertLog(t, extractLogs(pullResult.stdout), /Checked out project locally/i);
 
-  const yaml = await fs.readFile(path.resolve(tmpDir, 'openfn.yaml'), 'utf8');
+  const yaml = await fs.readFile(
+    path.resolve(tmpDir, '.openfn/checkout.yaml'),
+    'utf8'
+  );
   t.regex(yaml, /id\: test-project/);
 
   // modify expression to trigger a change
@@ -520,7 +526,7 @@ test.serial(
 );
 
 test.serial(
-  'deploy collections: add a collection via openfn.yaml',
+  'deploy collections: add a collection via resources.yaml',
   async (t) => {
     const projectId = 'gggggggg';
     server.addProject(makeProject(projectId) as any);
@@ -538,13 +544,13 @@ test.serial(
       /Checked out project locally/i
     );
 
-    const openfnPath = path.resolve(tmpDir, 'openfn.yaml');
-    const before: any = yamlToJson(await fs.readFile(openfnPath, 'utf8'));
-    t.falsy(before.project.collections);
+    const resourcesPath = path.resolve(tmpDir, 'resources.yaml');
+    const before: any = yamlToJson(await fs.readFile(resourcesPath, 'utf8'));
+    t.deepEqual(before.collections, {});
 
     // add a collection by hand - no workflow files are touched
-    before.project.collections = ['my-collection'];
-    await fs.writeFile(openfnPath, jsonToYaml(before));
+    before.collections = { 'my-collection': {} };
+    await fs.writeFile(resourcesPath, jsonToYaml(before));
 
     const { stdout, stderr } = await run(
       `openfn project deploy --no-confirm --log-json -l debug`
@@ -623,7 +629,7 @@ workflows:
 });
 
 test.serial(
-  'deploy collections: remove a collection via openfn.yaml',
+  'deploy collections: remove a collection via resources.yaml',
   async (t) => {
     const projectId = 'hhhhhhhh';
     server.addProject({
@@ -633,7 +639,7 @@ test.serial(
 
     t.is(server.state.projects[projectId].collections.length, 1);
 
-    // pull the project - openfn.yaml should list the fetched collection
+    // pull the project - resources.yaml should list the fetched collection
     const pullResult = await run(
       `openfn project pull ${projectId} --log-json -l debug`
     );
@@ -644,13 +650,13 @@ test.serial(
       /Checked out project locally/i
     );
 
-    const openfnPath = path.resolve(tmpDir, 'openfn.yaml');
-    const before: any = yamlToJson(await fs.readFile(openfnPath, 'utf8'));
-    t.deepEqual(before.project.collections, ['my-collection']);
+    const resourcesPath = path.resolve(tmpDir, 'resources.yaml');
+    const before: any = yamlToJson(await fs.readFile(resourcesPath, 'utf8'));
+    t.deepEqual(before.collections, { 'my-collection': {} });
 
     // remove the collection by hand - no workflow files are touched
-    before.project.collections = [];
-    await fs.writeFile(openfnPath, jsonToYaml(before));
+    before.collections = {};
+    await fs.writeFile(resourcesPath, jsonToYaml(before));
 
     const { stdout, stderr } = await run(
       `openfn project deploy --no-confirm --log-json -l debug`

integration-tests/cli/test/project-v2.test.ts

@@ -282,13 +282,12 @@ steps:
     );
 
     // add the credential map to the yaml
+    await writeFile(`${TMP_DIR}/openfn.yaml`, `credentials: creds.yaml`);
+
+    // point the checked out project at the mock server
     await writeFile(
-      `${TMP_DIR}/openfn.yaml`,
-      `
-project:
-  endpoint: http://localhost:1234
-workspace:
-  credentials: creds.yaml`
+      `${TMP_DIR}/.openfn/checkout.yaml`,
+      `endpoint: http://localhost:1234`
     );
 
     // write the credential map

integration-tests/cli/test/sync.test.ts

@@ -160,9 +160,9 @@ test('pull a new project', async (t) => {
   t.regex(proj_yaml, /id: patients/);
   t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
 
-  const openfn_yaml = read('openfn.yaml');
-  t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
-  t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`));
+  const checkout_yaml = read('.openfn/checkout.yaml');
+  t.regex(checkout_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
+  t.regex(checkout_yaml, new RegExp(`endpoint: ${endpoint}`));
 
   const job = read('workflows/workflow/job.js');
   t.is(job, 'fn()');
@@ -188,9 +188,9 @@ test('pull a new project with an alias', async (t) => {
   t.regex(proj_yaml, /id: patients/);
   t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
 
-  const openfn_yaml = read('openfn.yaml');
-  t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
-  t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`));
+  const checkout_yaml = read('.openfn/checkout.yaml');
+  t.regex(checkout_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
+  t.regex(checkout_yaml, new RegExp(`endpoint: ${endpoint}`));
 
   const job = read('workflows/workflow/job.js');
   t.is(job, 'fn()');
@@ -231,9 +231,9 @@ test('pull an update to project', async (t) => {
   t.regex(proj_yaml, /fn\(x\)/);
   t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
 
-  const openfn_yaml = read('openfn.yaml');
-  t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
-  t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`));
+  const checkout_yaml = read('.openfn/checkout.yaml');
+  t.regex(checkout_yaml, new RegExp(`uuid: ${project.openfn.uuid}`));
+  t.regex(checkout_yaml, new RegExp(`endpoint: ${endpoint}`));
 
   const job_updated = read('workflows/workflow/job.js');
   t.is(job_updated, 'fn()');

packages/cli/src/projects/checkout.ts

@@ -1,5 +1,6 @@
 import yargs from 'yargs';
 import Project, { Workspace } from '@openfn/project';
+import type l from '@openfn/lexicon';
 import path from 'path';
 import fs from 'fs';
 import { rimraf } from 'rimraf';
@@ -10,7 +11,11 @@ import * as o from '../options';
 import * as po from './options';
 
 import type { Opts } from './options';
-import { tidyWorkflowDir, updateForkedFrom } from './util';
+import {
+  ensureCheckoutIgnored,
+  tidyWorkflowDir,
+  updateForkedFrom,
+} from './util';
 import { createProjectCredentials } from './credentials-helpers';
 import abort from '../util/abort';
 
@@ -23,9 +28,19 @@ export type CheckoutOptions = Pick<
   | 'clean'
   | 'force'
   | 'createCredentials'
+  | 'branch'
+  | 'track'
 >;
 
-const options = [o.log, po.workspace, po.clean, o.force, po.creds];
+const options = [
+  o.log,
+  po.workspace,
+  po.branch,
+  po.track,
+  po.clean,
+  o.force,
+  po.creds,
+];
 
 const command: yargs.CommandModule = {
   command: 'checkout <project>',
@@ -43,7 +58,9 @@ export default command;
 export const handler = async (options: CheckoutOptions, logger?: Logger) => {
   const projectIdentifier = options.project!;
   const workspacePath = options.workspace ?? process.cwd();
-  const workspace = new Workspace(workspacePath, logger);
+  const workspace = new Workspace(workspacePath, logger, true, {
+    branch: options.branch,
+  });
 
   // get the config
   // TODO: try to retain the endpoint for the projects
@@ -109,18 +126,52 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => {
   }
   // Check whether the checked out project has diverged from its forked from versions
 
+  // On a git branch, the checkout file binds the branch to a project.
+  // Checking out some other project is an ad-hoc checkout: expand its files,
+  // but keep the branch tracking the original project
+  const tracked = workspace.activeProject;
+  const isAdHoc = Boolean(
+    options.branch &&
+      !options.track &&
+      tracked &&
+      !isSameProject(tracked, switchProject)
+  );
+  if (isAdHoc) {
+    const trackedName = tracked!.alias ?? tracked!.id ?? tracked!.uuid;
+    logger?.warn(
+      `Branch ${options.branch} tracks project ${trackedName}. Expanding files from ${switchProject.alias} without changing tracking: deploy will still target ${trackedName}`
+    );
+    logger?.warn(
+      `Pass --track to make ${options.branch} track ${switchProject.alias} instead`
+    );
+  }
+
   // delete workflow dir before expanding project
   if (options.clean) {
     await rimraf(workspace.workflowsPath);
   } else {
-    await tidyWorkflowDir(localProject, switchProject, false, workspacePath);
+    await tidyWorkflowDir(
+      localProject,
+      switchProject,
+      false,
+      workspacePath,
+      options.branch
+    );
   }
 
   // write the forked from map
   updateForkedFrom(switchProject);
 
   // expand project into directory
-  const files: any = switchProject.serialize('fs');
+  const files: any = switchProject.serialize('fs', {
+    branch: options.branch,
+  });
+  if (isAdHoc) {
+    // Don't touch the workspace or checkout metadata
+    delete files[switchProject.generateConfig().path];
+    delete files[switchProject.generateCheckout(options.branch).path];
+  }
+  await ensureCheckoutIgnored(workspacePath);
   for (const f in files) {
     if (files[f]) {
       fs.mkdirSync(path.join(workspacePath, path.dirname(f)), {
@@ -143,6 +194,13 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => {
   logger?.success(`Expanded project to ${workspacePath}`);
 };
 
+const isSameProject = (tracked: l.ProjectMeta, project: Project) => {
+  if (tracked.uuid && project.openfn?.uuid) {
+    return tracked.uuid === project.openfn.uuid;
+  }
+  return tracked.id === project.id;
+};
+
 // This function will tell us if the active/checked out project
 // has any changes compared to the tracked state file
 // It implies that changes will be lost on checkout

packages/cli/src/projects/clean.ts

@@ -4,6 +4,7 @@ import { rimraf } from 'rimraf';
 
 import { build, ensure } from '../util/command-builders';
 import { handler as checkout } from './checkout';
+import ensureCheckout from './ensure-checkout';
 import type { Logger } from '../util/logger';
 import * as o from '../options';
 import * as po from './options';
@@ -12,10 +13,10 @@ import type { Opts } from './options';
 
 export type CleanOptions = Pick<
   Opts,
-  'command' | 'workspace' | 'log' | 'confirm' | 'force'
+  'command' | 'workspace' | 'log' | 'confirm' | 'force' | 'branch'
 >;
 
-const options = [o.log, o.confirm, o.force, po.workspace];
+const options = [o.log, o.confirm, o.force, po.workspace, po.branch];
 
 const command: yargs.CommandModule = {
   command: 'clean',
@@ -29,7 +30,22 @@ export default command;
 
 export const handler = async (options: CleanOptions, logger: Logger) => {
   const workspacePath = options.workspace ?? process.cwd();
-  const workspace = new Workspace(workspacePath, logger);
+
+  // Find out what's checked out before deleting anything
+  const workspace = await ensureCheckout(
+    new Workspace(workspacePath, logger, true, {
+      branch: options.branch,
+    }),
+    logger
+  );
+
+  // This has to be checked before deleting anything
+  const activeProject = workspace.activeProject;
+  if (!activeProject) {
+    throw new Error(
+      'No active project found in workspace. Run `project pull` first.'
+    );
+  }
 
   const skip = options.force || options.confirm === false;
   const doIt = await logger.confirm(
@@ -42,13 +58,6 @@ export const handler = async (options: CleanOptions, logger: Logger) => {
 
   await rimraf(workspace.workflowsPath);
 
-  const activeProject = workspace.activeProject;
-  if (!activeProject) {
-    throw new Error(
-      'No active project found in workspace. Run `project pull` first.'
-    );
-  }
-
   const projectId = String(activeProject.uuid ?? (activeProject as any).id);
   await checkout({ ...options, project: projectId, force: true }, logger);
 };