From f1211c85e76bff35ac1b2b5c97ec581fd22875b3 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 28 Sep 2026 14:43:53 +0100 Subject: [PATCH 01/22] lexicon: remove trigger.enabled from spec --- packages/lexicon/core.d.ts | 3 +++ packages/lexicon/portability.d.ts | 2 -- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/lexicon/core.d.ts b/packages/lexicon/core.d.ts index 9bbfe20e8..f07ac1915 100644 --- a/packages/lexicon/core.d.ts +++ b/packages/lexicon/core.d.ts @@ -194,6 +194,9 @@ export interface WorkflowMeta { export interface NodeMeta { uuid?: UUID; + /** only meaningful on a trigger step */ + enabled?: boolean; + [key: string]: unknown; } diff --git a/packages/lexicon/portability.d.ts b/packages/lexicon/portability.d.ts index fa6d712de..3b76c48fc 100644 --- a/packages/lexicon/portability.d.ts +++ b/packages/lexicon/portability.d.ts @@ -81,8 +81,6 @@ export interface Trigger extends Step { /** cron schedule, only meaningful when type is 'cron' */ cron_expression?: string; - enabled?: boolean; - webhook_reply?: 'before_start' | 'after_completion'; webhook_response_config?: { error_code?: number; From d3811fa46cb366f4229e04dfccc5c918ee577286 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 28 Sep 2026 15:36:35 +0100 Subject: [PATCH 02/22] update handling of trigger.enabled --- packages/project/src/Workflow.ts | 4 +--- packages/project/src/parse/from-app-spec.ts | 14 +++++++++++++- packages/project/src/parse/from-app-state.ts | 2 -- packages/project/test/canonical.test.ts | 4 ---- packages/project/test/parse/from-app-spec.test.ts | 9 +++++++-- packages/project/test/parse/from-app-state.test.ts | 14 +++++++------- packages/project/test/serialize/to-fs.test.ts | 14 ++++++++------ 7 files changed, 36 insertions(+), 25 deletions(-) diff --git a/packages/project/src/Workflow.ts b/packages/project/src/Workflow.ts index c53cfa83c..c38c74931 100644 --- a/packages/project/src/Workflow.ts +++ b/packages/project/src/Workflow.ts @@ -4,9 +4,7 @@ import { generateHash, HashOptions } from './util/version'; const clone = (obj: any) => JSON.parse(JSON.stringify(obj)); -export type WithMeta = T & { - openfn?: l.NodeMeta; -}; +export type WithMeta = l.WithMeta; class Workflow { workflow: l.WorkflowState; // this is the raw workflow JSON representation diff --git a/packages/project/src/parse/from-app-spec.ts b/packages/project/src/parse/from-app-spec.ts index 2231a1261..371d930d2 100644 --- a/packages/project/src/parse/from-app-spec.ts +++ b/packages/project/src/parse/from-app-spec.ts @@ -56,8 +56,20 @@ const mapWorkflow = ( for (const [triggerKey, trigger] of Object.entries( workflow.triggers ?? {} ) as [string, any][]) { + const { enabled, ...rest } = trigger; + + // Handle enabled state + if (typeof enabled === 'boolean') { + rest.openfn = Object.assign({}, rest.openfn, { enabled }); + } // a trigger's id is its type, matching how v2 specs are written - addStep(triggerKey, omitNil({ ...trigger, id: trigger.type })); + addStep( + triggerKey, + omitNil({ + ...rest, + id: trigger.type, + }) + ); } for (const [jobKey, job] of Object.entries(workflow.jobs ?? {}) as [ diff --git a/packages/project/src/parse/from-app-state.ts b/packages/project/src/parse/from-app-state.ts index c039a0474..9b4c8e2b8 100644 --- a/packages/project/src/parse/from-app-state.ts +++ b/packages/project/src/parse/from-app-state.ts @@ -161,7 +161,6 @@ export const mapWorkflow = ( Object.values(triggers).forEach((trigger: Provisioner.Trigger) => { const { type, - enabled, custom_path, cron_expression, webhook_reply, @@ -193,7 +192,6 @@ export const mapWorkflow = ( omitNil({ id: type, type, - enabled, custom_path, cron_expression, cron_cursor_job_id, diff --git a/packages/project/test/canonical.test.ts b/packages/project/test/canonical.test.ts index a52956b1e..33a9305e5 100644 --- a/packages/project/test/canonical.test.ts +++ b/packages/project/test/canonical.test.ts @@ -36,7 +36,6 @@ const project: ProjectSpec = { id: 'webhook', name: 'Webhook Trigger', type: 'webhook', - enabled: true, webhook_reply: 'before_start', webhook_response_config: { success_code: 202, @@ -88,7 +87,6 @@ const project: ProjectSpec = { id: 'cron', name: 'Cron Trigger', type: 'cron', - enabled: false, cron_expression: '0 0 * * *', cron_cursor_job_id: 'cron-job', webhook_reply: 'after_completion', @@ -187,7 +185,6 @@ workflows: - id: webhook name: Webhook Trigger type: webhook - enabled: true webhook_reply: before_start webhook_response_config: success_code: 202 @@ -203,7 +200,6 @@ workflows: - id: cron name: Cron Trigger type: cron - enabled: false cron_expression: 0 0 * * * cron_cursor_job_id: cron-job webhook_reply: after_completion diff --git a/packages/project/test/parse/from-app-spec.test.ts b/packages/project/test/parse/from-app-spec.test.ts index bb0e5e559..d1a997d02 100644 --- a/packages/project/test/parse/from-app-spec.test.ts +++ b/packages/project/test/parse/from-app-spec.test.ts @@ -151,8 +151,10 @@ test('a trigger becomes a step, and sets the workflow start', (t) => { const trigger = getStep(wf, 'cron'); t.is(trigger.type, 'cron'); - t.is(trigger.enabled, false); t.is(trigger.cron_expression, '*/15 * * * *'); + + // Note that enabled is not part of the spec structure + t.is(trigger.openfn.enabled, false); }); test('edges hang off their source step, keyed by target', (t) => { @@ -199,7 +201,10 @@ test('a js_expression condition keeps its expression', (t) => { workflows: { wf: { name: 'wf', - jobs: { A: { name: 'A', body: 'fn()' }, B: { name: 'B', body: 'fn()' } }, + jobs: { + A: { name: 'A', body: 'fn()' }, + B: { name: 'B', body: 'fn()' }, + }, triggers: {}, edges: { 'A->B': { diff --git a/packages/project/test/parse/from-app-state.test.ts b/packages/project/test/parse/from-app-state.test.ts index c3252e6b3..9876af3ff 100644 --- a/packages/project/test/parse/from-app-state.test.ts +++ b/packages/project/test/parse/from-app-state.test.ts @@ -206,8 +206,10 @@ test('should create a Project from prov state with a workflow', (t) => { { id: 'webhook', type: 'webhook', - enabled: true, - openfn: { uuid: '4a06289c-15aa-4662-8dc6-f0aaacd8a058' }, + openfn: { + enabled: true, + uuid: '4a06289c-15aa-4662-8dc6-f0aaacd8a058', + }, next: { 'transform-data': { condition: 'always', @@ -243,9 +245,7 @@ test('should create a Project from prov state with a workflow', (t) => { test('should create a Project from prov state with cron_cursor_job_id mapped to a step id', (t) => { const project = fromAppState(state, meta); - const cronWorkflow = project.workflows.find( - (w) => w.id === 'cron-workflow' - )!; + const cronWorkflow = project.workflows.find((w) => w.id === 'cron-workflow')!; const trigger = cronWorkflow.steps.find((s: any) => s.type === 'cron') as any; t.is(trigger.cron_cursor_job_id, 'transform-data'); @@ -274,10 +274,10 @@ test('mapWorkflow: map a cron trigger', (t) => { id: 'cron', type: 'cron', next: {}, - enabled: true, cron_expression: '0 1 0 0', cron_cursor_job_id: 'x', openfn: { + enabled: true, uuid: '1234', }, }); @@ -337,7 +337,6 @@ test('mapWorkflow: map a webhook trigger', (t) => { t.deepEqual(trigger, { id: 'webhook', type: 'webhook', - enabled: true, webhook_reply: 'before_start', webhook_response_config: { success_code: 202, @@ -353,6 +352,7 @@ test('mapWorkflow: map a webhook trigger', (t) => { }, }, openfn: { + enabled: true, uuid: '4a06289c-15aa-4662-8dc6-f0aaacd8a058', }, }); diff --git a/packages/project/test/serialize/to-fs.test.ts b/packages/project/test/serialize/to-fs.test.ts index a4db902df..a84cf9bc1 100644 --- a/packages/project/test/serialize/to-fs.test.ts +++ b/packages/project/test/serialize/to-fs.test.ts @@ -156,7 +156,7 @@ test('extractWorkflow: single simple workflow with random edge property', (t) => }); }); -test('extractWorkflow: include trigger enabled state (true)', (t) => { +test('extractWorkflow: excludes trigger enabled state (true)', (t) => { const project = new Project( { workflows: [ @@ -167,7 +167,9 @@ test('extractWorkflow: include trigger enabled state (true)', (t) => { { id: 'webhook', type: 'webhook', - enabled: true, + openfn: { + enabled: true, + }, }, ], openfn: { @@ -188,11 +190,10 @@ test('extractWorkflow: include trigger enabled state (true)', (t) => { t.deepEqual(JSON.parse(content).steps[0], { id: 'webhook', type: 'webhook', - enabled: true, }); }); -test('extractWorkflow: include trigger enabled state (false)', (t) => { +test('extractWorkflow: excludes trigger enabled state (false)', (t) => { const project = new Project( { workflows: [ @@ -203,7 +204,9 @@ test('extractWorkflow: include trigger enabled state (false)', (t) => { { id: 'webhook', type: 'webhook', - enabled: false, + openfn: { + enabled: false, + }, }, ], openfn: { @@ -224,7 +227,6 @@ test('extractWorkflow: include trigger enabled state (false)', (t) => { t.deepEqual(JSON.parse(content).steps[0], { id: 'webhook', type: 'webhook', - enabled: false, }); }); From 154fd6f5ecb53d994a8bf6c661976f84bc41162d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 28 Sep 2026 15:53:14 +0100 Subject: [PATCH 03/22] update version hash --- packages/lexicon/core.d.ts | 19 ++++++++++++++++++- packages/project/src/util/version.ts | 1 - .../test/util/version-workflow.test.ts | 5 +++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/lexicon/core.d.ts b/packages/lexicon/core.d.ts index f07ac1915..04abbcc4b 100644 --- a/packages/lexicon/core.d.ts +++ b/packages/lexicon/core.d.ts @@ -1,7 +1,14 @@ import { SanitizePolicies } from '@openfn/logger'; import type { RawSourceMap } from 'source-map'; -import { Credential, Collection, Job, ProjectSpec, WorkflowSpec } from './portability'; +import { + Credential, + Collection, + Job, + Trigger, + ProjectSpec, + WorkflowSpec, +} from './portability'; export { Step, StepId, @@ -45,6 +52,9 @@ export interface ProjectState extends WithState { } export interface WorkflowState extends WithState { + // override steps to include per-step state (uuid, trigger enabled, etc) + steps: Array | WithMeta>; + /** holds version history information of a workflow **/ history?: string[]; @@ -200,6 +210,13 @@ export interface NodeMeta { [key: string]: unknown; } +/** + * Utility to append a .openfn state object to a step or edge + */ +export type WithMeta = T & { + openfn?: NodeMeta; +}; + /** * State is an object passed into a workflow and returned from a workflow */ diff --git a/packages/project/src/util/version.ts b/packages/project/src/util/version.ts index cc96bc33e..f97d00e03 100644 --- a/packages/project/src/util/version.ts +++ b/packages/project/src/util/version.ts @@ -56,7 +56,6 @@ export const generateHash = ( const triggerKeys = [ 'type', 'custom_path', - 'enabled', 'cron_expression', 'webhook_reply', 'webhook_response_config', diff --git a/packages/project/test/util/version-workflow.test.ts b/packages/project/test/util/version-workflow.test.ts index 554f73da0..bdd341e9f 100644 --- a/packages/project/test/util/version-workflow.test.ts +++ b/packages/project/test/util/version-workflow.test.ts @@ -26,7 +26,7 @@ const example = { target_job_id: '40b839bd-5ade-414e-8dde-ed3ae77239ea', }, ], - version_history: ['app:211291f6e6d5'], + version_history: ['app:6eb3b9b100a9'], inserted_at: '2025-12-19T15:26:49Z', jobs: [ { @@ -68,6 +68,7 @@ test('match lightning version', async (t) => { const wf = proj.workflows[0]; const hash = wf.getVersionHash(); + console.log(hash); t.log(expected); t.log(hash); t.is(parse(hash).hash, parse(expected).hash); @@ -221,7 +222,7 @@ test('ordering: trigger keys appear in sorted order', (t) => { ); // Trigger keys sorted: cron_expression, enabled, type const hash = workflow.getVersionHash({ sha: false }); - t.is(hash, 'cli:wf* * *falsecroncodextruecron-x'); + t.is(hash, 'cli:wf* * *croncodextruecron-x'); }); test('ordering: complete workflow with all elements', (t) => { From b765520e41dfccc0be5930226d3edfbb10c66fd5 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 28 Sep 2026 15:54:31 +0100 Subject: [PATCH 04/22] changeset --- .changeset/busy-needles-rhyme.md | 6 ++++++ .changeset/rich-clubs-shave.md | 5 +++++ 2 files changed, 11 insertions(+) create mode 100644 .changeset/busy-needles-rhyme.md create mode 100644 .changeset/rich-clubs-shave.md diff --git a/.changeset/busy-needles-rhyme.md b/.changeset/busy-needles-rhyme.md new file mode 100644 index 000000000..957f061eb --- /dev/null +++ b/.changeset/busy-needles-rhyme.md @@ -0,0 +1,6 @@ +--- +'@openfn/lexicon': patch +'@openfn/project': patch +--- + +Remove trigger.enabled from project spec representations diff --git a/.changeset/rich-clubs-shave.md b/.changeset/rich-clubs-shave.md new file mode 100644 index 000000000..a4248f8ca --- /dev/null +++ b/.changeset/rich-clubs-shave.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': patch +--- + +Remove trigger.enabled from workflow.yaml, so that it doesn't show up on git diffs. From 1d3009b13d74d7f608963b6a1774c1a944acee44 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 28 Sep 2026 16:09:46 +0100 Subject: [PATCH 05/22] remove log --- packages/project/test/util/version-workflow.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/project/test/util/version-workflow.test.ts b/packages/project/test/util/version-workflow.test.ts index bdd341e9f..81eec0f44 100644 --- a/packages/project/test/util/version-workflow.test.ts +++ b/packages/project/test/util/version-workflow.test.ts @@ -68,7 +68,6 @@ test('match lightning version', async (t) => { const wf = proj.workflows[0]; const hash = wf.getVersionHash(); - console.log(hash); t.log(expected); t.log(hash); t.is(parse(hash).hash, parse(expected).hash); From c4ccef224b6ed7d8a2a195f3159c374c73b734b1 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 28 Sep 2026 18:09:25 +0100 Subject: [PATCH 06/22] update version util and fix tests --- packages/cli/src/projects/version.ts | 81 +++++++++++++++++++-- packages/cli/test/projects/checkout.test.ts | 1 - packages/cli/test/projects/fetch.test.ts | 2 +- packages/cli/test/projects/fixtures.ts | 13 ++-- 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/packages/cli/src/projects/version.ts b/packages/cli/src/projects/version.ts index 94b20a893..46237d791 100644 --- a/packages/cli/src/projects/version.ts +++ b/packages/cli/src/projects/version.ts @@ -1,5 +1,6 @@ +import fs from 'node:fs/promises'; import yargs from 'yargs'; -import { Workspace } from '@openfn/project'; +import { Workspace, yamlToJson, generateVersionHash } from '@openfn/project'; import { ensure, build } from '../util/command-builders'; import type { Logger } from '../util/logger'; @@ -9,21 +10,91 @@ import * as po from './options'; import type { Opts } from './options'; export type VersionOptions = Required< - Pick + Pick< + Opts, + 'command' | 'workflow' | 'workspace' | 'workflowMappings' | 'json' | 'log' + > >; -const options = [o.workflow, po.workspace, po.workflowMappings]; +const options = [o.workflow, po.workspace, po.workflowMappings, o.log]; const command: yargs.CommandModule = { command: 'version [workflow]', - describe: 'Returns the version hash of a given workflow in a workspace', + describe: + 'Returns the version hash of a workflow. Pass a workflow id in the checked-out project, a path to a workflow yaml/json file, or pipe workflow content via stdin', handler: ensure('project-version', options), - builder: (yargs) => build(options, yargs), + builder: (yargs) => + build(options, yargs) + .example( + 'version', + 'Print version hashes for every workflow in the checked-out project' + ) + .example( + 'version my-workflow', + 'Print the version hash of workflow "my-workflow" in the checked-out project' + ) + .example( + 'version my-workflow.yaml', + 'Print the version hash of a standalone workflow yaml/json file' + ) + .example( + 'xclip -selection clipboard -o | openfn project version', + 'Print the version hash of workflow content piped in via stdin' + ), }; export default command; +// Some workflow files wrap the workflow in a { workflow, options } envelope +const asWorkflow = (json: any) => json.workflow ?? json; + +const hashWorkflow = (json: any) => { + const workflow = asWorkflow(json); + const name = workflow.name || workflow.id || 'workflow'; + return { name, hash: generateVersionHash(workflow) }; +}; + +const readStdin = () => + new Promise((resolve, reject) => { + let data = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk) => (data += chunk)); + process.stdin.on('end', () => resolve(data)); + process.stdin.on('error', reject); + }); + +const printHash = (logger: Logger, name: string, hash: string) => { + logger.success(`Workflow(s) and their hashes\n\n${name}\n${hash}`); +}; + export const handler = async (options: VersionOptions, logger: Logger) => { + // A workflow file passed directly, e.g. `openfn project version my-workflow.yaml` + if (options.workflow && /\.(ya?ml|json)$/i.test(options.workflow)) { + logger.debug(`Reading workflow from file: ${options.workflow}`); + const content = await fs.readFile(options.workflow, 'utf8'); + const json = options.workflow.toLowerCase().endsWith('.json') + ? JSON.parse(content) + : yamlToJson(content); + const { name, hash } = hashWorkflow(json); + printHash(logger, name, hash); + return; + } + + // Workflow content piped through stdin, e.g. `pbpaste | openfn project version` + if (!options.workflow && !process.stdin.isTTY) { + logger.debug('Reading workflow from stdin'); + const content = await readStdin(); + if (content.trim()) { + const json = content.trim().startsWith('{') + ? JSON.parse(content) + : yamlToJson(content); + const { name, hash } = hashWorkflow(json); + printHash(logger, name, hash); + return; + } + } + + logger.debug('Reading workflow(s) from checked-out workspace project'); const workspace = new Workspace(options.workspace); if (!workspace.valid) { logger.error('Command was run in an invalid openfn workspace'); diff --git a/packages/cli/test/projects/checkout.test.ts b/packages/cli/test/projects/checkout.test.ts index 015576ab6..2d838f22d 100644 --- a/packages/cli/test/projects/checkout.test.ts +++ b/packages/cli/test/projects/checkout.test.ts @@ -481,7 +481,6 @@ workspace: { id: 'webhook', type: 'webhook', - enabled: true, next: { 'transform-data-to-fhir-standard': { disabled: false, diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts index eb1402ed9..7b038bdee 100644 --- a/packages/cli/test/projects/fetch.test.ts +++ b/packages/cli/test/projects/fetch.test.ts @@ -433,8 +433,8 @@ test.serial( { id: 'webhook', type: 'webhook', - enabled: true, openfn: { + enabled: true, uuid: '4a06289c-15aa-4662-8dc6-f0aaacd8a058', }, next: { diff --git a/packages/cli/test/projects/fixtures.ts b/packages/cli/test/projects/fixtures.ts index c1c3c7f72..9824c3c9a 100644 --- a/packages/cli/test/projects/fixtures.ts +++ b/packages/cli/test/projects/fixtures.ts @@ -44,9 +44,7 @@ export const myProject_v1: Provisioner.Project = { }, lock_version: 1, deleted_at: null, - version_history: [ - 'cli:7126e08da251', // alterstate - ], + version_history: ['cli:57c1e804512a'], }, }, updated_at: '2025-04-23T11:15:59Z', @@ -85,8 +83,8 @@ workflows: uuid: 66add020-e6eb-4eec-836b-20008afca816 - id: webhook type: webhook - enabled: true openfn: + enabled: true uuid: 4a06289c-15aa-4662-8dc6-f0aaacd8a058 next: transform-data: @@ -95,7 +93,7 @@ workflows: openfn: uuid: a9a3adef-b394-4405-814d-3ac4323f4b4b history: - - cli:7126e08da251 + - cli:57c1e804512a openfn: uuid: 72ca3eb0-042c-47a0-a2a1-a545ed4a8406 inserted_at: 2025-04-23T11:19:32Z @@ -122,7 +120,6 @@ workflows: configuration: super@openfn.org|http1 - id: webhook type: webhook - enabled: true next: transform-data: disabled: false @@ -245,8 +242,8 @@ workflows: uuid: 3d4727b6-4052-4f58-a834-3a03e433ff1d - id: trigger-a type: webhook - enabled: true openfn: + enabled: true uuid: 1b1c1dd5-e8d9-432f-aeaf-4e09397cac98 next: job-a: @@ -270,8 +267,8 @@ workflows: uuid: 37e6e616-3840-4d71-b63c-a736ebc208b7 - id: trigger-b type: webhook - enabled: true openfn: + enabled: true uuid: d65ed915-7f39-428b-af57-57ed2ecf507e next: job-b: From 141defd2f50ed23b974a10e562dd1cc50f16ae4d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 28 Sep 2026 18:13:43 +0100 Subject: [PATCH 07/22] add notes to docs --- packages/cli/README.md | 16 ++++++++++++++++ packages/project/README.md | 14 ++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/packages/cli/README.md b/packages/cli/README.md index 3fda1316c..744b4ca38 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -268,6 +268,22 @@ openfn deploy - `-c, --config, --config-path` - path to the config file (defaults to `.config.json`) - `--no-confirm` - skip the confirmation prompt +## Checking a Workflow's Version Hash + +`openfn project version` prints the version hash used to detect drift between a local and remote workflow (see `@openfn/project`'s `src/util/version.ts`). It accepts three kinds of input: + +- No arguments: prints the hash for every workflow in the checked-out project (or pass a workflow id/name to hash just one). +- A path to a standalone workflow yaml/json file. +- Workflow content piped in via stdin, e.g. from your clipboard. + +```bash +openfn project version my-workflow.yaml +xclip -selection clipboard -o | openfn project version # Linux +pbpaste | openfn project version # macOS +``` + +This is particularly useful for regenerating hardcoded version-hash fixtures in tests (in this package and in `@openfn/project`) whenever the hashing logic or a workflow's spec/state shape changes - run the command against the fixture's workflow content instead of hand-computing the new hash. + ## Logging The CLI is actually a collection of packages, each of which will log with slightly different rules. To help understand where logs are coming from, each package prints a namespace or prefix at the start of its log. diff --git a/packages/project/README.md b/packages/project/README.md index 1150b254b..171cdf58a 100644 --- a/packages/project/README.md +++ b/packages/project/README.md @@ -93,3 +93,17 @@ a-b # can comment here to ``` Use special names `webhook` and `cron` to create trigger nodes (when converting into app state, the difference between a step and a trigger becomes important). + +## Version Hashes + +`Workflow.getVersionHash()` (and the underlying `generateHash` in `src/util/version.ts`) derive a content hash for a workflow, used to detect drift between a local and remote copy. A number of tests hardcode expected hash strings as fixtures (e.g. `test/util/version-workflow.test.ts`, and the `history`/hash values baked into fixtures under `packages/cli/test/projects/fixtures.ts`). + +If you change what the hash covers - add or remove a hashed field, move a field between the spec and state representations, change `triggerKeys`/`stepKeys`/`edgeKeys` in `version.ts` - every hardcoded hash in these fixtures will need regenerating. Don't hand-compute them: use `openfn project version` (see the CLI README), which accepts a standalone workflow yaml/json file or workflow content piped via stdin, and prints the same hash this package would compute: + +```bash +openfn project version path/to/workflow.yaml +xclip -selection clipboard -o | openfn project version # Linux +pbpaste | openfn project version # macOS +``` + +Paste the printed hash back into the test or fixture. From b632d11a01bdd91d61d13a0c4a6b4f8dcbbfb27b Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 28 Sep 2026 18:14:06 +0100 Subject: [PATCH 08/22] remove .only --- packages/project/test/merge/merge-project.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/project/test/merge/merge-project.test.ts b/packages/project/test/merge/merge-project.test.ts index 421547ce8..35e1c6798 100644 --- a/packages/project/test/merge/merge-project.test.ts +++ b/packages/project/test/merge/merge-project.test.ts @@ -701,7 +701,7 @@ test('remove a step from an existing workflow', (t) => { t.is(result.workflows[0].steps.length, 0); }); -test.only('removing a step also deletes its outgoing edges when serialized', (t) => { +test('removing a step also deletes its outgoing edges when serialized', (t) => { // create a base workflow with an edge x -> y const wf = { name: 'wf', From dffb182bd8302b732b4599529457638656c8f563 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 28 Sep 2026 18:28:59 +0100 Subject: [PATCH 09/22] fix test --- packages/project/test/merge/merge-project.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/project/test/merge/merge-project.test.ts b/packages/project/test/merge/merge-project.test.ts index 35e1c6798..aadef34e1 100644 --- a/packages/project/test/merge/merge-project.test.ts +++ b/packages/project/test/merge/merge-project.test.ts @@ -694,11 +694,14 @@ test('remove a step from an existing workflow', (t) => { const main = createProject(wf_a, 'a'); const staging = createProject(wf_b, 'b'); + t.is(main.workflows[0].steps.length, 1); + t.is(staging.workflows[0].steps.length, 0); + // merge staging into main const result: any = merge(staging, main); // The resulting project should have no steps - t.is(result.workflows[0].steps.length, 0); + t.true(result.workflows[0].isRemoved('x')); }); test('removing a step also deletes its outgoing edges when serialized', (t) => { From b74b499db2599fb6911f04f2166ad58effb982be Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 29 Sep 2026 16:50:42 +0100 Subject: [PATCH 10/22] update test --- packages/cli/test/projects/fetch.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/test/projects/fetch.test.ts b/packages/cli/test/projects/fetch.test.ts index 7b038bdee..5e227ea47 100644 --- a/packages/cli/test/projects/fetch.test.ts +++ b/packages/cli/test/projects/fetch.test.ts @@ -456,7 +456,7 @@ test.serial( lock_version: 1, }, id: 'my-workflow', - history: ['cli:7126e08da251'], + history: ['cli:57c1e804512a'], }, ], }; From 77e61a133c5ffb4c053432ad9544359d7c825f6f Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 29 Sep 2026 16:56:42 +0100 Subject: [PATCH 11/22] fix integration test --- integration-tests/cli/test/project-v1.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-tests/cli/test/project-v1.test.ts b/integration-tests/cli/test/project-v1.test.ts index 3865d6a54..61aad365a 100644 --- a/integration-tests/cli/test/project-v1.test.ts +++ b/integration-tests/cli/test/project-v1.test.ts @@ -147,7 +147,6 @@ start: webhook steps: - id: webhook type: webhook - enabled: true next: transform-data: disabled: false From e113a521c28b170b6043848fc6cbd9ba8c9ab7e4 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 30 Sep 2026 14:57:52 +0100 Subject: [PATCH 12/22] first swing at refactor to split up openfn.yaml --- packages/cli/src/projects/checkout.ts | 26 +++++++++++--- packages/cli/src/projects/clean.ts | 8 +++-- packages/cli/src/projects/deploy.ts | 32 +++++++++-------- packages/cli/src/projects/fetch.ts | 10 ++++-- packages/cli/src/projects/list.ts | 8 +++-- packages/cli/src/projects/merge.ts | 10 ++++-- packages/cli/src/projects/options.ts | 38 ++++++++++++++++++++ packages/cli/src/projects/pull.ts | 6 +++- packages/cli/src/projects/util.ts | 7 ++-- packages/cli/src/projects/version.ts | 11 ++++-- packages/lexicon/core.d.ts | 3 +- packages/project/src/Project.ts | 14 +++++--- packages/project/src/Workspace.ts | 24 +++++++++++-- packages/project/src/parse/from-fs.ts | 9 ++++- packages/project/src/serialize/to-fs.ts | 11 ++++-- packages/project/src/util/config.ts | 46 +++++++++++++++++++++++-- 16 files changed, 212 insertions(+), 51 deletions(-) diff --git a/packages/cli/src/projects/checkout.ts b/packages/cli/src/projects/checkout.ts index fb9e2cbeb..181d6145e 100644 --- a/packages/cli/src/projects/checkout.ts +++ b/packages/cli/src/projects/checkout.ts @@ -23,9 +23,17 @@ export type CheckoutOptions = Pick< | 'clean' | 'force' | 'createCredentials' + | 'branch' >; -const options = [o.log, po.workspace, po.clean, o.force, po.creds]; +const options = [ + o.log, + po.workspace, + po.branch, + po.clean, + o.force, + po.creds, +]; const command: yargs.CommandModule = { command: 'checkout ', @@ -43,7 +51,9 @@ export default command; export const handler = async (options: CheckoutOptions, logger?: Logger) => { const projectIdentifier = options.project!; const workspacePath = options.workspace ?? process.cwd(); - const workspace = new Workspace(workspacePath, logger); + const workspace = new Workspace(workspacePath, logger, true, { + branch: options.branch, + }); // get the config // TODO: try to retain the endpoint for the projects @@ -113,14 +123,22 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => { if (options.clean) { await rimraf(workspace.workflowsPath); } else { - await tidyWorkflowDir(localProject, switchProject, false, workspacePath); + await tidyWorkflowDir( + localProject, + switchProject, + false, + workspacePath, + options.branch + ); } // write the forked from map updateForkedFrom(switchProject); // expand project into directory - const files: any = switchProject.serialize('fs'); + const files: any = switchProject.serialize('fs', { + branch: options.branch, + }); for (const f in files) { if (files[f]) { fs.mkdirSync(path.join(workspacePath, path.dirname(f)), { diff --git a/packages/cli/src/projects/clean.ts b/packages/cli/src/projects/clean.ts index d919c53c3..152dd2573 100644 --- a/packages/cli/src/projects/clean.ts +++ b/packages/cli/src/projects/clean.ts @@ -12,10 +12,10 @@ import type { Opts } from './options'; export type CleanOptions = Pick< Opts, - 'command' | 'workspace' | 'log' | 'confirm' | 'force' + 'command' | 'workspace' | 'log' | 'confirm' | 'force' | 'branch' >; -const options = [o.log, o.confirm, o.force, po.workspace]; +const options = [o.log, o.confirm, o.force, po.workspace, po.branch]; const command: yargs.CommandModule = { command: 'clean', @@ -29,7 +29,9 @@ export default command; export const handler = async (options: CleanOptions, logger: Logger) => { const workspacePath = options.workspace ?? process.cwd(); - const workspace = new Workspace(workspacePath, logger); + const workspace = new Workspace(workspacePath, logger, true, { + branch: options.branch, + }); const skip = options.force || options.confirm === false; const doIt = await logger.confirm( diff --git a/packages/cli/src/projects/deploy.ts b/packages/cli/src/projects/deploy.ts index 7c5499be0..dca066869 100644 --- a/packages/cli/src/projects/deploy.ts +++ b/packages/cli/src/projects/deploy.ts @@ -4,7 +4,7 @@ import Project, { versionsEqual, Workspace, } from '@openfn/project'; -import { writeFile } from 'node:fs/promises'; +import { mkdir, writeFile } from 'node:fs/promises'; import path from 'node:path'; import * as o from '../options'; @@ -41,16 +41,15 @@ export type DeployOptions = Pick< | 'logJson' | 'confirm' > & { - // CLI positional args, rather than options - project?: string; - target?: string; - + project?: string; // this is a CLI positional arg, not an option alias?: string; + branch?: string | false; credentials?: CredentialsStrategy; dryRun?: boolean; jsonDiff?: boolean; name?: string; new?: boolean; + target?: string; workflow?: string[]; workspace?: string; }; @@ -59,6 +58,7 @@ const options = [ // local options o2.env, o2.workspace, + o2.branch, o2.dryRun, o2.new, o2.name, @@ -331,6 +331,12 @@ export async function handler(options: DeployOptions, logger: Logger) { let filePath: string | undefined; let targetIdentifier: string | undefined; + // TODO this is the hard way to load the local alias + // We need track alias in openfn.yaml to make this easier (and tracked in from fs) + let ws = new Workspace(options.workspace || '.', undefined, true, { + branch: options.branch, + }); + if (options.project && options.target) { // two positionals: `deploy ` - the first is always a file filePath = options.project; @@ -347,7 +353,6 @@ export async function handler(options: DeployOptions, logger: Logger) { // The local project that we want to actually deploy let localProject: Project; - let ws: Workspace | undefined; let alias = options.alias; if (filePath) { @@ -390,6 +395,7 @@ export async function handler(options: DeployOptions, logger: Logger) { localProject = await Project.from('fs', { root: options.workspace || '.', alias, + branch: options.branch, name: options.name, }); } @@ -591,15 +597,11 @@ export async function handler(options: DeployOptions, logger: Logger) { ); updateForkedFrom(finalProject); - const configData = finalProject.generateConfig(); - - // Write the updated openfn.yaml - // TODO: allow us to suppress writing this stuff - // (useful if posting from spec) - await writeFile( - path.resolve(options.workspace ?? process.cwd(), configData.path), - configData.content - ); + for (const configData of finalProject.generateConfig(options.branch)) { + const configPath = path.resolve(options.workspace!, configData.path); + await mkdir(path.dirname(configPath), { recursive: true }); + await writeFile(configPath, configData.content); + } // TODO if this was marked as new, we probably need to ensure a unique alias here const finalOutputPath = getSerializePath(finalProject, options.workspace!); diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index e87f0b3fa..0808cfe9e 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -31,6 +31,7 @@ export type FetchOptions = Pick< | 'outputPath' | 'project' | 'workspace' + | 'branch' >; const options = [ @@ -47,6 +48,7 @@ const options = [ po.outputPath, po.env, po.workspace, + po.branch, po.format, ]; @@ -75,7 +77,9 @@ const fetchV1 = async (options: FetchOptions, logger: Logger) => { const workspacePath = options.workspace ?? process.cwd(); logger.debug('Using workspace at', workspacePath); - const workspace = new Workspace(workspacePath, logger, false); + const workspace = new Workspace(workspacePath, logger, false, { + branch: options.branch, + }); // TODO we may need to resolve an alias to a UUID and endpoint const localProject = workspace.get(options.project!); if (localProject) { @@ -123,7 +127,9 @@ export const fetchV2 = async (options: FetchOptions, logger: Logger) => { const workspacePath = options.workspace ?? process.cwd(); logger.debug('Using workspace at', workspacePath); - const workspace = new Workspace(workspacePath, logger, false); + const workspace = new Workspace(workspacePath, logger, false, { + branch: options.branch, + }); const { outputPath } = options; const remoteProject = await fetchRemoteProject(workspace, options, logger); diff --git a/packages/cli/src/projects/list.ts b/packages/cli/src/projects/list.ts index 932ad182e..629b25407 100644 --- a/packages/cli/src/projects/list.ts +++ b/packages/cli/src/projects/list.ts @@ -9,9 +9,9 @@ import * as po from './options'; import type { Opts } from './options'; import abort from '../util/abort'; -export type ProjectListOptions = Pick; +export type ProjectListOptions = Pick; -const options = [o.log, po.workspace]; +const options = [o.log, po.workspace, po.branch]; const command: yargs.CommandModule = { command: 'list [project-path]', @@ -28,7 +28,9 @@ export const handler = async (options: ProjectListOptions, logger: Logger) => { logger.info(' ', options.workspace); logger.break(); - const workspace = new Workspace(options.workspace!); + const workspace = new Workspace(options.workspace!, undefined, true, { + branch: options.branch, + }); if (!workspace.valid) { // TODO how can we be more helpful here? diff --git a/packages/cli/src/projects/merge.ts b/packages/cli/src/projects/merge.ts index d036cff8d..90b0d3aec 100644 --- a/packages/cli/src/projects/merge.ts +++ b/packages/cli/src/projects/merge.ts @@ -17,13 +17,16 @@ export type MergeOptions = Required< 'command' | 'project' | 'workspace' | 'removeUnmapped' | 'workflowMappings' > > & - Pick & { base?: string }; + Pick & { + base?: string; + }; const options = [ po.removeUnmapped, po.workflowMappings, po.workflow, po.workspace, + po.branch, o.log, // custom output because we don't want defaults or anything { @@ -60,7 +63,9 @@ export default command; export const handler = async (options: MergeOptions, logger: Logger) => { const workspacePath = options.workspace; - const workspace = new Workspace(workspacePath); + const workspace = new Workspace(workspacePath, undefined, true, { + branch: options.branch, + }); if (!workspace.valid) { logger.error('Command was run in an invalid openfn workspace'); return; @@ -169,6 +174,7 @@ export const handler = async (options: MergeOptions, logger: Logger) => { await checkout( { workspace: workspacePath, + branch: options.branch, project: options.outputPath ? finalPath : final.id, log: options.log, // after the merge, we have to force the output to be checked out, ignoring divergence diff --git a/packages/cli/src/projects/options.ts b/packages/cli/src/projects/options.ts index 42ee35dc6..45652534e 100644 --- a/packages/cli/src/projects/options.ts +++ b/packages/cli/src/projects/options.ts @@ -1,3 +1,4 @@ +import { execSync } from 'node:child_process'; import resolvePath from '../util/resolve-path'; import { Opts as BaseOpts, CLIOption } from '../options'; import getCLIOptionObject from '../util/get-cli-option-object'; @@ -20,6 +21,7 @@ export type Opts = BaseOpts & { format?: 'yaml' | 'json' | 'state'; clean?: boolean; createCredentials?: boolean; + branch?: string | false; }; // project specific options @@ -148,6 +150,42 @@ export const workspace: CLIOption = { }, }; +// Returns the current git branch, or false if not on a branch +// (not a git repo, git not installed, detached HEAD) +const detectBranch = (cwd: string) => { + try { + const branch = execSync('git symbolic-ref --short -q HEAD', { + cwd, + stdio: ['ignore', 'pipe', 'ignore'], + }) + .toString() + .trim(); + return branch || false; + } catch (e) { + return false; + } +}; + +// Must come after the workspace option so that the workspace path is resolved +export const branch: CLIOption = { + name: 'branch', + yargs: { + description: + 'The git branch to track the checked out project against. Detected automatically if not set. Pass --no-branch to ignore git.', + }, + ensure: (opts: any) => { + let value = opts.branch ?? process.env.OPENFN_BRANCH; + if (typeof value === 'string' && /^(false|null)?$/.test(value)) { + value = false; + } + if (value === undefined || value === true) { + const cwd = opts.workspace ?? process.env.OPENFN_WORKSPACE ?? '.'; + value = detectBranch(resolvePath(cwd)); + } + opts.branch = value === false ? false : String(value); + }, +}; + const newProject: CLIOption = { name: 'new', yargs: { diff --git a/packages/cli/src/projects/pull.ts b/packages/cli/src/projects/pull.ts index 5de9bf09c..a3d3a85a7 100644 --- a/packages/cli/src/projects/pull.ts +++ b/packages/cli/src/projects/pull.ts @@ -25,6 +25,7 @@ export type PullOptions = Pick< | 'snapshots' | 'force' | 'createCredentials' + | 'branch' >; const options = [ @@ -33,6 +34,7 @@ const options = [ o2.alias, o2.env, o2.workspace, + o2.branch, o2.creds, // general options @@ -79,7 +81,9 @@ const ensureProjectId = (options: any, logger?: Logger) => { logger?.debug( 'No project ID specified: looking up checked out project in Workspace' ); - const ws = new Workspace(options.workspace); + const ws = new Workspace(options.workspace, undefined, true, { + branch: options.branch, + }); if (ws.activeProject) { options.project = ws.activeProject.uuid; logger?.info( diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index 64d5c7714..4e8b84d2a 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -215,14 +215,15 @@ export async function tidyWorkflowDir( currentProject: Project | undefined, incomingProject: Project | undefined, dryRun = false, - dirPath = '.' + dirPath = '.', + branch?: string | false ) { if (!currentProject || !incomingProject) { return []; } - const currentFiles = currentProject.serialize('fs'); - const newFiles = incomingProject.serialize('fs'); + const currentFiles = currentProject.serialize('fs', { branch }); + const newFiles = incomingProject.serialize('fs', { branch }); const toRemove: string[] = []; // any files not in the new list should be removed diff --git a/packages/cli/src/projects/version.ts b/packages/cli/src/projects/version.ts index 94b20a893..724158535 100644 --- a/packages/cli/src/projects/version.ts +++ b/packages/cli/src/projects/version.ts @@ -9,10 +9,13 @@ import * as po from './options'; import type { Opts } from './options'; export type VersionOptions = Required< - Pick + Pick< + Opts, + 'command' | 'workflow' | 'workspace' | 'workflowMappings' | 'json' | 'branch' + > >; -const options = [o.workflow, po.workspace, po.workflowMappings]; +const options = [o.workflow, po.workspace, po.branch, po.workflowMappings]; const command: yargs.CommandModule = { command: 'version [workflow]', @@ -24,7 +27,9 @@ const command: yargs.CommandModule = { export default command; export const handler = async (options: VersionOptions, logger: Logger) => { - const workspace = new Workspace(options.workspace); + const workspace = new Workspace(options.workspace, undefined, true, { + branch: options.branch, + }); if (!workspace.valid) { logger.error('Command was run in an invalid openfn workspace'); return; diff --git a/packages/lexicon/core.d.ts b/packages/lexicon/core.d.ts index 9bbfe20e8..edf54c9b5 100644 --- a/packages/lexicon/core.d.ts +++ b/packages/lexicon/core.d.ts @@ -154,7 +154,8 @@ export interface WorkspaceFileLegacy { // Structure of the new openfn.yaml file export interface WorkspaceFile { workspace: WorkspaceConfig; - project: ProjectMeta; + // legacy: project meta now lives in .openfn/checkout.yaml + project?: ProjectMeta; } export interface WorkspaceConfig { diff --git a/packages/project/src/Project.ts b/packages/project/src/Project.ts index 1808f11c1..18850a878 100644 --- a/packages/project/src/Project.ts +++ b/packages/project/src/Project.ts @@ -12,7 +12,11 @@ import { getUuidForEdge, getUuidForStep } from './util/uuid'; import { merge, MergeProjectOptions } from './merge/merge-project'; import { diff as projectDiff } from './util/project-diff'; import { Workspace } from './Workspace'; -import { buildConfig, extractConfig } from './util/config'; +import { + buildConfig, + extractCheckout, + extractConfig, +} from './util/config'; import { Provisioner } from '@openfn/lexicon/lightning'; import { SandboxMeta, @@ -301,11 +305,11 @@ export class Project { } /** - * Generates the contents of the openfn.yaml file, - * plus its file path + * Generates the contents of the openfn.yaml and checkout files, + * plus their file paths */ - generateConfig() { - return extractConfig(this); + generateConfig(branch?: string | false | null) { + return [extractConfig(this), extractCheckout(this, branch)]; } clone() { diff --git a/packages/project/src/Workspace.ts b/packages/project/src/Workspace.ts index f7e2bc3ec..018b21a78 100644 --- a/packages/project/src/Workspace.ts +++ b/packages/project/src/Workspace.ts @@ -9,12 +9,19 @@ import { buildConfig, loadWorkspaceFile, findWorkspaceFile, + loadCheckoutFile, } from './util/config'; import fromProject from './parse/from-project'; import type { Logger } from '@openfn/logger'; import matchProject from './util/match-project'; import { extractAliasFromFilename } from './parse/from-path'; +export type WorkspaceOptions = { + // The git branch the workspace is on, which determines which checkout file + // to use. Pass false or null (or leave unset) to use the default checkout + branch?: string | false | null; +}; + export class Workspace { // @ts-ignore config not definitely assigned - it sure is config: l.WorkspaceConfig; @@ -24,6 +31,8 @@ export class Workspace { root: string; + branch?: string | false | null; + private projects: Project[] = []; private projectPaths = new Map(); private isValid: boolean = false; @@ -31,8 +40,14 @@ export class Workspace { // Set validate to false to suppress warnings if a Workspace doesn't exist // This is appropriate if, say, fetching a project for the first time - constructor(workspacePath: string, logger?: Logger, validate = true) { + constructor( + workspacePath: string, + logger?: Logger, + validate = true, + options: WorkspaceOptions = {} + ) { this.root = workspacePath; + this.branch = options.branch; this.logger = logger ?? createLogger('Workspace', { level: 'info' }); let context = { workspace: undefined, project: undefined }; @@ -55,7 +70,11 @@ export class Workspace { // it's not super reliable // Actually would it not be better to find the ACTUAL project and just // reference that? - this.activeProject = context.project; + this.activeProject = loadCheckoutFile( + workspacePath, + this.branch, + context.project + ); const projectsPath = path.join(workspacePath, this.config.dirs.projects); // dealing with projects @@ -134,6 +153,7 @@ export class Workspace { return await Project.from('fs', { root: this.root, config: this.config, + branch: this.branch, // The checked out project can't meaningfully be said to have an alias // But we can force one if it makes sense from context alias: alias ?? null, diff --git a/packages/project/src/parse/from-fs.ts b/packages/project/src/parse/from-fs.ts index 5db753e3e..e3ee84c34 100644 --- a/packages/project/src/parse/from-fs.ts +++ b/packages/project/src/parse/from-fs.ts @@ -9,6 +9,7 @@ import { buildConfig, loadWorkspaceFile, findWorkspaceFile, + loadCheckoutFile, } from '../util/config'; import { omit } from 'lodash-es'; import { Logger } from '@openfn/logger'; @@ -21,6 +22,8 @@ export type FromFsConfig = { logger?: Logger; alias?: string | null; name?: string; + // git branch, used to locate the checkout file + branch?: string | false | null; }; // Parse a single project from a root folder @@ -33,11 +36,15 @@ export const parseProject = async (options: FromFsConfig) => { const { type, content } = findWorkspaceFile(root); const context = loadWorkspaceFile(content, type as any); const config = buildConfig(options.config ?? context.workspace); + const checkout = + loadCheckoutFile(root, options.branch, context.project) ?? {}; const proj: any = { id: options.name ? slugify(options.name) : context.project?.id, name: options.name ? slugify(options.name) : context.project?.name, openfn: omit(context.project, ['id', 'forked_from', 'collections']), + + // TOOD where do collections live now? // openfn.yaml only ever holds bare collection names - no uuids, those // belong to the server collections: context.project.collections?.map((name: string) => ({ @@ -46,7 +53,7 @@ export const parseProject = async (options: FromFsConfig) => { config: config, workflows: [], cli: omitNil({ - forked_from: context.project.forked_from, + forked_from: checkout.forked_from, }), }; diff --git a/packages/project/src/serialize/to-fs.ts b/packages/project/src/serialize/to-fs.ts index 220111b05..ae99912f6 100644 --- a/packages/project/src/serialize/to-fs.ts +++ b/packages/project/src/serialize/to-fs.ts @@ -3,20 +3,25 @@ import { omit } from 'lodash-es'; import { Project } from '../Project'; import { jsonToYaml } from '../util/yaml'; -import { extractConfig } from '../util/config'; +import { extractCheckout, extractConfig } from '../util/config'; const stringify = (json: any) => JSON.stringify(json, null, 2); type ToFsOptions = { // private: stamp schema_version on each workflow file. off by default includeSchemaVersion?: boolean; + // git branch, used to locate the checkout file + branch?: string | false | null; }; export default function (project: Project, options: ToFsOptions = {}) { const files: Record = {}; - const { path, content } = extractConfig(project); - files[path] = content; + const config = extractConfig(project); + files[config.path] = config.content; + + const checkout = extractCheckout(project, options.branch); + files[checkout.path] = checkout.content; for (const wf of project.workflows) { const { path, content } = extractWorkflow(project, wf.id, options); diff --git a/packages/project/src/util/config.ts b/packages/project/src/util/config.ts index bef3e625d..f2f2129c4 100644 --- a/packages/project/src/util/config.ts +++ b/packages/project/src/util/config.ts @@ -22,8 +22,19 @@ export const buildConfig = (config: Partial = {}) => ({ }, }); -// Generate a workspace config (openfn.yaml) file for a project -export const extractConfig = (source: Project, format?: 'yaml' | 'json') => { +// The checkout file tracks which project is expanded into the workflows dir, +// plus any transient sync state. When on a git branch, each branch gets its +// own checkout file so that merges don't clobber the target branch's checkout +export const getCheckoutPath = (branch?: string | false | null) => + branch + ? path.join('.openfn', 'branches', branch, 'checkout.yaml') + : path.join('.openfn', 'checkout.yaml'); + +// Generate the checkout file for a project +export const extractCheckout = ( + source: Project, + branch?: string | false | null +) => { const project: any = { ...(source.openfn || {}), id: source.id, @@ -42,11 +53,40 @@ export const extractConfig = (source: Project, format?: 'yaml' | 'json') => { project.collections = source.collections.map((c) => c.name); } + return { + path: getCheckoutPath(branch), + content: jsonToYaml(project), + }; +}; + +// Load project metadata from the checkout file +// If there's no checkout file, fall back to the legacy project block +// in openfn.yaml (which will be migrated on the next write) +export const loadCheckoutFile = ( + root: string = '.', + branch?: string | false | null, + legacyProject?: l.ProjectMeta +): l.ProjectMeta | undefined => { + try { + const content = readFileSync( + path.resolve(root, getCheckoutPath(branch)), + 'utf8' + ); + return (yamlToJson(content) as l.ProjectMeta) ?? {}; + } catch (e) { + if (legacyProject && Object.keys(legacyProject).length) { + return legacyProject; + } + } +}; + +// Generate a workspace config (openfn.yaml) file for a project +export const extractConfig = (source: Project, format?: 'yaml' | 'json') => { const workspace = { ...source.config, }; - const content = { project, workspace }; + const content = { workspace }; format = format ?? workspace.formats.openfn; if (format === 'yaml') { From 31edd5a07ccbdd4ccbd501ab573a4e9336b3daa6 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 30 Sep 2026 15:30:28 +0100 Subject: [PATCH 13/22] order keys --- packages/project/src/util/config.ts | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/packages/project/src/util/config.ts b/packages/project/src/util/config.ts index f2f2129c4..36c938f3d 100644 --- a/packages/project/src/util/config.ts +++ b/packages/project/src/util/config.ts @@ -5,6 +5,22 @@ import { pickBy, isNil } from 'lodash-es'; import { yamlToJson, jsonToYaml } from './yaml'; import Project from '../Project'; +// Recursively sort object keys so that config files serialize stably +const sortKeys = (value: any): any => { + if (Array.isArray(value)) { + return value.map(sortKeys); + } + if (value && typeof value === 'object') { + return Object.keys(value) + .sort() + .reduce((obj: any, key) => { + obj[key] = sortKeys(value[key]); + return obj; + }, {}); + } + return value; +}; + // Initialize and default Workspace (and Project) config export const buildConfig = (config: Partial = {}) => ({ @@ -55,7 +71,7 @@ export const extractCheckout = ( return { path: getCheckoutPath(branch), - content: jsonToYaml(project), + content: jsonToYaml(sortKeys(project)), }; }; @@ -86,18 +102,16 @@ export const extractConfig = (source: Project, format?: 'yaml' | 'json') => { ...source.config, }; - const content = { workspace }; - format = format ?? workspace.formats.openfn; if (format === 'yaml') { return { path: 'openfn.yaml', - content: jsonToYaml(content), + content: jsonToYaml(sortKeys(workspace)), }; } return { path: 'openfn.json', - content: JSON.stringify(content, null, 2), + content: JSON.stringify(sortKeys(workspace), null, 2), }; }; @@ -113,6 +127,8 @@ export const loadWorkspaceFile = ( json = JSON.parse(contents); } + // Flat format: top level keys are workspace config (plus a legacy project block) + // Nested format: { workspace, project } const legacy = !json.workspace && !json.projects; if (legacy) { project = json.project ?? {}; From e09d49d2429668ef5fbcc5a6e2feb6e0a5c86941 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 30 Sep 2026 16:06:35 +0100 Subject: [PATCH 14/22] fix ropey merge issues --- packages/cli/src/projects/deploy.ts | 23 ++++++++++++++--------- packages/project/src/parse/from-fs.ts | 9 ++++----- packages/project/src/util/config.ts | 23 ++++++++++++++--------- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/packages/cli/src/projects/deploy.ts b/packages/cli/src/projects/deploy.ts index dca066869..4393ef8ad 100644 --- a/packages/cli/src/projects/deploy.ts +++ b/packages/cli/src/projects/deploy.ts @@ -331,12 +331,6 @@ export async function handler(options: DeployOptions, logger: Logger) { let filePath: string | undefined; let targetIdentifier: string | undefined; - // TODO this is the hard way to load the local alias - // We need track alias in openfn.yaml to make this easier (and tracked in from fs) - let ws = new Workspace(options.workspace || '.', undefined, true, { - branch: options.branch, - }); - if (options.project && options.target) { // two positionals: `deploy ` - the first is always a file filePath = options.project; @@ -353,6 +347,7 @@ export async function handler(options: DeployOptions, logger: Logger) { // The local project that we want to actually deploy let localProject: Project; + let ws: Workspace | undefined; let alias = options.alias; if (filePath) { @@ -384,7 +379,9 @@ export async function handler(options: DeployOptions, logger: Logger) { logger.debug('Reading checked-out project from workspace'); // TODO this is the hard way to load the local alias // We need track alias in openfn.yaml to make this easier (and tracked in from fs) - ws = new Workspace(options.workspace || '.'); + ws = new Workspace(options.workspace || '.', undefined, true, { + branch: options.branch, + }); const active = ws.getTrackedProject(); @@ -417,7 +414,9 @@ export async function handler(options: DeployOptions, logger: Logger) { localProject.alias = alias ?? null; } } else { - ws ??= new Workspace(options.workspace || '.'); + ws ??= new Workspace(options.workspace || '.', undefined, true, { + branch: options.branch, + }); tracker = ws.get(targetIdentifier ?? localProject.uuid!); // A project loaded from a file already knows which remote it belongs @@ -597,8 +596,14 @@ export async function handler(options: DeployOptions, logger: Logger) { ); updateForkedFrom(finalProject); + // Write the updated openfn.yaml and checkout file + // TODO: allow us to suppress writing this stuff + // (useful if posting from spec) for (const configData of finalProject.generateConfig(options.branch)) { - const configPath = path.resolve(options.workspace!, configData.path); + const configPath = path.resolve( + options.workspace ?? process.cwd(), + configData.path + ); await mkdir(path.dirname(configPath), { recursive: true }); await writeFile(configPath, configData.content); } diff --git a/packages/project/src/parse/from-fs.ts b/packages/project/src/parse/from-fs.ts index e3ee84c34..6f04d0c37 100644 --- a/packages/project/src/parse/from-fs.ts +++ b/packages/project/src/parse/from-fs.ts @@ -40,14 +40,13 @@ export const parseProject = async (options: FromFsConfig) => { loadCheckoutFile(root, options.branch, context.project) ?? {}; const proj: any = { - id: options.name ? slugify(options.name) : context.project?.id, - name: options.name ? slugify(options.name) : context.project?.name, - openfn: omit(context.project, ['id', 'forked_from', 'collections']), + id: options.name ? slugify(options.name) : checkout.id, + name: options.name ? slugify(options.name) : checkout.name, + openfn: omit(checkout, ['id', 'forked_from', 'collections']), - // TOOD where do collections live now? // openfn.yaml only ever holds bare collection names - no uuids, those // belong to the server - collections: context.project.collections?.map((name: string) => ({ + collections: context.collections?.map((name: string) => ({ name, })), config: config, diff --git a/packages/project/src/util/config.ts b/packages/project/src/util/config.ts index 36c938f3d..8a659936c 100644 --- a/packages/project/src/util/config.ts +++ b/packages/project/src/util/config.ts @@ -63,12 +63,6 @@ export const extractCheckout = ( project.forked_from = source.cli.forked_from; } - if (source.collections?.length) { - // openfn.yaml only ever carries collection names - no ids/uuids, those - // belong to the server. - project.collections = source.collections.map((c) => c.name); - } - return { path: getCheckoutPath(branch), content: jsonToYaml(sortKeys(project)), @@ -98,10 +92,16 @@ export const loadCheckoutFile = ( // Generate a workspace config (openfn.yaml) file for a project export const extractConfig = (source: Project, format?: 'yaml' | 'json') => { - const workspace = { + const workspace: any = { ...source.config, }; + if (source.collections?.length) { + // openfn.yaml only ever carries collection names - no ids/uuids, those + // belong to the server + workspace.collections = source.collections.map((c) => c.name); + } + format = format ?? workspace.formats.openfn; if (format === 'yaml') { return { @@ -119,7 +119,7 @@ export const loadWorkspaceFile = ( contents: string | l.WorkspaceFile | l.WorkspaceFileLegacy, format: 'yaml' | 'json' = 'yaml' ) => { - let project, workspace; + let project, workspace, collections: string[] | undefined; let json: any = contents; if (format === 'yaml') { json = yamlToJson(contents as any) ?? {}; @@ -142,9 +142,13 @@ export const loadWorkspaceFile = ( dirs, project: _ /* ignore!*/, name, + collections: flatCollections, ...rest } = json; + // Collections live at the top level, but may be in a legacy project block + collections = flatCollections ?? project.collections; + workspace = pickBy( { ...rest, @@ -156,9 +160,10 @@ export const loadWorkspaceFile = ( } else { project = json.project ?? {}; workspace = json.workspace ?? {}; + collections = project.collections; } - return { project, workspace }; + return { project, workspace, collections }; }; export const findWorkspaceFile = (dir: string = '.') => { From b808a754fcd5633cb02bea34ca69a72b588eac10 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 30 Sep 2026 16:33:53 +0100 Subject: [PATCH 15/22] add logic to checkout on git without tracking --- packages/cli/src/projects/checkout.ts | 38 +++++++++++++++++++++++++++ packages/cli/src/projects/options.ts | 10 +++++++ 2 files changed, 48 insertions(+) diff --git a/packages/cli/src/projects/checkout.ts b/packages/cli/src/projects/checkout.ts index 181d6145e..060559213 100644 --- a/packages/cli/src/projects/checkout.ts +++ b/packages/cli/src/projects/checkout.ts @@ -1,5 +1,6 @@ import yargs from 'yargs'; import Project, { Workspace } from '@openfn/project'; +import type l from '@openfn/lexicon'; import path from 'path'; import fs from 'fs'; import { rimraf } from 'rimraf'; @@ -24,12 +25,14 @@ export type CheckoutOptions = Pick< | 'force' | 'createCredentials' | 'branch' + | 'track' >; const options = [ o.log, po.workspace, po.branch, + po.track, po.clean, o.force, po.creds, @@ -119,6 +122,26 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => { } // Check whether the checked out project has diverged from its forked from versions + // On a git branch, the checkout file binds the branch to a project. + // Checking out some other project is an ad-hoc checkout: expand its files, + // but keep the branch tracking the original project + const tracked = workspace.activeProject; + const isAdHoc = Boolean( + options.branch && + !options.track && + tracked && + !isSameProject(tracked, switchProject) + ); + if (isAdHoc) { + const trackedName = tracked!.alias ?? tracked!.id ?? tracked!.uuid; + logger?.warn( + `Branch ${options.branch} tracks project ${trackedName}. Expanding files from ${switchProject.alias} without changing tracking: deploy will still target ${trackedName}` + ); + logger?.warn( + `Pass --track to make ${options.branch} track ${switchProject.alias} instead` + ); + } + // delete workflow dir before expanding project if (options.clean) { await rimraf(workspace.workflowsPath); @@ -139,6 +162,14 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => { const files: any = switchProject.serialize('fs', { branch: options.branch, }); + if (isAdHoc) { + // Don't touch the workspace or checkout metadata + for (const { path: configPath } of switchProject.generateConfig( + options.branch + )) { + delete files[configPath]; + } + } for (const f in files) { if (files[f]) { fs.mkdirSync(path.join(workspacePath, path.dirname(f)), { @@ -156,6 +187,13 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => { logger?.success(`Expanded project to ${workspacePath}`); }; +const isSameProject = (tracked: l.ProjectMeta, project: Project) => { + if (tracked.uuid && project.openfn?.uuid) { + return tracked.uuid === project.openfn.uuid; + } + return tracked.id === project.id; +}; + // This function will tell us if the active/checked out project // has any changes compared to the tracked state file // It implies that changes will be lost on checkout diff --git a/packages/cli/src/projects/options.ts b/packages/cli/src/projects/options.ts index 45652534e..960273231 100644 --- a/packages/cli/src/projects/options.ts +++ b/packages/cli/src/projects/options.ts @@ -22,6 +22,7 @@ export type Opts = BaseOpts & { clean?: boolean; createCredentials?: boolean; branch?: string | false; + track?: boolean; }; // project specific options @@ -50,6 +51,15 @@ export const clean: CLIOption = { }, }; +export const track: CLIOption = { + name: 'track', + yargs: { + boolean: true, + description: + 'When on a git branch, make this branch track the checked out project (by default, checking out a different project only expands its files)', + }, +}; + export const creds: CLIOption = { name: 'create-credentials', yargs: { From 0d6e70794c13dfe5e946a6a8f41537504f339f74 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 30 Sep 2026 16:57:42 +0100 Subject: [PATCH 16/22] Update tests for checkout file and flat openfn.yaml Project metadata now lives in .openfn/checkout.yaml, openfn.yaml holds workspace config (plus collections) as flat, sorted keys. Integration tests pin OPENFN_BRANCH=false so they don't pick up the kit repo's branch. Co-Authored-By: Claude Sonnet 5.5 --- integration-tests/cli/src/run.ts | 2 + integration-tests/cli/test/deploy.test.ts | 5 +- integration-tests/cli/test/deploy.v2.test.ts | 18 ++- integration-tests/cli/test/sync.test.ts | 18 +-- packages/cli/test/projects/checkout.test.ts | 68 ++++++----- packages/cli/test/projects/deploy.test.ts | 2 +- packages/project/test/serialize/to-fs.test.ts | 53 ++++----- packages/project/test/util/config.test.ts | 110 +++++++++++------- 8 files changed, 161 insertions(+), 115 deletions(-) diff --git a/integration-tests/cli/src/run.ts b/integration-tests/cli/src/run.ts index fd00339e0..9735d1305 100644 --- a/integration-tests/cli/src/run.ts +++ b/integration-tests/cli/src/run.ts @@ -7,6 +7,8 @@ const getOptions = () => ({ env: { ...process.env, OPENFN_REPO_DIR: path.resolve('repo'), + // workspaces live inside the kit git repo: don't pick up its branch + OPENFN_BRANCH: 'false', }, }); diff --git a/integration-tests/cli/test/deploy.test.ts b/integration-tests/cli/test/deploy.test.ts index 302d02a68..7110cfde9 100644 --- a/integration-tests/cli/test/deploy.test.ts +++ b/integration-tests/cli/test/deploy.test.ts @@ -273,7 +273,10 @@ test.serial('redirect to v2 protocol if openfn.yaml is present', async (t) => { t.falsy(bootstrap.stderr); assertLog(t, extractLogs(bootstrap.stdout), /Detected openfn.yaml file/i); - const yaml = await fs.readFile(path.join(tmpDir, 'openfn.yaml'), 'utf8'); + const yaml = await fs.readFile( + path.join(tmpDir, '.openfn/checkout.yaml'), + 'utf8' + ); t.regex(yaml, new RegExp(`uuid\\: ${projectId}`)); const workflowYaml = await fs.readFile( diff --git a/integration-tests/cli/test/deploy.v2.test.ts b/integration-tests/cli/test/deploy.v2.test.ts index e7995e1d8..4876bdd1e 100644 --- a/integration-tests/cli/test/deploy.v2.test.ts +++ b/integration-tests/cli/test/deploy.v2.test.ts @@ -40,7 +40,10 @@ test.serial('pull a project', async (t) => { assertLog(t, extractLogs(stdout), /Checked out project locally/i); - const yaml = await fs.readFile(path.resolve(tmpDir, 'openfn.yaml'), 'utf8'); + const yaml = await fs.readFile( + path.resolve(tmpDir, '.openfn/checkout.yaml'), + 'utf8' + ); t.regex(yaml, /uuid\: a/); t.regex(yaml, /id\: test-project/); }); @@ -56,7 +59,10 @@ test.serial('pull, change and re-deploy', async (t) => { t.falsy(pullResult.stderr); assertLog(t, extractLogs(pullResult.stdout), /Checked out project locally/i); - const yaml = await fs.readFile(path.resolve(tmpDir, 'openfn.yaml'), 'utf8'); + const yaml = await fs.readFile( + path.resolve(tmpDir, '.openfn/checkout.yaml'), + 'utf8' + ); t.regex(yaml, /id\: test-project/); // modify expression to trigger a change @@ -540,10 +546,10 @@ test.serial( const openfnPath = path.resolve(tmpDir, 'openfn.yaml'); const before: any = yamlToJson(await fs.readFile(openfnPath, 'utf8')); - t.falsy(before.project.collections); + t.falsy(before.collections); // add a collection by hand - no workflow files are touched - before.project.collections = ['my-collection']; + before.collections = ['my-collection']; await fs.writeFile(openfnPath, jsonToYaml(before)); const { stdout, stderr } = await run( @@ -646,10 +652,10 @@ test.serial( const openfnPath = path.resolve(tmpDir, 'openfn.yaml'); const before: any = yamlToJson(await fs.readFile(openfnPath, 'utf8')); - t.deepEqual(before.project.collections, ['my-collection']); + t.deepEqual(before.collections, ['my-collection']); // remove the collection by hand - no workflow files are touched - before.project.collections = []; + before.collections = []; await fs.writeFile(openfnPath, jsonToYaml(before)); const { stdout, stderr } = await run( diff --git a/integration-tests/cli/test/sync.test.ts b/integration-tests/cli/test/sync.test.ts index 38179a93e..8291a6a64 100644 --- a/integration-tests/cli/test/sync.test.ts +++ b/integration-tests/cli/test/sync.test.ts @@ -160,9 +160,9 @@ test('pull a new project', async (t) => { t.regex(proj_yaml, /id: patients/); t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`)); - const openfn_yaml = read('openfn.yaml'); - t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`)); - t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`)); + const checkout_yaml = read('.openfn/checkout.yaml'); + t.regex(checkout_yaml, new RegExp(`uuid: ${project.openfn.uuid}`)); + t.regex(checkout_yaml, new RegExp(`endpoint: ${endpoint}`)); const job = read('workflows/workflow/job.js'); t.is(job, 'fn()'); @@ -188,9 +188,9 @@ test('pull a new project with an alias', async (t) => { t.regex(proj_yaml, /id: patients/); t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`)); - const openfn_yaml = read('openfn.yaml'); - t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`)); - t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`)); + const checkout_yaml = read('.openfn/checkout.yaml'); + t.regex(checkout_yaml, new RegExp(`uuid: ${project.openfn.uuid}`)); + t.regex(checkout_yaml, new RegExp(`endpoint: ${endpoint}`)); const job = read('workflows/workflow/job.js'); t.is(job, 'fn()'); @@ -231,9 +231,9 @@ test('pull an update to project', async (t) => { t.regex(proj_yaml, /fn\(x\)/); t.regex(proj_yaml, new RegExp(`uuid: ${project.openfn.uuid}`)); - const openfn_yaml = read('openfn.yaml'); - t.regex(openfn_yaml, new RegExp(`uuid: ${project.openfn.uuid}`)); - t.regex(openfn_yaml, new RegExp(`endpoint: ${endpoint}`)); + const checkout_yaml = read('.openfn/checkout.yaml'); + t.regex(checkout_yaml, new RegExp(`uuid: ${project.openfn.uuid}`)); + t.regex(checkout_yaml, new RegExp(`endpoint: ${endpoint}`)); const job_updated = read('workflows/workflow/job.js'); t.is(job_updated, 'fn()'); diff --git a/packages/cli/test/projects/checkout.test.ts b/packages/cli/test/projects/checkout.test.ts index 2d838f22d..a7166b664 100644 --- a/packages/cli/test/projects/checkout.test.ts +++ b/packages/cli/test/projects/checkout.test.ts @@ -233,8 +233,10 @@ test.serial( logger ); - const openfn = yamlToJson(fs.readFileSync('/ws/openfn.yaml', 'utf8')); - t.deepEqual(openfn.project.forked_from, { + const checkout = yamlToJson( + fs.readFileSync('/ws/.openfn/checkout.yaml', 'utf8') + ); + t.deepEqual(checkout.forked_from, { 'simple-workflow-main': 'a', 'another-workflow-main': 'b', }); @@ -451,19 +453,24 @@ test.serial('respect openfn.yaml settings', async (t) => { const yaml = fs.readFileSync('/ws1/openfn.yaml', 'utf8'); t.is( yaml, - `project: - uuid: - id: staging - name: Staging -workspace: - credentials: credentials.yaml - dirs: - projects: p - workflows: w - formats: - openfn: yaml - project: json - workflow: json + `credentials: credentials.yaml +dirs: + projects: p + workflows: w +formats: + openfn: yaml + project: json + workflow: json +` + ); + + // project metadata is written to the checkout file + const checkout = fs.readFileSync('/ws1/.openfn/checkout.yaml', 'utf8'); + t.is( + checkout, + `id: staging +name: Staging +uuid: ` ); @@ -756,10 +763,7 @@ test.serial( const openfn: any = yamlToJson(fs.readFileSync('/ws6/openfn.yaml', 'utf8')); // ids should never be written to disk - only bare names - t.deepEqual(openfn.project.collections, [ - 'my-collection', - 'another-collection', - ]); + t.deepEqual(openfn.collections, ['my-collection', 'another-collection']); } ); @@ -913,8 +917,10 @@ test.serial( ); // assert that staging was checked out ok - let openfn = yamlToJson(fs.readFileSync('/tmp/openfn.yaml', 'utf8')); - t.is(openfn.project.id, 'foo'); + let openfn = yamlToJson( + fs.readFileSync('/tmp/.openfn/checkout.yaml', 'utf8') + ); + t.is(openfn.id, 'foo'); let expression = fs.readFileSync('/tmp/workflows/a/aaa.js', 'utf8'); t.is(expression, '// abc'); @@ -931,8 +937,8 @@ test.serial( logger._reset(); // assert that main was checked out ok - openfn = yamlToJson(fs.readFileSync('/tmp/openfn.yaml', 'utf8')); - t.is(openfn.project.id, 'bar'); + openfn = yamlToJson(fs.readFileSync('/tmp/.openfn/checkout.yaml', 'utf8')); + t.is(openfn.id, 'bar'); expression = fs.readFileSync('/tmp/workflows/a/aaa.js', 'utf8'); t.is(expression, '// 2'); @@ -960,8 +966,10 @@ test.serial( logger._reset(); // assert that main was checked out ok - let openfn = yamlToJson(fs.readFileSync('/tmp/openfn.yaml', 'utf8')); - t.is(openfn.project.id, 'bar'); + let openfn = yamlToJson( + fs.readFileSync('/tmp/.openfn/checkout.yaml', 'utf8') + ); + t.is(openfn.id, 'bar'); let expression = fs.readFileSync('/tmp/workflows/a/aaa.js', 'utf8'); t.is(expression, '// 2'); @@ -977,8 +985,8 @@ test.serial( ); // assert that staging was checked out ok - openfn = yamlToJson(fs.readFileSync('/tmp/openfn.yaml', 'utf8')); - t.is(openfn.project.id, 'foo'); + openfn = yamlToJson(fs.readFileSync('/tmp/.openfn/checkout.yaml', 'utf8')); + t.is(openfn.id, 'foo'); expression = fs.readFileSync('/tmp/workflows/a/aaa.js', 'utf8'); t.is(expression, '// abc'); @@ -1005,8 +1013,10 @@ test.serial( logger._reset(); // assert that main was checked out ok - let openfn = yamlToJson(fs.readFileSync('/tmp/openfn.yaml', 'utf8')); - t.is(openfn.project.id, 'bar'); + let openfn = yamlToJson( + fs.readFileSync('/tmp/.openfn/checkout.yaml', 'utf8') + ); + t.is(openfn.id, 'bar'); // Now make a change - on checkout, this change will be lost (it is not saved anywhere) fs.writeFileSync('/tmp/workflows/a/aaa.js', 'foobar'); diff --git a/packages/cli/test/projects/deploy.test.ts b/packages/cli/test/projects/deploy.test.ts index 92e932a8e..cda7f2e07 100644 --- a/packages/cli/test/projects/deploy.test.ts +++ b/packages/cli/test/projects/deploy.test.ts @@ -567,7 +567,7 @@ test.serial( // user hand-edits openfn.yaml: keep one, drop one, add a new one - // no workflow files are touched const openfn: any = yamlToJson(fs.readFileSync('/ws/openfn.yaml', 'utf8')); - openfn.project.collections = ['keep-me', 'new-collection']; + openfn.collections = ['keep-me', 'new-collection']; await writeFile('/ws/openfn.yaml', jsonToYaml(openfn)); await deploy( diff --git a/packages/project/test/serialize/to-fs.test.ts b/packages/project/test/serialize/to-fs.test.ts index a84cf9bc1..1f3ac0ad3 100644 --- a/packages/project/test/serialize/to-fs.test.ts +++ b/packages/project/test/serialize/to-fs.test.ts @@ -1,6 +1,7 @@ import test from 'ava'; import { Project } from '../../src/Project'; import toFs, { extractWorkflow } from '../../src/serialize/to-fs'; +import { yamlToJson } from '../../src/util/yaml'; const step = { id: 'step', @@ -305,6 +306,7 @@ test('toFs: extract a project with 1 workflow and 1 step', (t) => { // Ensure that all the right files have been created t.deepEqual(Object.keys(files), [ 'openfn.json', + '.openfn/checkout.yaml', 'workflows/my-workflow/my-workflow.json', 'workflows/my-workflow/step.js', ]); @@ -313,15 +315,15 @@ test('toFs: extract a project with 1 workflow and 1 step', (t) => { // (this should be validated in more detail by each step) const config = JSON.parse(files['openfn.json']); t.deepEqual(config, { - workspace: { - credentials: 'credentials.yaml', - formats: { openfn: 'json', project: 'yaml', workflow: 'json' }, - dirs: { projects: '.projects', workflows: 'workflows' }, - }, - project: { - id: 'my-project', - name: 'My Project', - }, + credentials: 'credentials.yaml', + dirs: { projects: '.projects', workflows: 'workflows' }, + formats: { openfn: 'json', project: 'yaml', workflow: 'json' }, + }); + + const checkout = yamlToJson(files['.openfn/checkout.yaml']); + t.deepEqual(checkout, { + id: 'my-project', + name: 'My Project', }); const workflow = JSON.parse(files['workflows/my-workflow/my-workflow.json']); @@ -358,6 +360,7 @@ test('toFs: extract a project with forked_from meta', (t) => { // Ensure that all the right files have been created t.deepEqual(Object.keys(files), [ 'openfn.json', + '.openfn/checkout.yaml', 'workflows/my-workflow/my-workflow.json', 'workflows/my-workflow/step.js', ]); @@ -366,16 +369,16 @@ test('toFs: extract a project with forked_from meta', (t) => { // (this should be validated in more detail by each step) const config = JSON.parse(files['openfn.json']); t.deepEqual(config, { - workspace: { - credentials: 'credentials.yaml', - formats: { openfn: 'json', project: 'yaml', workflow: 'json' }, - dirs: { projects: '.projects', workflows: 'workflows' }, - }, - project: { - id: 'my-project', - name: 'My Project', - forked_from: 'abcd', - }, + credentials: 'credentials.yaml', + dirs: { projects: '.projects', workflows: 'workflows' }, + formats: { openfn: 'json', project: 'yaml', workflow: 'json' }, + }); + + const checkout = yamlToJson(files['.openfn/checkout.yaml']); + t.deepEqual(checkout, { + id: 'my-project', + name: 'My Project', + forked_from: 'abcd', }); const workflow = JSON.parse(files['workflows/my-workflow/my-workflow.json']); @@ -408,10 +411,7 @@ test('toFs: writes collection names into openfn.json (freshly authored, no uuid const files = toFs(project); const config = JSON.parse(files['openfn.json']); - t.deepEqual(config.project.collections, [ - 'my-collection', - 'another-collection', - ]); + t.deepEqual(config.collections, ['my-collection', 'another-collection']); }); test('toFs: strips uuids from fetched collections when writing to openfn.json', (t) => { @@ -442,10 +442,7 @@ test('toFs: strips uuids from fetched collections when writing to openfn.json', const files = toFs(project); const config = JSON.parse(files['openfn.json']); - t.deepEqual(config.project.collections, [ - 'my-collection', - 'another-collection', - ]); + t.deepEqual(config.collections, ['my-collection', 'another-collection']); }); test('toFs: omits collections key when there are none', (t) => { @@ -470,7 +467,7 @@ test('toFs: omits collections key when there are none', (t) => { const files = toFs(project); const config = JSON.parse(files['openfn.json']); - t.falsy(config.project.collections); + t.falsy(config.collections); }); // TODO we need many more tests on this, with options diff --git a/packages/project/test/util/config.test.ts b/packages/project/test/util/config.test.ts index 02cdb5ffd..c4d46f0c1 100644 --- a/packages/project/test/util/config.test.ts +++ b/packages/project/test/util/config.test.ts @@ -1,6 +1,8 @@ import test from 'ava'; import { + extractCheckout, extractConfig, + getCheckoutPath, findWorkspaceFile, loadWorkspaceFile, } from '../../src/util/config'; @@ -158,7 +160,7 @@ test('find openfn.json', (t) => { t.deepEqual(result.content, { x: 1 }); }); -test('generate openfn.yaml', (t) => { +test('generate openfn.yaml (workspace config only)', (t) => { const proj = new Project( { id: 'my-project', @@ -177,28 +179,65 @@ test('generate openfn.yaml', (t) => { } ); const result = extractConfig(proj); - t.is(result.path, 'openfn.yaml'), - t.deepEqual( - result.content, - `project: - uuid: 1234 - id: my-project - name: My Project - forked_from: abcd -workspace: - credentials: credentials.yaml - formats: - openfn: yaml - project: yaml - workflow: yaml - dirs: - projects: .projects - workflows: workflows + t.is(result.path, 'openfn.yaml'); + // keys are sorted alphabetically, and there's no project block + t.deepEqual( + result.content, + `credentials: credentials.yaml +dirs: + projects: .projects + workflows: workflows +formats: + openfn: yaml + project: yaml + workflow: yaml ` - ); + ); }); -test("exclude forked_from if it's not set", (t) => { +test('generate openfn.yaml with collection names', (t) => { + const proj = new Project({ + id: 'my-project', + collections: [{ uuid: 'remote-uuid', name: 'my-collection' }], + }); + const result = extractConfig(proj); + t.true(result.content.startsWith('collections:\n - my-collection\n')); +}); + +test('generate checkout file with forked_from', (t) => { + const proj = new Project({ + id: 'my-project', + name: 'My Project', + openfn: { + uuid: 1234, + }, + cli: { + forked_from: 'abcd', + }, + }); + const result = extractCheckout(proj); + t.is(result.path, '.openfn/checkout.yaml'); + t.deepEqual( + result.content, + `forked_from: abcd +id: my-project +name: My Project +uuid: 1234 +` + ); +}); + +test('getCheckoutPath: default and per-branch', (t) => { + t.is(getCheckoutPath(), '.openfn/checkout.yaml'); + t.is(getCheckoutPath(false), '.openfn/checkout.yaml'); + t.is(getCheckoutPath('dev'), '.openfn/branches/dev/checkout.yaml'); + t.is( + getCheckoutPath('feature/x'), + '.openfn/branches/feature/x/checkout.yaml' + ); +}); + +test("checkout file excludes forked_from if it's not set", (t) => { const proj = new Project( { id: 'my-project', @@ -214,25 +253,15 @@ test("exclude forked_from if it's not set", (t) => { }, } ); - const result = extractConfig(proj); - t.is(result.path, 'openfn.yaml'), - t.deepEqual( - result.content, - `project: - uuid: 1234 - id: my-project - name: My Project -workspace: - credentials: credentials.yaml - formats: - openfn: yaml - project: yaml - workflow: yaml - dirs: - projects: .projects - workflows: workflows + const result = extractCheckout(proj, 'dev'); + t.is(result.path, '.openfn/branches/dev/checkout.yaml'); + t.deepEqual( + result.content, + `id: my-project +name: My Project +uuid: 1234 ` - ); + ); }); test.todo('generate openfn.json'); @@ -242,9 +271,8 @@ test('include project name', (t) => { id: 'my-project', name: 'My Project', }); - const result = extractConfig(proj, 'json'); - const json = JSON.parse(result.content); - t.is(json.project.name, 'My Project'); + const result = extractCheckout(proj); + t.true(result.content.includes('name: My Project')); }); test.todo('include parent project name'); From 3b7003703be1da5c9abc7243044505b85284e425 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 5 Oct 2026 15:10:43 +0100 Subject: [PATCH 17/22] format --- packages/project/src/Project.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/project/src/Project.ts b/packages/project/src/Project.ts index 740471af4..48d970767 100644 --- a/packages/project/src/Project.ts +++ b/packages/project/src/Project.ts @@ -12,11 +12,7 @@ import { getUuidForEdge, getUuidForStep } from './util/uuid'; import { merge, MergeProjectOptions } from './merge/merge-project'; import { diff as projectDiff } from './util/project-diff'; import { Workspace } from './Workspace'; -import { - buildConfig, - extractCheckout, - extractConfig, -} from './util/config'; +import { buildConfig, extractCheckout, extractConfig } from './util/config'; import { Provisioner } from '@openfn/lexicon/lightning'; import { SandboxMeta, From 747f79c4d1653c750f22a89250c78951f5a81210 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 7 Oct 2026 09:12:17 +0100 Subject: [PATCH 18/22] move collections to resources.yaml --- integration-tests/cli/.state.json | 53 +++++++++ integration-tests/cli/test/deploy.v2.test.ts | 26 ++--- packages/cli/src/projects/deploy.ts | 27 ++++- packages/cli/src/projects/diff.ts | 26 ++++- packages/cli/test/projects/checkout.test.ts | 15 ++- packages/cli/test/projects/deploy.test.ts | 104 ++++++++++++++++- packages/project/src/Project.ts | 12 ++ packages/project/src/index.ts | 3 + packages/project/src/merge/merge-project.ts | 13 ++- packages/project/src/parse/from-fs.ts | 25 ++-- packages/project/src/serialize/to-fs.ts | 13 +-- packages/project/src/util/config.ts | 8 +- packages/project/src/util/project-diff.ts | 33 +++++- packages/project/src/util/resources.ts | 56 +++++++++ .../project/test/merge/merge-project.test.ts | 24 +++- packages/project/test/parse/from-fs.test.ts | 81 ++++++++++++- packages/project/test/serialize/to-fs.test.ts | 108 +++++++++--------- packages/project/test/util/config.test.ts | 4 +- .../project/test/util/project-diff.test.ts | 42 +++++++ packages/project/test/util/resources.test.ts | 65 +++++++++++ 20 files changed, 623 insertions(+), 115 deletions(-) create mode 100644 integration-tests/cli/.state.json create mode 100644 packages/project/test/util/resources.test.ts diff --git a/integration-tests/cli/.state.json b/integration-tests/cli/.state.json new file mode 100644 index 000000000..b9880f4ba --- /dev/null +++ b/integration-tests/cli/.state.json @@ -0,0 +1,53 @@ +{ + "workflows": { + "my-workflow": { + "id": "702bf73f-fde1-44ca-87c5-c9e508ec2407", + "name": "My Workflow", + "jobs": { + "transform-data": { + "id": "43d6474d-df15-4ce6-a1a3-52e6400a36a9", + "name": "Transform data", + "adaptor": "@openfn/language-common@latest", + "body": "fn(s => s)", + "project_credential_id": "0f750fd9-b34c-4cf1-a6a9-a97cc18a2f4c" + } + }, + "triggers": { + "webhook": { + "id": "ca8358c9-b47b-4ad6-ba45-7a37384b81af", + "type": "webhook", + "enabled": true + } + }, + "edges": { + "webhook->transform-data": { + "id": "7026ceb7-f841-4b4b-a182-b2470384a1c2", + "condition_type": null, + "target_job_id": "43d6474d-df15-4ce6-a1a3-52e6400a36a9", + "enabled": true, + "source_trigger_id": "ca8358c9-b47b-4ad6-ba45-7a37384b81af" + } + }, + "lock_version": 1, + "inserted_at": "2026-10-05T14:49:28.664Z", + "updated_at": "2026-10-05T14:49:28.664Z", + "deleted_at": null, + "version_history": [ + "app:d9287d0b8d8c" + ] + } + }, + "id": "9351c38a-18ca-48ee-af7f-e4a479d92294", + "name": "My Project", + "project_credentials": { + "super@openfn.org|http1": { + "id": "0f750fd9-b34c-4cf1-a6a9-a97cc18a2f4c", + "name": "http1", + "owner": "super@openfn.org" + } + }, + "collections": {}, + "inserted_at": "2026-10-05T14:49:28.664Z", + "updated_at": "2026-10-05T14:49:28.664Z", + "channels": {} +} \ No newline at end of file diff --git a/integration-tests/cli/test/deploy.v2.test.ts b/integration-tests/cli/test/deploy.v2.test.ts index 4876bdd1e..7e5139dda 100644 --- a/integration-tests/cli/test/deploy.v2.test.ts +++ b/integration-tests/cli/test/deploy.v2.test.ts @@ -526,7 +526,7 @@ test.serial( ); test.serial( - 'deploy collections: add a collection via openfn.yaml', + 'deploy collections: add a collection via resources.yaml', async (t) => { const projectId = 'gggggggg'; server.addProject(makeProject(projectId) as any); @@ -544,13 +544,13 @@ test.serial( /Checked out project locally/i ); - const openfnPath = path.resolve(tmpDir, 'openfn.yaml'); - const before: any = yamlToJson(await fs.readFile(openfnPath, 'utf8')); - t.falsy(before.collections); + const resourcesPath = path.resolve(tmpDir, 'resources.yaml'); + const before: any = yamlToJson(await fs.readFile(resourcesPath, 'utf8')); + t.deepEqual(before.collections, {}); // add a collection by hand - no workflow files are touched - before.collections = ['my-collection']; - await fs.writeFile(openfnPath, jsonToYaml(before)); + before.collections = { 'my-collection': {} }; + await fs.writeFile(resourcesPath, jsonToYaml(before)); const { stdout, stderr } = await run( `openfn project deploy --no-confirm --log-json -l debug` @@ -629,7 +629,7 @@ workflows: }); test.serial( - 'deploy collections: remove a collection via openfn.yaml', + 'deploy collections: remove a collection via resources.yaml', async (t) => { const projectId = 'hhhhhhhh'; server.addProject({ @@ -639,7 +639,7 @@ test.serial( t.is(server.state.projects[projectId].collections.length, 1); - // pull the project - openfn.yaml should list the fetched collection + // pull the project - resources.yaml should list the fetched collection const pullResult = await run( `openfn project pull ${projectId} --log-json -l debug` ); @@ -650,13 +650,13 @@ test.serial( /Checked out project locally/i ); - const openfnPath = path.resolve(tmpDir, 'openfn.yaml'); - const before: any = yamlToJson(await fs.readFile(openfnPath, 'utf8')); - t.deepEqual(before.collections, ['my-collection']); + const resourcesPath = path.resolve(tmpDir, 'resources.yaml'); + const before: any = yamlToJson(await fs.readFile(resourcesPath, 'utf8')); + t.deepEqual(before.collections, { 'my-collection': {} }); // remove the collection by hand - no workflow files are touched - before.collections = []; - await fs.writeFile(openfnPath, jsonToYaml(before)); + before.collections = {}; + await fs.writeFile(resourcesPath, jsonToYaml(before)); const { stdout, stderr } = await run( `openfn project deploy --no-confirm --log-json -l debug` diff --git a/packages/cli/src/projects/deploy.ts b/packages/cli/src/projects/deploy.ts index 3e8965b44..ed1054f36 100644 --- a/packages/cli/src/projects/deploy.ts +++ b/packages/cli/src/projects/deploy.ts @@ -1,12 +1,13 @@ import yargs from 'yargs'; import Project, { MergeProjectOptions, + RESOURCES_FILE, toResourceChannels, versionsEqual, Workspace, } from '@openfn/project'; import { isEqual } from 'lodash-es'; -import { mkdir, writeFile } from 'node:fs/promises'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; import path from 'node:path'; import * as o from '../options'; @@ -152,7 +153,11 @@ export const hasRemoteDiverged = ( return diverged; }; +// A missing collections key (local.collections undefined) means collections +// aren't managed locally, so there's nothing to change export const collectionsChanged = (local: Project, remote: Project) => { + if (!local.collections) return false; + const names = (project: Project) => new Set((project.collections ?? []).map((c) => c.name)); @@ -164,11 +169,13 @@ export const collectionsChanged = (local: Project, remote: Project) => { }; // Collections dropped from the merged project (ie, removed from -// openfn.yaml) need an explicit delete entry in the deploy payload +// resources.yaml) need an explicit delete entry in the deploy payload export const deletedCollections = ( merged: Project, remote: Project ): Provisioner.Collection[] => { + if (!merged.collections) return []; + const keptNames = new Set((merged.collections ?? []).map((c) => c.name)); return (remote.collections ?? []) .filter((c) => !keptNames.has(c.name) && c.uuid) @@ -624,6 +631,22 @@ export async function handler(options: DeployOptions, logger: Logger) { await writeFile(configPath, configData.content); } + // Sync collections into resources.yaml (anything else in there is left alone) + const resourcesPath = path.resolve( + options.workspace ?? process.cwd(), + RESOURCES_FILE + ); + const existingResources = await readFile(resourcesPath, 'utf8').catch( + (e) => { + if (e.code !== 'ENOENT') throw e; + return undefined; + } + ); + const resources = finalProject.generateResources(existingResources); + if (resources.content) { + await writeFile(resourcesPath, resources.content); + } + // TODO if this was marked as new, we probably need to ensure a unique alias here const finalOutputPath = getSerializePath( finalProject, diff --git a/packages/cli/src/projects/diff.ts b/packages/cli/src/projects/diff.ts index 7b72eb144..af32f8d25 100644 --- a/packages/cli/src/projects/diff.ts +++ b/packages/cli/src/projects/diff.ts @@ -1,6 +1,6 @@ import c from 'chalk'; import Project, { generateStepDiff, generateEdgeDiff } from '@openfn/project'; -import type { ChannelDiff } from '@openfn/project'; +import type { ChannelDiff, CollectionDiff } from '@openfn/project'; import type { StepChange, EdgeChange } from '@openfn/project'; import type { Logger } from '../util/logger'; @@ -81,8 +81,18 @@ const printChannelDiff = (channels: ChannelDiff[], logger: Logger) => { logger.break(); }; -// TODO need to include collection diffs -// https://github.com/OpenFn/kit/issues/1524 +const printCollectionDiff = (collections: CollectionDiff[], logger: Logger) => { + logger.always('Collections:'); + for (const { name, type } of collections) { + if (type === 'added') { + logger.always(c.green(` ${name}: added`)); + } else { + logger.always(c.red(` ${name}: removed`)); + } + } + logger.break(); +}; + export const printRichDiff = ( local: Project, remote: Project, @@ -93,7 +103,11 @@ export const printRichDiff = ( local, locallyChangedWorkflows ); - if (diffs.length === 0 && resources.channels.length === 0) { + if ( + diffs.length === 0 && + resources.channels.length === 0 && + resources.collections.length === 0 + ) { logger.info('No workflow changes detected'); return diffs; } @@ -140,5 +154,9 @@ export const printRichDiff = ( printChannelDiff(resources.channels, logger); } + if (resources.collections.length > 0) { + printCollectionDiff(resources.collections, logger); + } + return diffs; }; diff --git a/packages/cli/test/projects/checkout.test.ts b/packages/cli/test/projects/checkout.test.ts index a7166b664..d4762ec1f 100644 --- a/packages/cli/test/projects/checkout.test.ts +++ b/packages/cli/test/projects/checkout.test.ts @@ -736,7 +736,7 @@ test.serial( ); test.serial( - 'checkout: writes fetched collections into openfn.yaml', + 'checkout: writes fetched collections into resources.yaml', async (t) => { mock({ '/ws6/workflows': {}, @@ -761,9 +761,18 @@ test.serial( logger ); + const resources: any = yamlToJson( + fs.readFileSync('/ws6/resources.yaml', 'utf8') + ); + // ids should never be written to disk - only names + t.deepEqual(resources.collections, { + 'my-collection': {}, + 'another-collection': {}, + }); + + // collections don't belong in openfn.yaml const openfn: any = yamlToJson(fs.readFileSync('/ws6/openfn.yaml', 'utf8')); - // ids should never be written to disk - only bare names - t.deepEqual(openfn.collections, ['my-collection', 'another-collection']); + t.falsy(openfn.collections); } ); diff --git a/packages/cli/test/projects/deploy.test.ts b/packages/cli/test/projects/deploy.test.ts index 5321d34ea..f07fc0eb2 100644 --- a/packages/cli/test/projects/deploy.test.ts +++ b/packages/cli/test/projects/deploy.test.ts @@ -619,11 +619,12 @@ test.serial( await setup(projectYaml); - // user hand-edits openfn.yaml: keep one, drop one, add a new one - + // user hand-edits resources.yaml: keep one, drop one, add a new one - // no workflow files are touched - const openfn: any = yamlToJson(fs.readFileSync('/ws/openfn.yaml', 'utf8')); - openfn.collections = ['keep-me', 'new-collection']; - await writeFile('/ws/openfn.yaml', jsonToYaml(openfn)); + await writeFile( + '/ws/resources.yaml', + jsonToYaml({ collections: { 'keep-me': {}, 'new-collection': {} } }) + ); await deploy( { @@ -635,6 +636,11 @@ test.serial( logger ); + // the diff should say what's being deleted and added + t.truthy(logger._find('always', /Collections:/)); + t.truthy(logger._find('always', /remove-me: removed/)); + t.truthy(logger._find('always', /new-collection: added/)); + // a collections-only edit must not be treated as "nothing to deploy" t.falsy(logger._find('success', /Nothing to deploy/)); t.truthy(logger._find('success', /Updated project at/)); @@ -649,6 +655,12 @@ test.serial( ); t.truthy(created?.id); t.falsy(created?.delete); + + // resources.yaml is kept in sync after the deploy + const resources: any = yamlToJson( + fs.readFileSync('/ws/resources.yaml', 'utf8') + ); + t.deepEqual(resources.collections, { 'keep-me': {}, 'new-collection': {} }); } ); @@ -1143,6 +1155,34 @@ test('collectionsChanged: true when a name was removed locally', (t) => { t.true(collectionsChanged(local, remote)); }); +test('collectionsChanged: false when collections are not managed locally', (t) => { + // undefined means "leave alone", unlike an empty list + const local = {} as unknown as Project; + const remote = { + collections: [{ uuid: 'uuid-a', name: 'a' }], + } as unknown as Project; + + t.false(collectionsChanged(local, remote)); +}); + +test('collectionsChanged: true when every collection was removed locally', (t) => { + const local = { collections: [] } as unknown as Project; + const remote = { + collections: [{ uuid: 'uuid-a', name: 'a' }], + } as unknown as Project; + + t.true(collectionsChanged(local, remote)); +}); + +test('deletedCollections: nothing to delete when collections are not managed locally', (t) => { + const merged = {} as unknown as Project; + const remote = { + collections: [{ uuid: 'uuid-a', name: 'a' }], + } as unknown as Project; + + t.deepEqual(deletedCollections(merged, remote), []); +}); + test('deletedCollections: flags a remote name missing from the merged project', (t) => { const merged = { collections: [{ name: 'keep-me' }], @@ -1248,6 +1288,62 @@ test('printRichDiff: should report removed channels', (t) => { t.truthy(logger._find('always', /one: removed/)); }); +test('printRichDiff: should report added and removed collections', (t) => { + const local = new Project({ + name: 'local', + workflows: [], + collections: [{ name: 'keep-me' }, { name: 'new-one' }], + }); + const remote = new Project({ + name: 'remote', + workflows: [], + collections: [ + { uuid: 'a', name: 'keep-me' }, + { uuid: 'b', name: 'old-one' }, + ], + }); + + printRichDiff(local, remote, [], logger); + + t.truthy(logger._find('always', /following changes to the remote project/)); + t.truthy(logger._find('always', /Collections:/)); + t.truthy(logger._find('always', /new-one: added/)); + t.truthy(logger._find('always', /old-one: removed/)); +}); + +test('printRichDiff: should show every collection as removed when all are deleted', (t) => { + const local = new Project({ name: 'local', workflows: [], collections: [] }); + const remote = new Project({ + name: 'remote', + workflows: [], + collections: [ + { uuid: 'a', name: 'one' }, + { uuid: 'b', name: 'two' }, + ], + }); + + printRichDiff(local, remote, [], logger); + + t.truthy(logger._find('always', /one: removed/)); + t.truthy(logger._find('always', /two: removed/)); +}); + +test('printRichDiff: should not report collections that are not managed locally', (t) => { + const local = new Project({ name: 'local', workflows: [] }); + const remote = new Project({ + name: 'remote', + workflows: [], + collections: [{ uuid: 'a', name: 'one' }], + }); + + // use a private logger: an absence check isn't safe on the shared one + const privateLogger = createMockLogger(undefined, { level: 'debug' }); + printRichDiff(local, remote, [], privateLogger); + + t.falsy(privateLogger._find('always', /Collections:/)); + t.truthy(privateLogger._find('info', /No workflow changes detected/)); +}); + test('printRichDiff: should list changed channel fields', (t) => { const local = new Project({ name: 'local', diff --git a/packages/project/src/Project.ts b/packages/project/src/Project.ts index 48d970767..9a76a95ba 100644 --- a/packages/project/src/Project.ts +++ b/packages/project/src/Project.ts @@ -13,6 +13,7 @@ import { merge, MergeProjectOptions } from './merge/merge-project'; import { diff as projectDiff } from './util/project-diff'; import { Workspace } from './Workspace'; import { buildConfig, extractCheckout, extractConfig } from './util/config'; +import { RESOURCES_FILE, updateResourceCollections } from './util/resources'; import { Provisioner } from '@openfn/lexicon/lightning'; import { SandboxMeta, @@ -311,6 +312,17 @@ export class Project { return [extractConfig(this), extractCheckout(this, branch)]; } + /** + * Generates the contents of resources.yaml with this project's collections + * merged into an existing file (other keys are left alone) + */ + generateResources(existing?: string) { + return { + path: RESOURCES_FILE, + content: updateResourceCollections(this, existing), + }; + } + clone() { return new Project(this.serialize('project') as any); } diff --git a/packages/project/src/index.ts b/packages/project/src/index.ts index 2898043c6..44b1d428a 100644 --- a/packages/project/src/index.ts +++ b/packages/project/src/index.ts @@ -12,10 +12,13 @@ export { diff } from './util/project-diff'; export type { WorkflowDiff, ChannelDiff, + CollectionDiff, ProjectDiff, DiffType, } from './util/project-diff'; +export { RESOURCES_FILE } from './util/resources'; + export { generateStepDiff, generateEdgeDiff } from './util/workflow-diff'; export type { StepChange, diff --git a/packages/project/src/merge/merge-project.ts b/packages/project/src/merge/merge-project.ts index 9d1186189..af6b5f770 100644 --- a/packages/project/src/merge/merge-project.ts +++ b/packages/project/src/merge/merge-project.ts @@ -202,11 +202,16 @@ export function merge( return merged; } +// If the source has no collections defined, they aren't managed locally and +// the target's are left alone. An empty list means remove them all export function mergeCollections( - source: CollectionState[] = [], - target: CollectionState[] = [] -): CollectionState[] { - const targetByName = new Map(target.map((c) => [c.name, c])); + source?: CollectionState[], + target?: CollectionState[] +): CollectionState[] | undefined { + if (!source) { + return target; + } + const targetByName = new Map((target ?? []).map((c) => [c.name, c])); return source.map(({ name }) => ({ name, uuid: targetByName.get(name)?.uuid, diff --git a/packages/project/src/parse/from-fs.ts b/packages/project/src/parse/from-fs.ts index ac40b2402..e104702f4 100644 --- a/packages/project/src/parse/from-fs.ts +++ b/packages/project/src/parse/from-fs.ts @@ -15,7 +15,11 @@ import { omit } from 'lodash-es'; import { Logger } from '@openfn/logger'; import omitNil from '../util/omit-nil'; import slugify from '../util/slugify'; -import { RESOURCES_FILE, fromResourceChannels } from '../util/resources'; +import { + RESOURCES_FILE, + fromResourceChannels, + fromResourceCollections, +} from '../util/resources'; export type FromFsConfig = { root: string; @@ -45,8 +49,8 @@ export const parseProject = async (options: FromFsConfig) => { name: options.name ? slugify(options.name) : checkout.name, openfn: omit(checkout, ['id', 'forked_from', 'collections']), - // openfn.yaml only ever holds bare collection names - no uuids, those - // belong to the server + // Legacy: collections used to be a list of names in openfn.yaml. A + // collections key in resources.yaml takes precedence (see below) collections: context.collections?.map((name: string) => ({ name, })), @@ -57,16 +61,21 @@ export const parseProject = async (options: FromFsConfig) => { }), }; - // resources.yaml is optional: if it's missing, or has no channels key, - // channels stay undefined and are left untouched on merge/deploy + // resources.yaml is optional: if it's missing, or has no channels or + // collections key, they stay undefined and are left untouched on + // merge/deploy const resources = await fs .readFile(path.resolve(root, RESOURCES_FILE), 'utf-8') .catch((e) => { if (e.code !== 'ENOENT') throw e; }); - const channels = resources && yamlToJson(resources)?.channels; - if (channels) { - proj.channels = fromResourceChannels(channels); + const parsed = resources ? yamlToJson(resources) : undefined; + if (parsed?.channels) { + proj.channels = fromResourceChannels(parsed.channels); + } + if (parsed && 'collections' in parsed) { + // an empty key means no collections (not the same as no key!) + proj.collections = fromResourceCollections(parsed.collections); } // now find all the workflows diff --git a/packages/project/src/serialize/to-fs.ts b/packages/project/src/serialize/to-fs.ts index 44258a147..6293ce131 100644 --- a/packages/project/src/serialize/to-fs.ts +++ b/packages/project/src/serialize/to-fs.ts @@ -4,7 +4,7 @@ import { omit } from 'lodash-es'; import { Project } from '../Project'; import { jsonToYaml } from '../util/yaml'; import { extractCheckout, extractConfig } from '../util/config'; -import { RESOURCES_FILE, toResourceChannels } from '../util/resources'; +import { RESOURCES_FILE, toResources } from '../util/resources'; const stringify = (json: any) => JSON.stringify(json, null, 2); @@ -24,12 +24,11 @@ export default function (project: Project, options: ToFsOptions = {}) { const checkout = extractCheckout(project, options.branch); files[checkout.path] = checkout.content; - // Only write resources.yaml if the project knows about channels. A missing - // file means channels are not managed locally - if (project.channels) { - files[RESOURCES_FILE] = jsonToYaml({ - channels: toResourceChannels(project.channels, project.credentials), - }); + // Only write resources.yaml if the project knows about channels or + // collections. A missing key means they are not managed locally + const resources = toResources(project); + if (resources) { + files[RESOURCES_FILE] = resources; } for (const wf of project.workflows) { diff --git a/packages/project/src/util/config.ts b/packages/project/src/util/config.ts index 8a659936c..a3225906f 100644 --- a/packages/project/src/util/config.ts +++ b/packages/project/src/util/config.ts @@ -92,16 +92,10 @@ export const loadCheckoutFile = ( // Generate a workspace config (openfn.yaml) file for a project export const extractConfig = (source: Project, format?: 'yaml' | 'json') => { - const workspace: any = { + const workspace = { ...source.config, }; - if (source.collections?.length) { - // openfn.yaml only ever carries collection names - no ids/uuids, those - // belong to the server - workspace.collections = source.collections.map((c) => c.name); - } - format = format ?? workspace.formats.openfn; if (format === 'yaml') { return { diff --git a/packages/project/src/util/project-diff.ts b/packages/project/src/util/project-diff.ts index de0d5927b..f942bfad3 100644 --- a/packages/project/src/util/project-diff.ts +++ b/packages/project/src/util/project-diff.ts @@ -17,10 +17,16 @@ export type ChannelDiff = { changes?: Record; }; +export type CollectionDiff = { + name: string; + type: Exclude; +}; + export type ProjectDiff = { workflows: WorkflowDiff[]; resources: { channels: ChannelDiff[]; + collections: CollectionDiff[]; }; }; @@ -34,7 +40,7 @@ export type ProjectDiff = { * @param b - The comparison project (e.g., staging branch) * Channels are compared by their resources.yaml id. If either project has no * channels defined (undefined) they are not managed, so no channel changes are - * reported. + * reported. Collections are compared by name, and follow the same rule. * * @returns An object with `workflows` and `resources` indicating how B differs * from A. Each workflow diff is: @@ -87,9 +93,32 @@ export function diff( } } - return { workflows: diffs, resources: { channels: diffChannels(a, b) } }; + return { + workflows: diffs, + resources: { + channels: diffChannels(a, b), + collections: diffCollections(a, b), + }, + }; } +const diffCollections = (a: Project, b: Project): CollectionDiff[] => { + if (!a.collections || !b.collections) { + return []; + } + + const namesA = new Set(a.collections.map((c) => c.name)); + const namesB = new Set(b.collections.map((c) => c.name)); + return [ + ...Array.from(namesA) + .filter((name) => !namesB.has(name)) + .map((name) => ({ name, type: 'removed' as const })), + ...Array.from(namesB) + .filter((name) => !namesA.has(name)) + .map((name) => ({ name, type: 'added' as const })), + ]; +}; + const diffChannels = (a: Project, b: Project): ChannelDiff[] => { if (!a.channels || !b.channels) { return []; diff --git a/packages/project/src/util/resources.ts b/packages/project/src/util/resources.ts index 0495cea8a..6f21905a3 100644 --- a/packages/project/src/util/resources.ts +++ b/packages/project/src/util/resources.ts @@ -1,6 +1,8 @@ import type l from '@openfn/lexicon'; import getCredentialName from './get-credential-name'; import slugify from './slugify'; +import { jsonToYaml, yamlToJson } from './yaml'; +import type { Project } from '../Project'; // resources.yaml holds server-side resources, keyed by type (channels, ...) export const RESOURCES_FILE = 'resources.yaml'; @@ -65,3 +67,57 @@ export const fromResourceChannels = ( ...(c.credential && { destination_credential_id: c.credential }), }; }); + +// Collections are keyed by name. The values are empty for now, but this +// leaves room for per-collection settings later (no uuids: those belong to +// the server) +type ResourceCollections = Record>; + +export const toResourceCollections = ( + collections: l.CollectionState[] = [] +): ResourceCollections => + collections.reduce((obj: ResourceCollections, c) => { + obj[c.name] = {}; + return obj; + }, {}); + +// Also accepts a plain list of names, which is how collections used to be +// written in openfn.yaml +export const fromResourceCollections = ( + collections: ResourceCollections | string[] | null | undefined +): l.CollectionState[] => + (Array.isArray(collections) + ? collections + : Object.keys(collections ?? {}) + ).map((name) => ({ name })); + +// Build the contents of resources.yaml. Channels and collections are each only +// written if the project knows about them: a missing key means they are not +// managed locally +export const toResources = (project: Project) => { + const resources: Record = {}; + if (project.channels) { + resources.channels = toResourceChannels( + project.channels, + project.credentials + ); + } + if (project.collections) { + resources.collections = toResourceCollections(project.collections); + } + return Object.keys(resources).length ? jsonToYaml(resources) : undefined; +}; + +// Merge a project's collections into an existing resources.yaml, leaving +// everything else in the file (like channels) alone +export const updateResourceCollections = ( + project: Project, + existing?: string +) => { + if (!project.collections) { + return existing; + } + const resources = (existing && yamlToJson(existing)) || {}; + resources.collections = toResourceCollections(project.collections); + return jsonToYaml(resources); +}; diff --git a/packages/project/test/merge/merge-project.test.ts b/packages/project/test/merge/merge-project.test.ts index 2733dfd16..1a92f3ce0 100644 --- a/packages/project/test/merge/merge-project.test.ts +++ b/packages/project/test/merge/merge-project.test.ts @@ -345,7 +345,7 @@ test('replace mode: merged collections keep target uuid on a name match', (t) => t.falsy(result.collections?.find((c) => c.name === 'remove-me')); }); -test('replace mode: no local collections means none survive the merge', (t) => { +test('replace mode: an empty local collections list means none survive the merge', (t) => { const wf = { steps: [ { id: 'x', name: 'X', adaptor: 'common', expression: 'fn(s => s)' }, @@ -357,13 +357,33 @@ test('replace mode: no local collections means none survive the merge', (t) => { const targetCollections = [{ uuid: 'remote-uuid-1', name: 'my-collection' }]; const target = createProject(wf_a, 'a', { collections: targetCollections }); - const source = createProject(wf_b, 'b'); + const source = createProject(wf_b, 'b', { collections: [] }); const result = merge(source, target, { mode: REPLACE_MERGE }); t.deepEqual(result.collections, []); }); +test('replace mode: undefined local collections leave the target collections alone', (t) => { + const wf = { + steps: [ + { id: 'x', name: 'X', adaptor: 'common', expression: 'fn(s => s)' }, + ], + }; + const wf_a = assignUUIDs(wf); + const wf_b = assignUUIDs(wf); + + const targetCollections = [{ uuid: 'remote-uuid-1', name: 'my-collection' }]; + + const target = createProject(wf_a, 'a', { collections: targetCollections }); + // no collections key means they're not managed locally + const source = createProject(wf_b, 'b'); + + const result = merge(source, target, { mode: REPLACE_MERGE }); + + t.deepEqual(result.collections, targetCollections); +}); + test('sandbox mode: source collections fully replace target collections', (t) => { const wf = { steps: [ diff --git a/packages/project/test/parse/from-fs.test.ts b/packages/project/test/parse/from-fs.test.ts index cea19d373..8cd1b3c8a 100644 --- a/packages/project/test/parse/from-fs.test.ts +++ b/packages/project/test/parse/from-fs.test.ts @@ -1,7 +1,7 @@ import test from 'ava'; import mock from 'mock-fs'; import { parseProject } from '../../src/parse/from-fs'; -import { jsonToYaml } from '../../src/util/yaml'; +import { jsonToYaml, yamlToJson } from '../../src/util/yaml'; import { buildConfig } from '../../src/util/config'; test.afterEach(() => { @@ -191,7 +191,29 @@ test.serial('should load the name from config', async (t) => { t.is(project.id, 'peaches-for-me'); }); -test.serial('should load collections from config', async (t) => { +test.serial( + 'legacy collections move to resources.yaml when written back out', + async (t) => { + mockFile('/ws/openfn.yaml', { + workspace: buildConfig(), + project: { id: 'my-project', collections: ['my-collection'] }, + }); + mockFile('/ws/workflows/workflow-1/workflow-1.yaml', { + id: 'workflow-1', + name: 'Workflow 1', + }); + + const project = await parseProject({ root: '/ws' }); + const written = project.serialize('fs'); + + t.deepEqual(yamlToJson(written['resources.yaml']), { + collections: { 'my-collection': {} }, + }); + t.false(written['openfn.yaml'].includes('collections')); + } +); + +test.serial('should load legacy collections from openfn.yaml', async (t) => { mockFile('/ws/openfn.yaml', { workspace: buildConfig(), project: { @@ -217,6 +239,61 @@ test.serial('should load collections from config', async (t) => { t.falsy(project.openfn!.collections); }); +test.serial('should load collections from resources.yaml', async (t) => { + mockFile('/ws/openfn.yaml', buildConfig()); + mockFile('/ws/resources.yaml', { + collections: { 'my-collection': {}, 'another-collection': {} }, + }); + + const project = await parseProject({ root: '/ws' }); + // in-memory shape is {name, uuid?}, resources.yaml itself has no uuids + t.deepEqual(project.collections, [ + { name: 'my-collection' }, + { name: 'another-collection' }, + ]); +}); + +test.serial( + 'resources.yaml collections take precedence over legacy', + async (t) => { + mockFile('/ws/openfn.yaml', { + workspace: buildConfig(), + project: { id: 'my-project', collections: ['legacy-collection'] }, + }); + mockFile('/ws/resources.yaml', { collections: { 'new-collection': {} } }); + + const project = await parseProject({ root: '/ws' }); + t.deepEqual(project.collections, [{ name: 'new-collection' }]); + } +); + +test.serial( + 'an empty collections key in resources.yaml means no collections', + async (t) => { + mockFile('/ws/openfn.yaml', { + workspace: buildConfig(), + project: { id: 'my-project', collections: ['legacy-collection'] }, + }); + mockFile('/ws/resources.yaml', { collections: {} }); + + const project = await parseProject({ root: '/ws' }); + t.deepEqual(project.collections, []); + } +); + +test.serial( + 'should leave collections undefined if resources.yaml has no collections key', + async (t) => { + mockFile('/ws/openfn.yaml', buildConfig()); + mockFile('/ws/resources.yaml', { + channels: { c: { name: 'c', destination_url: 'https://x.org' } }, + }); + + const project = await parseProject({ root: '/ws' }); + t.is(project.collections, undefined); + } +); + test.serial( 'should return undefined collections when none are configured', async (t) => { diff --git a/packages/project/test/serialize/to-fs.test.ts b/packages/project/test/serialize/to-fs.test.ts index 5d8ceccea..d47277ff0 100644 --- a/packages/project/test/serialize/to-fs.test.ts +++ b/packages/project/test/serialize/to-fs.test.ts @@ -388,17 +388,12 @@ test('toFs: extract a project with forked_from meta', (t) => { t.is(files['workflows/my-workflow/step.js'], 'fn(s => s)'); }); -test('toFs: writes collection names into openfn.json (freshly authored, no uuid yet)', (t) => { +test('toFs: writes collections to resources.yaml keyed by name (freshly authored, no uuid yet)', (t) => { const project = new Project( { name: 'My Project', collections: [{ name: 'my-collection' }, { name: 'another-collection' }], - workflows: [ - { - id: 'my-workflow', - steps: [step], - }, - ], + workflows: [], }, { formats: { @@ -410,64 +405,67 @@ test('toFs: writes collection names into openfn.json (freshly authored, no uuid const files = toFs(project); - const config = JSON.parse(files['openfn.json']); - t.deepEqual(config.collections, ['my-collection', 'another-collection']); + t.deepEqual(yamlToJson(files['resources.yaml']), { + collections: { 'my-collection': {}, 'another-collection': {} }, + }); + // collections don't belong in openfn.json + t.falsy(JSON.parse(files['openfn.json']).collections); }); -test('toFs: strips uuids from fetched collections when writing to openfn.json', (t) => { - const project = new Project( - { - name: 'My Project', - // this is the shape a freshly-fetched project's collections are in - - // uuids should never be written to openfn.yaml - collections: [ - { uuid: 'remote-uuid-1', name: 'my-collection' }, - { uuid: 'remote-uuid-2', name: 'another-collection' }, - ], - workflows: [ - { - id: 'my-workflow', - steps: [step], - }, - ], - }, - { - formats: { - openfn: 'json', - workflow: 'json', - }, - } - ); +test('toFs: strips uuids from fetched collections when writing to resources.yaml', (t) => { + const project = new Project({ + name: 'My Project', + // this is the shape a freshly-fetched project's collections are in - + // uuids should never be written to disk + collections: [ + { uuid: 'remote-uuid-1', name: 'my-collection' }, + { uuid: 'remote-uuid-2', name: 'another-collection' }, + ], + workflows: [], + }); const files = toFs(project); - const config = JSON.parse(files['openfn.json']); - t.deepEqual(config.collections, ['my-collection', 'another-collection']); + t.deepEqual(yamlToJson(files['resources.yaml']), { + collections: { 'my-collection': {}, 'another-collection': {} }, + }); }); -test('toFs: omits collections key when there are none', (t) => { - const project = new Project( - { - name: 'My Project', - workflows: [ - { - id: 'my-workflow', - steps: [step], - }, - ], - }, - { - formats: { - openfn: 'json', - workflow: 'json', - }, - } - ); +test('toFs: writes an empty collections key when the project has none', (t) => { + const project = new Project({ + name: 'My Project', + collections: [], + workflows: [], + }); const files = toFs(project); - const config = JSON.parse(files['openfn.json']); - t.falsy(config.collections); + // an empty key means "no collections", which is not the same as no key + t.deepEqual(yamlToJson(files['resources.yaml']), { collections: {} }); +}); + +test('toFs: writes collections and channels to the same resources.yaml', (t) => { + const project = new Project({ + name: 'My Project', + collections: [{ name: 'my-collection' }], + channels: [ + { + name: 'My Channel', + destination_url: 'https://example.com', + enabled: true, + }, + ], + workflows: [], + }); + + const resources = yamlToJson(toFs(project)['resources.yaml']); + t.deepEqual(Object.keys(resources).sort(), ['channels', 'collections']); +}); + +test('toFs: does not write resources.yaml when collections are unknown', (t) => { + const project = new Project({ name: 'My Project', workflows: [] }); + + t.false('resources.yaml' in toFs(project)); }); test('toFs: writes channels to resources.yaml keyed by id, with credential names', (t) => { diff --git a/packages/project/test/util/config.test.ts b/packages/project/test/util/config.test.ts index c4d46f0c1..f0624ff97 100644 --- a/packages/project/test/util/config.test.ts +++ b/packages/project/test/util/config.test.ts @@ -195,13 +195,13 @@ formats: ); }); -test('generate openfn.yaml with collection names', (t) => { +test('generate openfn.yaml does not include collections', (t) => { const proj = new Project({ id: 'my-project', collections: [{ uuid: 'remote-uuid', name: 'my-collection' }], }); const result = extractConfig(proj); - t.true(result.content.startsWith('collections:\n - my-collection\n')); + t.false(result.content.includes('collections')); }); test('generate checkout file with forked_from', (t) => { diff --git a/packages/project/test/util/project-diff.test.ts b/packages/project/test/util/project-diff.test.ts index 5aa48d9f8..509527761 100644 --- a/packages/project/test/util/project-diff.test.ts +++ b/packages/project/test/util/project-diff.test.ts @@ -321,3 +321,45 @@ test('diff: should report workflow and channel changes together', (t) => { t.is(result.workflows.length, 1); t.is(result.resources.channels.length, 1); }); + +const withCollections = (collections?: string[]) => + new Project({ + name: 'p', + workflows: [], + collections: collections?.map((name) => ({ name })), + }); + +test('diff: resources.collections is empty for identical collections', (t) => { + const a = withCollections(['one']); + const b = withCollections(['one']); + + t.deepEqual(diff(a, b).resources.collections, []); +}); + +test('diff: should detect added and removed collections', (t) => { + const a = withCollections(['keep', 'remove']); + const b = withCollections(['keep', 'add']); + + t.deepEqual(diff(a, b).resources.collections, [ + { name: 'remove', type: 'removed' }, + { name: 'add', type: 'added' }, + ]); +}); + +test('diff: should report all collections removed when one side is emptied', (t) => { + const remote = withCollections(['one', 'two']); + const local = withCollections([]); + + t.deepEqual(diff(remote, local).resources.collections, [ + { name: 'one', type: 'removed' }, + { name: 'two', type: 'removed' }, + ]); +}); + +test('diff: should ignore collections if either project does not define them', (t) => { + const withOne = withCollections(['one']); + const none = withCollections(undefined); + + t.deepEqual(diff(withOne, none).resources.collections, []); + t.deepEqual(diff(none, withOne).resources.collections, []); +}); diff --git a/packages/project/test/util/resources.test.ts b/packages/project/test/util/resources.test.ts new file mode 100644 index 000000000..62dae7c83 --- /dev/null +++ b/packages/project/test/util/resources.test.ts @@ -0,0 +1,65 @@ +import test from 'ava'; +import { Project } from '../../src/Project'; +import { jsonToYaml, yamlToJson } from '../../src/util/yaml'; +import { + fromResourceCollections, + toResourceCollections, + updateResourceCollections, +} from '../../src/util/resources'; + +test('toResourceCollections: keyed by name, with no uuids', (t) => { + t.deepEqual( + toResourceCollections([{ uuid: 'a', name: 'one' }, { name: 'two' }]), + { one: {}, two: {} } + ); +}); + +test('fromResourceCollections: reads a map keyed by name', (t) => { + t.deepEqual(fromResourceCollections({ one: {}, two: {} }), [ + { name: 'one' }, + { name: 'two' }, + ]); +}); + +test('fromResourceCollections: also reads a plain list of names', (t) => { + t.deepEqual(fromResourceCollections(['one', 'two']), [ + { name: 'one' }, + { name: 'two' }, + ]); +}); + +test('fromResourceCollections: an empty or null key means no collections', (t) => { + t.deepEqual(fromResourceCollections({}), []); + t.deepEqual(fromResourceCollections(null), []); +}); + +test('updateResourceCollections: replaces collections and leaves other keys alone', (t) => { + const project = new Project({ name: 'p', collections: [{ name: 'new' }] }); + const existing = jsonToYaml({ + channels: { c: { name: 'c', destination_url: 'https://x.org' } }, + collections: { old: {} }, + }); + + const result = yamlToJson(updateResourceCollections(project, existing)!); + + t.deepEqual(result, { + channels: { c: { name: 'c', destination_url: 'https://x.org' } }, + collections: { new: {} }, + }); +}); + +test('updateResourceCollections: creates the content if there is no existing file', (t) => { + const project = new Project({ name: 'p', collections: [{ name: 'one' }] }); + + t.deepEqual(yamlToJson(updateResourceCollections(project)!), { + collections: { one: {} }, + }); +}); + +test('updateResourceCollections: leaves the file alone if collections are not managed', (t) => { + const project = new Project({ name: 'p' }); + const existing = jsonToYaml({ channels: {} }); + + t.is(updateResourceCollections(project, existing), existing); + t.is(updateResourceCollections(project), undefined); +}); From b440f857dcaeccb9db15b789e9d2d262bd9f73de Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 7 Oct 2026 17:51:16 +0100 Subject: [PATCH 19/22] Add prompt to reworked openfn yaml (#1577) * add util for checking if there's a valid checkout * add helpers for writing checkout metadata * refactoring * use new helpers in deploy * prompt user for checkout * add user prompt * simplify * fix tests * better fix --- .changeset/small-cases-buy.md | 5 + integration-tests/cli/.gitignore | 3 +- integration-tests/cli/.state.json | 53 ---- packages/cli/src/projects/checkout.ts | 7 +- packages/cli/src/projects/clean.ts | 27 +- packages/cli/src/projects/deploy.ts | 30 +- packages/cli/src/projects/ensure-checkout.ts | 125 ++++++++ packages/cli/src/projects/list.ts | 7 +- packages/cli/src/projects/merge.ts | 6 +- packages/cli/src/projects/pull.ts | 14 +- packages/cli/src/projects/util.ts | 17 + packages/cli/test/projects/clean.test.ts | 40 +++ packages/cli/test/projects/deploy.test.ts | 138 ++++++++ .../cli/test/projects/ensure-checkout.test.ts | 300 ++++++++++++++++++ packages/cli/test/projects/list.test.ts | 26 ++ packages/cli/test/projects/util.test.ts | 71 ++++- packages/project/src/Project.ts | 15 +- packages/project/src/index.ts | 1 + packages/project/src/util/config.ts | 19 ++ packages/project/test/util/config.test.ts | 52 +++ 20 files changed, 861 insertions(+), 95 deletions(-) create mode 100644 .changeset/small-cases-buy.md delete mode 100644 integration-tests/cli/.state.json create mode 100644 packages/cli/src/projects/ensure-checkout.ts create mode 100644 packages/cli/test/projects/ensure-checkout.test.ts diff --git a/.changeset/small-cases-buy.md b/.changeset/small-cases-buy.md new file mode 100644 index 000000000..a05e27a3e --- /dev/null +++ b/.changeset/small-cases-buy.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': patch +--- + +When running commands with v2 sync, if no checkout metadata is detected, prompt the user diff --git a/integration-tests/cli/.gitignore b/integration-tests/cli/.gitignore index ffdcec42c..f3bb1a002 100644 --- a/integration-tests/cli/.gitignore +++ b/integration-tests/cli/.gitignore @@ -1 +1,2 @@ -repo \ No newline at end of file +repo +.state.json diff --git a/integration-tests/cli/.state.json b/integration-tests/cli/.state.json deleted file mode 100644 index b9880f4ba..000000000 --- a/integration-tests/cli/.state.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "workflows": { - "my-workflow": { - "id": "702bf73f-fde1-44ca-87c5-c9e508ec2407", - "name": "My Workflow", - "jobs": { - "transform-data": { - "id": "43d6474d-df15-4ce6-a1a3-52e6400a36a9", - "name": "Transform data", - "adaptor": "@openfn/language-common@latest", - "body": "fn(s => s)", - "project_credential_id": "0f750fd9-b34c-4cf1-a6a9-a97cc18a2f4c" - } - }, - "triggers": { - "webhook": { - "id": "ca8358c9-b47b-4ad6-ba45-7a37384b81af", - "type": "webhook", - "enabled": true - } - }, - "edges": { - "webhook->transform-data": { - "id": "7026ceb7-f841-4b4b-a182-b2470384a1c2", - "condition_type": null, - "target_job_id": "43d6474d-df15-4ce6-a1a3-52e6400a36a9", - "enabled": true, - "source_trigger_id": "ca8358c9-b47b-4ad6-ba45-7a37384b81af" - } - }, - "lock_version": 1, - "inserted_at": "2026-10-05T14:49:28.664Z", - "updated_at": "2026-10-05T14:49:28.664Z", - "deleted_at": null, - "version_history": [ - "app:d9287d0b8d8c" - ] - } - }, - "id": "9351c38a-18ca-48ee-af7f-e4a479d92294", - "name": "My Project", - "project_credentials": { - "super@openfn.org|http1": { - "id": "0f750fd9-b34c-4cf1-a6a9-a97cc18a2f4c", - "name": "http1", - "owner": "super@openfn.org" - } - }, - "collections": {}, - "inserted_at": "2026-10-05T14:49:28.664Z", - "updated_at": "2026-10-05T14:49:28.664Z", - "channels": {} -} \ No newline at end of file diff --git a/packages/cli/src/projects/checkout.ts b/packages/cli/src/projects/checkout.ts index 10158530f..c22a7b375 100644 --- a/packages/cli/src/projects/checkout.ts +++ b/packages/cli/src/projects/checkout.ts @@ -164,11 +164,8 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => { }); if (isAdHoc) { // Don't touch the workspace or checkout metadata - for (const { path: configPath } of switchProject.generateConfig( - options.branch - )) { - delete files[configPath]; - } + delete files[switchProject.generateConfig().path]; + delete files[switchProject.generateCheckout(options.branch).path]; } for (const f in files) { if (files[f]) { diff --git a/packages/cli/src/projects/clean.ts b/packages/cli/src/projects/clean.ts index 152dd2573..9c9b63f97 100644 --- a/packages/cli/src/projects/clean.ts +++ b/packages/cli/src/projects/clean.ts @@ -4,6 +4,7 @@ import { rimraf } from 'rimraf'; import { build, ensure } from '../util/command-builders'; import { handler as checkout } from './checkout'; +import ensureCheckout from './ensure-checkout'; import type { Logger } from '../util/logger'; import * as o from '../options'; import * as po from './options'; @@ -29,9 +30,22 @@ export default command; export const handler = async (options: CleanOptions, logger: Logger) => { const workspacePath = options.workspace ?? process.cwd(); - const workspace = new Workspace(workspacePath, logger, true, { - branch: options.branch, - }); + + // Find out what's checked out before deleting anything + const workspace = await ensureCheckout( + new Workspace(workspacePath, logger, true, { + branch: options.branch, + }), + logger + ); + + // This has to be checked before deleting anything + const activeProject = workspace.activeProject; + if (!activeProject) { + throw new Error( + 'No active project found in workspace. Run `project pull` first.' + ); + } const skip = options.force || options.confirm === false; const doIt = await logger.confirm( @@ -44,13 +58,6 @@ export const handler = async (options: CleanOptions, logger: Logger) => { await rimraf(workspace.workflowsPath); - const activeProject = workspace.activeProject; - if (!activeProject) { - throw new Error( - 'No active project found in workspace. Run `project pull` first.' - ); - } - const projectId = String(activeProject.uuid ?? (activeProject as any).id); await checkout({ ...options, project: projectId, force: true }, logger); }; diff --git a/packages/cli/src/projects/deploy.ts b/packages/cli/src/projects/deploy.ts index ed1054f36..88c4fd8f3 100644 --- a/packages/cli/src/projects/deploy.ts +++ b/packages/cli/src/projects/deploy.ts @@ -18,12 +18,13 @@ import { fetchProject, serialize, getSerializePath, - updateForkedFrom, + writeCheckoutFile, findLocallyChangedWorkflows, AuthOptions, } from './util'; import { build, ensure } from '../util/command-builders'; import { printRichDiff } from './diff'; +import ensureCheckout from './ensure-checkout'; import { getCredentialsVisitor, remapCredentials } from './credentials-helpers'; import type { Provisioner } from '@openfn/lexicon/lightning'; @@ -404,6 +405,13 @@ export async function handler(options: DeployOptions, logger: Logger) { branch: options.branch, }); + // We only need to know what's checked out if we're going to write the + // checkout back after deploying, and if the user hasn't told us which + // project to deploy to + if (options.checkout !== false && !targetIdentifier) { + ws = await ensureCheckout(ws, logger); + } + const active = ws.getTrackedProject(); if (!alias && active?.alias) { @@ -617,25 +625,19 @@ export async function handler(options: DeployOptions, logger: Logger) { ); if (options.checkout !== false) { - updateForkedFrom(finalProject); + const workspacePath = options.workspace ?? process.cwd(); // Write the updated openfn.yaml and checkout file // TODO: allow us to suppress writing this stuff // (useful if posting from spec) - for (const configData of finalProject.generateConfig(options.branch)) { - const configPath = path.resolve( - options.workspace ?? process.cwd(), - configData.path - ); - await mkdir(path.dirname(configPath), { recursive: true }); - await writeFile(configPath, configData.content); - } + const configData = finalProject.generateConfig(); + const configPath = path.resolve(workspacePath, configData.path); + await mkdir(path.dirname(configPath), { recursive: true }); + await writeFile(configPath, configData.content); + await writeCheckoutFile(workspacePath, finalProject, options.branch); // Sync collections into resources.yaml (anything else in there is left alone) - const resourcesPath = path.resolve( - options.workspace ?? process.cwd(), - RESOURCES_FILE - ); + const resourcesPath = path.resolve(workspacePath, RESOURCES_FILE); const existingResources = await readFile(resourcesPath, 'utf8').catch( (e) => { if (e.code !== 'ENOENT') throw e; diff --git a/packages/cli/src/projects/ensure-checkout.ts b/packages/cli/src/projects/ensure-checkout.ts new file mode 100644 index 000000000..c14327665 --- /dev/null +++ b/packages/cli/src/projects/ensure-checkout.ts @@ -0,0 +1,125 @@ +import { select } from '@inquirer/prompts'; +import { Workspace, hasCheckoutMeta } from '@openfn/project'; +import type Project from '@openfn/project'; + +import abort from '../util/abort'; +import type { Logger } from '../util/logger'; +import { writeCheckoutFile } from './util'; + +// Resolves to the chosen project, or null if the user skipped the question +type Prompt = ( + projects: Project[], + defaultProject: Project +) => Promise; + +type EnsureCheckoutOptions = { + // Can we ask the user questions? Defaults to whether stdin is a terminal + interactive?: boolean; + prompt?: Prompt; + // If false, carry on without a checkout when we can't ask the user, rather + // than aborting (for commands that can work without one) + required?: boolean; +}; + +const name = (project: Project) => project.alias ?? project.id; + +// eg `staging (my-project | 1234-abcd)`. Either id may be missing +export const describeProject = (project: Project) => { + const ids = [project.id, project.openfn?.uuid].filter(Boolean).join(' | '); + return `${project.alias || '(no alias)'}${ids ? ` (${ids})` : ''}`; +}; + +const promptForProject: Prompt = (projects, defaultProject) => + select({ + message: + 'No checked out project found. Select the project which is currently tracked.', + choices: [ + ...projects.map((project) => ({ + name: describeProject(project), + value: project as Project | null, + description: project.openfn?.endpoint, + })), + { + name: 'Skip checkout', + value: null, + description: 'Continue without setting the checked out project', + }, + ], + default: defaultProject, + }); + +// Guess which project is checked out: one with an alias matching the git +// branch (or main, if there's no branch), else just the first. +// Pass undefined, not null or false, if there is no branch +export const getDefaultProject = (projects: Project[], branch = 'main') => + projects.find((p) => p.alias === branch) ?? projects[0]; + +/** + * Make sure the workspace knows which project is checked out. + * + * If there's no checkout metadata (say, after a fresh git clone, since it's + * not tracked in git), work out which project the files belong to and set + * up the checkout file. The files themselves are not touched. + * + * Returns a workspace that reflects the checkout. + */ +export default async ( + workspace: Workspace, + logger: Logger, + { + interactive = Boolean(process.stdin.isTTY), + prompt = promptForProject, + required = true, + }: EnsureCheckoutOptions = {} +) => { + if (!workspace.valid || hasCheckoutMeta(workspace.root, workspace.branch)) { + return workspace; + } + + const projects = workspace.list(); + if (!projects.length) { + // Nothing to choose from: let the command report that in its own way + return workspace; + } + + if (projects.length > 1 && !interactive && !required) { + logger.debug('No checked out project found: carrying on without one'); + return workspace; + } + + let project: Project | null = null; + if (projects.length === 1) { + project = projects[0]; + logger.info(`No checked out project found: using ${name(project)}`); + } else if (interactive) { + logger.info( + 'Failed to find a checkout.yaml file, which describes which project is currently checked out. This usually happens when pulling a project from git.' + ); + logger.info( + "No problem - just pick which project is checked out. This won't affect your working tree, it'll just update local metadata and tell the CLI which remote project instance to track\n." + ); + project = await prompt( + projects, + getDefaultProject(projects, workspace.branch || undefined) + ); + } else { + abort(logger, 'No checked out project found', { + details: `This workspace tracks several projects (${projects + .map(name) + .join(', ')}) and it's not known which one is checked out`, + fix: 'Run this command in an interactive terminal to choose one', + }); + } + + if (!project) { + logger.info('Skipped: no checked out project has been set'); + return workspace; + } + + await writeCheckoutFile(workspace.root, project, workspace.branch); + logger.success(`Set the checked out project to ${name(project)}`); + + return new Workspace(workspace.root, logger, true, { + branch: workspace.branch, + }); +}; diff --git a/packages/cli/src/projects/list.ts b/packages/cli/src/projects/list.ts index 629b25407..2b0576f88 100644 --- a/packages/cli/src/projects/list.ts +++ b/packages/cli/src/projects/list.ts @@ -8,6 +8,7 @@ import * as po from './options'; import type { Opts } from './options'; import abort from '../util/abort'; +import ensureCheckout from './ensure-checkout'; export type ProjectListOptions = Pick; @@ -28,7 +29,7 @@ export const handler = async (options: ProjectListOptions, logger: Logger) => { logger.info(' ', options.workspace); logger.break(); - const workspace = new Workspace(options.workspace!, undefined, true, { + let workspace = new Workspace(options.workspace!, undefined, true, { branch: options.branch, }); @@ -42,6 +43,10 @@ export const handler = async (options: ProjectListOptions, logger: Logger) => { }); } + // Listing works fine without a checkout: it just can't mark the active + // project, so there's no need to abort if we can't ask which it is + workspace = await ensureCheckout(workspace, logger, { required: false }); + logger.always(`Available openfn projects\n\n${workspace .list() .map((p) => describeProject(p, p === workspace.getTrackedProject())) diff --git a/packages/cli/src/projects/merge.ts b/packages/cli/src/projects/merge.ts index 90b0d3aec..08634ce06 100644 --- a/packages/cli/src/projects/merge.ts +++ b/packages/cli/src/projects/merge.ts @@ -10,6 +10,7 @@ import * as po from './options'; import type { Opts } from './options'; import { handler as checkout } from './checkout'; +import ensureCheckout from './ensure-checkout'; export type MergeOptions = Required< Pick< @@ -63,13 +64,16 @@ export default command; export const handler = async (options: MergeOptions, logger: Logger) => { const workspacePath = options.workspace; - const workspace = new Workspace(workspacePath, undefined, true, { + let workspace = new Workspace(workspacePath, undefined, true, { branch: options.branch, }); if (!workspace.valid) { logger.error('Command was run in an invalid openfn workspace'); return; } + if (!options.base) { + workspace = await ensureCheckout(workspace, logger); + } let targetProject: Project; if (options.base) { diff --git a/packages/cli/src/projects/pull.ts b/packages/cli/src/projects/pull.ts index a3d3a85a7..06d183ef2 100644 --- a/packages/cli/src/projects/pull.ts +++ b/packages/cli/src/projects/pull.ts @@ -4,6 +4,7 @@ import { Workspace } from '@openfn/project'; import { build, ensure, override } from '../util/command-builders'; import { handler as fetch } from './fetch'; import { handler as checkout } from './checkout'; +import ensureCheckout from './ensure-checkout'; import * as o from '../options'; import * as o2 from './options'; @@ -67,7 +68,7 @@ export const command: yargs.CommandModule = { export async function handler(options: PullOptions, logger: Logger) { options.workspace ??= process.cwd(); - ensureProjectId(options, logger); + await ensureProjectId(options, logger); await fetch(options, logger); logger.info(`Downloaded latest project version`); @@ -76,14 +77,17 @@ export async function handler(options: PullOptions, logger: Logger) { logger.success(`Checked out project locally`); } -const ensureProjectId = (options: any, logger?: Logger) => { +const ensureProjectId = async (options: any, logger: Logger) => { if (!options.project) { logger?.debug( 'No project ID specified: looking up checked out project in Workspace' ); - const ws = new Workspace(options.workspace, undefined, true, { - branch: options.branch, - }); + const ws = await ensureCheckout( + new Workspace(options.workspace, undefined, true, { + branch: options.branch, + }), + logger + ); if (ws.activeProject) { options.project = ws.activeProject.uuid; logger?.info( diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index 4e8b84d2a..a34e8d7d2 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -260,6 +260,23 @@ export const updateForkedFrom = (proj: Project) => { return proj; }; +// Set up the checkout file for a project without touching anything else +// (use this when the workflows are already on disk, like after a git clone) +export const writeCheckoutFile = async ( + workspacePath: string, + project: Project, + branch?: string | false | null +) => { + updateForkedFrom(project); + + const { path: checkoutPath, content } = project.generateCheckout(branch); + const filePath = path.resolve(workspacePath, checkoutPath); + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, content); + + return filePath; +}; + // Compare a project to its version hashed when forked // This tells us whether the project was edited since it was created export const findLocallyChangedWorkflows = async ( diff --git a/packages/cli/test/projects/clean.test.ts b/packages/cli/test/projects/clean.test.ts index f953c3244..1d0c646cb 100644 --- a/packages/cli/test/projects/clean.test.ts +++ b/packages/cli/test/projects/clean.test.ts @@ -115,3 +115,43 @@ test.serial( ); } ); + +test.serial( + 'does not delete anything if it is not known which project is checked out', + async (t) => { + // two tracked projects, but no checkout metadata anywhere + mock({ + '/ws/workflows/old-workflow': { 'old-job.js': 'fn(s => s)' }, + '/ws/openfn.yaml': '', + '/ws/.projects/main@app.openfn.org.yaml': projectStateFile, + '/ws/.projects/staging@app.openfn.org.yaml': projectStateFile, + }); + const exitCode = process.exitCode; + + await t.throwsAsync( + cleanHandler({ workspace: '/ws', force: true } as any, logger), + { message: 'No checked out project found' } + ); + process.exitCode = exitCode; + + t.true(fs.existsSync('/ws/workflows/old-workflow/old-job.js')); + } +); + +test.serial( + 'does not delete anything if there is no active project to restore', + async (t) => { + // a workspace with no projects and no checkout metadata at all + mock({ + '/ws/workflows/old-workflow': { 'old-job.js': 'fn(s => s)' }, + '/ws/openfn.yaml': '', + }); + + await t.throwsAsync( + cleanHandler({ workspace: '/ws', force: true } as any, logger), + { message: /No active project found/ } + ); + + t.true(fs.existsSync('/ws/workflows/old-workflow/old-job.js')); + } +); diff --git a/packages/cli/test/projects/deploy.test.ts b/packages/cli/test/projects/deploy.test.ts index f07fc0eb2..f0f545e10 100644 --- a/packages/cli/test/projects/deploy.test.ts +++ b/packages/cli/test/projects/deploy.test.ts @@ -30,6 +30,7 @@ import { myProject_v1_spec, } from './fixtures'; import { checkout } from '../../src/projects'; +import { rimraf } from 'rimraf'; let server: any; let strictServer: any; @@ -1361,3 +1362,140 @@ test('printRichDiff: should list changed channel fields', (t) => { t.truthy(logger._find('always', /one: changed/)); t.truthy(logger._find('always', /enabled: "true" -> "false"/)); }); + +const twoProjectsNoCheckout = () => + mockFs({ + '/ws/.projects/main@localhost.yaml': projectYaml, + '/ws/.projects/staging@localhost.yaml': projectYaml, + '/ws/openfn.yaml': '', + }); + +test.serial( + 'deploy: aborts if it is not known which project is checked out', + async (t) => { + twoProjectsNoCheckout(); + const exitCode = process.exitCode; + + await t.throwsAsync( + deploy( + { + endpoint: ENDPOINT, + apiKey: 'test-api-key', + workspace: '/ws', + confirm: false, + } as any, + logger + ), + { message: 'No checked out project found' } + ); + process.exitCode = exitCode; + } +); + +test.serial( + 'deploy: does not look for the checked out project with checkout: false', + async (t) => { + twoProjectsNoCheckout(); + // use a private logger: an absence check isn't safe on the shared one + const privateLogger = createMockLogger(undefined, { level: 'debug' }); + + // this deploy doesn't go anywhere, but it must not stop to ask + await deploy( + { + endpoint: ENDPOINT, + apiKey: 'test-api-key', + workspace: '/ws', + confirm: false, + checkout: false, + } as any, + privateLogger + ).catch(() => {}); + + t.falsy(privateLogger._find('error', /No checked out project found/)); + } +); + +test.serial( + 'deploy: does not look for the checked out project if there is an explicit target', + async (t) => { + twoProjectsNoCheckout(); + const privateLogger = createMockLogger(undefined, { level: 'debug' }); + + // this deploy doesn't go anywhere, but it must not stop to ask + await deploy( + { + endpoint: ENDPOINT, + apiKey: 'test-api-key', + workspace: '/ws', + confirm: false, + project: 'staging', + } as any, + privateLogger + ).catch(() => {}); + + t.falsy(privateLogger._find('error', /No checked out project found/)); + } +); + +test.serial( + 'deploy: merges the workspace into an explicit target on a fresh clone, without a checkout', + async (t) => { + await setup(projectYaml); + + // simulate a fresh git clone: the workflows are there, nothing is checked + // out, and there's more than one tracked project + // (fs.rmSync is a no-op under mock-fs, hence rimraf) + await rimraf('/ws/.openfn'); + t.false(fs.existsSync('/ws/.openfn')); + fs.writeFileSync('/ws/.projects/other@localhost.yaml', projectYaml); + await writeFile('/ws/workflows/my-workflow/transform-data.js', 'log()'); + + await deploy( + { + endpoint: ENDPOINT, + apiKey: 'test-api-key', + workspace: '/ws', + project: 'main', + checkout: false, + force: true, + confirm: false, + } as any, + logger + ); + + t.truthy(logger._find('success', /Updated project at/)); + t.is( + server.state.projects[UUID].workflows['my-workflow'].jobs[ + 'transform-data' + ].body, + 'log()' + ); + // deploying doesn't set up a checkout + t.false(fs.existsSync('/ws/.openfn')); + } +); + +test.serial( + 'deploy: deploys a project file on a fresh clone, without a checkout', + async (t) => { + mockFs({ + '/ws/project.yaml': myProject_spec, + '/ws/openfn.yaml': '', + '/ws/.projects/main@localhost.yaml': projectYaml, + '/ws/.projects/other@localhost.yaml': projectYaml, + }); + + await deploy( + { + endpoint: ENDPOINT, + apiKey: 'test-api-key', + workspace: '/ws', + project: '/ws/project.yaml', + } as any, + logger + ); + + t.truthy(logger._find('success', /Created new project at/)); + t.is(Object.keys(server.state.projects).length, 2); + } +); diff --git a/packages/cli/test/projects/ensure-checkout.test.ts b/packages/cli/test/projects/ensure-checkout.test.ts new file mode 100644 index 000000000..e9ead3df6 --- /dev/null +++ b/packages/cli/test/projects/ensure-checkout.test.ts @@ -0,0 +1,300 @@ +import test from 'ava'; +import fs from 'node:fs'; +import mock from 'mock-fs'; +import { createMockLogger } from '@openfn/logger'; +import Project, { Workspace, yamlToJson } from '@openfn/project'; + +import ensureCheckout, { + describeProject, + getDefaultProject, +} from '../../src/projects/ensure-checkout'; + +const logger = createMockLogger(undefined, { level: 'debug' }); + +test.afterEach(() => { + mock.restore(); + logger._reset(); +}); + +const projectFile = (id: string) => + new Project({ + id, + name: id, + openfn: { uuid: `uuid-${id}`, endpoint: 'https://app.openfn.org' }, + workflows: [{ id: 'wf', name: 'wf', history: ['abc'], steps: [] }], + }).serialize('project', { format: 'yaml' }) as string; + +const setup = (ids = ['main', 'staging'], extra = {}) => + mock({ + '/ws/openfn.yaml': '', + '/ws/workflows/wf/wf.yaml': 'id: wf', + ...Object.fromEntries( + ids.map((id) => [ + `/ws/.projects/${id}@app.openfn.org.yaml`, + projectFile(id), + ]) + ), + ...extra, + }); + +const readCheckout = (path = '/ws/.openfn/checkout.yaml') => + yamlToJson(fs.readFileSync(path, 'utf8')); + +// A prompt that records what it was asked and picks a project by alias +const pickPrompt = (alias: string) => { + const calls: any[] = []; + const prompt = async (projects: Project[], defaultProject: Project) => { + calls.push({ projects, defaultProject }); + return projects.find((p) => p.alias === alias)!; + }; + return { prompt, calls }; +}; + +const neverPrompt = async () => { + throw new Error('should not have prompted'); +}; + +test.serial('does nothing if there is already checkout metadata', async (t) => { + setup(['main', 'staging'], { '/ws/.openfn/checkout.yaml': 'id: main' }); + const workspace = new Workspace('/ws', logger); + + const result = await ensureCheckout(workspace, logger, { + interactive: true, + prompt: neverPrompt, + }); + + t.is(result, workspace); + t.is(readCheckout().id, 'main'); +}); + +test.serial('does nothing if there are no tracked projects', async (t) => { + setup([]); + const workspace = new Workspace('/ws', logger, false); + + const result = await ensureCheckout(workspace, logger, { + interactive: true, + prompt: neverPrompt, + }); + + t.is(result, workspace); + t.false(fs.existsSync('/ws/.openfn')); +}); + +test.serial('uses the only tracked project without prompting', async (t) => { + setup(['main']); + + const result = await ensureCheckout(new Workspace('/ws', logger), logger, { + interactive: true, + prompt: neverPrompt, + }); + + t.is(readCheckout().uuid, 'uuid-main'); + t.is(result.activeProject?.uuid, 'uuid-main'); +}); + +test.serial('prompts if there are several tracked projects', async (t) => { + setup(); + const { prompt, calls } = pickPrompt('staging'); + + const result = await ensureCheckout(new Workspace('/ws', logger), logger, { + interactive: true, + prompt, + }); + + // explain why we're asking + t.truthy(logger._find('info', /Failed to find a checkout.yaml file/)); + t.truthy(logger._find('info', /just pick which project is checked out/)); + + t.is(calls.length, 1); + t.deepEqual(calls[0].projects.map((p: Project) => p.alias).sort(), [ + 'main', + 'staging', + ]); + + // the checkout file has been set up, and the workspace knows about it + t.is(readCheckout().uuid, 'uuid-staging'); + t.is(result.activeProject?.uuid, 'uuid-staging'); +}); + +test.serial( + 'does not set anything up if the user skips the prompt', + async (t) => { + setup(); + const workspace = new Workspace('/ws', logger); + + const result = await ensureCheckout(workspace, logger, { + interactive: true, + prompt: async () => null, + }); + + t.is(result, workspace); + t.false(fs.existsSync('/ws/.openfn')); + t.truthy(logger._find('info', /Skipped/)); + } +); + +test.serial( + 'does not touch the workflows when setting up the checkout', + async (t) => { + setup(); + const { prompt } = pickPrompt('main'); + + await ensureCheckout(new Workspace('/ws', logger), logger, { + interactive: true, + prompt, + }); + + t.deepEqual(fs.readdirSync('/ws').sort(), [ + '.openfn', + '.projects', + 'openfn.yaml', + 'workflows', + ]); + t.is(fs.readFileSync('/ws/workflows/wf/wf.yaml', 'utf8'), 'id: wf'); + t.is(fs.readFileSync('/ws/openfn.yaml', 'utf8'), ''); + } +); + +test.serial('suggests the project matching the git branch', async (t) => { + setup(['main', 'staging']); + const { prompt, calls } = pickPrompt('main'); + + await ensureCheckout( + new Workspace('/ws', logger, true, { branch: 'staging' }), + logger, + { interactive: true, prompt } + ); + + t.is(calls[0].defaultProject.alias, 'staging'); +}); + +test.serial( + 'suggests main if there is no git branch, even if it is not first', + async (t) => { + // projects are listed alphabetically, so main is not first here + setup(['alpha', 'main']); + const { prompt, calls } = pickPrompt('alpha'); + + await ensureCheckout( + new Workspace('/ws', logger, true, { branch: false }), + logger, + { interactive: true, prompt } + ); + + t.is(calls[0].defaultProject.alias, 'main'); + } +); + +test.serial('writes the checkout file for the branch', async (t) => { + setup(); + const { prompt } = pickPrompt('staging'); + + await ensureCheckout( + new Workspace('/ws', logger, true, { branch: 'dev' }), + logger, + { interactive: true, prompt } + ); + + t.is( + readCheckout('/ws/.openfn/branches/dev/checkout.yaml').uuid, + 'uuid-staging' + ); + t.false(fs.existsSync('/ws/.openfn/checkout.yaml')); +}); + +test.serial( + 'aborts without prompting if several projects and not interactive', + async (t) => { + setup(); + const exitCode = process.exitCode; + + await t.throwsAsync( + ensureCheckout(new Workspace('/ws', logger), logger, { + interactive: false, + prompt: neverPrompt, + }), + { message: 'No checked out project found' } + ); + + // abort() flags the process as failed: don't let that leak out of the test + process.exitCode = exitCode; + + t.false(fs.existsSync('/ws/.openfn')); + t.truthy(logger._find('error', /No checked out project found/)); + } +); + +test.serial( + 'carries on without a checkout if not interactive and not required', + async (t) => { + setup(); + const workspace = new Workspace('/ws', logger); + + const result = await ensureCheckout(workspace, logger, { + interactive: false, + required: false, + prompt: neverPrompt, + }); + + t.is(result, workspace); + t.false(fs.existsSync('/ws/.openfn')); + } +); + +test.serial('still prompts if interactive, even if not required', async (t) => { + setup(); + const { prompt, calls } = pickPrompt('staging'); + + const result = await ensureCheckout(new Workspace('/ws', logger), logger, { + interactive: true, + required: false, + prompt, + }); + + t.is(calls.length, 1); + t.is(result.activeProject?.uuid, 'uuid-staging'); +}); + +const projects = (aliases: string[]) => + aliases.map((alias) => ({ alias })) as unknown as Project[]; + +test('getDefaultProject: prefers the alias matching the branch', (t) => { + const result = getDefaultProject(projects(['main', 'staging']), 'staging'); + t.is(result.alias, 'staging'); +}); + +test('getDefaultProject: defaults to main if there is no branch', (t) => { + t.is(getDefaultProject(projects(['a', 'main', 'b'])).alias, 'main'); +}); + +test('getDefaultProject: falls back to the first project', (t) => { + t.is(getDefaultProject(projects(['a', 'b']), 'feature').alias, 'a'); +}); + +const described = (props: object) => describeProject(props as Project); + +test('describeProject: alias with id and uuid', (t) => { + t.is( + described({ alias: 'staging', id: 'my-project', openfn: { uuid: 'abcd' } }), + 'staging (my-project | abcd)' + ); +}); + +test('describeProject: copes with a missing uuid or id', (t) => { + t.is( + described({ alias: 'staging', id: 'my-project' }), + 'staging (my-project)' + ); + t.is( + described({ alias: 'staging', openfn: { uuid: 'abcd' } }), + 'staging (abcd)' + ); +}); + +test('describeProject: just the alias if there is no id or uuid', (t) => { + t.is(described({ alias: 'staging' }), 'staging'); +}); + +test('describeProject: copes with a missing alias', (t) => { + t.is(described({ id: 'my-project' }), '(no alias) (my-project)'); +}); diff --git a/packages/cli/test/projects/list.test.ts b/packages/cli/test/projects/list.test.ts index 0891e5038..aab504a98 100644 --- a/packages/cli/test/projects/list.test.ts +++ b/packages/cli/test/projects/list.test.ts @@ -4,8 +4,14 @@ import { createMockLogger } from '@openfn/logger'; import mock from 'mock-fs'; import { jsonToYaml } from '@openfn/project'; +// a workspace with several projects but no checkout metadata +const noCheckoutProject = jsonToYaml({ id: 'my-project', workflows: [] }); + mock({ 'no-ws/': { 'some.yaml': 'name: smth' }, + '/no-checkout/openfn.yaml': '', + '/no-checkout/.projects/main@app.openfn.org.yaml': noCheckoutProject, + '/no-checkout/.projects/staging@app.openfn.org.yaml': noCheckoutProject, '/ws/openfn.yaml': jsonToYaml({ project: { id: 'my-project', @@ -172,3 +178,23 @@ staging | my-project message as string ); }); + +test.serial( + 'lists projects without a checkout, without prompting or aborting', + async (t) => { + const privateLogger = createMockLogger('', { level: 'debug' }); + + await list( + { command: 'projects', workspace: '/no-checkout' }, + privateLogger + ); + + const { message } = privateLogger._find( + 'always', + /available openfn projects/i + ); + t.regex(message as string, /^main \|/m); + t.regex(message as string, /^staging \|/m); + t.notRegex(message as string, /\(active\)/); + } +); diff --git a/packages/cli/test/projects/util.test.ts b/packages/cli/test/projects/util.test.ts index b4486ee17..b5a7fb44e 100644 --- a/packages/cli/test/projects/util.test.ts +++ b/packages/cli/test/projects/util.test.ts @@ -1,10 +1,17 @@ import test from 'ava'; -import Project, { generateWorkflow } from '@openfn/project'; +import fs from 'node:fs'; +import mock from 'mock-fs'; +import Project, { generateWorkflow, yamlToJson } from '@openfn/project'; import { findLocallyChangedWorkflows, tidyWorkflowDir, + writeCheckoutFile, } from '../../src/projects/util'; +test.afterEach(() => { + mock.restore(); +}); + test('tidyWorkflowDir: removes workflows that no longer exist', async (t) => { const currentProject = new Project({ name: 'current', @@ -271,3 +278,65 @@ test('findLocallyChangedWorkflows: detect 1 locally removed workflow', async (t) const changed = await findLocallyChangedWorkflows(workspace, project); t.deepEqual(changed, ['b']); }); + +const trackedProject = () => + new Project({ + id: 'my-project', + name: 'My Project', + openfn: { uuid: 'abcd', endpoint: 'https://app.openfn.org' }, + workflows: [ + { id: 'wf', name: 'wf', history: ['old', 'latest'], steps: [] }, + ], + }); + +test.serial( + 'writeCheckoutFile: writes the project meta and forked_from to the checkout file', + async (t) => { + mock({ '/ws': {} }); + + const filePath = await writeCheckoutFile('/ws', trackedProject()); + + t.is(filePath, '/ws/.openfn/checkout.yaml'); + t.deepEqual(yamlToJson(fs.readFileSync(filePath, 'utf8')), { + endpoint: 'https://app.openfn.org', + forked_from: { wf: 'latest' }, + id: 'my-project', + name: 'My Project', + uuid: 'abcd', + }); + } +); + +test.serial( + 'writeCheckoutFile: writes to the checkout file for a branch', + async (t) => { + mock({ '/ws': {} }); + + const filePath = await writeCheckoutFile('/ws', trackedProject(), 'dev'); + + t.is(filePath, '/ws/.openfn/branches/dev/checkout.yaml'); + t.true(fs.existsSync(filePath)); + t.false(fs.existsSync('/ws/.openfn/checkout.yaml')); + } +); + +test.serial( + 'writeCheckoutFile: does not touch the workspace config or workflows', + async (t) => { + mock({ + '/ws/openfn.yaml': 'credentials: creds.yaml', + '/ws/workflows/wf/wf.yaml': 'id: wf', + }); + + await writeCheckoutFile('/ws', trackedProject()); + + t.deepEqual(fs.readdirSync('/ws').sort(), [ + '.openfn', + 'openfn.yaml', + 'workflows', + ]); + t.is(fs.readFileSync('/ws/openfn.yaml', 'utf8'), 'credentials: creds.yaml'); + t.is(fs.readFileSync('/ws/workflows/wf/wf.yaml', 'utf8'), 'id: wf'); + t.deepEqual(fs.readdirSync('/ws/workflows/wf'), ['wf.yaml']); + } +); diff --git a/packages/project/src/Project.ts b/packages/project/src/Project.ts index 9a76a95ba..b81136b31 100644 --- a/packages/project/src/Project.ts +++ b/packages/project/src/Project.ts @@ -305,11 +305,18 @@ export class Project { } /** - * Generates the contents of the openfn.yaml and checkout files, - * plus their file paths + * Generates the contents of the openfn.yaml file, + * plus its file path */ - generateConfig(branch?: string | false | null) { - return [extractConfig(this), extractCheckout(this, branch)]; + generateConfig() { + return extractConfig(this); + } + + /** + * Generates the contents of the checkout file, plus its file path + */ + generateCheckout(branch?: string | false | null) { + return extractCheckout(this, branch); } /** diff --git a/packages/project/src/index.ts b/packages/project/src/index.ts index 44b1d428a..f7a87bf1c 100644 --- a/packages/project/src/index.ts +++ b/packages/project/src/index.ts @@ -18,6 +18,7 @@ export type { } from './util/project-diff'; export { RESOURCES_FILE } from './util/resources'; +export { hasCheckoutMeta } from './util/config'; export { generateStepDiff, generateEdgeDiff } from './util/workflow-diff'; export type { diff --git a/packages/project/src/util/config.ts b/packages/project/src/util/config.ts index a3225906f..77bbf3a78 100644 --- a/packages/project/src/util/config.ts +++ b/packages/project/src/util/config.ts @@ -181,3 +181,22 @@ export const findWorkspaceFile = (dir: string = '.') => { } return { content, type }; }; + +// Does this workspace know which project is checked out? +// True if there's checkout metadata in the checkout file, or in a legacy +// project block in openfn.yaml. An empty checkout file doesn't count +export const hasCheckoutMeta = ( + root: string = '.', + branch?: string | false | null +) => { + let legacyProject: l.ProjectMeta | undefined; + try { + const { type, content } = findWorkspaceFile(root); + legacyProject = loadWorkspaceFile(content, type as any).project; + } catch (e) { + // No workspace file: there can't be a legacy project block + } + + const meta = loadCheckoutFile(root, branch, legacyProject); + return !!meta && Object.keys(meta).length > 0; +}; diff --git a/packages/project/test/util/config.test.ts b/packages/project/test/util/config.test.ts index f0624ff97..6866e406f 100644 --- a/packages/project/test/util/config.test.ts +++ b/packages/project/test/util/config.test.ts @@ -3,6 +3,7 @@ import { extractCheckout, extractConfig, getCheckoutPath, + hasCheckoutMeta, findWorkspaceFile, loadWorkspaceFile, } from '../../src/util/config'; @@ -276,3 +277,54 @@ test('include project name', (t) => { }); test.todo('include parent project name'); + +test('hasCheckoutMeta: false if there is no workspace at all', (t) => { + mock({ '/ws': {} }); + + t.false(hasCheckoutMeta('/ws')); +}); + +test('hasCheckoutMeta: false if there is a workspace but no checkout file', (t) => { + mock({ '/ws/openfn.yaml': 'credentials: creds.yaml' }); + + t.false(hasCheckoutMeta('/ws')); +}); + +test('hasCheckoutMeta: true if there is a checkout file', (t) => { + mock({ + '/ws/openfn.yaml': 'credentials: creds.yaml', + '/ws/.openfn/checkout.yaml': 'id: my-project', + }); + + t.true(hasCheckoutMeta('/ws')); +}); + +test('hasCheckoutMeta: false if the checkout file is empty', (t) => { + mock({ + '/ws/openfn.yaml': 'credentials: creds.yaml', + '/ws/.openfn/checkout.yaml': '', + }); + + t.false(hasCheckoutMeta('/ws')); +}); + +test('hasCheckoutMeta: true if there is a legacy project block in openfn.yaml', (t) => { + mock({ + '/ws/openfn.yaml': + 'project:\n id: my-project\nworkspace:\n credentials: creds.yaml', + }); + + t.true(hasCheckoutMeta('/ws')); +}); + +test('hasCheckoutMeta: looks for the checkout file for the given branch', (t) => { + mock({ + '/ws/openfn.yaml': 'credentials: creds.yaml', + '/ws/.openfn/branches/dev/checkout.yaml': 'id: my-project', + }); + + t.true(hasCheckoutMeta('/ws', 'dev')); + // other branches, and no branch, don't have any + t.false(hasCheckoutMeta('/ws', 'main')); + t.false(hasCheckoutMeta('/ws')); +}); From 2fd16a118b1dfbfe78ef3a052683ee9ea828c235 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Wed, 7 Oct 2026 18:04:15 +0100 Subject: [PATCH 20/22] ignore checkout meta (#1578) --- packages/cli/src/projects/checkout.ts | 7 ++- packages/cli/src/projects/util.ts | 16 +++++++ packages/cli/test/projects/checkout.test.ts | 9 ++++ packages/cli/test/projects/util.test.ts | 50 +++++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/projects/checkout.ts b/packages/cli/src/projects/checkout.ts index c22a7b375..d004773cf 100644 --- a/packages/cli/src/projects/checkout.ts +++ b/packages/cli/src/projects/checkout.ts @@ -11,7 +11,11 @@ import * as o from '../options'; import * as po from './options'; import type { Opts } from './options'; -import { tidyWorkflowDir, updateForkedFrom } from './util'; +import { + ensureCheckoutIgnored, + tidyWorkflowDir, + updateForkedFrom, +} from './util'; import { createProjectCredentials } from './credentials-helpers'; import abort from '../util/abort'; @@ -167,6 +171,7 @@ export const handler = async (options: CheckoutOptions, logger?: Logger) => { delete files[switchProject.generateConfig().path]; delete files[switchProject.generateCheckout(options.branch).path]; } + await ensureCheckoutIgnored(workspacePath); for (const f in files) { if (files[f]) { fs.mkdirSync(path.join(workspacePath, path.dirname(f)), { diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index a34e8d7d2..b77d7dd97 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -260,6 +260,21 @@ export const updateForkedFrom = (proj: Project) => { return proj; }; +// The checkout state in .openfn is local to each machine, so it shouldn't be +// committed. Make git ignore everything in there. +// Set OPENFN_IGNORE_CHECKOUT_META=true to skip this +export const ensureCheckoutIgnored = async (workspacePath: string) => { + if (process.env.OPENFN_IGNORE_CHECKOUT_META === 'true') { + return; + } + + const filePath = path.resolve(workspacePath, '.openfn', '.gitignore'); + if (!fs.existsSync(filePath)) { + await mkdir(path.dirname(filePath), { recursive: true }); + await writeFile(filePath, '*\n'); + } +}; + // Set up the checkout file for a project without touching anything else // (use this when the workflows are already on disk, like after a git clone) export const writeCheckoutFile = async ( @@ -268,6 +283,7 @@ export const writeCheckoutFile = async ( branch?: string | false | null ) => { updateForkedFrom(project); + await ensureCheckoutIgnored(workspacePath); const { path: checkoutPath, content } = project.generateCheckout(branch); const filePath = path.resolve(workspacePath, checkoutPath); diff --git a/packages/cli/test/projects/checkout.test.ts b/packages/cli/test/projects/checkout.test.ts index d4762ec1f..fb6d69e7a 100644 --- a/packages/cli/test/projects/checkout.test.ts +++ b/packages/cli/test/projects/checkout.test.ts @@ -222,6 +222,15 @@ test.serial('checkout: same id as active', async (t) => { ); }); +test.serial('checkout: makes git ignore the checkout state', async (t) => { + await checkoutHandler( + { command: 'project-checkout', project: 'my-project', workspace: '/ws' }, + logger + ); + + t.is(fs.readFileSync('/ws/.openfn/.gitignore', 'utf8'), '*\n'); +}); + test.serial( 'checkout: writes forked_from based on version history', async (t) => { diff --git a/packages/cli/test/projects/util.test.ts b/packages/cli/test/projects/util.test.ts index b5a7fb44e..3f5b36ce0 100644 --- a/packages/cli/test/projects/util.test.ts +++ b/packages/cli/test/projects/util.test.ts @@ -6,6 +6,7 @@ import { findLocallyChangedWorkflows, tidyWorkflowDir, writeCheckoutFile, + ensureCheckoutIgnored, } from '../../src/projects/util'; test.afterEach(() => { @@ -340,3 +341,52 @@ test.serial( t.deepEqual(fs.readdirSync('/ws/workflows/wf'), ['wf.yaml']); } ); + +test.serial( + 'ensureCheckoutIgnored: makes git ignore everything in .openfn', + async (t) => { + mock({ '/ws': {} }); + + await ensureCheckoutIgnored('/ws'); + + t.is(fs.readFileSync('/ws/.openfn/.gitignore', 'utf8'), '*\n'); + } +); + +test.serial( + 'ensureCheckoutIgnored: leaves an existing .gitignore alone', + async (t) => { + mock({ '/ws/.openfn/.gitignore': 'custom\n' }); + + await ensureCheckoutIgnored('/ws'); + + t.is(fs.readFileSync('/ws/.openfn/.gitignore', 'utf8'), 'custom\n'); + } +); + +test.serial( + 'ensureCheckoutIgnored: does nothing if OPENFN_IGNORE_CHECKOUT_META is set', + async (t) => { + mock({ '/ws': {} }); + process.env.OPENFN_IGNORE_CHECKOUT_META = 'true'; + + try { + await ensureCheckoutIgnored('/ws'); + } finally { + delete process.env.OPENFN_IGNORE_CHECKOUT_META; + } + + t.false(fs.existsSync('/ws/.openfn')); + } +); + +test.serial( + 'writeCheckoutFile: makes git ignore the checkout state', + async (t) => { + mock({ '/ws': {} }); + + await writeCheckoutFile('/ws', trackedProject()); + + t.is(fs.readFileSync('/ws/.openfn/.gitignore', 'utf8'), '*\n'); + } +); From 131196a7e58f08053449d24527c1766d58c92fde Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 8 Oct 2026 10:35:30 +0100 Subject: [PATCH 21/22] changeset --- .changeset/eager-houses-call.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .changeset/eager-houses-call.md diff --git a/.changeset/eager-houses-call.md b/.changeset/eager-houses-call.md new file mode 100644 index 000000000..41929b61e --- /dev/null +++ b/.changeset/eager-houses-call.md @@ -0,0 +1,10 @@ +--- +'@openfn/project': minor +'@openfn/cli': minor +--- + +Restructure `openfn.yaml` to make it safer for merging on github. + +Any checkout state (basically anything under the project key) have been moved into a new `.openfn` folder, which should not be tracked. + +`openfn.yaml` now just contains workspace-wide configuration. From 677589102fd2696af17df6d92cb1a905342ca1bf Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Thu, 8 Oct 2026 11:18:59 +0100 Subject: [PATCH 22/22] changesets and docs --- .changeset/busy-needles-rhyme.md | 6 ------ .changeset/hip-actors-switch.md | 5 +++++ .changeset/orange-lions-talk.md | 5 +++++ .changeset/rich-clubs-shave.md | 5 ----- .changeset/true-dolls-stay.md | 6 ++++++ packages/cli/src/projects/options.ts | 4 +++- packages/project/src/util/config.ts | 8 ++++++-- 7 files changed, 25 insertions(+), 14 deletions(-) delete mode 100644 .changeset/busy-needles-rhyme.md create mode 100644 .changeset/hip-actors-switch.md create mode 100644 .changeset/orange-lions-talk.md delete mode 100644 .changeset/rich-clubs-shave.md create mode 100644 .changeset/true-dolls-stay.md diff --git a/.changeset/busy-needles-rhyme.md b/.changeset/busy-needles-rhyme.md deleted file mode 100644 index 957f061eb..000000000 --- a/.changeset/busy-needles-rhyme.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -'@openfn/lexicon': patch -'@openfn/project': patch ---- - -Remove trigger.enabled from project spec representations diff --git a/.changeset/hip-actors-switch.md b/.changeset/hip-actors-switch.md new file mode 100644 index 000000000..33e091794 --- /dev/null +++ b/.changeset/hip-actors-switch.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': patch +--- + +Fix an issue where `project clean` can try to delete stuff even if nothing is checked out diff --git a/.changeset/orange-lions-talk.md b/.changeset/orange-lions-talk.md new file mode 100644 index 000000000..3a458c822 --- /dev/null +++ b/.changeset/orange-lions-talk.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': patch +--- + +Include collections and channels in sync diffs diff --git a/.changeset/rich-clubs-shave.md b/.changeset/rich-clubs-shave.md deleted file mode 100644 index a4248f8ca..000000000 --- a/.changeset/rich-clubs-shave.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@openfn/cli': patch ---- - -Remove trigger.enabled from workflow.yaml, so that it doesn't show up on git diffs. diff --git a/.changeset/true-dolls-stay.md b/.changeset/true-dolls-stay.md new file mode 100644 index 000000000..3e537705b --- /dev/null +++ b/.changeset/true-dolls-stay.md @@ -0,0 +1,6 @@ +--- +'@openfn/project': patch +'@openfn/cli': patch +--- + +Track collections in resources.yaml diff --git a/packages/cli/src/projects/options.ts b/packages/cli/src/projects/options.ts index 0e29f7da4..1b4f8cb9e 100644 --- a/packages/cli/src/projects/options.ts +++ b/packages/cli/src/projects/options.ts @@ -192,10 +192,12 @@ const detectBranch = (cwd: string) => { } }; -// Must come after the workspace option so that the workspace path is resolved +// --branch is used to force the CLI to load checkout metadata from a different +// path in the .openfn folder. It is not likely to be used by users. export const branch: CLIOption = { name: 'branch', yargs: { + hidden: true, description: 'The git branch to track the checked out project against. Detected automatically if not set. Pass --no-branch to ignore git.', }, diff --git a/packages/project/src/util/config.ts b/packages/project/src/util/config.ts index 77bbf3a78..e4fe7dbe7 100644 --- a/packages/project/src/util/config.ts +++ b/packages/project/src/util/config.ts @@ -39,8 +39,12 @@ export const buildConfig = (config: Partial = {}) => ({ }); // The checkout file tracks which project is expanded into the workflows dir, -// plus any transient sync state. When on a git branch, each branch gets its -// own checkout file so that merges don't clobber the target branch's checkout +// plus any transient sync state (like forked_from, which helps track divergence). +// +// It's local to each machine and git-ignored, so git can't switch it when you +// change branch. To keep the CLI aware of what's checked out after a branch +// switch (without the user having to re-run `openfn checkout`), each git +// branch gets its own checkout file. export const getCheckoutPath = (branch?: string | false | null) => branch ? path.join('.openfn', 'branches', branch, 'checkout.yaml')