diff --git a/.github/actions/cli/action.yml b/.github/actions/cli/action.yml index bccea892..b8565bf2 100644 --- a/.github/actions/cli/action.yml +++ b/.github/actions/cli/action.yml @@ -2,8 +2,7 @@ name: CLI artifact description: >- Produces the DataWeave `dw` CLI distro zip; optionally runs the master-only native-cli regression/TCK suites; optionally publishes the distro as a CI - artifact (main) or a release asset (release). Requires build-foundation to - have run earlier in the same job. + artifact. Requires build-foundation to have run earlier in the same job. inputs: native-version: description: -PnativeVersion value; empty omits the flag. @@ -14,13 +13,9 @@ inputs: required: false default: 'false' publish: - description: "'none' | 'artifact' | 'release'." + description: "'none' | 'artifact'." required: false default: 'none' - repo-token: - description: Token for svenstaro release upload (publish=release). - required: false - default: '' arch: description: Runtime arch token for artifact/asset names. required: false @@ -33,10 +28,6 @@ inputs: description: Gradle distro classifier (linux/windows/osx) for the source zip name. required: false default: '' - tag: - description: Release tag (publish=release). - required: false - default: '' runs: using: composite steps: @@ -58,15 +49,6 @@ runs: if: inputs.publish == 'artifact' uses: actions/upload-artifact@v7.0.1 with: + name: cli-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }} path: native-cli/build/distributions/dw-cli-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }}.zip archive: false - - - name: Upload binaries to release - if: inputs.publish == 'release' - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ inputs.repo-token }} - file: native-cli/build/distributions/native-cli-${{ inputs.native-version }}-native-distro-${{ inputs.distro-os }}.zip - asset_name: dw-cli-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }}.zip - tag: ${{ inputs.tag }} - overwrite: true diff --git a/.github/actions/native-lib/action.yml b/.github/actions/native-lib/action.yml index e644da2b..7e42868d 100644 --- a/.github/actions/native-lib/action.yml +++ b/.github/actions/native-lib/action.yml @@ -2,21 +2,16 @@ name: Native shared library artifact description: >- Stages and publishes the raw dwlib shared library (dwlib.so/.dll/.dylib) and its C header (dwlib.h) — the FFI surface both the Python and Node bindings - embed, published here as standalone downloads. Requires build-foundation - earlier in the same job. + embed — as CI artifacts. Requires build-foundation earlier in the same job. inputs: native-version: description: -PnativeVersion value; empty omits the flag. required: false default: '' publish: - description: "'none' | 'artifact' | 'release'." + description: "'none' | 'artifact'." required: false default: 'none' - repo-token: - description: Token for svenstaro release upload (publish=release). - required: false - default: '' arch: description: Runtime arch token for artifact/asset names. required: false @@ -25,10 +20,6 @@ inputs: description: OS naming token (linux/windows/macos) for artifact/asset names. required: false default: '' - tag: - description: Release tag (publish=release). - required: false - default: '' runs: using: composite steps: @@ -46,43 +37,3 @@ runs: native-lib/python/src/dataweave/native/dwlib.so native-lib/python/src/dataweave/native/dwlib.dll native-lib/python/src/dataweave/native/dwlib.h - - - name: Upload native shared library to release (Linux) - if: inputs.publish == 'release' && runner.os == 'Linux' - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ inputs.repo-token }} - file: native-lib/python/src/dataweave/native/dwlib.so - asset_name: dwlib-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }}.so - tag: ${{ inputs.tag }} - overwrite: true - - - name: Upload native shared library to release (Windows) - if: inputs.publish == 'release' && runner.os == 'Windows' - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ inputs.repo-token }} - file: native-lib/python/src/dataweave/native/dwlib.dll - asset_name: dwlib-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }}.dll - tag: ${{ inputs.tag }} - overwrite: true - - - name: Upload native shared library to release (macOS) - if: inputs.publish == 'release' && runner.os == 'macOS' - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ inputs.repo-token }} - file: native-lib/python/src/dataweave/native/dwlib.dylib - asset_name: dwlib-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }}.dylib - tag: ${{ inputs.tag }} - overwrite: true - - - name: Upload native library header to release - if: inputs.publish == 'release' - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ inputs.repo-token }} - file: native-lib/python/src/dataweave/native/dwlib.h - asset_name: dwlib-${{ inputs.native-version }}.h - tag: ${{ inputs.tag }} - overwrite: true diff --git a/.github/actions/node/action.yml b/.github/actions/node/action.yml index 4d8bd007..9780ef6e 100644 --- a/.github/actions/node/action.yml +++ b/.github/actions/node/action.yml @@ -2,9 +2,8 @@ name: Node artifact description: >- Sets up Node.js, builds the DataWeave Node package (which embeds dwlib), and runs the Node unit/integration tests (always). Optionally runs the master-only - Node TCK conformance lane, and optionally publishes the .tgz as a CI artifact - (main) or a release asset (release). Requires build-foundation earlier in the - same job. + Node TCK conformance lane and optionally publishes the .tgz as a CI artifact. + Requires build-foundation earlier in the same job. inputs: native-version: description: -PnativeVersion value; empty omits the flag. @@ -15,13 +14,9 @@ inputs: required: false default: 'false' publish: - description: "'none' | 'artifact' | 'release'." + description: "'none' | 'artifact'." required: false default: 'none' - repo-token: - description: Token for svenstaro release upload (publish=release). - required: false - default: '' arch: description: Runtime arch token for artifact/asset names. required: false @@ -30,10 +25,6 @@ inputs: description: OS naming token (linux/windows/macos) for artifact/asset names. required: false default: '' - tag: - description: Release tag (publish=release). - required: false - default: '' runs: using: composite steps: @@ -80,6 +71,7 @@ runs: if: inputs.publish == 'artifact' uses: actions/upload-artifact@v7.0.1 with: + name: node-platform-${{ inputs.native-version }}-${{ inputs.script-name }}-${{ inputs.arch }} path: ${{ steps.node-package.outputs.platform }} archive: false @@ -87,25 +79,6 @@ runs: if: inputs.publish == 'artifact' && inputs.script-name == 'linux' uses: actions/upload-artifact@v7.0.1 with: + name: node-meta-${{ inputs.native-version }} path: ${{ steps.node-package.outputs.meta }} archive: false - - - name: Upload platform Node package to release - if: inputs.publish == 'release' - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ inputs.repo-token }} - file: ${{ steps.node-package.outputs.platform }} - asset_name: ${{ steps.node-package.outputs.platform_name }} - tag: ${{ inputs.tag }} - overwrite: true - - - name: Upload meta Node package to release - if: inputs.publish == 'release' && inputs.script-name == 'linux' - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ inputs.repo-token }} - file: ${{ steps.node-package.outputs.meta }} - asset_name: ${{ steps.node-package.outputs.meta_name }} - tag: ${{ inputs.tag }} - overwrite: true diff --git a/.github/actions/python/action.yml b/.github/actions/python/action.yml index 345062ec..73d4b7d6 100644 --- a/.github/actions/python/action.yml +++ b/.github/actions/python/action.yml @@ -3,8 +3,8 @@ description: >- Installs Python build dependencies and builds the DataWeave Python wheel (which embeds dwlib), runs the Python unit/integration tests, and optionally runs the master-only Python TCK conformance lane before publishing the wheel - as a CI artifact (main) or a release asset (release). Requires - build-foundation to have run earlier in the same job. + as a CI artifact. Requires build-foundation to have run earlier in the same + job. inputs: native-version: description: -PnativeVersion value; empty omits the flag. @@ -22,21 +22,13 @@ inputs: required: false default: 'false' platform: - description: Platform token for matrix-qualified TCK JUnit artifact names. + description: Platform token for CI artifact names. required: false default: '' publish: - description: "'none' | 'artifact' | 'release'." + description: "'none' | 'artifact'." required: false default: 'none' - repo-token: - description: Token for svenstaro release upload (publish=release). - required: false - default: '' - tag: - description: Release tag (publish=release). - required: false - default: '' runs: using: composite steps: @@ -60,19 +52,10 @@ runs: if: inputs.publish == 'artifact' uses: actions/upload-artifact@v7.0.1 with: + name: python-wheel-${{ inputs.native-version }}-${{ inputs.platform }} path: native-lib/python/dist/dataweave_native-0.0.1-py3-*.whl archive: false - - name: Upload Python wheel to release - if: inputs.publish == 'release' - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ inputs.repo-token }} - file: native-lib/python/dist/dataweave_native-0.0.1-py3-none-*.whl - file_glob: true - tag: ${{ inputs.tag }} - overwrite: true - - name: Run Python TCK Conformance if: always() && inputs.run-tck == 'true' run: ./gradlew --stacktrace --no-problems-report native-lib:pythonTck diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 729c3e2e..ca389f45 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,9 +50,7 @@ jobs: uses: ./.github/actions/cli with: native-version: ${{ env.NATIVE_VERSION }} - publish: 'release' - repo-token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ github.ref }} + publish: 'artifact' arch: ${{ env.ARCH }} script-name: ${{ matrix.script_name }} distro-os: ${{ matrix.distro_os }} @@ -62,17 +60,14 @@ jobs: with: native-version: ${{ env.NATIVE_VERSION }} break-system-packages: 'true' - publish: 'release' - repo-token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ github.ref }} + platform: ${{ matrix.script_name }}-${{ env.ARCH }} + publish: 'artifact' - name: Node uses: ./.github/actions/node with: native-version: ${{ env.NATIVE_VERSION }} - publish: 'release' - repo-token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ github.ref }} + publish: 'artifact' arch: ${{ env.ARCH }} script-name: ${{ matrix.script_name }} @@ -80,8 +75,62 @@ jobs: uses: ./.github/actions/native-lib with: native-version: ${{ env.NATIVE_VERSION }} - publish: 'release' - repo-token: ${{ secrets.GITHUB_TOKEN }} - tag: ${{ github.ref }} + publish: 'artifact' arch: ${{ env.ARCH }} script-name: ${{ matrix.script_name }} + + publish-release: + needs: RELEASE_EXTENSION + runs-on: mulesoft-ubuntu + permissions: + contents: write + steps: + - name: Download release artifacts + uses: actions/download-artifact@v8 + with: + path: release-assets + - name: Create release and upload assets + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + TAG: ${{ github.ref_name }} + run: | + set -euo pipefail + VERSION="${TAG#v}" + HEADER=$(find "release-assets/dwlib-${VERSION}-linux-x86_64" -maxdepth 1 -name 'dwlib.h' -print -quit) + if [ -z "$HEADER" ]; then + echo "dwlib.h is missing" + exit 1 + fi + mv "$HEADER" "$RUNNER_TEMP/dwlib.h" + find release-assets -type f -name 'dwlib.h' -delete + mv "$RUNNER_TEMP/dwlib.h" "release-assets/dwlib-${VERSION}.h" + for platform in linux-x86_64 windows-x86_64 macos-arm64; do + case "$platform" in + linux-x86_64) extension=so ;; + windows-x86_64) extension=dll ;; + macos-arm64) extension=dylib ;; + esac + source="release-assets/dwlib-${VERSION}-${platform}/dwlib.${extension}" + if [ ! -f "$source" ]; then + echo "native library is missing: $source" + exit 1 + fi + mv "$source" "release-assets/dwlib-${VERSION}-${platform}.${extension}" + done + gh release view "$TAG" || gh release create "$TAG" --generate-notes + shopt -s globstar nullglob + assets=( + release-assets/**/*.zip + release-assets/**/*.whl + release-assets/**/*.tgz + release-assets/dwlib-*.so + release-assets/dwlib-*.dll + release-assets/dwlib-*.dylib + release-assets/dwlib-*.h + ) + if [ "${#assets[@]}" -eq 0 ]; then + echo "release artifacts are missing" + exit 1 + fi + gh release upload "$TAG" "${assets[@]}" --clobber diff --git a/docs/superpowers/specs/2026-09-02-release-artifact-publication-design.md b/docs/superpowers/specs/2026-09-02-release-artifact-publication-design.md new file mode 100644 index 00000000..e5cb9edb --- /dev/null +++ b/docs/superpowers/specs/2026-09-02-release-artifact-publication-design.md @@ -0,0 +1,103 @@ +# Release Artifact Publication Design + +**Status:** Approved +**Date:** 2026-09-02 +**Scope:** Move GitHub Release publication out of the cross-platform build jobs +in `.github/workflows/release.yml`. Preserve the existing Linux, Windows, and +macOS artifact matrix. + +## Goal + +Publish release assets from a MuleSoft-managed Ubuntu runner after all native +platform builds succeed, so the GitHub-hosted macOS runner never accesses the +GitHub Releases API and is not blocked by the organization IP allow list. + +## Background + +Release run `33542023719` built the macOS ARM64 native artifacts successfully, +then failed in the CLI composite action while `svenstaro/upload-release-action` +requested release `v1.0.37`. GitHub rejected the request because the +GitHub-hosted macOS runner IP is outside the `mulesoft` organization allow list. + +The existing `mulesoft-ubuntu` and `mulesoft-windows` jobs completed. The +normal CI workflow already demonstrates artifact-only output from every matrix +job. + +## Design + +The existing tag-triggered release workflow remains a single workflow with two +phases: + +1. A three-platform matrix builds native CLI, Python, Node, and native library + outputs. Each job uses `publish: artifact`; none uploads to a GitHub Release. +2. A non-matrix `publish-release` job runs on `mulesoft-ubuntu`, requires the + entire build matrix to succeed, downloads all produced artifacts, creates a + GitHub Release when its tag has no release, and uploads the downloaded files. + +The publishing job uses `GITHUB_TOKEN` and `permissions: contents: write`. +It runs only on a MuleSoft-managed runner, whose IP is accepted by the +organization allow list. + +## Artifact Contract + +All artifacts must have explicit, unique names that include the package version, +platform, and architecture where applicable. The artifact names must be stable +enough for the publishing job to download them without inspecting a prior job's +output. + +The publication job uploads only the packaged distribution files, not the +artifact archive wrappers. It preserves the release asset names currently +published by each composite action: + +- CLI: `dw-cli---.zip` +- Node: `dataweave-native--.tgz`, plus the Linux-produced + `dataweave-native-.tgz` meta package +- Python: the existing platform-qualified built wheel filename +- Native library: platform-specific `dwlib---` file, + plus the shared header only once + +The Python packaging already produces platform-qualified wheel filenames. The +publisher must retain and upload each wheel, so Linux, Windows, and macOS +customers receive their matching native-library build. The release publisher +must upload each distinct asset exactly once. + +## Publish Semantics + +For a tag `v`, `publish-release` must: + +1. Download all release artifacts produced by the build matrix. +2. Test whether the matching GitHub Release exists. +3. Create it with generated notes only when absent. +4. Upload every asset with replacement enabled, making reruns idempotent. + +The composite actions expose only `none` and `artifact` publication modes. No +platform build job may invoke `svenstaro/upload-release-action`, `gh release`, +or any GitHub Releases API endpoint. A failed matrix job prevents +`publish-release` from starting, so a partial release is never published. + +## Non-goals + +- Changing the set of supported operating systems or architectures. +- Replacing the GitHub-hosted macOS runner. +- Publishing to Homebrew, npm, PyPI, or any other registry. +- Modifying the manual Homebrew promotion workflow. +- Publishing a release before every platform build completes. + +## Testing + +- Validate workflow syntax with the available YAML parser and `actionlint` when + available. +- Inspect the generated workflow to verify build jobs use artifact publication + only and the sole release API consumer runs on `mulesoft-ubuntu`. +- Trigger a test tag in a controlled repository or perform a release dry run if + repository policy permits it; verify macOS build artifacts are downloaded and + attached from the Ubuntu publisher. + +## Success Criteria + +- A tag build produces artifacts on Linux, Windows, and macOS without calling + the GitHub Releases API from GitHub-hosted macOS. +- The `mulesoft-ubuntu` publisher creates the GitHub Release if it is missing. +- A rerun replaces assets on an existing release rather than failing. +- A failed build matrix prevents release creation or asset upload. +- Release assets retain their existing external filenames. diff --git a/native-lib/python/tests/unit/test_ci_structure.py b/native-lib/python/tests/unit/test_ci_structure.py index c2d32dec..7d6eacab 100644 --- a/native-lib/python/tests/unit/test_ci_structure.py +++ b/native-lib/python/tests/unit/test_ci_structure.py @@ -67,7 +67,8 @@ def test_python_artifact_owns_test_dependencies_and_tck_junit_upload(): assert "python-tck-junit-${{ inputs.platform }}" in action assert action.index("Create Native Lib Python Wheel") < action.index("Run Python TCK Conformance") assert action.index("Upload Python wheel (artifact)") < action.index("Run Python TCK Conformance") - assert action.index("Upload Python wheel to release") < action.index("Run Python TCK Conformance") + assert "Upload Python wheel to release" not in action + assert "svenstaro/upload-release-action" not in action assert action.index("Run Python TCK Conformance") < action.index("Upload Python TCK JUnit") assert named_step_if(action, "Run Python TCK Conformance") == "always() && inputs.run-tck == 'true'" assert "native-lib/build/test-results/pythonTck.xml" in action diff --git a/scripts/release-artifacts.test.mjs b/scripts/release-artifacts.test.mjs new file mode 100644 index 00000000..18627df2 --- /dev/null +++ b/scripts/release-artifacts.test.mjs @@ -0,0 +1,58 @@ +import { readFileSync } from "node:fs"; +import assert from "node:assert/strict"; +import { test } from "node:test"; + +const files = [ + ".github/actions/cli/action.yml", + ".github/actions/python/action.yml", + ".github/actions/node/action.yml", + ".github/actions/native-lib/action.yml", +]; + +test("artifact publishing names every uploaded release input", () => { + for (const file of files) { + const action = readFileSync(file, "utf8"); + const uploads = action.matchAll( + /uses: actions\/upload-artifact@v7\.0\.1\n\s+with:\n(?(?:\s+.+\n?)+)/g, + ); + for (const upload of uploads) { + assert.match(upload.groups.options, /^\s+name: .+/m); + } + } +}); + +test("Python wheel artifact retains its platform-qualified wheel filename", () => { + const action = readFileSync(".github/actions/python/action.yml", "utf8"); + assert.match(action, /path: native-lib\/python\/dist\/dataweave_native-0\.0\.1-py3-\*\.whl/); +}); + +test("composite actions do not expose direct release publishing", () => { + for (const file of files) { + const action = readFileSync(file, "utf8"); + assert.doesNotMatch(action, /'release'/); + assert.doesNotMatch(action, /repo-token:/); + assert.doesNotMatch(action, /tag:/); + assert.doesNotMatch(action, /svenstaro\/upload-release-action/); + } +}); + +test("release publication happens only on internal Ubuntu after the matrix", () => { + const workflow = readFileSync(".github/workflows/release.yml", "utf8"); + assert.match(workflow, /publish: 'artifact'/); + assert.match(workflow, /publish-release:/); + assert.match(workflow, /needs: RELEASE_EXTENSION/); + assert.match(workflow, /runs-on: mulesoft-ubuntu/); + assert.match(workflow, /publish-release:\n(?:.|\n)*?permissions:\n\s+contents: write/); + assert.match(workflow, /GH_REPO: \$\{\{ github\.repository \}\}/); + assert.match(workflow, /gh release view "\$TAG" \|\| gh release create "\$TAG" --generate-notes/); + assert.match(workflow, /gh release upload "\$TAG"/); + assert.doesNotMatch(workflow, /publish: 'release'/); +}); + +test("publisher retains the versioned native library release filenames", () => { + const workflow = readFileSync(".github/workflows/release.yml", "utf8"); + assert.doesNotMatch(workflow, /merge-multiple: true/); + assert.match(workflow, /dwlib-\$\{VERSION\}-\$\{platform\}\.\$\{extension\}/); + assert.match(workflow, /release-assets\/dwlib-\$\{VERSION\}-linux-x86_64/); + assert.match(workflow, /shopt -s globstar nullglob/); +});