From 575f6bc6e4f36a6f4243b244363c57c8eadb1175 Mon Sep 17 00:00:00 2001 From: Volv G Date: Sat, 3 Oct 2026 21:50:00 -0700 Subject: [PATCH] Relocate root pipeline relative refs to the output dir (v0.1.32) The root @pipeline's relative file:// / resolve:// refs were emitted verbatim and then validated against the output dir, so compiling anywhere but the source dir failed. Apply the existing child relocation to the root too. --- README.md | 3 + .../tangle-cli/src/tangle_cli/__init__.py | 2 +- .../src/tangle_cli/pipeline_compiler.py | 63 +++++++-------- pyproject.toml | 2 +- tests/test_packaging.py | 2 +- tests/test_pipeline_compiler.py | 81 +++++++++++++++---- tests/test_pipeline_decompiler.py | 7 +- uv.lock | 2 +- 8 files changed, 103 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index c6dea38..bdc4d1b 100644 --- a/README.md +++ b/README.md @@ -529,6 +529,9 @@ uv run tangle sdk pipelines compile pipeline.py -o pipeline.yaml uv run tangle sdk pipelines compile pipeline.py -o pipeline.yaml --pipeline pipeline_fn_name ``` +Relative `file://` and `resolve://` refs are written relative to the `.py` file. +The compiler rewrites them so they still resolve from wherever `-o` points. + To compile and submit in one step, without keeping the compiled YAML around, use `pipeline-runs submit-from-python`: diff --git a/packages/tangle-cli/src/tangle_cli/__init__.py b/packages/tangle-cli/src/tangle_cli/__init__.py index d203387..6b837cb 100644 --- a/packages/tangle-cli/src/tangle_cli/__init__.py +++ b/packages/tangle-cli/src/tangle_cli/__init__.py @@ -14,6 +14,6 @@ try: __version__ = metadata_version("tangle-cli") except PackageNotFoundError: - __version__ = "0.1.31" + __version__ = "0.1.32" __all__ = ["TangleDynamicDiscoveryClient", "__version__"] diff --git a/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py b/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py index 34257b6..2e05366 100644 --- a/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py +++ b/packages/tangle-cli/src/tangle_cli/pipeline_compiler.py @@ -874,22 +874,21 @@ def _compile_pipeline_fn( components_yaml_name=components_path.name, ) - # 4b. A CHILD artifact is written under ``.subgraphs/``, away from - # its own source directory. Its author-written relative local refs - # (plain ``ref(url="file://./leaf.yaml")``) point at files next to - # the child SOURCE, so relocate them to be relative to the child - # SIDECAR directory — the URL still resolves to the SAME original - # file (no copying), just from the sidecar's location. Compiler- - # managed refs (``@task`` resolver + subpipeline) already point at - # bundle files and are skipped. The root is never relocated: its - # output dir is its bundle root (in-place compile contract). - if not is_root: - _relocate_child_local_refs( - body_dict=body_dict, - builder=builder, - source_dir=base_dir, - sidecar_dir=output_path.parent, - ) + # 4b. An artifact may be written away from its own source directory (a + # CHILD under ``.subgraphs/``; the ROOT wherever ``--output`` + # points). Its author-written relative local refs (plain + # ``ref(url="file://./leaf.yaml")``) point at files next to the + # SOURCE, so relocate them to be relative to the artifact's OUTPUT + # directory — the URL still resolves to the SAME original file (no + # copying), just from the output's location. Compiler-managed refs + # (``@task`` resolver + subpipeline) already point at bundle files + # and are skipped. Compiling in place leaves the URLs unchanged. + _relocate_local_refs( + body_dict=body_dict, + builder=builder, + source_dir=base_dir, + sidecar_dir=output_path.parent, + ) # 4c. @registered refs point at an EXISTING gen_config.yaml (the # operation is registered/published elsewhere), so there is no @@ -1468,16 +1467,15 @@ def _validate_artifact(artifact: SubgraphArtifact, ctx: CompileContext) -> None: # Asset policy. EVERY # artifact's relative local refs are validated relative to THAT # artifact's own output directory: - # * the ROOT body relative to the root output dir (``label`` is None, - # preserving the verbatim single-pipeline error message); + # * the ROOT body relative to the root output dir (``label`` is None); # * each CHILD body relative to ITS child-sidecar dir (``label`` adds # child-sidecar + task context to the error). # Generated bundle files the compiler is about to write — child graph # sidecars and child ``@task`` components sidecars — are in # ``ctx.planned_files`` and count as present, so the compiler-managed - # parent→child / child→child / child @task refs pass. A child's + # parent→child / child→child / child @task refs pass. An artifact's # author-written relative leaf ref was relocated to be relative to - # the child-sidecar dir, so it is validated against the real source-side + # its output dir, so it is validated against the real source-side # file via ``../``; a missing external leaf fails clearly here, before # any file is written. _validate_local_component_refs_for_artifact( @@ -1924,19 +1922,19 @@ def _rewrite_registered_componentref_urls( tasks[task_id]["componentRef"] = {"url": url} -def _relocate_child_local_refs( +def _relocate_local_refs( *, body_dict: dict[str, Any], builder: Any, source_dir: Path, sidecar_dir: Path, ) -> None: - """Rewrite a child's author-written relative local componentRefs from - being relative to its SOURCE dir to relative to its SIDECAR dir. + """Rewrite an artifact's author-written relative local componentRefs from + being relative to its SOURCE dir to relative to its output (``sidecar_dir``). - A child compiles into ``.subgraphs/`` but its ``ref(url=...)`` - URLs were authored relative to the child's own source file. Rewriting - them keeps each ref pointing at the SAME original file (no copying) so + A child compiles into ``.subgraphs/`` and the root into any + ``--output`` dir, but their ``ref(url=...)`` URLs were authored relative + to their own source file. Rewriting them keeps each ref pointing at the SAME original file (no copying) so the existing hydrator — which resolves child refs relative to the loaded sidecar's directory — still finds it. @@ -2578,8 +2576,7 @@ def _validate_local_component_refs_for_artifact( External relative refs must already exist on disk. Args: - artifact_label: ``None`` for the ROOT (uses the legacy error wording - relative to the OUTPUT directory); a ``child pipeline '' + artifact_label: ``None`` for the ROOT; a ``child pipeline '' ()`` label for a child sidecar (uses child-context wording relative to the child sidecar's directory). @@ -2608,13 +2605,9 @@ def _validate_local_component_refs_for_artifact( if artifact_label is None: raise CompileError( f"task {task_id!r} references local component {url!r}, but the " - f"target does not exist relative to the output directory: " - f"{target}. Hydrate resolves componentRef URLs relative to the " - "compiled YAML's location, so the referenced file must sit " - "next to the compiled output. Fix options: compile into the " - "pipeline source directory so referenced files are colocated " - "with the output; place the referenced component next to the " - "compiled YAML; or use an absolute file:///… / gs://… URL or a " + f"target does not exist: {target}. Relative componentRef URLs " + "are authored relative to the pipeline source file's directory. " + "Fix the path, or use an absolute file:///… / gs://… URL or a " "published name: ref." ) raise CompileError( diff --git a/pyproject.toml b/pyproject.toml index 61264eb..6dde619 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tangle-cli" -version = "0.1.31" +version = "0.1.32" description = "CLI for Tangle, the open-source ML pipeline orchestration platform" readme = "README.md" authors = [ diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 990d27b..bca0db8 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -183,7 +183,7 @@ def test_tangle_cli_wheel_supports_expert_no_deps_import_path_without_tangle_api requires_dist = [line for line in metadata.splitlines() if line.startswith("Requires-Dist: ")] assert not any(name.startswith("tangle_api/") for name in names) assert "tangle_cli/openapi/openapi.json" not in names - assert "Version: 0.1.31" in metadata + assert "Version: 0.1.32" in metadata assert "Requires-Dist: tangle-api==0.1.1" in requires_dist assert not any("extra == 'native'" in line for line in requires_dist) assert "Provides-Extra: native" in metadata diff --git a/tests/test_pipeline_compiler.py b/tests/test_pipeline_compiler.py index 618dc89..c9565b5 100644 --- a/tests/test_pipeline_compiler.py +++ b/tests/test_pipeline_compiler.py @@ -11,6 +11,7 @@ import shutil import sys from pathlib import Path +from unittest.mock import MagicMock import pytest import yaml @@ -25,6 +26,7 @@ compile_pipeline, register_image_id, ) +from tangle_cli.pipeline_hydrator import PipelineHydrator from tangle_cli.pipelines import PipelineValidationError, compile_pipeline_file from tangle_cli.python_pipeline.errors import CompileError from tangle_cli.schema_validation import validate_dehydrated_data @@ -41,12 +43,11 @@ def run_app(app, args: list[str]) -> None: def _provide_noop(out: Path) -> None: - """Colocate the referenced ``noop.yaml`` component next to the output. + """Place the ``noop.yaml`` component in the output's directory. - Fixtures like ``pipeline.py`` / ``multi_arg_pipeline.py`` reference - ``file://./noop.yaml``; the compiler validates that relative local - componentRef targets exist relative to the OUTPUT directory, so the - referenced component must sit next to the compiled YAML. + Pipelines written next to the output reference ``file://./noop.yaml``, + which resolves from the pipeline SOURCE directory, so the component must + exist there. """ out.parent.mkdir(parents=True, exist_ok=True) shutil.copy(FIXTURES / "noop.yaml", out.parent / "noop.yaml") @@ -82,8 +83,12 @@ def test_compile_writes_single_dehydrated_yaml(tmp_path): task_body = tasks["Wait For Noop"] # componentRef is a PURE ref — no inline spec / text. + # The source-relative ref is relocated to resolve from the output dir. cref = task_body["componentRef"] - assert cref == {"url": "file://./noop.yaml"} + assert list(cref) == ["url"] + assert (out.parent / cref["url"].removeprefix("file://")).resolve() == ( + FIXTURES / "noop.yaml" + ).resolve() assert "spec" not in cref assert "text" not in cref @@ -147,23 +152,66 @@ def test_compile_pipeline_does_not_leak_sys_state(tmp_path): # Free-function compile: user-facing failure modes. +def _script_missing_noop(tmp_path: Path) -> Path: + """Copy ``pipeline.py`` (and its config) to a dir WITHOUT ``noop.yaml``, + so its relative ``file://./noop.yaml`` ref is unresolvable.""" + src = tmp_path / "src" + src.mkdir() + for name in ("pipeline.py", "config.yaml"): + shutil.copy(FIXTURES / name, src / name) + return src / "pipeline.py" + + def test_compile_unresolvable_local_ref_fails(tmp_path): - """A relative file:// componentRef whose target is NOT colocated with - the output fails clearly with guidance, and writes no output.""" + """A relative file:// componentRef whose target does NOT exist next to + the pipeline source fails clearly with guidance, and writes no output.""" out = tmp_path / "compiled.yaml" - # Deliberately do NOT colocate noop.yaml next to the output. with pytest.raises(CompileError) as exc: - compile_pipeline(FIXTURES / "pipeline.py", out) + compile_pipeline(_script_missing_noop(tmp_path), out) msg = str(exc.value) - assert "file://./noop.yaml" in msg - assert "output directory" in msg + assert "noop.yaml" in msg # Guidance for the user. - assert "compile into the pipeline source directory" in msg.lower() + assert "relative to the pipeline source file's directory" in msg # No output written on failure (neither pipeline nor sidecar). assert not out.exists() assert not (tmp_path / "compiled.components.yaml").exists() +@pytest.mark.parametrize("out_rel", ["pipe/compiled.yaml", "pipe/temp/compiled.yaml", "other/deep/compiled.yaml"]) +def test_compile_relocates_root_local_refs_to_any_output_dir(tmp_path, out_rel): + """The ROOT's source-relative ``file://`` / ``resolve://`` refs are + rewritten to resolve from wherever the output lands: in place, a deeper + dir, or an unrelated one. Each compiled YAML then hydrates.""" + comp = tmp_path / "comp" + comp.mkdir() + shutil.copy(FIXTURES / "noop.yaml", comp / "noop.yaml") + (comp / "resolve.yaml").write_text(yaml.safe_dump({"frag": {"local": "./noop.yaml"}})) + src = tmp_path / "pipe" + src.mkdir() + (src / "pipeline.py").write_text( + "from tangle_cli.python_pipeline import Out, pipeline, ref\n" + "\n" + "@pipeline('Relocated')\n" + "def relocated() -> Out[str]:\n" + " ref(url='file://../comp/noop.yaml').named('By File')()\n" + " return ref(url='resolve://../comp/resolve.yaml#frag').named('By Resolve')()\n" + ) + out = tmp_path / out_rel + + compile_pipeline(src / "pipeline.py", out) + + tasks = yaml.safe_load(out.read_text())["implementation"]["graph"]["tasks"] + urls = {task_id: task["componentRef"]["url"] for task_id, task in tasks.items()} + if out.parent == src: # in place: authored URLs are kept verbatim + assert urls == { + "By File": "file://../comp/noop.yaml", + "By Resolve": "resolve://../comp/resolve.yaml#frag", + } + hydrated = PipelineHydrator(client=MagicMock()).hydrate_file(out).data + for task in hydrated["implementation"]["graph"]["tasks"].values(): + assert task["componentRef"]["spec"]["name"] == "Noop" + + def test_compile_empty_graph_fails(tmp_path): """A pipeline whose body calls no ref(...) -> CompileError.""" out = tmp_path / "compiled.yaml" @@ -1643,10 +1691,9 @@ def test_compile_pipeline_file_wraps_compile_error(tmp_path): """The facade translates the compiler's CompileError into the CLI's uniform PipelineValidationError (mirrors hydrate_pipeline_file).""" out = tmp_path / "compiled.yaml" - # noop.yaml deliberately not colocated -> unresolvable local ref. with pytest.raises(PipelineValidationError) as exc: - compile_pipeline_file(FIXTURES / "pipeline.py", out) - assert "file://./noop.yaml" in str(exc.value) + compile_pipeline_file(_script_missing_noop(tmp_path), out) + assert "noop.yaml" in str(exc.value) assert not out.exists() @@ -1857,7 +1904,7 @@ def test_compile_cli_unresolvable_local_ref_exit_nonzero(tmp_path): "sdk", "pipelines", "compile", - str(FIXTURES / "pipeline.py"), + str(_script_missing_noop(tmp_path)), "-o", str(out), ] diff --git a/tests/test_pipeline_decompiler.py b/tests/test_pipeline_decompiler.py index a8b22b0..abde892 100644 --- a/tests/test_pipeline_decompiler.py +++ b/tests/test_pipeline_decompiler.py @@ -478,14 +478,15 @@ def test_compiling_a_decompiled_example_reproduces_the_original_bytes(example: P """compile -> decompile -> compile is a byte fixpoint on compiler output. Uses the in-memory API: the file command re-pins every component, which - rewrites locators by design. Both compiles share one directory because a - ``@task`` compiles to a relative sidecar ref resolved against the output. + rewrites locators by design. The generated source sits next to the + compiled YAML because it keeps that YAML's relative refs, which resolve + from the source file's directory. """ workdir = tmp_path / "out" workdir.mkdir() compile_pipeline(example, workdir / "p.yaml") original = (workdir / "p.yaml").read_text(encoding="utf-8") - generated = tmp_path / "generated.py" + generated = workdir / "generated.py" generated.write_text(decompile_pipeline(parse_yaml_string(original)).source, encoding="utf-8") compile_pipeline(generated, workdir / "p2.yaml") assert (workdir / "p2.yaml").read_text(encoding="utf-8") == original diff --git a/uv.lock b/uv.lock index 50b6e9c..25a212b 100644 --- a/uv.lock +++ b/uv.lock @@ -2083,7 +2083,7 @@ requires-dist = [{ name = "pydantic", specifier = ">=2.0" }] [[package]] name = "tangle-cli" -version = "0.1.31" +version = "0.1.32" source = { editable = "." } dependencies = [ { name = "cloud-pipelines" },