Relocate root pipeline relative refs to the output dir (v0.1.32)

README.md

@@ -529,6 +529,9 @@ uv run tangle sdk pipelines compile pipeline.py -o pipeline.yaml
 uv run tangle sdk pipelines compile pipeline.py -o pipeline.yaml --pipeline pipeline_fn_name
 ```
 
+Relative `file://` and `resolve://` refs are written relative to the `.py` file.
+The compiler rewrites them so they still resolve from wherever `-o` points.
+
 To compile and submit in one step, without keeping the compiled YAML around, use
 `pipeline-runs submit-from-python`:
 

packages/tangle-cli/src/tangle_cli/__init__.py

@@ -14,6 +14,6 @@
 try:
     __version__ = metadata_version("tangle-cli")
 except PackageNotFoundError:
-    __version__ = "0.1.31"
+    __version__ = "0.1.32"
 
 __all__ = ["TangleDynamicDiscoveryClient", "__version__"]

packages/tangle-cli/src/tangle_cli/pipeline_compiler.py

@@ -874,22 +874,21 @@ def _compile_pipeline_fn(
             components_yaml_name=components_path.name,
         )
 
-    # 4b. A CHILD artifact is written under ``<root>.subgraphs/``, away from
-    #     its own source directory. Its author-written relative local refs
-    #     (plain ``ref(url="file://./leaf.yaml")``) point at files next to
-    #     the child SOURCE, so relocate them to be relative to the child
-    #     SIDECAR directory — the URL still resolves to the SAME original
-    #     file (no copying), just from the sidecar's location. Compiler-
-    #     managed refs (``@task`` resolver + subpipeline) already point at
-    #     bundle files and are skipped. The root is never relocated: its
-    #     output dir is its bundle root (in-place compile contract).
-    if not is_root:
-        _relocate_child_local_refs(
-            body_dict=body_dict,
-            builder=builder,
-            source_dir=base_dir,
-            sidecar_dir=output_path.parent,
-        )
+    # 4b. An artifact may be written away from its own source directory (a
+    #     CHILD under ``<root>.subgraphs/``; the ROOT wherever ``--output``
+    #     points). Its author-written relative local refs (plain
+    #     ``ref(url="file://./leaf.yaml")``) point at files next to the
+    #     SOURCE, so relocate them to be relative to the artifact's OUTPUT
+    #     directory — the URL still resolves to the SAME original file (no
+    #     copying), just from the output's location. Compiler-managed refs
+    #     (``@task`` resolver + subpipeline) already point at bundle files
+    #     and are skipped. Compiling in place leaves the URLs unchanged.
+    _relocate_local_refs(
+        body_dict=body_dict,
+        builder=builder,
+        source_dir=base_dir,
+        sidecar_dir=output_path.parent,
+    )
 
     # 4c. @registered refs point at an EXISTING gen_config.yaml (the
     #     operation is registered/published elsewhere), so there is no
@@ -1468,16 +1467,15 @@ def _validate_artifact(artifact: SubgraphArtifact, ctx: CompileContext) -> None:
     # Asset policy. EVERY
     # artifact's relative local refs are validated relative to THAT
     # artifact's own output directory:
-    #   * the ROOT body relative to the root output dir (``label`` is None,
-    #     preserving the verbatim single-pipeline error message);
+    #   * the ROOT body relative to the root output dir (``label`` is None);
     #   * each CHILD body relative to ITS child-sidecar dir (``label`` adds
     #     child-sidecar + task context to the error).
     # Generated bundle files the compiler is about to write — child graph
     # sidecars and child ``@task`` components sidecars — are in
     # ``ctx.planned_files`` and count as present, so the compiler-managed
-    # parent→child / child→child / child @task refs pass. A child's
+    # parent→child / child→child / child @task refs pass. An artifact's
     # author-written relative leaf ref was relocated to be relative to
-    # the child-sidecar dir, so it is validated against the real source-side
+    # its output dir, so it is validated against the real source-side
     # file via ``../``; a missing external leaf fails clearly here, before
     # any file is written.
     _validate_local_component_refs_for_artifact(
@@ -1924,19 +1922,19 @@ def _rewrite_registered_componentref_urls(
         tasks[task_id]["componentRef"] = {"url": url}
 
 
-def _relocate_child_local_refs(
+def _relocate_local_refs(
     *,
     body_dict: dict[str, Any],
     builder: Any,
     source_dir: Path,
     sidecar_dir: Path,
 ) -> None:
-    """Rewrite a child's author-written relative local componentRefs from
-    being relative to its SOURCE dir to relative to its SIDECAR dir.
+    """Rewrite an artifact's author-written relative local componentRefs from
+    being relative to its SOURCE dir to relative to its output (``sidecar_dir``).
 
-    A child compiles into ``<root>.subgraphs/`` but its ``ref(url=...)``
-    URLs were authored relative to the child's own source file. Rewriting
-    them keeps each ref pointing at the SAME original file (no copying) so
+    A child compiles into ``<root>.subgraphs/`` and the root into any
+    ``--output`` dir, but their ``ref(url=...)`` URLs were authored relative
+    to their own source file. Rewriting them keeps each ref pointing at the SAME original file (no copying) so
     the existing hydrator — which resolves child refs relative to the
     loaded sidecar's directory — still finds it.
 
@@ -2578,8 +2576,7 @@ def _validate_local_component_refs_for_artifact(
     External relative refs must already exist on disk.
 
     Args:
-        artifact_label: ``None`` for the ROOT (uses the legacy error wording
-            relative to the OUTPUT directory); a ``child pipeline '<name>'
+        artifact_label: ``None`` for the ROOT; a ``child pipeline '<name>'
             (<file>)`` label for a child sidecar (uses child-context wording
             relative to the child sidecar's directory).
 
@@ -2608,13 +2605,9 @@ def _validate_local_component_refs_for_artifact(
         if artifact_label is None:
             raise CompileError(
                 f"task {task_id!r} references local component {url!r}, but the "
-                f"target does not exist relative to the output directory: "
-                f"{target}. Hydrate resolves componentRef URLs relative to the "
-                "compiled YAML's location, so the referenced file must sit "
-                "next to the compiled output. Fix options: compile into the "
-                "pipeline source directory so referenced files are colocated "
-                "with the output; place the referenced component next to the "
-                "compiled YAML; or use an absolute file:///… / gs://… URL or a "
+                f"target does not exist: {target}. Relative componentRef URLs "
+                "are authored relative to the pipeline source file's directory. "
+                "Fix the path, or use an absolute file:///… / gs://… URL or a "
                 "published name: ref."
             )
         raise CompileError(

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "tangle-cli"
-version = "0.1.31"
+version = "0.1.32"
 description = "CLI for Tangle, the open-source ML pipeline orchestration platform"
 readme = "README.md"
 authors = [

tests/test_packaging.py

@@ -183,7 +183,7 @@ def test_tangle_cli_wheel_supports_expert_no_deps_import_path_without_tangle_api
     requires_dist = [line for line in metadata.splitlines() if line.startswith("Requires-Dist: ")]
     assert not any(name.startswith("tangle_api/") for name in names)
     assert "tangle_cli/openapi/openapi.json" not in names
-    assert "Version: 0.1.31" in metadata
+    assert "Version: 0.1.32" in metadata
     assert "Requires-Dist: tangle-api==0.1.1" in requires_dist
     assert not any("extra == 'native'" in line for line in requires_dist)
     assert "Provides-Extra: native" in metadata

tests/test_pipeline_compiler.py

@@ -11,6 +11,7 @@
 import shutil
 import sys
 from pathlib import Path
+from unittest.mock import MagicMock
 
 import pytest
 import yaml
@@ -25,6 +26,7 @@
     compile_pipeline,
     register_image_id,
 )
+from tangle_cli.pipeline_hydrator import PipelineHydrator
 from tangle_cli.pipelines import PipelineValidationError, compile_pipeline_file
 from tangle_cli.python_pipeline.errors import CompileError
 from tangle_cli.schema_validation import validate_dehydrated_data
@@ -41,12 +43,11 @@ def run_app(app, args: list[str]) -> None:
 
 
 def _provide_noop(out: Path) -> None:
-    """Colocate the referenced ``noop.yaml`` component next to the output.
+    """Place the ``noop.yaml`` component in the output's directory.
 
-    Fixtures like ``pipeline.py`` / ``multi_arg_pipeline.py`` reference
-    ``file://./noop.yaml``; the compiler validates that relative local
-    componentRef targets exist relative to the OUTPUT directory, so the
-    referenced component must sit next to the compiled YAML.
+    Pipelines written next to the output reference ``file://./noop.yaml``,
+    which resolves from the pipeline SOURCE directory, so the component must
+    exist there.
     """
     out.parent.mkdir(parents=True, exist_ok=True)
     shutil.copy(FIXTURES / "noop.yaml", out.parent / "noop.yaml")
@@ -82,8 +83,12 @@ def test_compile_writes_single_dehydrated_yaml(tmp_path):
     task_body = tasks["Wait For Noop"]
 
     # componentRef is a PURE ref — no inline spec / text.
+    # The source-relative ref is relocated to resolve from the output dir.
     cref = task_body["componentRef"]
-    assert cref == {"url": "file://./noop.yaml"}
+    assert list(cref) == ["url"]
+    assert (out.parent / cref["url"].removeprefix("file://")).resolve() == (
+        FIXTURES / "noop.yaml"
+    ).resolve()
     assert "spec" not in cref
     assert "text" not in cref
 
@@ -147,23 +152,66 @@ def test_compile_pipeline_does_not_leak_sys_state(tmp_path):
 # Free-function compile: user-facing failure modes.
 
 
+def _script_missing_noop(tmp_path: Path) -> Path:
+    """Copy ``pipeline.py`` (and its config) to a dir WITHOUT ``noop.yaml``,
+    so its relative ``file://./noop.yaml`` ref is unresolvable."""
+    src = tmp_path / "src"
+    src.mkdir()
+    for name in ("pipeline.py", "config.yaml"):
+        shutil.copy(FIXTURES / name, src / name)
+    return src / "pipeline.py"
+
+
 def test_compile_unresolvable_local_ref_fails(tmp_path):
-    """A relative file:// componentRef whose target is NOT colocated with
-    the output fails clearly with guidance, and writes no output."""
+    """A relative file:// componentRef whose target does NOT exist next to
+    the pipeline source fails clearly with guidance, and writes no output."""
     out = tmp_path / "compiled.yaml"
-    # Deliberately do NOT colocate noop.yaml next to the output.
     with pytest.raises(CompileError) as exc:
-        compile_pipeline(FIXTURES / "pipeline.py", out)
+        compile_pipeline(_script_missing_noop(tmp_path), out)
     msg = str(exc.value)
-    assert "file://./noop.yaml" in msg
-    assert "output directory" in msg
+    assert "noop.yaml" in msg
     # Guidance for the user.
-    assert "compile into the pipeline source directory" in msg.lower()
+    assert "relative to the pipeline source file's directory" in msg
     # No output written on failure (neither pipeline nor sidecar).
     assert not out.exists()
     assert not (tmp_path / "compiled.components.yaml").exists()
 
 
+@pytest.mark.parametrize("out_rel", ["pipe/compiled.yaml", "pipe/temp/compiled.yaml", "other/deep/compiled.yaml"])
+def test_compile_relocates_root_local_refs_to_any_output_dir(tmp_path, out_rel):
+    """The ROOT's source-relative ``file://`` / ``resolve://`` refs are
+    rewritten to resolve from wherever the output lands: in place, a deeper
+    dir, or an unrelated one. Each compiled YAML then hydrates."""
+    comp = tmp_path / "comp"
+    comp.mkdir()
+    shutil.copy(FIXTURES / "noop.yaml", comp / "noop.yaml")
+    (comp / "resolve.yaml").write_text(yaml.safe_dump({"frag": {"local": "./noop.yaml"}}))
+    src = tmp_path / "pipe"
+    src.mkdir()
+    (src / "pipeline.py").write_text(
+        "from tangle_cli.python_pipeline import Out, pipeline, ref\n"
+        "\n"
+        "@pipeline('Relocated')\n"
+        "def relocated() -> Out[str]:\n"
+        "    ref(url='file://../comp/noop.yaml').named('By File')()\n"
+        "    return ref(url='resolve://../comp/resolve.yaml#frag').named('By Resolve')()\n"
+    )
+    out = tmp_path / out_rel
+
+    compile_pipeline(src / "pipeline.py", out)
+
+    tasks = yaml.safe_load(out.read_text())["implementation"]["graph"]["tasks"]
+    urls = {task_id: task["componentRef"]["url"] for task_id, task in tasks.items()}
+    if out.parent == src:  # in place: authored URLs are kept verbatim
+        assert urls == {
+            "By File": "file://../comp/noop.yaml",
+            "By Resolve": "resolve://../comp/resolve.yaml#frag",
+        }
+    hydrated = PipelineHydrator(client=MagicMock()).hydrate_file(out).data
+    for task in hydrated["implementation"]["graph"]["tasks"].values():
+        assert task["componentRef"]["spec"]["name"] == "Noop"
+
+
 def test_compile_empty_graph_fails(tmp_path):
     """A pipeline whose body calls no ref(...) -> CompileError."""
     out = tmp_path / "compiled.yaml"
@@ -1643,10 +1691,9 @@ def test_compile_pipeline_file_wraps_compile_error(tmp_path):
     """The facade translates the compiler's CompileError into the CLI's
     uniform PipelineValidationError (mirrors hydrate_pipeline_file)."""
     out = tmp_path / "compiled.yaml"
-    # noop.yaml deliberately not colocated -> unresolvable local ref.
     with pytest.raises(PipelineValidationError) as exc:
-        compile_pipeline_file(FIXTURES / "pipeline.py", out)
-    assert "file://./noop.yaml" in str(exc.value)
+        compile_pipeline_file(_script_missing_noop(tmp_path), out)
+    assert "noop.yaml" in str(exc.value)
     assert not out.exists()
 
 
@@ -1857,7 +1904,7 @@ def test_compile_cli_unresolvable_local_ref_exit_nonzero(tmp_path):
                 "sdk",
                 "pipelines",
                 "compile",
-                str(FIXTURES / "pipeline.py"),
+                str(_script_missing_noop(tmp_path)),
                 "-o",
                 str(out),
             ]

tests/test_pipeline_decompiler.py

@@ -478,14 +478,15 @@ def test_compiling_a_decompiled_example_reproduces_the_original_bytes(example: P
     """compile -> decompile -> compile is a byte fixpoint on compiler output.
 
     Uses the in-memory API: the file command re-pins every component, which
-    rewrites locators by design. Both compiles share one directory because a
-    ``@task`` compiles to a relative sidecar ref resolved against the output.
+    rewrites locators by design. The generated source sits next to the
+    compiled YAML because it keeps that YAML's relative refs, which resolve
+    from the source file's directory.
     """
     workdir = tmp_path / "out"
     workdir.mkdir()
     compile_pipeline(example, workdir / "p.yaml")
     original = (workdir / "p.yaml").read_text(encoding="utf-8")
-    generated = tmp_path / "generated.py"
+    generated = workdir / "generated.py"
     generated.write_text(decompile_pipeline(parse_yaml_string(original)).source, encoding="utf-8")
     compile_pipeline(generated, workdir / "p2.yaml")
     assert (workdir / "p2.yaml").read_text(encoding="utf-8") == original