From 456e443b97a0a717fbb3efafee1ed8f83e78aaa7 Mon Sep 17 00:00:00 2001 From: Justin Flick Date: Tue, 29 Sep 2026 17:13:37 -0400 Subject: [PATCH 1/2] fix(tangle-cli): shrink bundle-mode payload with bz2 and Base85 Bundle mode embeds every local module in one container command-line argument. Linux rejects any single argument longer than MAX_ARG_STRLEN (128 KiB) with E2BIG, so a component whose bundle crosses that size fails at exec before any Python runs, and nothing warns at generation time. Encode the payload with bz2 instead of zlib, and Base85 instead of Base64. Both are in the standard library, so generated components gain no dependency. On a real 32-module, ~400 KB bundle, the argument drops from 132,638 bytes (over the limit) to 94,952 bytes, with identical module sources that decode and execute. Base85 contains no quotes or backslashes, so the blob stays safe inside the emitted Python string literal, and the program reaches the shell only as $0 written out by printf, so it is never shell-interpreted. Every generated component embeds its own decoder, so previously generated components keep working unchanged. Very small bundles grow by a few dozen bytes of bz2 header, which does not matter for the limit. The bundled_modules_b64 keyword keeps its name because callers pass it by keyword; only its documentation changes. --- .../src/tangle_cli/component_from_func.py | 3 +- .../src/tangle_cli/module_bundler.py | 33 ++++++++++++----- .../bundle_mode.expected.yaml | 4 +- tests/test_component_from_func.py | 22 +++++------ tests/test_module_bundler.py | 37 ++++++++++++++++++- 5 files changed, 74 insertions(+), 25 deletions(-) diff --git a/packages/tangle-cli/src/tangle_cli/component_from_func.py b/packages/tangle-cli/src/tangle_cli/component_from_func.py index a286cde..359d702 100644 --- a/packages/tangle-cli/src/tangle_cli/component_from_func.py +++ b/packages/tangle-cli/src/tangle_cli/component_from_func.py @@ -1937,7 +1937,8 @@ def build_component_dict( dependencies: List of pip dependencies annotations: Metadata annotations dict mode: Generation mode - bundled_modules_b64: Base64-encoded pickled modules (bundle mode only) + bundled_modules_b64: Encoded module sources from ``ModuleBundler.encode`` + (bundle mode only) Returns: Dict representing the full component YAML structure. diff --git a/packages/tangle-cli/src/tangle_cli/module_bundler.py b/packages/tangle-cli/src/tangle_cli/module_bundler.py index bf796ce..e1b2a66 100644 --- a/packages/tangle-cli/src/tangle_cli/module_bundler.py +++ b/packages/tangle-cli/src/tangle_cli/module_bundler.py @@ -240,7 +240,20 @@ def collect_sources( @staticmethod def encode(module_sources: dict[str, str]) -> str | None: - """Compress and base64-encode a dict of module sources for embedding. + """Compress and Base85-encode a dict of module sources for embedding. + + The encoded blob travels inside a single container command-line + argument, and Linux rejects any single argument longer than + ``MAX_ARG_STRLEN`` (128 KiB) with ``E2BIG`` before the program starts. + bz2 compresses Python source markedly better than zlib, and Base85 + expands bytes by 25% rather than Base64's 33%. Both are in the + standard library, so the generated component gains no dependency. + The Base85 alphabet contains no quotes or backslashes, so the blob is + safe inside the Python string literal ``build_injection`` emits. + The generated program is passed to ``sh`` as ``$0`` and written out + with ``printf``, so its characters are never shell-interpreted. + Each generated component embeds its own decoder, so no older + component depends on this format. Modules are sorted so that dependencies execute before dependents. We perform a topological sort over the module-level import graph @@ -259,27 +272,29 @@ def encode(module_sources: dict[str, str]) -> str | None: module_sources: ``{module_name: source_text}`` dict. Returns: - Base64-encoded string, or ``None`` if *module_sources* is empty. + Base85-encoded bz2 string, or ``None`` if *module_sources* is empty. """ if not module_sources: return None - import zlib + import bz2 ordered_names = _topological_order(module_sources) ordered = {name: module_sources[name] for name in ordered_names} sources_json = json.dumps(ordered) - compressed = zlib.compress(sources_json.encode(), level=9) - return base64.b64encode(compressed).decode("ascii") + compressed = bz2.compress(sources_json.encode(), compresslevel=9) + return base64.b85encode(compressed).decode("ascii") @staticmethod def build_injection(bundled_modules_b64: str) -> str: """Return a Python snippet that decodes and injects bundled modules into ``sys.modules``. The snippet is self-contained: it imports ``sys``, ``types``, ``base64``, - ``json``, and ``zlib``, then decompresses the embedded blob and registers + ``json``, and ``bz2``, then decompresses the embedded blob and registers each module via ``types.ModuleType`` + ``exec``. Args: - bundled_modules_b64: Base64 string produced by ``encode``. + bundled_modules_b64: Encoded string produced by ``encode``. The + name predates the Base85 encoding and is kept because callers + pass it by keyword. """ return textwrap.dedent(f"""\ # --- Inject local dependency modules from embedded source --- @@ -287,9 +302,9 @@ def build_injection(bundled_modules_b64: str) -> str: import types import base64 import json - import zlib + import bz2 - _EMBEDDED_MODULES = json.loads(zlib.decompress(base64.b64decode({repr(bundled_modules_b64)}))) + _EMBEDDED_MODULES = json.loads(bz2.decompress(base64.b85decode({repr(bundled_modules_b64)}))) # Pass 1: register all modules in sys.modules (without executing source) # so transitive imports between bundled modules can resolve in any order. _module_objs = {{}} diff --git a/tests/snapshots/component_generator/bundle_mode.expected.yaml b/tests/snapshots/component_generator/bundle_mode.expected.yaml index 932fe71..0e38e82 100644 --- a/tests/snapshots/component_generator/bundle_mode.expected.yaml +++ b/tests/snapshots/component_generator/bundle_mode.expected.yaml @@ -48,9 +48,9 @@ implementation: import types import base64 import json - import zlib + import bz2 - _EMBEDDED_MODULES = json.loads(zlib.decompress(base64.b64decode('eNqrVspIzSlILSpWslJQUtJRgHH1Sksyc8CCMXkpqWkKyTmpiXkaJakVJZpWMXkKQFCUWlJalKcAEtIrLinKLNDQ1MvJL08t0tCMyVOqBQAjHx2s'))) + _EMBEDDED_MODULES = json.loads(bz2.decompress(base64.b85decode('LRx4!F+o`-Q(18$BEL(s;08nKB5C8-(W!}ml08pbtKmY&@Q`1D*LnBQ#lOTYi2+ZHDgR)PvJA0By)WPshuE^e4Vq$U&uyMDL7ibmAPazAg3?vD18nI+p str: with open(output_file) as f: component = yaml.safe_load(f) python_source = component["implementation"]["container"]["command"][-1] - match = _re.search(r"base64\.b64decode\('([A-Za-z0-9+/=]+)'\)", python_source) + match = _re.search(r"base64\.b85decode\('([^']+)'\)", python_source) assert match is not None - embedded = json.loads(zlib.decompress(base64.b64decode(match.group(1)))) + embedded = json.loads(bz2.decompress(base64.b85decode(match.group(1)))) assert "runtime_helper" in embedded assert "authoring_envs" not in embedded @@ -704,7 +704,7 @@ def test_bundle_yaml_orders_dependencies_before_dependents(self, tmp_path): ``AttributeError`` at component runtime. """ import base64 - import zlib + import bz2 (tmp_path / "aaa.py").write_text(textwrap.dedent("""\ import bbb @@ -749,11 +749,11 @@ def my_component() -> str: # quoted via ``repr`` in the source). import re as _re - # The injection emits ``base64.b64decode('')`` — the b64 - # alphabet is ``[A-Za-z0-9+/=]``, never a single quote. - match = _re.search(r"base64\.b64decode\('([A-Za-z0-9+/=]+)'\)", python_source) - assert match is not None, "injection snippet must contain a b64 blob" - embedded = json.loads(zlib.decompress(base64.b64decode(match.group(1)))) + # The injection emits ``base64.b85decode('')`` — the Base85 + # alphabet never contains a single quote. + match = _re.search(r"base64\.b85decode\('([^']+)'\)", python_source) + assert match is not None, "injection snippet must contain an encoded blob" + embedded = json.loads(bz2.decompress(base64.b85decode(match.group(1)))) order = list(embedded.keys()) assert order.index("bbb") < order.index("aaa"), f"bbb must execute before aaa (got order: {order})" @@ -1086,7 +1086,7 @@ def test_bundle_follows_transitive_imports_in_parent_init(self, tmp_path): the bundle crashes at runtime with ImportError. """ import base64 - import zlib + import bz2 # mylib/__init__.py imports helpers; component only imports mylib.core (tmp_path / "mylib").mkdir() @@ -1108,7 +1108,7 @@ def test_bundle_follows_transitive_imports_in_parent_init(self, tmp_path): # dependency in the embedded dict (issue #30197). b64 = ModuleBundler.encode(sources) assert b64 is not None - order = list(json.loads(zlib.decompress(base64.b64decode(b64))).keys()) + order = list(json.loads(bz2.decompress(base64.b85decode(b64))).keys()) assert order.index("mylib.helpers") < order.index( "mylib" ), f"mylib.helpers must execute before mylib (got order: {order})" diff --git a/tests/test_module_bundler.py b/tests/test_module_bundler.py index aeb0d98..c7f8153 100644 --- a/tests/test_module_bundler.py +++ b/tests/test_module_bundler.py @@ -9,10 +9,13 @@ from __future__ import annotations import base64 +import bz2 import json import textwrap import zlib +from pathlib import Path +import tangle_cli from tangle_cli.module_bundler import ( ModuleBundler, _import_node_targets, @@ -22,9 +25,39 @@ ) -def _decode(b64: str) -> dict[str, str]: +def _decode(encoded: str) -> dict[str, str]: """Mirror of the runtime injection's decompress step.""" - return json.loads(zlib.decompress(base64.b64decode(b64))) + return json.loads(bz2.decompress(base64.b85decode(encoded))) + + +class TestEncodedPayload: + """The payload rides in one command-line argument, capped at 128 KiB by Linux.""" + + @staticmethod + def _realistic_sources() -> dict[str, str]: + """This package's own modules: a large, real body of Python source.""" + root = Path(tangle_cli.__file__).parent + return {f"tangle_cli.{p.stem}": p.read_text(encoding="utf-8") for p in sorted(root.glob("*.py"))} + + def test_round_trips_realistic_sources(self): + sources = self._realistic_sources() + encoded = ModuleBundler.encode(sources) + assert encoded is not None + assert _decode(encoded) == {name: sources[name] for name in _topological_order(sources)} + + def test_is_safe_inside_the_emitted_string_literal(self): + encoded = ModuleBundler.encode(self._realistic_sources()) + assert encoded is not None + assert not set(encoded) & {"'", '"', "\\", "\n"} + assert repr(encoded) == f"'{encoded}'" + + def test_is_smaller_than_zlib_base64(self): + sources = self._realistic_sources() + encoded = ModuleBundler.encode(sources) + assert encoded is not None + ordered = {name: sources[name] for name in _topological_order(sources)} + legacy = base64.b64encode(zlib.compress(json.dumps(ordered).encode(), level=9)) + assert len(encoded) < 0.85 * len(legacy) class TestTopologicalOrder: From 2ad045c4a908e327c3968958d31e1d3491f4a64b Mon Sep 17 00:00:00 2001 From: Justin Flick Date: Wed, 30 Sep 2026 22:31:26 -0700 Subject: [PATCH 2/2] fix(tangle-cli): make compact bundling opt-in and Jinja-safe --- README.md | 4 +- .../src/tangle_cli/component_from_func.py | 26 ++--- .../src/tangle_cli/component_generator.py | 5 +- .../src/tangle_cli/components_cli.py | 4 +- .../src/tangle_cli/module_bundler.py | 73 ++++++++----- .../src/tangle_cli/python_pipeline/task.py | 7 +- .../src/tangle_cli/version_manager.py | 2 +- .../bundle_mode.expected.yaml | 4 +- tests/test_bundle_hydration.py | 100 ++++++++++++++++++ tests/test_component_from_func.py | 22 ++-- tests/test_component_generator.py | 14 ++- tests/test_module_bundler.py | 51 ++++++--- tests/test_pipeline_compiler.py | 9 +- tests/test_pipelines_cli.py | 6 +- tests/test_version_manager.py | 9 +- 15 files changed, 245 insertions(+), 91 deletions(-) create mode 100644 tests/test_bundle_hydration.py diff --git a/README.md b/README.md index 50db428..4cefb74 100644 --- a/README.md +++ b/README.md @@ -416,7 +416,9 @@ uv run tangle api published-components experimental-search \ ### Local components -`generate from-python` converts a local Python function into a component YAML using inline source by default, or `--mode bundle` to embed local dependency modules. Common options include `--function`, `--output`, `--name`, `--image`, `--dependencies-from`, `--strip-code`, `--use-legacy-naming`, and `--resolve-root`. +`generate from-python` converts a local Python function into a component YAML using inline source by default, or `--mode bundle` to embed local dependency modules with zlib/Base64. Common options include `--function`, `--output`, `--name`, `--image`, `--dependencies-from`, `--strip-code`, `--use-legacy-naming`, and `--resolve-root`. + +Opt in to `--mode bundle-bz2` for bz2/Base85 compression, which can reduce large Python bundles. The same mode is accepted by `@task(mode="bundle-bz2")` and `local_from_python.mode` during hydration. It requires Python's `_bz2` extension in the runtime image. Encoded braces are escaped in the generated Python literal so Jinja hydration cannot interpret the payload as a template. Both bundle modes still use one command-line argument, so sufficiently large bundles can still exceed Linux's per-argument limit. `bump-version` increments or sets component version metadata in YAML and updates/regenerates a referenced Python source when the component contains `python_original_code_path` annotations. diff --git a/packages/tangle-cli/src/tangle_cli/component_from_func.py b/packages/tangle-cli/src/tangle_cli/component_from_func.py index 359d702..6355a74 100644 --- a/packages/tangle-cli/src/tangle_cli/component_from_func.py +++ b/packages/tangle-cli/src/tangle_cli/component_from_func.py @@ -1,11 +1,12 @@ """ Component YAML generator from Python functions. -Converts Python functions into Tangle component YAML files. Supports two modes: +Converts Python functions into Tangle component YAML files. Supports three modes: - **inline** (default): Single-file components with source code embedded directly. - **bundle**: Multi-file components with local dependency modules serialized via - zlib-compressed source text and injected into sys.modules at runtime. + zlib/Base64 source text and injected into sys.modules at runtime. +- **bundle-bz2**: Opt-in bz2/Base85 variant for smaller embedded payloads. Key functions: - generate_component_yaml() - Top-level entry point for YAML generation @@ -1856,7 +1857,7 @@ def _build_pip_install_command(deps: list[str]) -> list[str]: def _build_python_source( spec: FunctionSpec, - mode: Literal["inline", "bundle"], + mode: Literal["inline", "bundle", "bundle-bz2"], bundled_modules_b64: str | None = None, ) -> str: """Build the full Python source code to embed in the YAML. @@ -1881,8 +1882,8 @@ def _build_python_source( parts.append(_SERIALIZE_STR_HELPER) # For bundle mode: add sys.modules injection from compressed embedded source text - if mode == "bundle" and bundled_modules_b64: - parts.append(ModuleBundler.build_injection(bundled_modules_b64)) + if mode in {"bundle", "bundle-bz2"} and bundled_modules_b64: + parts.append(ModuleBundler.build_injection(bundled_modules_b64, mode=mode)) # Add the source code (type-hint-stripped) # Use full module source when available — this preserves helper functions defined @@ -1926,7 +1927,7 @@ def build_component_dict( container_image: str, dependencies: list[str], annotations: dict[str, str], - mode: Literal["inline", "bundle"] = "inline", + mode: Literal["inline", "bundle", "bundle-bz2"] = "inline", bundled_modules_b64: str | None = None, ) -> dict[str, Any]: """Build the complete component YAML dict. @@ -1938,7 +1939,7 @@ def build_component_dict( annotations: Metadata annotations dict mode: Generation mode bundled_modules_b64: Encoded module sources from ``ModuleBundler.encode`` - (bundle mode only) + using the same mode (bundle modes only) Returns: Dict representing the full component YAML structure. @@ -2036,7 +2037,7 @@ def generate_component_yaml( container_image: str, function_name: str | None = None, dependencies_from: Path | None = None, - mode: Literal["inline", "bundle"] = "inline", + mode: Literal["inline", "bundle", "bundle-bz2"] = "inline", custom_name: str | None = None, custom_annotations: dict[str, str] | None = None, strip_code: bool = False, @@ -2063,7 +2064,8 @@ def generate_component_yaml( container_image: Docker image reference function_name: Function to extract (auto-detected if None) dependencies_from: Path to pyproject.toml with pip dependencies - mode: "inline" for single-file, "bundle" for multi-file + mode: "inline" for single-file, "bundle" for zlib/Base64 multi-file, + "bundle-bz2" for opt-in bz2/Base85 multi-file custom_name: Override the component name custom_annotations: Additional annotations to merge strip_code: Omit python_original_code annotation @@ -2099,7 +2101,7 @@ def generate_component_yaml( # Only add resolve_root to sys.path in bundle mode — in inline mode the # sibling modules won't be embedded, so letting the import succeed would # produce YAML that fails at runtime in the container. - extra_paths = [resolve_root] if resolve_root and mode == "bundle" else None + extra_paths = [resolve_root] if resolve_root and mode in {"bundle", "bundle-bz2"} else None module = load_python_module(file_path, extra_sys_path=extra_paths) func = get_function_from_module(module, resolved_func_name) @@ -2227,7 +2229,7 @@ def _path_annotation(path: Path) -> str: # 5. Handle bundle mode — embed source text of local modules # (not bytecode, which is Python-version-specific) bundled_modules_b64: str | None = None - if mode == "bundle": + if mode in {"bundle", "bundle-bz2"}: module_sources = ModuleBundler.collect_sources( file_path, resolve_root=resolve_root, @@ -2235,7 +2237,7 @@ def _path_annotation(path: Path) -> str: source=spec.module_source_stripped, ) if module_sources: - bundled_modules_b64 = ModuleBundler.encode(module_sources) + bundled_modules_b64 = ModuleBundler.encode(module_sources, mode=mode) if bundled_modules_b64: sorted_names = sorted(module_sources.keys(), key=lambda k: (k.count("."), k)) annotations["bundled_modules"] = json.dumps(sorted_names) diff --git a/packages/tangle-cli/src/tangle_cli/component_generator.py b/packages/tangle-cli/src/tangle_cli/component_generator.py index 0a19cf2..1ca65a2 100644 --- a/packages/tangle-cli/src/tangle_cli/component_generator.py +++ b/packages/tangle-cli/src/tangle_cli/component_generator.py @@ -120,7 +120,7 @@ def generate_component_yaml( container_image: str, function_name: str | None = None, dependencies_from: Path | None = None, - mode: Literal["inline", "bundle"] = "inline", + mode: Literal["inline", "bundle", "bundle-bz2"] = "inline", custom_name: str | None = None, custom_annotations: dict[str, str] | None = None, strip_code: bool = False, @@ -138,7 +138,8 @@ def generate_component_yaml( container_image: Container image to place in the component spec. function_name: Function to generate, or ``None`` to auto-detect. dependencies_from: Optional dependency file for pip installs. - mode: ``"inline"`` or ``"bundle"`` generation mode. + mode: ``"inline"``, ``"bundle"`` (zlib/Base64), or opt-in + ``"bundle-bz2"`` (bz2/Base85) generation mode. custom_name: Optional component name override. custom_annotations: Optional metadata annotations to merge. strip_code: Omit original source annotations when true. diff --git a/packages/tangle-cli/src/tangle_cli/components_cli.py b/packages/tangle-cli/src/tangle_cli/components_cli.py index 0aa7a02..63ed2bd 100644 --- a/packages/tangle-cli/src/tangle_cli/components_cli.py +++ b/packages/tangle-cli/src/tangle_cli/components_cli.py @@ -128,8 +128,8 @@ def _components_generate_from_python_impl( generator = ComponentGenerator(logger=logger, verbose=True) selected_mode = args.mode or "inline" - if selected_mode not in {"inline", "bundle"}: - raise SystemExit("--mode must be 'inline' or 'bundle'") + if selected_mode not in {"inline", "bundle", "bundle-bz2"}: + raise SystemExit("--mode must be 'inline', 'bundle', or 'bundle-bz2'") python_path = pathlib.Path(args.python_file) output_path = generator.determine_output_path( python_path, diff --git a/packages/tangle-cli/src/tangle_cli/module_bundler.py b/packages/tangle-cli/src/tangle_cli/module_bundler.py index e1b2a66..f7ee20c 100644 --- a/packages/tangle-cli/src/tangle_cli/module_bundler.py +++ b/packages/tangle-cli/src/tangle_cli/module_bundler.py @@ -239,21 +239,18 @@ def collect_sources( return result @staticmethod - def encode(module_sources: dict[str, str]) -> str | None: - """Compress and Base85-encode a dict of module sources for embedding. - - The encoded blob travels inside a single container command-line - argument, and Linux rejects any single argument longer than - ``MAX_ARG_STRLEN`` (128 KiB) with ``E2BIG`` before the program starts. - bz2 compresses Python source markedly better than zlib, and Base85 - expands bytes by 25% rather than Base64's 33%. Both are in the - standard library, so the generated component gains no dependency. - The Base85 alphabet contains no quotes or backslashes, so the blob is - safe inside the Python string literal ``build_injection`` emits. - The generated program is passed to ``sh`` as ``$0`` and written out - with ``printf``, so its characters are never shell-interpreted. - Each generated component embeds its own decoder, so no older - component depends on this format. + def encode( + module_sources: dict[str, str], + *, + mode: Literal["bundle", "bundle-bz2"] = "bundle", + ) -> str | None: + """Compress and encode module sources for embedding. + + ``bundle`` retains the zlib/Base64 format. Opt-in ``bundle-bz2`` uses + bz2/Base85 to reduce the single command-line argument carrying the + payload. This postpones, but does not remove, Linux's per-argument + size limit. Both codecs are in the standard library; ``bundle-bz2`` + additionally requires Python's optional ``_bz2`` extension at runtime. Modules are sorted so that dependencies execute before dependents. We perform a topological sort over the module-level import graph @@ -270,41 +267,59 @@ def encode(module_sources: dict[str, str]) -> str | None: Args: module_sources: ``{module_name: source_text}`` dict. + mode: Bundle format, also passed to ``build_injection``. Returns: - Base85-encoded bz2 string, or ``None`` if *module_sources* is empty. + Encoded string, or ``None`` if *module_sources* is empty. """ if not module_sources: return None - import bz2 ordered_names = _topological_order(module_sources) ordered = {name: module_sources[name] for name in ordered_names} sources_json = json.dumps(ordered) - compressed = bz2.compress(sources_json.encode(), compresslevel=9) - return base64.b85encode(compressed).decode("ascii") + if mode == "bundle": + import zlib - @staticmethod - def build_injection(bundled_modules_b64: str) -> str: - """Return a Python snippet that decodes and injects bundled modules into ``sys.modules``. + return base64.b64encode(zlib.compress(sources_json.encode(), level=9)).decode("ascii") + if mode == "bundle-bz2": + import bz2 - The snippet is self-contained: it imports ``sys``, ``types``, ``base64``, - ``json``, and ``bz2``, then decompresses the embedded blob and registers - each module via ``types.ModuleType`` + ``exec``. + return base64.b85encode(bz2.compress(sources_json.encode(), compresslevel=9)).decode("ascii") + raise ValueError(f"Unsupported bundle mode: {mode}") + + @staticmethod + def build_injection( + bundled_modules_b64: str, + *, + mode: Literal["bundle", "bundle-bz2"] = "bundle", + ) -> str: + """Return a self-contained snippet that decodes and injects bundled modules. Args: bundled_modules_b64: Encoded string produced by ``encode``. The - name predates the Base85 encoding and is kept because callers - pass it by keyword. + name is kept for keyword-call compatibility. + mode: Bundle format used by ``encode`` (defaults to zlib/Base64). """ + if mode == "bundle": + compression, decoder = "zlib", "b64decode" + elif mode == "bundle-bz2": + compression, decoder = "bz2", "b85decode" + else: + raise ValueError(f"Unsupported bundle mode: {mode}") + + # Hydration may parse the generated YAML as Jinja before Python runs. + # Python hex escapes preserve Base85 bytes without exposing any Jinja + # opening delimiters ({{, {%, {#}), even through repeated rendering. + payload_literal = repr(bundled_modules_b64).replace("{", "\\x7b") return textwrap.dedent(f"""\ # --- Inject local dependency modules from embedded source --- import sys import types import base64 import json - import bz2 + import {compression} - _EMBEDDED_MODULES = json.loads(bz2.decompress(base64.b85decode({repr(bundled_modules_b64)}))) + _EMBEDDED_MODULES = json.loads({compression}.decompress(base64.{decoder}({payload_literal}))) # Pass 1: register all modules in sys.modules (without executing source) # so transitive imports between bundled modules can resolve in any order. _module_objs = {{}} diff --git a/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py b/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py index 4c882dc..b52561d 100644 --- a/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py +++ b/packages/tangle-cli/src/tangle_cli/python_pipeline/task.py @@ -110,7 +110,8 @@ def task( mode: Optional local-from-python generation mode. ``None`` preserves the hydrator default (currently ``inline``). Use ``"bundle"`` to ask hydrate-time codegen to embed - first-party imports using the existing module bundler. + first-party imports using zlib/Base64, or ``"bundle-bz2"`` for + the opt-in bz2/Base85 format. resolve_root: Optional module resolution root for bundle mode. Relative strings are resolved relative to the task source file, then emitted into @@ -170,8 +171,8 @@ def task( # still resolved relative to the @task source file. raw_dependencies_from = effective_deps_raw raw_resolve_root = resolve_root - if mode is not None and mode not in {"inline", "bundle"}: - raise ValueError("@task(mode=...) must be 'inline', 'bundle', or None") + if mode is not None and mode not in {"inline", "bundle", "bundle-bz2"}: + raise ValueError("@task(mode=...) must be 'inline', 'bundle', 'bundle-bz2', or None") if unwrap is None: unwrap_names: tuple[str, ...] = () diff --git a/packages/tangle-cli/src/tangle_cli/version_manager.py b/packages/tangle-cli/src/tangle_cli/version_manager.py index 0860889..6eabf29 100644 --- a/packages/tangle-cli/src/tangle_cli/version_manager.py +++ b/packages/tangle-cli/src/tangle_cli/version_manager.py @@ -382,7 +382,7 @@ def bump_version( generation_mode = annotations.get("tangle_cli_generation_mode") or ( "bundle" if annotations.get("bundled_modules") else "inline" ) - if generation_mode not in {"inline", "bundle"}: + if generation_mode not in {"inline", "bundle", "bundle-bz2"}: error = f"Unsupported generation mode: {generation_mode}" log.error(f"❌ {error}") return {"status": "failed", "yaml_file": str(yaml_path), "error": error} diff --git a/tests/snapshots/component_generator/bundle_mode.expected.yaml b/tests/snapshots/component_generator/bundle_mode.expected.yaml index 0e38e82..932fe71 100644 --- a/tests/snapshots/component_generator/bundle_mode.expected.yaml +++ b/tests/snapshots/component_generator/bundle_mode.expected.yaml @@ -48,9 +48,9 @@ implementation: import types import base64 import json - import bz2 + import zlib - _EMBEDDED_MODULES = json.loads(bz2.decompress(base64.b85decode('LRx4!F+o`-Q(18$BEL(s;08nKB5C8-(W!}ml08pbtKmY&@Q`1D*LnBQ#lOTYi2+ZHDgR)PvJA0By)WPshuE^e4Vq$U&uyMDL7ibmAPazAg3?vD18nI+p template_file -> render_template path, + # rather than directly instantiating a different Jinja environment in a test. + (tmp_path / "component-config.yaml").write_text( + dump_yaml({"template_file": template.name, "component_name": "Rendered bundle"}), + encoding="utf-8", + ) + pipeline = tmp_path / "pipeline.yaml" + pipeline.write_text( + dump_yaml({ + "name": "Bundle hydration", + "implementation": {"graph": {"tasks": { + "report": {"componentRef": {"url": "file://./component-config.yaml"}}, + }}}, + }), + encoding="utf-8", + ) + hydrated = PipelineHydrator(error_policy="raise").hydrate_file(pipeline) + component = hydrated.data["implementation"]["graph"]["tasks"]["report"]["componentRef"]["spec"] + assert component["name"] == "Rendered bundle" + assert component["implementation"]["container"]["command"][-1] == program + + # Hex escapes stay inert across another real hydration pass, unlike Jinja + # raw blocks that disappear on the first pass. + template.write_text(dump_yaml(component), encoding="utf-8") + rehydrated = PipelineHydrator(error_policy="raise").hydrate_file(pipeline) + component = rehydrated.data["implementation"]["graph"]["tasks"]["report"]["componentRef"]["spec"] + assert component["implementation"]["container"]["command"][-1] == program + + # No source files or project import path are available to the subprocess. + # Execute the generated sh bootstrap and argparse wrapper, not just the codec. + helper.unlink() + component_source.unlink() + runtime = tmp_path / "runtime" + runtime.mkdir() + command = component["implementation"]["container"]["command"] + completed = subprocess.run( + command + ["--prefix", "hydrated:"], + cwd=runtime, + env={"PATH": os.pathsep.join([os.path.dirname(sys.executable), os.defpath]), "TMPDIR": str(runtime)}, + capture_output=True, + text=True, + timeout=15, + ) + assert completed.returncode == 0, completed.stderr + assert completed.stdout.strip() == f"hydrated:{value}" + assert all(opener not in program for opener in ("{{", "{%", "{#")) + if mode == "bundle-bz2": + assert r"\x7b" in program + assert "base64.b85decode" in program + else: + assert "base64.b64decode" in program + assert "import bz2" not in program diff --git a/tests/test_component_from_func.py b/tests/test_component_from_func.py index 4da3a33..9cd0806 100644 --- a/tests/test_component_from_func.py +++ b/tests/test_component_from_func.py @@ -642,7 +642,7 @@ def func(x: str): def test_bundle_collects_imports_from_stripped_runtime_source_only(self, tmp_path): import base64 import re as _re - import bz2 + import zlib (tmp_path / "runtime_helper.py").write_text('VALUE = "runtime"\n', encoding="utf-8") (tmp_path / "tangle_deploy" / "python_pipeline").mkdir(parents=True) @@ -686,9 +686,9 @@ def my_component() -> str: with open(output_file) as f: component = yaml.safe_load(f) python_source = component["implementation"]["container"]["command"][-1] - match = _re.search(r"base64\.b85decode\('([^']+)'\)", python_source) + match = _re.search(r"base64\.b64decode\('([A-Za-z0-9+/=]+)'\)", python_source) assert match is not None - embedded = json.loads(bz2.decompress(base64.b85decode(match.group(1)))) + embedded = json.loads(zlib.decompress(base64.b64decode(match.group(1)))) assert "runtime_helper" in embedded assert "authoring_envs" not in embedded @@ -704,7 +704,7 @@ def test_bundle_yaml_orders_dependencies_before_dependents(self, tmp_path): ``AttributeError`` at component runtime. """ import base64 - import bz2 + import zlib (tmp_path / "aaa.py").write_text(textwrap.dedent("""\ import bbb @@ -749,11 +749,11 @@ def my_component() -> str: # quoted via ``repr`` in the source). import re as _re - # The injection emits ``base64.b85decode('')`` — the Base85 - # alphabet never contains a single quote. - match = _re.search(r"base64\.b85decode\('([^']+)'\)", python_source) - assert match is not None, "injection snippet must contain an encoded blob" - embedded = json.loads(bz2.decompress(base64.b85decode(match.group(1)))) + # The injection emits ``base64.b64decode('')`` — the b64 + # alphabet is ``[A-Za-z0-9+/=]``, never a single quote. + match = _re.search(r"base64\.b64decode\('([A-Za-z0-9+/=]+)'\)", python_source) + assert match is not None, "injection snippet must contain a b64 blob" + embedded = json.loads(zlib.decompress(base64.b64decode(match.group(1)))) order = list(embedded.keys()) assert order.index("bbb") < order.index("aaa"), f"bbb must execute before aaa (got order: {order})" @@ -1086,7 +1086,7 @@ def test_bundle_follows_transitive_imports_in_parent_init(self, tmp_path): the bundle crashes at runtime with ImportError. """ import base64 - import bz2 + import zlib # mylib/__init__.py imports helpers; component only imports mylib.core (tmp_path / "mylib").mkdir() @@ -1108,7 +1108,7 @@ def test_bundle_follows_transitive_imports_in_parent_init(self, tmp_path): # dependency in the embedded dict (issue #30197). b64 = ModuleBundler.encode(sources) assert b64 is not None - order = list(json.loads(bz2.decompress(base64.b85decode(b64))).keys()) + order = list(json.loads(zlib.decompress(base64.b64decode(b64))).keys()) assert order.index("mylib.helpers") < order.index( "mylib" ), f"mylib.helpers must execute before mylib (got order: {order})" diff --git a/tests/test_component_generator.py b/tests/test_component_generator.py index 73ac3ea..534277d 100644 --- a/tests/test_component_generator.py +++ b/tests/test_component_generator.py @@ -259,7 +259,9 @@ def bad_authoring() -> str: assert not (tmp_path / "bad-authoring.yaml").exists() -def test_bundle_mode_with_local_imports(monkeypatch, tmp_path: Path): +@pytest.mark.parametrize("mode", ["bundle", "bundle-bz2"]) +@pytest.mark.parametrize("use_cli", [False, True]) +def test_bundle_mode_with_local_imports(monkeypatch, tmp_path: Path, mode, use_cli): monkeypatch.setattr("tangle_cli.utils._fill_from_ci_env", lambda info: None) helpers_dir = tmp_path / "helpers" helpers_dir.mkdir() @@ -278,11 +280,19 @@ def my_component(name: str) -> str: ''', encoding="utf-8") (tmp_path / "pyproject.toml").write_text('[project]\nname = "test"\ndependencies = []\n', encoding="utf-8") - assert regenerate_yaml(py_file, image="python:3.12", function_name="my_component", mode="bundle") is True + if use_cli: + run_app(cli.build_app(), [ + "sdk", "components", "generate", "from-python", str(py_file), + "--image", "python:3.12", "--function", "my_component", "--mode", mode, + ]) + else: + assert regenerate_yaml(py_file, image="python:3.12", function_name="my_component", mode=mode) is True generated = yaml.safe_load((tmp_path / "my-component.yaml").read_text(encoding="utf-8")) program = generated["implementation"]["container"]["command"][-1] assert generated["name"] == "My component" + assert generated["metadata"]["annotations"]["tangle_cli_generation_mode"] == mode + assert ("base64.b85decode" if mode == "bundle-bz2" else "base64.b64decode") in program assert "_EMBEDDED_MODULES" in program assert "helpers.utils" in program diff --git a/tests/test_module_bundler.py b/tests/test_module_bundler.py index c7f8153..981b8d3 100644 --- a/tests/test_module_bundler.py +++ b/tests/test_module_bundler.py @@ -15,6 +15,7 @@ import zlib from pathlib import Path +import pytest import tangle_cli from tangle_cli.module_bundler import ( ModuleBundler, @@ -25,9 +26,9 @@ ) -def _decode(encoded: str) -> dict[str, str]: +def _decode(b64: str) -> dict[str, str]: """Mirror of the runtime injection's decompress step.""" - return json.loads(bz2.decompress(base64.b85decode(encoded))) + return json.loads(zlib.decompress(base64.b64decode(b64))) class TestEncodedPayload: @@ -41,23 +42,39 @@ def _realistic_sources() -> dict[str, str]: def test_round_trips_realistic_sources(self): sources = self._realistic_sources() - encoded = ModuleBundler.encode(sources) + encoded = ModuleBundler.encode(sources, mode="bundle-bz2") assert encoded is not None - assert _decode(encoded) == {name: sources[name] for name in _topological_order(sources)} - - def test_is_safe_inside_the_emitted_string_literal(self): - encoded = ModuleBundler.encode(self._realistic_sources()) - assert encoded is not None - assert not set(encoded) & {"'", '"', "\\", "\n"} - assert repr(encoded) == f"'{encoded}'" - - def test_is_smaller_than_zlib_base64(self): + decoded = json.loads(bz2.decompress(base64.b85decode(encoded))) + assert decoded == {name: sources[name] for name in _topological_order(sources)} + + def test_default_retains_zlib_base64(self): + sources = {"helper": "VALUE = 42\n"} + legacy = base64.b64encode(zlib.compress(json.dumps(sources).encode(), level=9)).decode("ascii") + assert ModuleBundler.encode(sources) == legacy + assert ModuleBundler.encode(sources, mode="bundle") == legacy + injection = ModuleBundler.build_injection(bundled_modules_b64=legacy) + assert f"zlib.decompress(base64.b64decode({legacy!r}))" in injection + assert "bz2" not in injection + + def test_escaped_injection_is_smaller_than_zlib_base64(self): sources = self._realistic_sources() - encoded = ModuleBundler.encode(sources) - assert encoded is not None - ordered = {name: sources[name] for name in _topological_order(sources)} - legacy = base64.b64encode(zlib.compress(json.dumps(ordered).encode(), level=9)) - assert len(encoded) < 0.85 * len(legacy) + encoded = ModuleBundler.encode(sources, mode="bundle-bz2") + legacy = ModuleBundler.encode(sources) + assert encoded is not None and legacy is not None + # Measure the actual emitted source, including escape overhead. + compact_injection = ModuleBundler.build_injection(encoded, mode="bundle-bz2") + legacy_injection = ModuleBundler.build_injection(legacy) + assert len(compact_injection) < 0.85 * len(legacy_injection) + + @pytest.mark.parametrize("mode", ["bundle", "bundle-bz2"]) + def test_empty_sources(self, mode): + assert ModuleBundler.encode({}, mode=mode) is None + + def test_unknown_mode_is_rejected(self): + with pytest.raises(ValueError, match="Unsupported bundle mode"): + ModuleBundler.encode({"helper": "VALUE = 42\n"}, mode="unknown") + with pytest.raises(ValueError, match="Unsupported bundle mode"): + ModuleBundler.build_injection("encoded", mode="unknown") class TestTopologicalOrder: diff --git a/tests/test_pipeline_compiler.py b/tests/test_pipeline_compiler.py index 618dc89..10ab0e6 100644 --- a/tests/test_pipeline_compiler.py +++ b/tests/test_pipeline_compiler.py @@ -541,8 +541,9 @@ def test_compile_task_decorator_rejects_unwrap_on_non_dict_annotation(tmp_path): compile_pipeline(pipeline_path, project / "compiled.yaml") -def test_compile_task_decorator_emits_bundle_mode_and_resolve_root(tmp_path): - """@task(mode="bundle") is carried into the auto-emitted sidecar.""" +@pytest.mark.parametrize("mode", ["bundle", "bundle-bz2"]) +def test_compile_task_decorator_emits_bundle_mode_and_resolve_root(tmp_path, mode): + """The selected bundle mode is carried into the auto-emitted sidecar.""" project = tmp_path / "project" src = project / "src" pipeline_path = src / "pipeline.py" @@ -551,7 +552,7 @@ def test_compile_task_decorator_emits_bundle_mode_and_resolve_root(tmp_path): pipeline_path.write_text( "from tangle_cli.python_pipeline import Out, pipeline, task\n" "from helpers import MESSAGE\n\n" - "@task(image='python:3.12', mode='bundle', resolve_root='.')\n" + f"@task(image='python:3.12', mode={mode!r}, resolve_root='.')\n" "def bundled_task() -> str:\n" " return MESSAGE\n\n" "@pipeline('Bundle Pipeline')\n" @@ -566,7 +567,7 @@ def test_compile_task_decorator_emits_bundle_mode_and_resolve_root(tmp_path): sidecar = yaml.safe_load(result.components_path.read_text()) local_from_python = sidecar["bundled-task"]["local_from_python"] - assert local_from_python["mode"] == "bundle" + assert local_from_python["mode"] == mode assert local_from_python["resolve_root"] == "./src" assert local_from_python["file"] == "./src/pipeline.py" assert local_from_python["function"] == "bundled_task" diff --git a/tests/test_pipelines_cli.py b/tests/test_pipelines_cli.py index e026ab0..7168fe1 100644 --- a/tests/test_pipelines_cli.py +++ b/tests/test_pipelines_cli.py @@ -1284,9 +1284,11 @@ def fake_regenerate_yaml(**kwargs): assert regenerated == [python_file.resolve()] +@pytest.mark.parametrize("mode", ["bundle", "bundle-bz2"]) def test_pipelines_hydrate_local_from_python_forwards_bundle_mode_and_resolve_root( monkeypatch, tmp_path: Path, + mode, ): from tangle_cli import pipeline_hydrator as hydrator_module from tangle_cli.pipelines import hydrate_pipeline_file @@ -1300,7 +1302,7 @@ def test_pipelines_hydrate_local_from_python_forwards_bundle_mode_and_resolve_ro pipeline_path = _write_local_from_python_pipeline( project_dir, "./src/component.py", - mode="bundle", + mode=mode, resolve_root="./src", ) calls: list[dict[str, object]] = [] @@ -1318,7 +1320,7 @@ def fake_regenerate_yaml(**kwargs): hydrate_pipeline_file(pipeline_path) assert calls[0]["python_file"] == python_file.resolve() - assert calls[0]["mode"] == "bundle" + assert calls[0]["mode"] == mode assert calls[0]["resolve_root"] == src_dir.resolve() diff --git a/tests/test_version_manager.py b/tests/test_version_manager.py index 404d42f..35694fc 100644 --- a/tests/test_version_manager.py +++ b/tests/test_version_manager.py @@ -4,6 +4,7 @@ import tempfile from pathlib import Path +import pytest import yaml from tangle_cli.component_from_func import generate_component_yaml @@ -413,7 +414,8 @@ def test_bump_generated_yaml_preserves_custom_name(tmp_path: Path): assert data["metadata"]["annotations"]["version"] == "1.1" -def test_bump_generated_yaml_preserves_bundle_mode(tmp_path: Path): +@pytest.mark.parametrize("mode", ["bundle", "bundle-bz2"]) +def test_bump_generated_yaml_preserves_bundle_mode(tmp_path: Path, mode): helpers_dir = tmp_path / "helpers" helpers_dir.mkdir() (helpers_dir / "__init__.py").write_text("", encoding="utf-8") @@ -442,7 +444,7 @@ def component(value: str) -> str: yaml_file, container_image="python:3.12", function_name="component", - mode="bundle", + mode=mode, ) result = bump_version(yaml_file) @@ -452,7 +454,8 @@ def component(value: str) -> str: data = yaml.safe_load(yaml_file.read_text()) annotations = data["metadata"]["annotations"] command = data["implementation"]["container"]["command"][-1] - assert annotations["tangle_cli_generation_mode"] == "bundle" + assert annotations["tangle_cli_generation_mode"] == mode + assert ("base64.b85decode" if mode == "bundle-bz2" else "base64.b64decode") in command assert "helpers.utils" in annotations["bundled_modules"] assert "_EMBEDDED_MODULES" in command assert "helpers.utils" in command