Track the ExecuTorch pin and move it to new nightlies automatically

.github/scripts/install-torch-tensorrt.sh

@@ -52,9 +52,24 @@ fi
 
 # Install Torch-TensorRT
 if [[ ${PLATFORM} == win32 ]]; then
+    # pin-check: no-nightly -- this glob also matches the Linux-only ExecuTorch runtime wheel, but
+    # ExecuTorch publishes no win32 nightly and the [executorch] extra is Linux-only, so this
+    # platform's plain torch-tensorrt install needs no nightly index.
     python -m pip install ${RUNNER_ARTIFACT_DIR}/torch_tensorrt*.whl
 else
-    python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl --use-deprecated=legacy-resolver
+    # The nightly channel is needed because this glob also matches the ExecuTorch runtime wheel,
+    # whose install_requires names an ExecuTorch dev build that is published only there.
+    # Hardcoded rather than ${CHANNEL} like the lines above: a .dev wheel exists on no other
+    # channel, so deriving it would break this install on exactly the test and release runs the
+    # index was added for. It is an extra index, not a replacement, and torch is already
+    # force-reinstalled from ${INDEX_URL} above, so the pinned torch is not at risk from it.
+    # || exit 1 because line 1's `set -exou pipefail` is commented out and linux-test.yml
+    # concatenates this file ahead of the user script, so a failure here would otherwise be
+    # discarded and the job would die later with an unrelated-looking ImportError. Scoped to the
+    # line this change is responsible for; re-enabling set -e for the whole file is a
+    # pre-existing hazard worth a separate change.
+    python -m pip install /opt/torch-tensorrt-builds/torch_tensorrt*.whl --use-deprecated=legacy-resolver \
+        --extra-index-url "https://download.pytorch.org/whl/nightly/${CU_VERSION}" || exit 1
 fi
 
 echo -e "Running test script";

.github/scripts/update_executorch_pin.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python3
+"""Move the ExecuTorch pin to the newest wheel published on an index.
+
+The pin is two coupled facts, spread across the tree but sourced from
+``dev_dep_versions.yml``: ``__executorch_version__`` selects the wheel the delegate is
+built to sit beside, and ``__executorch_commit__`` selects the tree it compiles from.
+They must name one ExecuTorch, so this script never guesses the commit: it reads it from
+the chosen wheel's own ``executorch/version.py``, which is the same provenance
+``tests/py/dynamo/executorch/test_executorch_pin.py`` checks the pins against.
+
+The daily workflow runs this, then opens a pull request when the pin moved. The existing
+pin guards and the executorch end-to-end lane run on that pull request, so "the newest
+wheel that actually works" is decided by the same gate a human bump goes through, not
+re-implemented here. A nightly that did not publish leaves the newest version unchanged,
+so the run rewrites nothing and opens nothing.
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import subprocess
+import sys
+import tempfile
+import zipfile
+from pathlib import Path
+
+from packaging.version import InvalidVersion, Version
+
+_REPO_ROOT = Path(__file__).resolve().parents[2]
+_VERSIONS_FILE = _REPO_ROOT / "dev_dep_versions.yml"
+
+# A wheel version on the PyTorch index carries a local label naming its CUDA build, for
+# example ``1.5.0.devYYYYMMDD+cu130``. The pin omits it so one pin serves every CUDA row.
+_LOCAL_LABEL = re.compile(r"\+.*$")
+
+
+def _run(cmd: list[str]) -> str:
+    return subprocess.run(cmd, check=True, capture_output=True, text=True).stdout
+
+
+def read_pin(field: str) -> str:
+    """Return a pinned value from ``dev_dep_versions.yml``."""
+    text = _VERSIONS_FILE.read_text(encoding="utf-8")
+    match = re.search(rf'^{field}:\s*"?([^"\s]+)"?\s*$', text, re.MULTILINE)
+    if match is None:
+        raise SystemExit(f"{field} is not set in {_VERSIONS_FILE.name}")
+    return match.group(1)
+
+
+def available_versions(index_args: list[str]) -> list[str]:
+    """Every ExecuTorch version the index offers.
+
+    ``pip index versions`` prints one ``Available versions:`` line. Parsing that is stable
+    and needs no network code here; the workflow passes the index the same way every other
+    install in the tree does.
+    """
+    out = _run(
+        [sys.executable, "-m", "pip", "index", "versions", "executorch", *index_args]
+    )
+    match = re.search(r"^\s*Available versions:\s*(.+)$", out, re.MULTILINE)
+    if match is None:
+        raise SystemExit("pip index versions printed no Available versions line")
+    return [v.strip() for v in match.group(1).split(",") if v.strip()]
+
+
+def pick_target(versions: list[str], track: str) -> str:
+    """The newest version on the wanted track.
+
+    ``nightly`` takes the newest dated dev build; ``stable`` takes the newest final
+    release, ignoring dev builds and release candidates. Ordering is PEP 440, not string
+    order, so a newer dev date on the same line sorts above an older one correctly.
+    """
+    parsed: list[tuple[Version, str]] = []
+    for raw in versions:
+        try:
+            version = Version(raw)
+        except InvalidVersion:
+            continue
+        # A nightly is a dated dev build. is_prerelease is also true for release
+        # candidates, and an rc sorts above every dev of the same line under PEP 440, so
+        # filtering on it would let the first rc on the index silently become the nightly
+        # pin. Match the dev segment itself instead.
+        if track == "nightly" and version.dev is None:
+            continue
+        if track == "stable" and (version.is_prerelease or version.is_devrelease):
+            continue
+        parsed.append((version, raw))
+    if not parsed:
+        raise SystemExit(f"no executorch version on the index matches track {track!r}")
+    newest = max(parsed, key=lambda pair: pair[0])[1]
+    return _LOCAL_LABEL.sub("", newest)
+
+
+def wheel_git_version(version: str, index_args: list[str]) -> str:
+    """The source commit the chosen wheel records for itself.
+
+    Every published wheel writes ``git_version`` into ``executorch/version.py``. Reading it
+    from the wheel is what keeps the two pins naming one ExecuTorch. A wheel built without
+    git provenance records ``None`` and must not become a pin, so that is an error, not a
+    guess.
+    """
+    with tempfile.TemporaryDirectory() as tmp:
+        _run(
+            [
+                sys.executable,
+                "-m",
+                "pip",
+                "download",
+                "--no-deps",
+                "--only-binary=:all:",
+                "--dest",
+                tmp,
+                f"executorch=={version}",
+                *index_args,
+            ]
+        )
+        wheels = list(Path(tmp).glob("executorch-*.whl"))
+        if not wheels:
+            raise SystemExit(
+                f"pip download produced no wheel for executorch=={version}"
+            )
+        with zipfile.ZipFile(wheels[0]) as archive:
+            source = archive.read("executorch/version.py").decode("utf-8")
+    match = re.search(r"""git_version[^=]*=\s*['"]([0-9a-f]{40})['"]""", source)
+    if match is None:
+        raise SystemExit(
+            f"executorch=={version} records no source commit, so the pin would name a "
+            "wheel whose provenance cannot be checked"
+        )
+    return match.group(1)
+
+
+def _upper_bound(version: str) -> str:
+    """The exclusive upper bound a range site pairs with the pin, next minor of its line.
+
+    ``tests/py/dynamo/executorch/test_executorch_pin.py`` derives the same bound from the
+    same two fields, so a range this writes and the range the guard expects agree by the
+    same rule rather than by coincidence.
+    """
+    major, minor = version.split(".")[:2]
+    return f"{major}.{int(minor) + 1}"
+
+
+# The only files that carry the pin as a real pin. A tree-wide literal replace was safe while the
+# pin was a dated dev string, because "1.5.0.dev20260825" appears nowhere else, but it corrupts the
+# tree the moment the pin is a plain release like "1.4.1": that token also lives in unrelated
+# requirements (for example pandocfilters>=1.4.1 in committed notebooks) and, worst, in uv.lock,
+# whose entries are content addressed, so rewriting the version inside a wheel URL while its hash and
+# size stay behind is a guaranteed install failure. So restrict the rewrite to the sites the guard in
+# tests/py/dynamo/executorch/test_executorch_pin.py enumerates, which are the only sites that are
+# actually pins. A new legitimate site must be added here and to that guard together.
+_PIN_SITES = (
+    ".github/workflows/executorch-build-linux.yml",
+    ".github/workflows/executorch-test-linux.yml",
+    "MODULE.bazel",
+    "docker/MODULE.bazel.docker",
+    "docker/MODULE.bazel.ngc",
+    "justfile",
+    "py/torch-tensorrt-executorch-runtime/README.md",
+    "py/torch-tensorrt-executorch-runtime/pyproject.toml",
+    "toolchains/ci_workspaces/MODULE.bazel.tmpl",
+    "examples/executorch_reference_runner/README.md",
+)
+
+
+def _pin_site_paths() -> list[Path]:
+    tracked = set(_run(["git", "-C", str(_REPO_ROOT), "ls-files"]).splitlines())
+    paths = [_REPO_ROOT / name for name in _PIN_SITES if name in tracked]
+    # dev_dep_versions.yml is the source of truth and is rewritten too; it is not in _PIN_SITES
+    # because the guard reads it rather than counting it as a downstream pin site.
+    paths.append(_VERSIONS_FILE)
+    return paths
+
+
+def write_pins(new_version: str, new_commit: str) -> bool:
+    """Rewrite the pin to the new version and commit at the known pin sites.
+
+    The rewrite is restricted to the sites the guard enumerates (see _PIN_SITES). It is NOT a
+    tree-wide literal replace: the old version is a unique token only while it is a dated dev build,
+    and a plain release like "1.4.1" also appears in unrelated requirements and in content-addressed
+    uv.lock entries, so replacing it everywhere would corrupt files that are not pin sites. Neither
+    setup.py nor uv.lock is a pin site; they are left for their own tooling to regenerate. Returns
+    whether anything changed.
+    """
+    old_version = read_pin("__executorch_version__")
+    old_commit = read_pin("__executorch_commit__")
+    if (new_version, new_commit) == (old_version, old_commit):
+        return False
+
+    # Anchored on >= and ,< rather than on a package name, so this rewrites the range wherever it
+    # appears, for example the executorch>=X,<Y in a workflow install line. The range carries the old
+    # version too, so it is rewritten in the same pass as the bare version below, not before it.
+    old_range = f">={old_version},<{_upper_bound(old_version)}"
+    new_range = f">={new_version},<{_upper_bound(new_version)}"
+
+    # A single regex pass, so a freshly written new version cannot be matched again. A plain
+    # text.replace of the bare version doubles the tail when the old version is a prefix of the new
+    # one (1.5.0 -> 1.5.0.post1 would yield 1.5.0.post1.post1), because the second replace re-hits
+    # what the first just wrote. Matching the old version only when it is not followed by another
+    # version character avoids that, and the range is handled by the same alternation so its own
+    # version is not rewritten twice.
+    version_token = re.compile(
+        re.escape(old_range) + r"|" + re.escape(old_version) + r"(?![0-9A-Za-z.+_-])"
+    )
+
+    def _sub_version(match: re.Match[str]) -> str:
+        return new_range if match.group(0) == old_range else new_version
+
+    changed = False
+    for path in _pin_site_paths():
+        try:
+            text = path.read_text(encoding="utf-8")
+        except (UnicodeDecodeError, FileNotFoundError):
+            continue
+        if old_version not in text and old_commit not in text:
+            continue
+        updated = version_token.sub(_sub_version, text)
+        updated = updated.replace(old_commit, new_commit)
+        if updated != text:
+            path.write_text(updated, encoding="utf-8")
+            changed = True
+
+    if read_pin("__executorch_version__") != new_version:
+        raise SystemExit(
+            "the version pin did not take; dev_dep_versions.yml is unchanged"
+        )
+    return changed
+
+
+def _index_args(track: str, channel: str) -> list[str]:
+    if track == "nightly":
+        return [
+            "--pre",
+            "--index-url",
+            f"https://download.pytorch.org/whl/nightly/{channel}",
+        ]
+    return []
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--track", choices=("nightly", "stable"), default="nightly")
+    parser.add_argument(
+        "--channel",
+        default="cu130",
+        help="nightly CUDA channel to read versions and provenance from",
+    )
+    parser.add_argument(
+        "--allow-downgrade",
+        action="store_true",
+        help=(
+            "move the pin even when the target sorts below the current pin. Off by default so a "
+            "regressed index, or switching --track from nightly to stable, cannot silently walk "
+            "the pin backwards onto a version this delegate cannot build against."
+        ),
+    )
+    args = parser.parse_args(argv)
+
+    index_args = _index_args(args.track, args.channel)
+    target = pick_target(available_versions(index_args), args.track)
+    current = read_pin("__executorch_version__")
+    if target == current:
+        print(f"executorch pin is already at the newest {args.track} version {current}")
+        return 0
+    # Never move backwards unless asked. pick_target returns the newest on the track, but "newest"
+    # regresses when the index drops the current line or when --track flips from nightly to stable,
+    # whose newest final release can sort below a dated nightly. The pin then lands on a version with
+    # no standalone libexecutorch.so and no CUDA build, which the delegate cannot link, and the run
+    # would still report success. Refuse it, and require an explicit opt-in for a deliberate re-pin.
+    if not args.allow_downgrade and Version(target) < Version(current):
+        print(
+            f"target {target} sorts below the current pin {current}; refusing to move the pin "
+            "backwards. Pass --allow-downgrade to override for a deliberate re-pin."
+        )
+        return 0
+
+    commit = wheel_git_version(target, index_args)
+    if write_pins(target, commit):
+        print(f"moved executorch pin {current} -> {target} (commit {commit})")
+    else:
+        print("nothing to write")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())