diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index 123680e5275..117d1bcf107 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -94,12 +94,35 @@ case "${IMAGE_NAME}" in exit 1 esac -TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt) BUILD_DOCS=1 -if [[ "${GCC_VERSION:-}" == "11" && -z "${SKIP_PYTORCH:-}" ]]; then - PYTORCH_BUILD_MAX_JOBS=6 -fi +# Pull channel + spec/url helpers out of torch_pin.py so install_pytorch.sh +# can install the selected PyTorch wheel channel. +# Self-hosted runners often have python3 but not the unversioned python alias. +PYTHON_BIN=$(command -v python3 || command -v python) +TORCH_PIN_HELPERS=$( + cd ../.. && + "$PYTHON_BIN" - <<'PY' +from torch_pin import ( + CHANNEL, + torch_index_url_base, + torch_spec, + torchaudio_spec, + torchvision_spec, +) + +print(CHANNEL) +print(torch_spec()) +print(torchaudio_spec()) +print(torchvision_spec()) +print(torch_index_url_base()) +PY +) +TORCH_CHANNEL=$(echo "${TORCH_PIN_HELPERS}" | sed -n '1p') +TORCH_SPEC=$(echo "${TORCH_PIN_HELPERS}" | sed -n '2p') +TORCHAUDIO_SPEC=$(echo "${TORCH_PIN_HELPERS}" | sed -n '3p') +TORCHVISION_SPEC=$(echo "${TORCH_PIN_HELPERS}" | sed -n '4p') +TORCH_INDEX_URL=$(echo "${TORCH_PIN_HELPERS}" | sed -n '5p') # Copy requirements-lintrunner.txt from root to here cp ../../requirements-lintrunner.txt ./ @@ -112,8 +135,11 @@ docker build \ --build-arg "GCC_VERSION=${GCC_VERSION}" \ --build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \ --build-arg "MINICONDA_VERSION=${MINICONDA_VERSION}" \ - --build-arg "TORCH_VERSION=${TORCH_VERSION}" \ - --build-arg "PYTORCH_BUILD_MAX_JOBS=${PYTORCH_BUILD_MAX_JOBS:-}" \ + --build-arg "TORCH_CHANNEL=${TORCH_CHANNEL}" \ + --build-arg "TORCH_SPEC=${TORCH_SPEC}" \ + --build-arg "TORCHAUDIO_SPEC=${TORCHAUDIO_SPEC}" \ + --build-arg "TORCHVISION_SPEC=${TORCHVISION_SPEC}" \ + --build-arg "TORCH_INDEX_URL=${TORCH_INDEX_URL}" \ --build-arg "BUCK2_VERSION=${BUCK2_VERSION}" \ --build-arg "LINTRUNNER=${LINTRUNNER:-}" \ --build-arg "BUILD_DOCS=${BUILD_DOCS}" \ diff --git a/.ci/docker/common/install_pytorch.sh b/.ci/docker/common/install_pytorch.sh index 3c80d093ab2..fd09b60ebd6 100755 --- a/.ci/docker/common/install_pytorch.sh +++ b/.ci/docker/common/install_pytorch.sh @@ -10,47 +10,47 @@ set -ex # shellcheck source=/dev/null source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" -install_domains() { - echo "Install torchvision and torchaudio" - pip_install --no-build-isolation --user "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}" - pip_install --no-build-isolation --user "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION}" -} - install_pytorch_and_domains() { - git clone https://github.com/pytorch/pytorch.git - - # Fetch the target commit - pushd pytorch || true - git checkout "${TORCH_VERSION}" - git submodule update --init --recursive - - chown -R ci-user . - - export _GLIBCXX_USE_CXX11_ABI=1 - if [[ "$(uname -m)" == "aarch64" ]]; then - export BUILD_IGNORE_SVE_UNAVAILABLE=1 - fi - if [[ -n "${PYTORCH_BUILD_MAX_JOBS:-}" ]]; then - export MAX_JOBS="${PYTORCH_BUILD_MAX_JOBS}" + local cache_args=() + if [ "${TORCH_CHANNEL}" = "test" ]; then + cache_args=("--no-cache-dir") fi - # Then build and install PyTorch - conda_run python setup.py bdist_wheel - pip_install "$(echo dist/*.whl)" - # Grab the pinned audio and vision commits from PyTorch - TORCHAUDIO_VERSION=release/2.11 - export TORCHAUDIO_VERSION - TORCHVISION_VERSION=release/0.27 - export TORCHVISION_VERSION + local wheelhouse + wheelhouse=$(mktemp -d) + chown ci-user:ci-user "${wheelhouse}" + + local system_name + local system_arch + local python_version + system_name=$(uname) + system_arch=$(uname -m) + python_version=$(conda_run python -c 'import platform; v=platform.python_version_tuple(); print(f"{v[0]}{v[1]}")') + local torch_wheel_cache_path="cached_artifacts/pytorch/executorch/pytorch_wheels/${system_name}/${system_arch}/${python_version}/cpu/${TORCH_CHANNEL}" + local torch_wheel_cache_uri="s3://gha-artifacts/${torch_wheel_cache_path}" + + # Do not cache test-channel wheels in S3: RC artifacts may be re-uploaded + # under the same package version. + if [[ "${TORCH_CHANNEL}" != "test" ]] && command -v aws >/dev/null 2>&1; then + aws s3 sync "${torch_wheel_cache_uri}" "${wheelhouse}" || true + fi - install_domains + conda_run python -m pip download --progress-bar off --no-deps "${cache_args[@]}" \ + --dest "${wheelhouse}" \ + --find-links "${wheelhouse}" \ + "${TORCH_SPEC}" "${TORCHVISION_SPEC}" "${TORCHAUDIO_SPEC}" \ + --index-url "${TORCH_INDEX_URL}/cpu" - popd || true - # Clean up the cloned PyTorch repo to reduce the Docker image size - rm -rf pytorch + if [[ "${TORCH_CHANNEL}" != "test" && -z "${GITHUB_RUNNER:-}" ]] && command -v aws >/dev/null 2>&1; then + aws s3 sync "${wheelhouse}" "${torch_wheel_cache_uri}" \ + --exclude "*" --include "*.whl" || true + fi - # Print sccache stats for debugging - as_ci_user sccache --show-stats + pip_install --force-reinstall "${cache_args[@]}" \ + --find-links "${wheelhouse}" \ + "${TORCH_SPEC}" "${TORCHVISION_SPEC}" "${TORCHAUDIO_SPEC}" \ + --index-url "${TORCH_INDEX_URL}/cpu" + rm -rf "${wheelhouse}" } install_pytorch_and_domains diff --git a/.ci/docker/ubuntu/Dockerfile b/.ci/docker/ubuntu/Dockerfile index 9a5b2536df0..8c2e731eaa9 100644 --- a/.ci/docker/ubuntu/Dockerfile +++ b/.ci/docker/ubuntu/Dockerfile @@ -54,7 +54,7 @@ COPY ./common/install_conda.sh install_conda.sh COPY ./common/utils.sh utils.sh RUN bash ./install_conda.sh && rm install_conda.sh utils.sh /opt/conda/requirements-ci.txt /opt/conda/conda-env-ci.txt -# Install sccache before building torch +# Install sccache before building ExecuTorch. COPY ./common/install_cache.sh install_cache.sh ENV PATH /opt/cache/bin:$PATH COPY ./common/utils.sh utils.sh @@ -65,7 +65,11 @@ ENV SCCACHE_REGION us-east-1 ENV AWS_REGION us-east-1 ENV AWS_DEFAULT_REGION us-east-1 -ARG TORCH_VERSION +ARG TORCH_CHANNEL +ARG TORCH_SPEC +ARG TORCHAUDIO_SPEC +ARG TORCHVISION_SPEC +ARG TORCH_INDEX_URL ARG SKIP_PYTORCH ARG PYTORCH_BUILD_MAX_JOBS COPY ./common/install_pytorch.sh install_pytorch.sh diff --git a/.ci/scripts/setup-linux.sh b/.ci/scripts/setup-linux.sh index feb8a128b17..05d5a5c4405 100755 --- a/.ci/scripts/setup-linux.sh +++ b/.ci/scripts/setup-linux.sh @@ -13,16 +13,13 @@ source "$(dirname "${BASH_SOURCE[0]}")/utils.sh" read -r BUILD_TOOL BUILD_MODE EDITABLE < <(parse_args "$@") echo "Build tool: $BUILD_TOOL, Mode: $BUILD_MODE" -# As Linux job is running inside a Docker container, all of its dependencies -# have already been installed, so we use PyTorch build from source here instead -# of nightly. This allows CI to test against latest commits from PyTorch if [[ "${EDITABLE:-false}" == "true" ]]; then - install_executorch --use-pt-pinned-commit --editable + install_executorch --editable else - install_executorch --use-pt-pinned-commit + install_executorch fi build_executorch_runner "${BUILD_TOOL}" "${BUILD_MODE}" if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then - do_not_use_nightly_on_ci + verify_torch_matches_pin_on_ci fi diff --git a/.ci/scripts/setup-macos.sh b/.ci/scripts/setup-macos.sh index 38863597224..1a9993a1a7e 100755 --- a/.ci/scripts/setup-macos.sh +++ b/.ci/scripts/setup-macos.sh @@ -126,22 +126,19 @@ fi # Install pinned torch before requirements-ci.txt so torchsr's transitive # torch dep is satisfied by the existing install and pip does not pull a -# separate copy from PyPI. sccache is initialized above so source-build -# cache misses still hit the cache. +# separate copy from PyPI. print_cmake_info install_pytorch_and_domains install_pip_dependencies -# install_executorch's --use-pt-pinned-commit skips re-installing torch since -# install_pytorch_and_domains already installed the pinned build above. if [[ "$EDITABLE" == "true" ]]; then - install_executorch --use-pt-pinned-commit --editable + install_executorch --editable else - install_executorch --use-pt-pinned-commit + install_executorch fi build_executorch_runner "${BUILD_TOOL}" "${BUILD_MODE}" if [[ "${GITHUB_BASE_REF:-}" == *main* || "${GITHUB_BASE_REF:-}" == *gh* ]]; then - do_not_use_nightly_on_ci + verify_torch_matches_pin_on_ci fi diff --git a/.ci/scripts/test_model_e2e.sh b/.ci/scripts/test_model_e2e.sh index e1ba976b0cc..afa15730c6d 100755 --- a/.ci/scripts/test_model_e2e.sh +++ b/.ci/scripts/test_model_e2e.sh @@ -284,7 +284,9 @@ elif [[ "$MODEL_NAME" == *whisper* ]] || [ "$MODEL_NAME" = "voxtral_realtime" ]; fi fi pip install datasets soundfile - pip install torchcodec==0.11.0 --extra-index-url https://download.pytorch.org/whl/test/cpu + TORCHCODEC_PKG=$(python -c "from torch_pin import torchcodec_spec; print(torchcodec_spec())") + TORCHCODEC_INDEX=$(python -c "from torch_pin import torch_index_url_base; print(torch_index_url_base())") + pip install "$TORCHCODEC_PKG" --index-url "${TORCHCODEC_INDEX}/cpu" python -c "from datasets import load_dataset;import soundfile as sf;sample = load_dataset('distil-whisper/librispeech_long', 'clean', split='validation')[0]['audio'];sf.write('${MODEL_DIR}/$AUDIO_FILE', sample['array'][:sample['sampling_rate']*30], sample['sampling_rate'])" fi diff --git a/.ci/scripts/test_wheel_package_qnn.sh b/.ci/scripts/test_wheel_package_qnn.sh index 763bd8733c1..22198d2a4e7 100644 --- a/.ci/scripts/test_wheel_package_qnn.sh +++ b/.ci/scripts/test_wheel_package_qnn.sh @@ -150,25 +150,37 @@ run_core_tests () { echo "=== [$LABEL] Installing wheel & deps ===" "$PIPBIN" install --upgrade pip "$PIPBIN" install "$WHEEL_FILE" - TORCH_VERSION=$( + # runpy.run_path uses a relative path, so the caller must run this script + # from the executorch repo root (where torch_pin.py lives). + TORCH_SPEC=$( "$PYBIN" - <<'PY' import runpy module_vars = runpy.run_path("torch_pin.py") -print(module_vars["TORCH_VERSION"]) +print(module_vars["torch_spec"]()) PY ) + TORCH_INDEX=$( + "$PYBIN" - <<'PY' +import runpy +module_vars = runpy.run_path("torch_pin.py") +print(module_vars["torch_index_url_base"]()) +PY +) + TORCH_CACHE_ARGS_OUTPUT=$( + "$PYBIN" - <<'PY' +import runpy +module_vars = runpy.run_path("torch_pin.py") +print(" ".join(module_vars["pip_cache_args"]())) +PY +) + TORCH_CACHE_ARGS=() + if [[ -n "${TORCH_CACHE_ARGS_OUTPUT}" ]]; then + read -r -a TORCH_CACHE_ARGS <<< "${TORCH_CACHE_ARGS_OUTPUT}" + fi + echo "=== [$LABEL] Install $TORCH_SPEC from ${TORCH_INDEX}/cpu ===" -# NIGHTLY_VERSION=$( -# "$PYBIN" - <<'PY' -# import runpy -# module_vars = runpy.run_path("torch_pin.py") -# print(module_vars["NIGHTLY_VERSION"]) -# PY -# ) - echo "=== [$LABEL] Install torch==${TORCH_VERSION} ===" - - # Install torch based on the pinned PyTorch version, preferring the PyTorch test index - "$PIPBIN" install torch=="${TORCH_VERSION}" --extra-index-url "https://download.pytorch.org/whl/test" + # Install torch based on the pinned PyTorch version from the channel index. + "$PIPBIN" install "${TORCH_CACHE_ARGS[@]}" "$TORCH_SPEC" --index-url "${TORCH_INDEX}/cpu" "$PIPBIN" install wheel # Install torchao based on the pinned commit from third-party/ao submodule diff --git a/.ci/scripts/tests/test_torch_pin.py b/.ci/scripts/tests/test_torch_pin.py new file mode 100644 index 00000000000..59004285d9c --- /dev/null +++ b/.ci/scripts/tests/test_torch_pin.py @@ -0,0 +1,72 @@ +import importlib + +import pytest + + +@pytest.fixture +def pin(): + import torch_pin + + yield torch_pin + importlib.reload(torch_pin) + + +@pytest.mark.parametrize( + "channel, expected_torch, expected_url", + [ + ( + "nightly", + "torch=={TORCH_VERSION}.{NIGHTLY_VERSION}", + "https://download.pytorch.org/whl/nightly", + ), + ("test", "torch=={TORCH_VERSION}", "https://download.pytorch.org/whl/test"), + ("release", "torch=={TORCH_VERSION}", "https://download.pytorch.org/whl"), + ], +) +def test_channel_resolution(pin, channel, expected_torch, expected_url): + pin.CHANNEL = channel + expected = expected_torch.format( + TORCH_VERSION=pin.TORCH_VERSION, NIGHTLY_VERSION=pin.NIGHTLY_VERSION + ) + assert pin.torch_spec() == expected + assert pin.torch_index_url_base() == expected_url + + +def test_all_specs_share_nightly_suffix(pin): + pin.CHANNEL = "nightly" + suffix = f".{pin.NIGHTLY_VERSION}" + assert pin.torch_spec().endswith(suffix) + assert pin.torchaudio_spec().endswith(suffix) + assert pin.torchcodec_spec().endswith(suffix) + assert pin.torchvision_spec().endswith(suffix) + + +def test_specs_drop_suffix_off_nightly(pin): + pin.CHANNEL = "release" + assert pin.torch_spec() == f"torch=={pin.TORCH_VERSION}" + assert pin.torchaudio_spec() == f"torchaudio=={pin.TORCHAUDIO_VERSION}" + assert pin.torchcodec_spec() == f"torchcodec=={pin.TORCHCODEC_VERSION}" + assert pin.torchvision_spec() == f"torchvision=={pin.TORCHVISION_VERSION}" + + +@pytest.mark.parametrize( + "channel, expected", + [ + ("nightly", []), + ("test", ["--no-cache-dir"]), + ("release", []), + ], +) +def test_cache_args_only_disable_cache_for_test_channel(pin, channel, expected): + pin.CHANNEL = channel + assert pin.pip_cache_args() == expected + + +def test_release_branches_derived_from_versions(pin): + assert pin.torch_branch() == f"release/{pin.TORCH_VERSION.rsplit('.', 1)[0]}" + assert pin.torchaudio_branch() == ( + f"release/{pin.TORCHAUDIO_VERSION.rsplit('.', 1)[0]}" + ) + assert pin.torchvision_branch() == ( + f"release/{pin.TORCHVISION_VERSION.rsplit('.', 1)[0]}" + ) diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index b312d0ede83..32b832f81b9 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -82,110 +82,60 @@ dedupe_macos_loader_path_rpaths() { done } -install_domains() { - echo "Install torchvision and torchaudio" - pip install --no-build-isolation --user "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}" - pip install --no-build-isolation --user "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION}" -} - install_pytorch_and_domains() { - pushd .ci/docker || return - TORCH_VERSION=$(cat ci_commit_pins/pytorch.txt) - popd || return - - git clone https://github.com/pytorch/pytorch.git - - # Fetch the target commit - pushd pytorch || return - git checkout "${TORCH_VERSION}" - - local system_name=$(uname) - if [[ "${system_name}" == "Darwin" ]]; then - local platform=$(python -c 'import sysconfig; import platform; v=platform.mac_ver()[0].split(".")[0]; platform=sysconfig.get_platform().split("-"); platform[1]=f"{v}_0"; print("_".join(platform))') + # CWD is the executorch repo root, where torch_pin.py lives. + local torch_channel + local torch_spec + local torchvision_spec + local torchaudio_spec + local torch_index_url + local torch_cache_args_output + torch_channel=$(python -c "from torch_pin import CHANNEL; print(CHANNEL)") + torch_spec=$(python -c "from torch_pin import torch_spec; print(torch_spec())") + torchvision_spec=$(python -c "from torch_pin import torchvision_spec; print(torchvision_spec())") + torchaudio_spec=$(python -c "from torch_pin import torchaudio_spec; print(torchaudio_spec())") + torch_index_url=$(python -c "from torch_pin import torch_index_url_base; print(torch_index_url_base())") + torch_cache_args_output=$(python -c "from torch_pin import pip_cache_args; print(' '.join(pip_cache_args()))") + local torch_cache_args=() + if [[ -n "${torch_cache_args_output}" ]]; then + read -r -a torch_cache_args <<< "${torch_cache_args_output}" fi - local python_version=$(python -c 'import platform; v=platform.python_version_tuple(); print(f"{v[0]}{v[1]}")') - local torch_release=$(cat version.txt) - # Download key must match the upload key below (basename of dist/*.whl, - # which always carries setup.py's resolved +gitHASH). Branch-ref pins - # like `release/2.12` would otherwise produce `+gitrelease` here and - # never hit the cache. - local torch_short_hash=$(git rev-parse --short=7 HEAD) - local torch_wheel_path="cached_artifacts/pytorch/executorch/pytorch_wheels/${system_name}/${python_version}" - local torch_wheel_name="torch-${torch_release}%2Bgit${torch_short_hash}-cp${python_version}-cp${python_version}-${platform:-}.whl" - - local cached_torch_wheel="https://gha-artifacts.s3.us-east-1.amazonaws.com/${torch_wheel_path}/${torch_wheel_name}" - # Cache PyTorch wheel is only needed on MacOS, Linux CI already has this as part - # of the Docker image - local torch_wheel_not_found=0 - if [[ "${system_name}" == "Darwin" ]]; then - pip install "${cached_torch_wheel}" || torch_wheel_not_found=1 - else - torch_wheel_not_found=1 + + local wheelhouse + wheelhouse=$(mktemp -d) + + local system_name + local system_arch + local python_version + system_name=$(uname) + system_arch=$(uname -m) + python_version=$(python -c 'import platform; v=platform.python_version_tuple(); print(f"{v[0]}{v[1]}")') + local torch_wheel_cache_path="cached_artifacts/pytorch/executorch/pytorch_wheels/${system_name}/${system_arch}/${python_version}/cpu/${torch_channel}" + local torch_wheel_cache_uri="s3://gha-artifacts/${torch_wheel_cache_path}" + + # Do not cache test-channel wheels in S3: RC artifacts may be re-uploaded + # under the same package version. + if [[ "${torch_channel}" != "test" ]] && command -v aws >/dev/null 2>&1; then + aws s3 sync "${torch_wheel_cache_uri}" "${wheelhouse}" || true fi - # Found no such wheel, we will build it from source then - if [[ "${torch_wheel_not_found}" == "1" ]]; then - echo "No cached wheel found, continue with building PyTorch at ${TORCH_VERSION}" - - # Install PyTorch's own build-time deps so the source build does not - # silently inherit them from whatever else happens to be in the env - # (e.g. executorch's requirements-ci.txt). - pip install -r requirements-build.txt - git submodule update --init --recursive - if [[ "$(uname -m)" == "aarch64" ]]; then - export BUILD_IGNORE_SVE_UNAVAILABLE=1 - fi - USE_DISTRIBUTED=1 python setup.py bdist_wheel - pip install "$(echo dist/*.whl)" - - # Invariant: the basename setup.py just produced must match the cache - # URL we'd reconstruct on the next run. If they diverge (someone edits - # torch_wheel_name above, or PyTorch renames its wheels), the cache - # will silently miss and every macOS run will fall back to a ~30-min - # source build. Fail loudly so the regression is caught immediately. - shopt -s nullglob - local built_wheels=(dist/*.whl) - shopt -u nullglob - if [[ ${#built_wheels[@]} -ne 1 ]]; then - echo "ERROR: expected exactly 1 wheel in dist/, found ${#built_wheels[@]}" >&2 - exit 1 - fi - local built_wheel_name - built_wheel_name=$(basename "${built_wheels[0]}") - local expected_wheel_name="${torch_wheel_name//\%2B/+}" - if [[ "${built_wheel_name}" != "${expected_wheel_name}" ]]; then - echo "ERROR: built torch wheel name does not match cache URL key:" >&2 - echo " built: ${built_wheel_name}" >&2 - echo " expected: ${expected_wheel_name}" >&2 - echo "Fix torch_wheel_name construction in install_pytorch_and_domains" >&2 - echo "in .ci/scripts/utils.sh" >&2 - exit 1 - fi + python -m pip download --no-deps "${torch_cache_args[@]}" \ + --dest "${wheelhouse}" \ + --find-links "${wheelhouse}" \ + "${torch_spec}" "${torchvision_spec}" "${torchaudio_spec}" \ + --index-url "${torch_index_url}/cpu" - # Only AWS runners have access to S3 - if command -v aws && [[ -z "${GITHUB_RUNNER:-}" ]]; then - for wheel_path in dist/*.whl; do - local wheel_name=$(basename "${wheel_path}") - echo "Caching ${wheel_name}" - aws s3 cp "${wheel_path}" "s3://gha-artifacts/${torch_wheel_path}/${wheel_name}" - done - fi - else - echo "Use cached wheel at ${cached_torch_wheel}" + if [[ "${torch_channel}" != "test" && -z "${GITHUB_RUNNER:-}" ]] && command -v aws >/dev/null 2>&1; then + aws s3 sync "${wheelhouse}" "${torch_wheel_cache_uri}" \ + --exclude "*" --include "*.whl" || true fi + pip install --force-reinstall "${torch_cache_args[@]}" \ + --find-links "${wheelhouse}" \ + "${torch_spec}" "${torchvision_spec}" "${torchaudio_spec}" \ + --index-url "${torch_index_url}/cpu" dedupe_macos_loader_path_rpaths - # Grab the pinned audio and vision commits from PyTorch - TORCHAUDIO_VERSION=release/2.11 - export TORCHAUDIO_VERSION - TORCHVISION_VERSION=release/0.27 - export TORCHVISION_VERSION - - install_domains - - popd || return - # Print sccache stats for debugging - sccache --show-stats || true + rm -rf "${wheelhouse}" } build_executorch_runner_buck2() { @@ -252,18 +202,27 @@ download_stories_model_artifacts() { echo '{"dim": 768, "multiple_of": 32, "n_heads": 12, "n_layers": 12, "norm_eps": 1e-05, "vocab_size": 32000}' > params.json } -do_not_use_nightly_on_ci() { - # An assert to make sure that we are not using PyTorch nightly on CI to prevent - # regression as documented in https://github.com/pytorch/executorch/pull/6564 - TORCH_VERSION=$(pip list | grep -w 'torch ' | awk -F ' ' {'print $2'} | tr -d '\n') - - # The version of PyTorch building from source looks like 2.6.0a0+gitc8a648d that - # includes the commit while nightly (2.6.0.dev20241019+cpu) or release (2.6.0) - # won't have that. Note that we couldn't check for the exact commit from the pin - # ci_commit_pins/pytorch.txt here because the value will be different when running - # this on PyTorch CI - if [[ "${TORCH_VERSION}" != *"+git"* ]]; then - echo "Unexpected torch version. Expected binary built from source, got ${TORCH_VERSION}" +verify_torch_matches_pin_on_ci() { + local expected_torch_version + local installed_torch_version + expected_torch_version=$(python - <<'PY' +from torch_pin import CHANNEL, NIGHTLY_VERSION, TORCH_VERSION + +if CHANNEL == "nightly": + print(f"{TORCH_VERSION}.{NIGHTLY_VERSION}") +else: + print(TORCH_VERSION) +PY +) + installed_torch_version=$(python - <<'PY' +import torch + +print(torch.__version__.split("+", 1)[0]) +PY +) + + if [[ "${installed_torch_version}" != "${expected_torch_version}" ]]; then + echo "Unexpected torch version. Expected ${expected_torch_version}, got ${installed_torch_version}" exit 1 fi } diff --git a/.github/scripts/update_pytorch_pin.py b/.github/scripts/update_pytorch_pin.py index dbc48552d9b..ac332ad010a 100644 --- a/.github/scripts/update_pytorch_pin.py +++ b/.github/scripts/update_pytorch_pin.py @@ -1,13 +1,16 @@ #!/usr/bin/env python3 import base64 -import hashlib import json import re import sys import urllib.request from pathlib import Path +_REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(_REPO_ROOT)) +from torch_pin import CHANNEL, NIGHTLY_VERSION, torch_branch # noqa: E402 + def parse_nightly_version(nightly_version): """ @@ -27,23 +30,6 @@ def parse_nightly_version(nightly_version): return f"{year}-{month}-{day}" -def get_torch_nightly_version(): - """ - Read NIGHTLY_VERSION from torch_pin.py. - - Returns: - NIGHTLY_VERSION string - """ - with open("torch_pin.py", "r") as f: - content = f.read() - - match = re.search(r'NIGHTLY_VERSION\s*=\s*["\']([^"\']+)["\']', content) - if not match: - raise ValueError("Could not find NIGHTLY_VERSION in torch_pin.py") - - return match.group(1) - - def get_commit_hash_for_nightly(date_str): """ Fetch commit hash from PyTorch nightly branch for a given date. @@ -91,17 +77,16 @@ def extract_hash_from_title(title): return match.group(1) -def update_pytorch_pin(commit_hash): +def update_pytorch_pin(ref): """ - Update .ci/docker/ci_commit_pins/pytorch.txt with the new commit hash. + Update .ci/docker/ci_commit_pins/pytorch.txt with the new ref. Args: - commit_hash: Commit hash to write + ref: Either a commit SHA (nightly) or a branch name (test/release). """ - pin_file = ".ci/docker/ci_commit_pins/pytorch.txt" - with open(pin_file, "w") as f: - f.write(f"{commit_hash}\n") - print(f"Updated {pin_file} with commit hash: {commit_hash}") + pin_file = _REPO_ROOT / ".ci/docker/ci_commit_pins/pytorch.txt" + pin_file.write_text(f"{ref}\n") + print(f"Updated {pin_file} with ref: {ref}") def should_skip_file(filename): @@ -118,18 +103,20 @@ def should_skip_file(filename): return filename in skip_files -def fetch_file_content(commit_hash, file_path): +def fetch_file_content(ref, file_path): """ Fetch file content from GitHub API. Args: - commit_hash: Commit hash to fetch from + ref: Commit SHA or branch name to fetch from file_path: File path in the repository Returns: File content as bytes """ - api_url = f"https://api.github.com/repos/pytorch/pytorch/contents/{file_path}?ref={commit_hash}" + api_url = ( + f"https://api.github.com/repos/pytorch/pytorch/contents/{file_path}?ref={ref}" + ) req = urllib.request.Request(api_url) req.add_header("Accept", "application/vnd.github.v3+json") @@ -146,7 +133,7 @@ def fetch_file_content(commit_hash, file_path): raise -def sync_directory(et_dir, pt_path, commit_hash): +def sync_directory(et_dir, pt_path, ref): """ Sync files from PyTorch to ExecuTorch using GitHub API. Only syncs files that already exist in ExecuTorch - does not add new files. @@ -154,7 +141,7 @@ def sync_directory(et_dir, pt_path, commit_hash): Args: et_dir: ExecuTorch directory path pt_path: PyTorch directory path in the repository (e.g., "c10") - commit_hash: Commit hash to fetch from + ref: Commit SHA or branch name to fetch from Returns: Number of files grafted @@ -181,12 +168,12 @@ def sync_directory(et_dir, pt_path, commit_hash): # Fetch content from PyTorch and compare try: - pt_content = fetch_file_content(commit_hash, pt_file_path) + pt_content = fetch_file_content(ref, pt_file_path) et_content = et_file.read_bytes() if pt_content != et_content: print(f"āš ļø Difference detected in {rel_path}") - print(f"šŸ“‹ Grafting from PyTorch commit {commit_hash}...") + print(f"šŸ“‹ Grafting from PyTorch ref {ref}...") et_file.write_bytes(pt_content) print(f"āœ… Grafted {et_file}") @@ -201,37 +188,34 @@ def sync_directory(et_dir, pt_path, commit_hash): return files_grafted -def sync_c10_directories(commit_hash): +def sync_c10_directories(ref): """ Sync c10 and torch/headeronly directories from PyTorch to ExecuTorch using GitHub API. Args: - commit_hash: PyTorch commit hash to sync from + ref: PyTorch commit SHA or branch name to sync from Returns: Total number of files grafted """ print("\nšŸ”„ Syncing c10 directories from PyTorch via GitHub API...") - # Get repository root - repo_root = Path.cwd() - # Define directory pairs to sync (from check_c10_sync.sh) # Format: (executorch_dir, pytorch_path_in_repo) dir_pairs = [ ( - repo_root / "runtime/core/portable_type/c10/c10", + _REPO_ROOT / "runtime/core/portable_type/c10/c10", "c10", ), ( - repo_root / "runtime/core/portable_type/c10/torch/headeronly", + _REPO_ROOT / "runtime/core/portable_type/c10/torch/headeronly", "torch/headeronly", ), ] total_grafted = 0 for et_dir, pt_path in dir_pairs: - files_grafted = sync_directory(et_dir, pt_path, commit_hash) + files_grafted = sync_directory(et_dir, pt_path, ref) total_grafted += files_grafted if total_grafted > 0: @@ -244,27 +228,23 @@ def sync_c10_directories(commit_hash): def main(): try: - # Read NIGHTLY_VERSION from torch_pin.py - nightly_version = get_torch_nightly_version() - print(f"Found NIGHTLY_VERSION: {nightly_version}") - - # Parse to date string - date_str = parse_nightly_version(nightly_version) - print(f"Parsed date: {date_str}") - - # Fetch commit hash from PyTorch nightly branch - commit_hash = get_commit_hash_for_nightly(date_str) - print(f"Found commit hash: {commit_hash}") + print(f"CHANNEL: {CHANNEL}") + if CHANNEL == "nightly": + print(f"Found NIGHTLY_VERSION: {NIGHTLY_VERSION}") + date_str = parse_nightly_version(NIGHTLY_VERSION) + print(f"Parsed date: {date_str}") + pin_ref = get_commit_hash_for_nightly(date_str) + else: + pin_ref = torch_branch() + print(f"Pin ref: {pin_ref}") # Update the pin file - update_pytorch_pin(commit_hash) + update_pytorch_pin(pin_ref) # Sync c10 directories from PyTorch - sync_c10_directories(commit_hash) + sync_c10_directories(pin_ref) - print( - "\nāœ… Successfully updated PyTorch commit pin and synced c10 directories!" - ) + print("\nāœ… Successfully updated PyTorch pin and synced c10 directories!") except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/.github/workflows/weekly-pytorch-pin-bump.yml b/.github/workflows/weekly-pytorch-pin-bump.yml index 30579c77701..33cc08b4c0c 100644 --- a/.github/workflows/weekly-pytorch-pin-bump.yml +++ b/.github/workflows/weekly-pytorch-pin-bump.yml @@ -22,29 +22,51 @@ jobs: with: python-version: '3.11' + - name: Check torch_pin channel + id: channel + run: | + CHANNEL=$(python -c "from torch_pin import CHANNEL; print(CHANNEL)") + echo "channel=${CHANNEL}" >> "$GITHUB_OUTPUT" + if [ "${CHANNEL}" != "nightly" ]; then + echo "torch_pin.py CHANNEL is '${CHANNEL}'; weekly nightly bump only runs when CHANNEL == 'nightly'." + fi + - name: Determine nightly version + if: steps.channel.outputs.channel == 'nightly' id: nightly run: | NIGHTLY_DATE=$(date -u -d 'yesterday' '+%Y%m%d') NIGHTLY_VERSION="dev${NIGHTLY_DATE}" echo "version=${NIGHTLY_VERSION}" >> "$GITHUB_OUTPUT" - - name: Read current TORCH_VERSION - id: torch - run: | - TORCH_VERSION=$(python -c "exec(open('torch_pin.py').read()); print(TORCH_VERSION)") - echo "version=${TORCH_VERSION}" >> "$GITHUB_OUTPUT" - - name: Update torch_pin.py with new NIGHTLY_VERSION + if: steps.channel.outputs.channel == 'nightly' + env: + NIGHTLY_VERSION: ${{ steps.nightly.outputs.version }} run: | - printf 'TORCH_VERSION = "%s"\nNIGHTLY_VERSION = "%s"\n' \ - "${{ steps.torch.outputs.version }}" \ - "${{ steps.nightly.outputs.version }}" > torch_pin.py + python - <<'PY' + import os + import pathlib + import re + + path = pathlib.Path("torch_pin.py") + path.write_text( + re.sub( + r'^NIGHTLY_VERSION\s*=\s*".*"$', + f'NIGHTLY_VERSION = "{os.environ["NIGHTLY_VERSION"]}"', + path.read_text(), + count=1, + flags=re.MULTILINE, + ) + ) + PY - name: Run pin bump script + if: steps.channel.outputs.channel == 'nightly' run: python .github/scripts/update_pytorch_pin.py - name: Create branch and PR + if: steps.channel.outputs.channel == 'nightly' env: GH_TOKEN: ${{ secrets.UPDATEBOT_TOKEN }} run: | diff --git a/install_executorch.py b/install_executorch.py index a1358418a14..dfe146d887b 100644 --- a/install_executorch.py +++ b/install_executorch.py @@ -174,7 +174,9 @@ def _parse_args() -> argparse.Namespace: parser.add_argument( "--use-pt-pinned-commit", action="store_true", - help="build from the pinned PyTorch commit instead of nightly", + help="legacy option: install plain `torch` (whatever pip resolves by " + "default). Without this flag, install the specific pinned version from " + "the channel selected in torch_pin.py (nightly / test / release).", ) parser.add_argument( "--editable", @@ -217,13 +219,13 @@ def main(args): return check_and_update_submodules() - # This option is used in CI to make sure that PyTorch build from the pinned commit - # is used instead of nightly. CI jobs wouldn't be able to catch regression from the - # latest PT commit otherwise - use_pytorch_nightly = not args.use_pt_pinned_commit + # By default install the specific pinned version from the channel selected + # in torch_pin.py. With --use-pt-pinned-commit, install plain `torch` using + # pip's default resolution. + install_pinned_version = not args.use_pt_pinned_commit # Step 1: Install core dependencies first - install_requirements(use_pytorch_nightly) + install_requirements(install_pinned_version) # Step 2: Install build dependencies for optional dependencies # They need to be installed before optional dependencies due to --no-build-isolation @@ -261,7 +263,7 @@ def main(args): # Step 4: Extra (optional) packages that is only useful for running examples. if not args.minimal: - install_optional_example_requirements(use_pytorch_nightly) + install_optional_example_requirements(install_pinned_version) if __name__ == "__main__": diff --git a/install_requirements.py b/install_requirements.py index 53204ffd3ee..2d97b1a31d7 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -11,29 +11,29 @@ import sys from install_utils import determine_torch_url, is_intel_mac_os, python_is_compatible - -# The pip repository that hosts nightly torch packages. -# This will be dynamically set based on CUDA availability and CUDA backend enabled/disabled. -TORCH_URL_BASE = "https://download.pytorch.org/whl/test" +from torch_pin import ( + pip_cache_args, + torch_index_url_base, + torch_spec, + torchaudio_spec, + torchvision_spec, +) # Since ExecuTorch often uses main-branch features of pytorch, only the nightly # pip versions will have the required features. # -# NOTE: If a newly-fetched version of the executorch repo changes the value of -# NIGHTLY_VERSION, you should re-run this script to install the necessary -# package versions. -# -# NOTE: If you're changing, make the corresponding change in .ci/docker/ci_commit_pins/pytorch.txt -# by picking the hash from the same date in -# https://hud.pytorch.org/hud/pytorch/pytorch/nightly/ @lint-ignore +# NOTE: If you change torch_pin.py, the pre-commit hook runs +# .github/scripts/update_pytorch_pin.py to refresh +# .ci/docker/ci_commit_pins/pytorch.txt and the c10 grafted headers. +# If you bypass the hook, run that script manually. # # NOTE: If you're changing, make the corresponding supported CUDA versions in # SUPPORTED_CUDA_VERSIONS in install_utils.py if needed. -def install_requirements(use_pytorch_nightly): - # Skip pip install on Intel macOS if using nightly. - if use_pytorch_nightly and is_intel_mac_os(): +def install_requirements(install_pinned_version): + # No prebuilt wheels are available for Intel macOS, regardless of channel. + if install_pinned_version and is_intel_mac_os(): print( "ERROR: Prebuilt PyTorch wheels are no longer available for Intel-based macOS.\n" "Please build from source by following https://docs.pytorch.org/executorch/main/using-executorch-building-from-source.html", @@ -42,14 +42,14 @@ def install_requirements(use_pytorch_nightly): sys.exit(1) # Determine the appropriate PyTorch URL based on CUDA delegate status - torch_url = determine_torch_url(TORCH_URL_BASE) + torch_url = determine_torch_url(torch_index_url_base()) # pip packages needed by exir. TORCH_PACKAGE = [ - # Setting use_pytorch_nightly to false to test the pinned PyTorch commit. Note - # that we don't need to set any version number there because they have already - # been installed on CI before this step, so pip won't reinstall them - ("torch==2.12.0" if use_pytorch_nightly else "torch"), + # Default: install the specific pinned version from the channel selected + # in torch_pin.py. With --use-pt-pinned-commit, pass plain "torch" and + # let pip resolve its default. + (torch_spec() if install_pinned_version else "torch"), ] # Install the requirements for core ExecuTorch package. @@ -61,6 +61,7 @@ def install_requirements(use_pytorch_nightly): "-m", "pip", "install", + *pip_cache_args(), "-r", "requirements-dev.txt", *TORCH_PACKAGE, @@ -106,14 +107,14 @@ def install_requirements(use_pytorch_nightly): ) -def install_optional_example_requirements(use_pytorch_nightly): +def install_optional_example_requirements(install_pinned_version): # Determine the appropriate PyTorch URL based on CUDA delegate status - torch_url = determine_torch_url(TORCH_URL_BASE) + torch_url = determine_torch_url(torch_index_url_base()) print("Installing torch domain libraries") DOMAIN_LIBRARIES = [ - ("torchvision==0.27.0" if use_pytorch_nightly else "torchvision"), - ("torchaudio==2.11.0" if use_pytorch_nightly else "torchaudio"), + (torchvision_spec() if install_pinned_version else "torchvision"), + (torchaudio_spec() if install_pinned_version else "torchaudio"), ] # Then install domain libraries subprocess.run( @@ -122,6 +123,7 @@ def install_optional_example_requirements(use_pytorch_nightly): "-m", "pip", "install", + *pip_cache_args(), *DOMAIN_LIBRARIES, "--extra-index-url", torch_url, @@ -152,7 +154,9 @@ def main(args): parser.add_argument( "--use-pt-pinned-commit", action="store_true", - help="build from the pinned PyTorch commit instead of nightly", + help="legacy option: install plain `torch` (whatever pip resolves by " + "default). Without this flag, install the specific pinned version from " + "the channel selected in torch_pin.py (nightly / test / release).", ) parser.add_argument( "--example", @@ -160,10 +164,10 @@ def main(args): help="Also installs required packages for running example scripts.", ) args = parser.parse_args(args) - use_pytorch_nightly = not bool(args.use_pt_pinned_commit) - install_requirements(use_pytorch_nightly) + install_pinned_version = not bool(args.use_pt_pinned_commit) + install_requirements(install_pinned_version) if args.example: - install_optional_example_requirements(use_pytorch_nightly) + install_optional_example_requirements(install_pinned_version) if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index bb3beda32b1..f9d0a67cfc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ dependencies=[ "packaging", "pandas>=2.2.2; python_version >= '3.10'", "parameterized", - "pytorch-tokenizers", + "pytorch-tokenizers>=1.2.0", "pyyaml", "ruamel.yaml", "sympy", @@ -74,6 +74,8 @@ dependencies=[ "scikit-learn==1.7.1", "hydra-core>=1.3.0", "omegaconf>=2.3.0", + "torch>=2.12.0", + "torchao>=0.17.0", ] [project.optional-dependencies] diff --git a/requirements-examples.txt b/requirements-examples.txt index f1579edcb14..9d14d826b51 100644 --- a/requirements-examples.txt +++ b/requirements-examples.txt @@ -4,4 +4,6 @@ datasets == 3.6.0 # 4.0.0 deprecates trust_remote_code and load scripts. For now timm == 1.0.7 torchsr == 1.0.4 torchtune @ git+https://github.com/pytorch/torchtune.git@6f2aa7254458145f99d7004cbd6ebc8e53a06404 +torchaudio >= 2.11.0 +torchvision >= 0.27.0 transformers == 5.0.0rc1 diff --git a/torch_pin.py b/torch_pin.py index 0c5cd50fe6d..0e14e00721c 100644 --- a/torch_pin.py +++ b/torch_pin.py @@ -1,2 +1,74 @@ +# CHANNEL selects the wheel source for torch and its domain libraries. +# "nightly" - dev builds from /whl/nightly. NIGHTLY_VERSION is appended to +# every package spec. +# "test" - release candidates from /whl/test. +# "release" - stable releases from /whl. +# For "test" and "release", NIGHTLY_VERSION is ignored and CI installs the +# published wheels directly. +# +# Example: pinning to a release candidate when nightly is broken: +# 1. Set CHANNEL = "test". +# 2. Set the four version constants to the RC's major.minor.patch +# (look up matching versions on https://download.pytorch.org/whl/test/). +# 3. Re-run install_requirements.sh; commit. The pre-commit hook calls +# .github/scripts/update_pytorch_pin.py, which writes torch_branch() +# (e.g. "release/2.12") into .ci/docker/ci_commit_pins/pytorch.txt and +# re-syncs grafted c10 headers. +CHANNEL = "release" + TORCH_VERSION = "2.12.0" -# NIGHTLY_VERSION = "dev20260318" Temporarily pinning to stable release candidate. Revert https://github.com/pytorch/executorch/pull/18287 +TORCHAUDIO_VERSION = "2.11.0" +TORCHCODEC_VERSION = "0.13.0" +TORCHVISION_VERSION = "0.27.0" + +NIGHTLY_VERSION = "dev20260318" + + +def _spec(name: str, version: str) -> str: + if CHANNEL == "nightly": + return f"{name}=={version}.{NIGHTLY_VERSION}" + return f"{name}=={version}" + + +def torch_spec() -> str: + return _spec("torch", TORCH_VERSION) + + +def torchaudio_spec() -> str: + return _spec("torchaudio", TORCHAUDIO_VERSION) + + +def torchcodec_spec() -> str: + return _spec("torchcodec", TORCHCODEC_VERSION) + + +def torchvision_spec() -> str: + return _spec("torchvision", TORCHVISION_VERSION) + + +def torch_index_url_base() -> str: + if CHANNEL == "release": + return "https://download.pytorch.org/whl" + return f"https://download.pytorch.org/whl/{CHANNEL}" + + +def pip_cache_args() -> list[str]: + if CHANNEL == "test": + return ["--no-cache-dir"] + return [] + + +def _release_branch(version: str) -> str: + return f"release/{version.rsplit('.', 1)[0]}" + + +def torch_branch() -> str: + return _release_branch(TORCH_VERSION) + + +def torchaudio_branch() -> str: + return _release_branch(TORCHAUDIO_VERSION) + + +def torchvision_branch() -> str: + return _release_branch(TORCHVISION_VERSION)