From 53e76b8ab606b3d6c3b66e2f970e95d116723375 Mon Sep 17 00:00:00 2001 From: Ashay Rane Date: Wed, 28 Sep 2022 15:38:30 -0500 Subject: [PATCH] build: create RollPyTorch to update PyTorch version in Torch-MLIR (#1419) This patch fetches the most recent nightly (binary) build of PyTorch, before pinning it in pytorch-requirements.txt, which is referenced in the top-level requirements.txt file. This way, end users will continue to be able to run `pip -r requirements.txt` without worrying whether doing so will break their Torch-MLIR build. This patch also fetches the git commit hash that corresponds to the nightly release, and this hash is passed to the out-of-tree build so that it can build PyTorch from source. If we were to sort the torch versions as numbers (in the usual descending order), then 1.9 appears before 1.13. To fix this problem, we use the `--version-sort` flag (along with `--reverse` for specifying a descending order). We also filter out lines that don't contain version numbers by only considering lines that start with a digit. As a matter of slight clarity, this patch renames the variable `torch_from_src` to `torch_from_bin`, since that variable is initialized to `TM_USE_PYTORCH_BINARY`. Co-authored-by: powderluv --- .github/workflows/RollPyTorch.yml | 65 +++++++++++++++++++ build_tools/build_libtorch.sh | 17 ++++- .../python_deploy/build_linux_packages.sh | 31 ++++++--- pytorch-requirements.txt | 3 + pytorch-version.txt | 1 + requirements.txt | 4 +- 6 files changed, 105 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/RollPyTorch.yml create mode 100644 pytorch-requirements.txt create mode 100644 pytorch-version.txt diff --git a/.github/workflows/RollPyTorch.yml b/.github/workflows/RollPyTorch.yml new file mode 100644 index 000000000..470191440 --- /dev/null +++ b/.github/workflows/RollPyTorch.yml @@ -0,0 +1,65 @@ +name: Roll PyTorch + +on: + workflow_dispatch: + +jobs: + build_linux: + name: Manylinux Build + runs-on: ubuntu-latest + steps: + - name: Get torch-mlir + uses: actions/checkout@v2 + with: + submodules: 'true' + - name: Setup ccache + uses: ./.github/actions/setup-build + with: + cache-suffix: x86_64-out-of-tree-OFF + - name: Determine nightly PyTorch version + run: | + cd ${GITHUB_WORKSPACE} + python -m pip install wheel + # Fetch the most recent nightly PyTorch release + PT_RELEASE=$(python -m pip index versions -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre torch | grep "Available versions" | tr ' ' '\n' | grep "^[0-9]" | sort --version-sort --reverse | head -n1 | tr -d ',' | sed 's/\([^+]*\).*/\1/') + printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt + # Fetch the whl file associated with the nightly release + rm -f torch-"${PT_RELEASE}"*.whl + python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre "torch==${PT_RELEASE}" + # Read the commit hash from the downloaded whl file without extracting it + PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | awk '{ print $3 }' | tr -d "'") + echo "${PT_HASH}" > pytorch-version.txt + rm torch-"${PT_RELEASE}"*.whl + # Write the release and hash to the environment file so that we can + # retrieve them when creating a PR + echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV} + echo "PT_RELEASE=${PT_RELEASE}" >> ${GITHUB_ENV} + - name: Build and test + run: | + cd ${GITHUB_WORKSPACE} + TM_PACKAGES="out-of-tree" TM_USE_PYTORCH_BINARY="OFF" \ + TORCH_MLIR_SRC_PYTORCH_BRANCH="${{ env.PT_HASH }}" \ + TORCH_MLIR_SRC_PYTORCH_RELEASE="${{ env.PT_RELEASE }}" \ + ./build_tools/python_deploy/build_linux_packages.sh + - name: Push changes to new branch + run: | + BRANCH="merge/pytorch-update-${{ env.PT_RELEASE }}" + TITLE="update PyTorch version to ${{ env.PT_RELEASE }}" + echo "BRANCH=${BRANCH}" >> ${GITHUB_ENV} + echo "TITLE=${TITLE}" >> ${GITHUB_ENV} + cd ${GITHUB_WORKSPACE} + git config user.email "torch-mlir@users.noreply.github.com" + git config user.name "Roll PyTorch Action" + git checkout -b "${BRANCH}" + git add pytorch-version.txt pytorch-requirements.txt + git commit -m "${TITLE}" + git push --set-upstream origin "${BRANCH}" + - name: Create PR to push new PyTorch version + run: | + URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + BODY="PyTorch commit hash: \`${{ env.PT_HASH }}\` -- CI link: ${URL}" + cd ${GITHUB_WORKSPACE} + gh pr create -H "${{ env.BRANCH }}" -B main --title "${{ env.TITLE }}" \ + --body "${BODY}" --reviewer powderluv + env: + GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }} diff --git a/build_tools/build_libtorch.sh b/build_tools/build_libtorch.sh index bc3ec12bb..024ba836b 100755 --- a/build_tools/build_libtorch.sh +++ b/build_tools/build_libtorch.sh @@ -45,10 +45,21 @@ install_requirements() { checkout_pytorch() { if [[ ! -d "$PYTORCH_ROOT" ]]; then - git clone --depth 1 --single-branch --branch "${TORCH_MLIR_SRC_PYTORCH_BRANCH}" https://github.com/"$TORCH_MLIR_SRC_PYTORCH_REPO" "$PYTORCH_ROOT" + # ${TORCH_MLIR_SRC_PYTORCH_BRANCH} could be a branch name or a commit hash. + # Althought `git clone` can accept a branch name, the same command does not + # accept a commit hash, so we instead use `git fetch`. The alternative is + # to clone the entire repository and then `git checkout` the requested + # branch or commit hash, but that's too expensive. + mkdir "${PYTORCH_ROOT}" + cd "${PYTORCH_ROOT}" + git init + git remote add origin "https://github.com/${TORCH_MLIR_SRC_PYTORCH_REPO}" + git fetch --depth=1 origin "${TORCH_MLIR_SRC_PYTORCH_BRANCH}" + git reset --hard FETCH_HEAD + else + cd "${PYTORCH_ROOT}" + git reset --hard HEAD fi - cd "$PYTORCH_ROOT" - git reset --hard HEAD git clean -df git submodule update --init --depth 1 --recursive } diff --git a/build_tools/python_deploy/build_linux_packages.sh b/build_tools/python_deploy/build_linux_packages.sh index 63b19c313..c47fe3d9f 100755 --- a/build_tools/python_deploy/build_linux_packages.sh +++ b/build_tools/python_deploy/build_linux_packages.sh @@ -55,9 +55,14 @@ TM_USE_PYTORCH_BINARY="${TM_USE_PYTORCH_BINARY:-ON}" TM_SKIP_TESTS="${TM_SKIP_TESTS:-OFF}" PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE" -export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}" +TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}" echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" +TORCH_MLIR_SRC_PYTORCH_REPO="${TORCH_MLIR_SRC_PYTORCH_REPO:-pytorch/pytorch}" +echo "Setting torch-mlir PyTorch Repo for source builds to: ${TORCH_MLIR_SRC_PYTORCH_REPO}" +TORCH_MLIR_SRC_PYTORCH_BRANCH="${TORCH_MLIR_SRC_PYTORCH_BRANCH:-master}" +echo "Setting torch-mlir PyTorch version for source builds to: ${TORCH_MLIR_SRC_PYTORCH_BRANCH}" + function run_on_host() { echo "Running on host for $1:$@" echo "Outputting to ${TM_OUTPUT_DIR}" @@ -105,6 +110,8 @@ function run_on_host() { -e "TM_PACKAGES=${package}" \ -e "TM_SKIP_TESTS=${TM_SKIP_TESTS}" \ -e "TM_USE_PYTORCH_BINARY=${TM_USE_PYTORCH_BINARY}" \ + -e "TORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO}" \ + -e "TORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH}" \ -e "CCACHE_DIR=/main_checkout/torch-mlir/.ccache" \ "${TM_CURRENT_DOCKER_IMAGE}" \ /bin/bash /main_checkout/torch-mlir/build_tools/python_deploy/build_linux_packages.sh @@ -160,9 +167,9 @@ function run_in_docker() { function build_in_tree() { - local torch_from_src="$1" + local torch_from_bin="$1" local python_version="$2" - echo ":::: Build in-tree Torch from source: $torch_from_src with Python: $python_version" + echo ":::: Build in-tree Torch from binary: $torch_from_bin with Python: $python_version" cmake -GNinja -B/main_checkout/torch-mlir/build \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_C_COMPILER=clang \ @@ -178,7 +185,9 @@ function build_in_tree() { -DLLVM_TARGETS_TO_BUILD=host \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ -DTORCH_MLIR_ENABLE_LTC=ON \ - -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_src" \ + -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \ + -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \ + -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \ -DPython3_EXECUTABLE="$(which python3)" \ /main_checkout/torch-mlir/externals/llvm-project/llvm cmake --build /main_checkout/torch-mlir/build @@ -249,15 +258,15 @@ function setup_venv() { source /main_checkout/torch-mlir/docker_venv/bin/activate echo ":::: pip installing dependencies" - python3 -m pip install -r /main_checkout/torch-mlir/externals/llvm-project/mlir/python/requirements.txt - python3 -m pip install -r /main_checkout/torch-mlir/requirements.txt + python3 -m pip install --upgrade -r /main_checkout/torch-mlir/externals/llvm-project/mlir/python/requirements.txt + python3 -m pip install --upgrade -r /main_checkout/torch-mlir/requirements.txt } function build_out_of_tree() { - local torch_from_src="$1" + local torch_from_bin="$1" local python_version="$2" - echo ":::: Build out-of-tree Torch from source: $torch_from_src with Python: $python_version" + echo ":::: Build out-of-tree Torch from binary: $torch_from_bin with Python: $python_version" if [ ! -d "/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" ] then @@ -289,7 +298,9 @@ function build_out_of_tree() { -DMLIR_DIR="/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" \ -DMLIR_ENABLE_BINDINGS_PYTHON=OFF \ -DTORCH_MLIR_ENABLE_LTC=ON \ - -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_src" \ + -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \ + -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \ + -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \ -DPython3_EXECUTABLE="$(which python3)" \ /main_checkout/torch-mlir cmake --build /main_checkout/torch-mlir/build_oot @@ -310,7 +321,7 @@ function clean_build() { } function build_torch_mlir() { - python -m pip install -r /main_checkout/torch-mlir/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu + python -m pip install --upgrade -r /main_checkout/torch-mlir/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu CMAKE_GENERATOR=Ninja \ TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \ python -m pip wheel -v -w /wheelhouse /main_checkout/torch-mlir/ \ diff --git a/pytorch-requirements.txt b/pytorch-requirements.txt new file mode 100644 index 000000000..73baf2805 --- /dev/null +++ b/pytorch-requirements.txt @@ -0,0 +1,3 @@ +-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html +--pre +torch==1.13.0.dev20220927 diff --git a/pytorch-version.txt b/pytorch-version.txt new file mode 100644 index 000000000..6b1e6487e --- /dev/null +++ b/pytorch-version.txt @@ -0,0 +1 @@ +04bb9533d516919190b80a0682e93dd44da9ef6d diff --git a/requirements.txt b/requirements.txt index 07a76ab6f..060ef671f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,4 @@ --f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html ---pre -torch +-r pytorch-requirements.txt numpy torchvision