build: create RollPyTorch to update PyTorch version in Torch-MLIR (#1419)

This patch fetches the most recent nightly (binary) build of PyTorch,
before pinning it in pytorch-requirements.txt, which is referenced in
the top-level requirements.txt file.  This way, end users will continue
to be able to run `pip -r requirements.txt` without worrying whether
doing so will break their Torch-MLIR build.

This patch also fetches the git commit hash that corresponds to the
nightly release, and this hash is passed to the out-of-tree build so
that it can build PyTorch from source.

If we were to sort the torch versions as numbers (in the usual
descending order), then 1.9 appears before 1.13.  To fix this problem,
we use the `--version-sort` flag (along with `--reverse` for specifying
a descending order).  We also filter out lines that don't contain
version numbers by only considering lines that start with a digit.

As a matter of slight clarity, this patch renames the variable
`torch_from_src` to `torch_from_bin`, since that variable is initialized
to `TM_USE_PYTORCH_BINARY`.

Co-authored-by: powderluv <powderluv@users.noreply.github.com>

.github/workflows/RollPyTorch.yml

@@ -0,0 +1,65 @@
+name: Roll PyTorch
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build_linux:
+    name: Manylinux Build
+    runs-on: ubuntu-latest
+    steps:
+    - name: Get torch-mlir
+      uses: actions/checkout@v2
+      with:
+        submodules: 'true'
+    - name: Setup ccache
+      uses: ./.github/actions/setup-build
+      with:
+        cache-suffix: x86_64-out-of-tree-OFF
+    - name: Determine nightly PyTorch version
+      run: |
+        cd ${GITHUB_WORKSPACE}
+        python -m pip install wheel
+        # Fetch the most recent nightly PyTorch release
+        PT_RELEASE=$(python -m pip index versions -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre torch | grep "Available versions" | tr ' ' '\n' | grep "^[0-9]" | sort --version-sort --reverse | head -n1 | tr -d ',' | sed 's/\([^+]*\).*/\1/')
+        printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt
+        # Fetch the whl file associated with the nightly release
+        rm -f torch-"${PT_RELEASE}"*.whl
+        python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre "torch==${PT_RELEASE}"
+        # Read the commit hash from the downloaded whl file without extracting it
+        PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | awk '{ print $3 }' | tr -d "'")
+        echo "${PT_HASH}" > pytorch-version.txt
+        rm torch-"${PT_RELEASE}"*.whl
+        # Write the release and hash to the environment file so that we can
+        # retrieve them when creating a PR
+        echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV}
+        echo "PT_RELEASE=${PT_RELEASE}" >> ${GITHUB_ENV}
+    - name: Build and test
+      run: |
+        cd ${GITHUB_WORKSPACE}
+        TM_PACKAGES="out-of-tree" TM_USE_PYTORCH_BINARY="OFF" \
+        TORCH_MLIR_SRC_PYTORCH_BRANCH="${{ env.PT_HASH }}" \
+        TORCH_MLIR_SRC_PYTORCH_RELEASE="${{ env.PT_RELEASE }}" \
+        ./build_tools/python_deploy/build_linux_packages.sh
+    - name: Push changes to new branch
+      run: |
+        BRANCH="merge/pytorch-update-${{ env.PT_RELEASE }}"
+        TITLE="update PyTorch version to ${{ env.PT_RELEASE }}"
+        echo "BRANCH=${BRANCH}" >> ${GITHUB_ENV}
+        echo "TITLE=${TITLE}" >> ${GITHUB_ENV}
+        cd ${GITHUB_WORKSPACE}
+        git config user.email "torch-mlir@users.noreply.github.com"
+        git config user.name "Roll PyTorch Action"
+        git checkout -b "${BRANCH}"
+        git add pytorch-version.txt pytorch-requirements.txt
+        git commit -m "${TITLE}"
+        git push --set-upstream origin "${BRANCH}"
+    - name: Create PR to push new PyTorch version
+      run: |
+        URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+        BODY="PyTorch commit hash: \`${{ env.PT_HASH }}\` -- CI link: ${URL}"
+        cd ${GITHUB_WORKSPACE}
+        gh pr create -H "${{ env.BRANCH }}" -B main --title "${{ env.TITLE }}" \
+          --body "${BODY}" --reviewer powderluv
+      env:
+        GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}

build_tools/build_libtorch.sh

@@ -45,10 +45,21 @@ install_requirements() {
 
 checkout_pytorch() {
   if [[ ! -d "$PYTORCH_ROOT" ]]; then
-    git clone --depth 1 --single-branch --branch "${TORCH_MLIR_SRC_PYTORCH_BRANCH}" https://github.com/"$TORCH_MLIR_SRC_PYTORCH_REPO" "$PYTORCH_ROOT"
+    # ${TORCH_MLIR_SRC_PYTORCH_BRANCH} could be a branch name or a commit hash.
-  fi
+    # Althought `git clone` can accept a branch name, the same command does not
-  cd "$PYTORCH_ROOT"
+    # accept a commit hash, so we instead use `git fetch`.  The alternative is
+    # to clone the entire repository and then `git checkout` the requested
+    # branch or commit hash, but that's too expensive.
+    mkdir "${PYTORCH_ROOT}"
+    cd "${PYTORCH_ROOT}"
+    git init
+    git remote add origin "https://github.com/${TORCH_MLIR_SRC_PYTORCH_REPO}"
+    git fetch --depth=1 origin "${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
+    git reset --hard FETCH_HEAD
+  else
+    cd "${PYTORCH_ROOT}"
     git reset --hard HEAD
+  fi
   git clean -df
   git submodule update --init --depth 1 --recursive
 }

build_tools/python_deploy/build_linux_packages.sh

@@ -55,9 +55,14 @@ TM_USE_PYTORCH_BINARY="${TM_USE_PYTORCH_BINARY:-ON}"
 TM_SKIP_TESTS="${TM_SKIP_TESTS:-OFF}"
 
 PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE"
-export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
+TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
 echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}"
 
+TORCH_MLIR_SRC_PYTORCH_REPO="${TORCH_MLIR_SRC_PYTORCH_REPO:-pytorch/pytorch}"
+echo "Setting torch-mlir PyTorch Repo for source builds to: ${TORCH_MLIR_SRC_PYTORCH_REPO}"
+TORCH_MLIR_SRC_PYTORCH_BRANCH="${TORCH_MLIR_SRC_PYTORCH_BRANCH:-master}"
+echo "Setting torch-mlir PyTorch version for source builds to: ${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
+
 function run_on_host() {
   echo "Running on host for $1:$@"
   echo "Outputting to ${TM_OUTPUT_DIR}"
@@ -105,6 +110,8 @@ function run_on_host() {
     -e "TM_PACKAGES=${package}" \
     -e "TM_SKIP_TESTS=${TM_SKIP_TESTS}" \
     -e "TM_USE_PYTORCH_BINARY=${TM_USE_PYTORCH_BINARY}" \
+    -e "TORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO}" \
+    -e "TORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH}" \
     -e "CCACHE_DIR=/main_checkout/torch-mlir/.ccache" \
     "${TM_CURRENT_DOCKER_IMAGE}" \
     /bin/bash /main_checkout/torch-mlir/build_tools/python_deploy/build_linux_packages.sh
@@ -160,9 +167,9 @@ function run_in_docker() {
 
 
 function build_in_tree() {
-  local torch_from_src="$1"
+  local torch_from_bin="$1"
   local python_version="$2"
-  echo ":::: Build in-tree Torch from source: $torch_from_src with Python: $python_version"
+  echo ":::: Build in-tree Torch from binary: $torch_from_bin with Python: $python_version"
   cmake -GNinja -B/main_checkout/torch-mlir/build \
       -DCMAKE_BUILD_TYPE=Release \
       -DCMAKE_C_COMPILER=clang \
@@ -178,7 +185,9 @@ function build_in_tree() {
       -DLLVM_TARGETS_TO_BUILD=host \
       -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
       -DTORCH_MLIR_ENABLE_LTC=ON \
-      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_src" \
+      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
+      -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
+      -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
       -DPython3_EXECUTABLE="$(which python3)" \
       /main_checkout/torch-mlir/externals/llvm-project/llvm
   cmake --build /main_checkout/torch-mlir/build
@@ -249,15 +258,15 @@ function setup_venv() {
   source /main_checkout/torch-mlir/docker_venv/bin/activate
 
   echo ":::: pip installing dependencies"
-  python3 -m pip install -r /main_checkout/torch-mlir/externals/llvm-project/mlir/python/requirements.txt
+  python3 -m pip install --upgrade -r /main_checkout/torch-mlir/externals/llvm-project/mlir/python/requirements.txt
-  python3 -m pip install -r /main_checkout/torch-mlir/requirements.txt
+  python3 -m pip install --upgrade -r /main_checkout/torch-mlir/requirements.txt
 
 }
 
 function build_out_of_tree() {
-  local torch_from_src="$1"
+  local torch_from_bin="$1"
   local python_version="$2"
-  echo ":::: Build out-of-tree Torch from source: $torch_from_src with Python: $python_version"
+  echo ":::: Build out-of-tree Torch from binary: $torch_from_bin with Python: $python_version"
 
   if [ ! -d "/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" ]
   then
@@ -289,7 +298,9 @@ function build_out_of_tree() {
       -DMLIR_DIR="/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" \
       -DMLIR_ENABLE_BINDINGS_PYTHON=OFF \
       -DTORCH_MLIR_ENABLE_LTC=ON \
-      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_src" \
+      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
+      -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
+      -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
       -DPython3_EXECUTABLE="$(which python3)" \
       /main_checkout/torch-mlir
   cmake --build /main_checkout/torch-mlir/build_oot
@@ -310,7 +321,7 @@ function clean_build() {
 }
 
 function build_torch_mlir() {
-  python -m pip install -r /main_checkout/torch-mlir/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
+  python -m pip install --upgrade -r /main_checkout/torch-mlir/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
   CMAKE_GENERATOR=Ninja \
   TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \
   python -m pip wheel -v -w /wheelhouse /main_checkout/torch-mlir/ \

pytorch-requirements.txt

@@ -0,0 +1,3 @@
+-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+--pre
+torch==1.13.0.dev20220927

pytorch-version.txt

@@ -0,0 +1 @@
+04bb9533d516919190b80a0682e93dd44da9ef6d

requirements.txt

@@ -1,6 +1,4 @@
--f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
+-r pytorch-requirements.txt
---pre
-torch
 
 numpy
 torchvision