build: create RollPyTorch to update PyTorch version in Torch-MLIR (#1419)

This patch fetches the most recent nightly (binary) build of PyTorch,
before pinning it in pytorch-requirements.txt, which is referenced in
the top-level requirements.txt file.  This way, end users will continue
to be able to run `pip -r requirements.txt` without worrying whether
doing so will break their Torch-MLIR build.

This patch also fetches the git commit hash that corresponds to the
nightly release, and this hash is passed to the out-of-tree build so
that it can build PyTorch from source.

If we were to sort the torch versions as numbers (in the usual
descending order), then 1.9 appears before 1.13.  To fix this problem,
we use the `--version-sort` flag (along with `--reverse` for specifying
a descending order).  We also filter out lines that don't contain
version numbers by only considering lines that start with a digit.

As a matter of slight clarity, this patch renames the variable
`torch_from_src` to `torch_from_bin`, since that variable is initialized
to `TM_USE_PYTORCH_BINARY`.

Co-authored-by: powderluv <powderluv@users.noreply.github.com>
pull/1433/head snapshot-20220929.611
Ashay Rane 2022-09-28 15:38:30 -05:00 committed by GitHub
parent 8d3ca887df
commit 53e76b8ab6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 105 additions and 16 deletions

View File

@ -0,0 +1,65 @@
name: Roll PyTorch
on:
workflow_dispatch:
jobs:
build_linux:
name: Manylinux Build
runs-on: ubuntu-latest
steps:
- name: Get torch-mlir
uses: actions/checkout@v2
with:
submodules: 'true'
- name: Setup ccache
uses: ./.github/actions/setup-build
with:
cache-suffix: x86_64-out-of-tree-OFF
- name: Determine nightly PyTorch version
run: |
cd ${GITHUB_WORKSPACE}
python -m pip install wheel
# Fetch the most recent nightly PyTorch release
PT_RELEASE=$(python -m pip index versions -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre torch | grep "Available versions" | tr ' ' '\n' | grep "^[0-9]" | sort --version-sort --reverse | head -n1 | tr -d ',' | sed 's/\([^+]*\).*/\1/')
printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt
# Fetch the whl file associated with the nightly release
rm -f torch-"${PT_RELEASE}"*.whl
python -m pip download -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre "torch==${PT_RELEASE}"
# Read the commit hash from the downloaded whl file without extracting it
PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | awk '{ print $3 }' | tr -d "'")
echo "${PT_HASH}" > pytorch-version.txt
rm torch-"${PT_RELEASE}"*.whl
# Write the release and hash to the environment file so that we can
# retrieve them when creating a PR
echo "PT_HASH=${PT_HASH}" >> ${GITHUB_ENV}
echo "PT_RELEASE=${PT_RELEASE}" >> ${GITHUB_ENV}
- name: Build and test
run: |
cd ${GITHUB_WORKSPACE}
TM_PACKAGES="out-of-tree" TM_USE_PYTORCH_BINARY="OFF" \
TORCH_MLIR_SRC_PYTORCH_BRANCH="${{ env.PT_HASH }}" \
TORCH_MLIR_SRC_PYTORCH_RELEASE="${{ env.PT_RELEASE }}" \
./build_tools/python_deploy/build_linux_packages.sh
- name: Push changes to new branch
run: |
BRANCH="merge/pytorch-update-${{ env.PT_RELEASE }}"
TITLE="update PyTorch version to ${{ env.PT_RELEASE }}"
echo "BRANCH=${BRANCH}" >> ${GITHUB_ENV}
echo "TITLE=${TITLE}" >> ${GITHUB_ENV}
cd ${GITHUB_WORKSPACE}
git config user.email "torch-mlir@users.noreply.github.com"
git config user.name "Roll PyTorch Action"
git checkout -b "${BRANCH}"
git add pytorch-version.txt pytorch-requirements.txt
git commit -m "${TITLE}"
git push --set-upstream origin "${BRANCH}"
- name: Create PR to push new PyTorch version
run: |
URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
BODY="PyTorch commit hash: \`${{ env.PT_HASH }}\` -- CI link: ${URL}"
cd ${GITHUB_WORKSPACE}
gh pr create -H "${{ env.BRANCH }}" -B main --title "${{ env.TITLE }}" \
--body "${BODY}" --reviewer powderluv
env:
GITHUB_TOKEN: ${{ secrets.WORKFLOW_INVOCATION_TOKEN }}

View File

@ -45,10 +45,21 @@ install_requirements() {
checkout_pytorch() {
if [[ ! -d "$PYTORCH_ROOT" ]]; then
git clone --depth 1 --single-branch --branch "${TORCH_MLIR_SRC_PYTORCH_BRANCH}" https://github.com/"$TORCH_MLIR_SRC_PYTORCH_REPO" "$PYTORCH_ROOT"
# ${TORCH_MLIR_SRC_PYTORCH_BRANCH} could be a branch name or a commit hash.
# Althought `git clone` can accept a branch name, the same command does not
# accept a commit hash, so we instead use `git fetch`. The alternative is
# to clone the entire repository and then `git checkout` the requested
# branch or commit hash, but that's too expensive.
mkdir "${PYTORCH_ROOT}"
cd "${PYTORCH_ROOT}"
git init
git remote add origin "https://github.com/${TORCH_MLIR_SRC_PYTORCH_REPO}"
git fetch --depth=1 origin "${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
git reset --hard FETCH_HEAD
else
cd "${PYTORCH_ROOT}"
git reset --hard HEAD
fi
cd "$PYTORCH_ROOT"
git reset --hard HEAD
git clean -df
git submodule update --init --depth 1 --recursive
}

View File

@ -55,9 +55,14 @@ TM_USE_PYTORCH_BINARY="${TM_USE_PYTORCH_BINARY:-ON}"
TM_SKIP_TESTS="${TM_SKIP_TESTS:-OFF}"
PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE"
export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}"
TORCH_MLIR_SRC_PYTORCH_REPO="${TORCH_MLIR_SRC_PYTORCH_REPO:-pytorch/pytorch}"
echo "Setting torch-mlir PyTorch Repo for source builds to: ${TORCH_MLIR_SRC_PYTORCH_REPO}"
TORCH_MLIR_SRC_PYTORCH_BRANCH="${TORCH_MLIR_SRC_PYTORCH_BRANCH:-master}"
echo "Setting torch-mlir PyTorch version for source builds to: ${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
function run_on_host() {
echo "Running on host for $1:$@"
echo "Outputting to ${TM_OUTPUT_DIR}"
@ -105,6 +110,8 @@ function run_on_host() {
-e "TM_PACKAGES=${package}" \
-e "TM_SKIP_TESTS=${TM_SKIP_TESTS}" \
-e "TM_USE_PYTORCH_BINARY=${TM_USE_PYTORCH_BINARY}" \
-e "TORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO}" \
-e "TORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH}" \
-e "CCACHE_DIR=/main_checkout/torch-mlir/.ccache" \
"${TM_CURRENT_DOCKER_IMAGE}" \
/bin/bash /main_checkout/torch-mlir/build_tools/python_deploy/build_linux_packages.sh
@ -160,9 +167,9 @@ function run_in_docker() {
function build_in_tree() {
local torch_from_src="$1"
local torch_from_bin="$1"
local python_version="$2"
echo ":::: Build in-tree Torch from source: $torch_from_src with Python: $python_version"
echo ":::: Build in-tree Torch from binary: $torch_from_bin with Python: $python_version"
cmake -GNinja -B/main_checkout/torch-mlir/build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER=clang \
@ -178,7 +185,9 @@ function build_in_tree() {
-DLLVM_TARGETS_TO_BUILD=host \
-DMLIR_ENABLE_BINDINGS_PYTHON=ON \
-DTORCH_MLIR_ENABLE_LTC=ON \
-DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_src" \
-DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
-DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
-DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
-DPython3_EXECUTABLE="$(which python3)" \
/main_checkout/torch-mlir/externals/llvm-project/llvm
cmake --build /main_checkout/torch-mlir/build
@ -249,15 +258,15 @@ function setup_venv() {
source /main_checkout/torch-mlir/docker_venv/bin/activate
echo ":::: pip installing dependencies"
python3 -m pip install -r /main_checkout/torch-mlir/externals/llvm-project/mlir/python/requirements.txt
python3 -m pip install -r /main_checkout/torch-mlir/requirements.txt
python3 -m pip install --upgrade -r /main_checkout/torch-mlir/externals/llvm-project/mlir/python/requirements.txt
python3 -m pip install --upgrade -r /main_checkout/torch-mlir/requirements.txt
}
function build_out_of_tree() {
local torch_from_src="$1"
local torch_from_bin="$1"
local python_version="$2"
echo ":::: Build out-of-tree Torch from source: $torch_from_src with Python: $python_version"
echo ":::: Build out-of-tree Torch from binary: $torch_from_bin with Python: $python_version"
if [ ! -d "/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" ]
then
@ -289,7 +298,9 @@ function build_out_of_tree() {
-DMLIR_DIR="/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" \
-DMLIR_ENABLE_BINDINGS_PYTHON=OFF \
-DTORCH_MLIR_ENABLE_LTC=ON \
-DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_src" \
-DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
-DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
-DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
-DPython3_EXECUTABLE="$(which python3)" \
/main_checkout/torch-mlir
cmake --build /main_checkout/torch-mlir/build_oot
@ -310,7 +321,7 @@ function clean_build() {
}
function build_torch_mlir() {
python -m pip install -r /main_checkout/torch-mlir/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
python -m pip install --upgrade -r /main_checkout/torch-mlir/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
CMAKE_GENERATOR=Ninja \
TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \
python -m pip wheel -v -w /wheelhouse /main_checkout/torch-mlir/ \

View File

@ -0,0 +1,3 @@
-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
--pre
torch==1.13.0.dev20220927

View File

@ -0,0 +1 @@
04bb9533d516919190b80a0682e93dd44da9ef6d

View File

@ -1,6 +1,4 @@
-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
--pre
torch
-r pytorch-requirements.txt
numpy
torchvision