From 67ab708b636d300db003fbec902411315e222bfb Mon Sep 17 00:00:00 2001 From: Ashay Rane Date: Mon, 13 Feb 2023 21:22:09 -0600 Subject: [PATCH] python: separate build- and test-related pip dependencies (#1874) We want to ensure that pip packages required for building torch-mlir should be included in the dependencies of torch-mlir, but we don't want the pip packages required for _testing_ of torch-mlir to be included among the dependencies. To be able to specify and install one set of dependencies and not the other, this patch separates the pip packages into two files: build-requirements.txt and test-requirements.txt. This patch also updates references to the requirements.txt file so that CI builds that run end-to-end tests install test-related pip dependencies while everything else (including WHL builds) sticks to just the build-related pip dependencies. Despite this change, this patch should not affect a torch-mlir developer's workflow. More precisely, since this patch makes the top-level requirements.txt file refer to both build-requirements.txt and test-requirements.txt files, a torch-mlir developer should be able to continue referring to the requirements.txt file without any impact. --- .github/actions/setup-build/action.yml | 3 ++- .github/workflows/RollPyTorch.yml | 5 +++-- build_tools/python_deploy/build_macos_packages.sh | 6 ++++-- build_tools/python_deploy/build_windows.ps1 | 3 ++- pytorch-requirements.txt | 1 - requirements.txt | 8 ++------ test-requirements.txt | 5 +++++ torchvision-requirements.txt | 3 +++ 8 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 test-requirements.txt create mode 100644 torchvision-requirements.txt diff --git a/.github/actions/setup-build/action.yml b/.github/actions/setup-build/action.yml index f53c6933b..85c3f7516 100644 --- a/.github/actions/setup-build/action.yml +++ b/.github/actions/setup-build/action.yml @@ -26,7 +26,8 @@ runs: - name: Install PyTorch nightly depends run: | - python -m pip install -r requirements.txt + python -m pip install -r pytorch-requirements.txt + python -m pip install -r build-requirements.txt shell: bash - name: Install prerequisites (Linux) diff --git a/.github/workflows/RollPyTorch.yml b/.github/workflows/RollPyTorch.yml index 24e64922f..cc8a8ea09 100644 --- a/.github/workflows/RollPyTorch.yml +++ b/.github/workflows/RollPyTorch.yml @@ -44,7 +44,8 @@ jobs: # Read the version from the downloaded whl file without extracting it PT_RELEASE=$(unzip -p torch-*.whl 'torch-*/METADATA' | grep "^Version:" | awk '{ print $2 }' | sed 's/\([^+]*\).*/\1/') echo "Found torch release ${PT_RELEASE}" - printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\ntorchvision==%s\n" "${PT_RELEASE}" "${VISION_RELEASE}" > pytorch-requirements.txt + printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorch==%s\n" "${PT_RELEASE}" > pytorch-requirements.txt + printf -- "-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html\n--pre\ntorchvision==%s\n" "${VISION_RELEASE}" > torchvision-requirements.txt # Read the commit hash from the downloaded whl file without extracting it PT_HASH=$(unzip -p torch-"${PT_RELEASE}"*.whl torch/version.py | grep git_version | awk '{ print $3 }' | tr -d "'") @@ -96,7 +97,7 @@ jobs: git fetch --recurse-submodules=no git checkout main git pull origin main - git add pytorch-hash.txt pytorch-requirements.txt lib/Dialect/Torch/Transforms/AbstractInterpLibrary.cpp include/torch-mlir/Dialect/Torch/IR/GeneratedTorchOps.td + git add pytorch-hash.txt pytorch-requirements.txt torchvision-requirements.txt lib/Dialect/Torch/Transforms/AbstractInterpLibrary.cpp include/torch-mlir/Dialect/Torch/IR/GeneratedTorchOps.td git diff --cached --exit-code || (git commit -m "update PyTorch version to ${{ env.PT_RELEASE }}" && git push --set-upstream origin main) - name: Update PyTorch Build Cache (if running on main branch) diff --git a/build_tools/python_deploy/build_macos_packages.sh b/build_tools/python_deploy/build_macos_packages.sh index 9a859527e..083d575ce 100755 --- a/build_tools/python_deploy/build_macos_packages.sh +++ b/build_tools/python_deploy/build_macos_packages.sh @@ -77,7 +77,8 @@ function build_torch_mlir() { python"${python_version}" -m venv "$output_dir"/build_venv source "$output_dir"/build_venv/bin/activate python"${python_version}" -m pip install -U pip - python"${python_version}" -m pip install -r "$repo_root"/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu + python"${python_version}" -m pip install -r "$repo_root"/pytorch-requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu + python"${python_version}" -m pip install -r "$repo_root"/build-requirements.txt CMAKE_GENERATOR=Ninja \ TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \ MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ @@ -107,7 +108,8 @@ function run_audit_wheel() { python"${python_version}" -m venv "$output_dir"/test_venv source "$output_dir"/test_venv/bin/activate python"${python_version}" -m pip install -U pip - python"${python_version}" -m pip install -r "$repo_root"/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu + python"${python_version}" -m pip install -r "$repo_root"/pytorch-requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu + python"${python_version}" -m pip install -r "$repo_root"/build-requirements.txt python"${python_version}" -m pip install "$generic_wheel" --extra-index-url https://download.pytorch.org/whl/nightly/cpu DYLD_LIBRARY_PATH="$output_dir"/test_venv/lib/python"${python_version}"/site-packages/torch/lib delocate-wheel -v "$generic_wheel" deactivate diff --git a/build_tools/python_deploy/build_windows.ps1 b/build_tools/python_deploy/build_windows.ps1 index 44c767875..808a16cb1 100644 --- a/build_tools/python_deploy/build_windows.ps1 +++ b/build_tools/python_deploy/build_windows.ps1 @@ -13,7 +13,8 @@ Write-Host "Installing Build Dependencies" python -m venv .\mlir_venv\ .\mlir_venv\Scripts\Activate.PS1 -pip install -r .\requirements.txt +pip install -r .\pytorch-requirements.txt +pip install -r .\build-requirements.txt pip install delvewheel Write-Host "Build Deps installation completed successfully" diff --git a/pytorch-requirements.txt b/pytorch-requirements.txt index 32f2c638d..51a519083 100644 --- a/pytorch-requirements.txt +++ b/pytorch-requirements.txt @@ -1,4 +1,3 @@ -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre torch==2.0.0.dev20230212 -torchvision==0.15.0.dev20230213 diff --git a/requirements.txt b/requirements.txt index f47a87da7..f346b53da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,3 @@ --r build-requirements.txt -r pytorch-requirements.txt - -# Test Requirements -pillow -dill -multiprocess +-r build-requirements.txt +-r test-requirements.txt diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 000000000..e752531e2 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,5 @@ +-r torchvision-requirements.txt + +pillow +dill +multiprocess diff --git a/torchvision-requirements.txt b/torchvision-requirements.txt new file mode 100644 index 000000000..e3fccd906 --- /dev/null +++ b/torchvision-requirements.txt @@ -0,0 +1,3 @@ +-f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html +--pre +torchvision==0.15.0.dev20230213