llvm
/
torch-mlir
mirror of https://github.com/llvm/torch-mlir
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ci: build PyTorch before building Torch-MLIR (#1542) 

This patch updates the build_linux_packages.sh script so that when
PyTorch needs to be built from source, it is built _before_ building
LLVM and before building Torch-MLIR.  The rationale behind this change
is that previously, when the PyTorch build was triggered through the
Torch-MLIR build, the PyTorch compilation added more entries to the
ccache artifacts.  However, since we cache the PyTorch _binary_ (i.e.
the WHL file), there is no need to add the PyTorch compilation to the
ccache artifacts.  By removing the PyTorch compilation files, we keep
the ccache artifact size small, thus reducing the number of evictions
when we exceed GitHub's allowed limit.
pull/1545/head
Ashay Rane 2022-11-01 17:03:58 -05:00 committed by GitHub
parent 0409595ccc
commit 805d728194
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 22 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
build_tools/python_deploy/build_linux_packages.sh
View File

@ -187,7 +187,17 @@ function run_in_docker() {
function build_in_tree() {
local torch_from_bin="$1"
local python_version="$2"
echo ":::: Build in-tree Torch from binary: $torch_from_bin with Python: $python_version"
if [ "${torch_from_bin}" == "OFF" ]
then
echo ":::: Installing PyTorch from source"
echo ":::: repo: ${TORCH_MLIR_SRC_PYTORCH_REPO}"
echo ":::: branch / commit: ${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
echo ":::: install without rebuild: ${TM_PYTORCH_INSTALL_WITHOUT_REBUILD}"
/main_checkout/torch-mlir/build_tools/build_libtorch.sh
fi
echo ":::: Build in-tree Torch with Python: $python_version"
cmake -GNinja -B/main_checkout/torch-mlir/build \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_COMPILER=clang \
@ -205,10 +215,7 @@ function build_in_tree() {
-DLLVM_TARGETS_TO_BUILD=host \
-DMLIR_ENABLE_BINDINGS_PYTHON=ON \
-DTORCH_MLIR_ENABLE_LTC=ON \
-DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
-DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
-DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
-DTM_PYTORCH_INSTALL_WITHOUT_REBUILD=${TM_PYTORCH_INSTALL_WITHOUT_REBUILD} \
-DTORCH_MLIR_USE_INSTALLED_PYTORCH="ON" \
-DPython3_EXECUTABLE="$(which python3)" \
/main_checkout/torch-mlir/externals/llvm-project/llvm
cmake --build /main_checkout/torch-mlir/build
@ -289,6 +296,15 @@ function build_out_of_tree() {
local python_version="$2"
echo ":::: Build out-of-tree Torch from binary: $torch_from_bin with Python: $python_version"
if [ "${torch_from_bin}" == "OFF" ]
then
echo ":::: Installing PyTorch from source"
echo ":::: repo: ${TORCH_MLIR_SRC_PYTORCH_REPO}"
echo ":::: branch / commit: ${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
echo ":::: install without rebuild: ${TM_PYTORCH_INSTALL_WITHOUT_REBUILD}"
/main_checkout/torch-mlir/build_tools/build_libtorch.sh
fi
if [ ! -d "/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" ]
then
echo ":::: LLVM / MLIR is not built so building it first.."
@ -323,10 +339,7 @@ function build_out_of_tree() {
-DMLIR_DIR="/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" \
-DMLIR_ENABLE_BINDINGS_PYTHON=OFF \
-DTORCH_MLIR_ENABLE_LTC=ON \
-DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
-DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
-DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
-DTM_PYTORCH_INSTALL_WITHOUT_REBUILD=${TM_PYTORCH_INSTALL_WITHOUT_REBUILD} \
-DTORCH_MLIR_USE_INSTALLED_PYTORCH="ON" \
-DPython3_EXECUTABLE="$(which python3)" \
/main_checkout/torch-mlir
cmake --build /main_checkout/torch-mlir/build_oot