From 805d728194f25f886d39bf6d290f773d69b2f6f4 Mon Sep 17 00:00:00 2001
From: Ashay Rane <ashay@users.noreply.github.com>
Date: Tue, 1 Nov 2022 17:03:58 -0500
Subject: [PATCH] ci: build PyTorch before building Torch-MLIR (#1542)

This patch updates the build_linux_packages.sh script so that when
PyTorch needs to be built from source, it is built _before_ building
LLVM and before building Torch-MLIR.  The rationale behind this change
is that previously, when the PyTorch build was triggered through the
Torch-MLIR build, the PyTorch compilation added more entries to the
ccache artifacts.  However, since we cache the PyTorch _binary_ (i.e.
the WHL file), there is no need to add the PyTorch compilation to the
ccache artifacts.  By removing the PyTorch compilation files, we keep
the ccache artifact size small, thus reducing the number of evictions
when we exceed GitHub's allowed limit.
---
 .../python_deploy/build_linux_packages.sh     | 31 +++++++++++++------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/build_tools/python_deploy/build_linux_packages.sh b/build_tools/python_deploy/build_linux_packages.sh
index c0367fca1..36e1e8761 100755
--- a/build_tools/python_deploy/build_linux_packages.sh
+++ b/build_tools/python_deploy/build_linux_packages.sh
@@ -187,7 +187,17 @@ function run_in_docker() {
 function build_in_tree() {
   local torch_from_bin="$1"
   local python_version="$2"
-  echo ":::: Build in-tree Torch from binary: $torch_from_bin with Python: $python_version"
+
+  if [ "${torch_from_bin}" == "OFF" ]
+  then
+    echo ":::: Installing PyTorch from source"
+    echo ":::: repo: ${TORCH_MLIR_SRC_PYTORCH_REPO}"
+    echo ":::: branch / commit: ${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
+    echo ":::: install without rebuild: ${TM_PYTORCH_INSTALL_WITHOUT_REBUILD}"
+    /main_checkout/torch-mlir/build_tools/build_libtorch.sh
+  fi
+
+  echo ":::: Build in-tree Torch with Python: $python_version"
   cmake -GNinja -B/main_checkout/torch-mlir/build \
       -DCMAKE_BUILD_TYPE=Release \
       -DCMAKE_C_COMPILER=clang \
@@ -205,10 +215,7 @@ function build_in_tree() {
       -DLLVM_TARGETS_TO_BUILD=host \
       -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
       -DTORCH_MLIR_ENABLE_LTC=ON \
-      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
-      -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
-      -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
-      -DTM_PYTORCH_INSTALL_WITHOUT_REBUILD=${TM_PYTORCH_INSTALL_WITHOUT_REBUILD} \
+      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="ON" \
       -DPython3_EXECUTABLE="$(which python3)" \
       /main_checkout/torch-mlir/externals/llvm-project/llvm
   cmake --build /main_checkout/torch-mlir/build
@@ -289,6 +296,15 @@ function build_out_of_tree() {
   local python_version="$2"
   echo ":::: Build out-of-tree Torch from binary: $torch_from_bin with Python: $python_version"
 
+  if [ "${torch_from_bin}" == "OFF" ]
+  then
+    echo ":::: Installing PyTorch from source"
+    echo ":::: repo: ${TORCH_MLIR_SRC_PYTORCH_REPO}"
+    echo ":::: branch / commit: ${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
+    echo ":::: install without rebuild: ${TM_PYTORCH_INSTALL_WITHOUT_REBUILD}"
+    /main_checkout/torch-mlir/build_tools/build_libtorch.sh
+  fi
+
   if [ ! -d "/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" ]
   then
   echo ":::: LLVM / MLIR is not built so building it first.."
@@ -323,10 +339,7 @@ function build_out_of_tree() {
       -DMLIR_DIR="/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" \
       -DMLIR_ENABLE_BINDINGS_PYTHON=OFF \
       -DTORCH_MLIR_ENABLE_LTC=ON \
-      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
-      -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
-      -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
-      -DTM_PYTORCH_INSTALL_WITHOUT_REBUILD=${TM_PYTORCH_INSTALL_WITHOUT_REBUILD} \
+      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="ON" \
       -DPython3_EXECUTABLE="$(which python3)" \
       /main_checkout/torch-mlir
   cmake --build /main_checkout/torch-mlir/build_oot