From 8f28d933e120e0c579e182d6a837c4a5be2bb841 Mon Sep 17 00:00:00 2001
From: Ashay Rane <ashay@users.noreply.github.com>
Date: Wed, 23 Aug 2023 11:11:17 -0500
Subject: [PATCH] CI: disable LTC e2e tests in stable PyTorch builds (#2414)

This way, we can keep CI green without being forced to ignore _all_
errors that arise in stable PyTorch builds
---
 .github/workflows/buildAndTest.yml            |  1 -
 .../python_deploy/build_linux_packages.sh     | 34 ++++++++++++++-----
 2 files changed, 25 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/buildAndTest.yml b/.github/workflows/buildAndTest.yml
index 28bf4a4d0..aa63625b9 100644
--- a/.github/workflows/buildAndTest.yml
+++ b/.github/workflows/buildAndTest.yml
@@ -22,7 +22,6 @@ concurrency:
 # macos  - arm64  - llvm in-tree     - pytorch binary - build only    # cross compile, can't test arm64
 jobs:
   build-test:
-    continue-on-error: ${{ matrix.torch-version == 'stable' }}
     strategy:
       fail-fast: true
       matrix:
diff --git a/build_tools/python_deploy/build_linux_packages.sh b/build_tools/python_deploy/build_linux_packages.sh
index f00423ffc..ac0e171a2 100755
--- a/build_tools/python_deploy/build_linux_packages.sh
+++ b/build_tools/python_deploy/build_linux_packages.sh
@@ -177,14 +177,14 @@ function run_in_docker() {
           ;;
         out-of-tree)
           setup_venv "$python_version" "$TM_TORCH_VERSION"
-          build_out_of_tree "$TM_USE_PYTORCH_BINARY" "$python_version"
+          build_out_of_tree "$TM_USE_PYTORCH_BINARY" "$python_version" "$TM_TORCH_VERSION"
           if [ "${TM_SKIP_TESTS}" == "OFF" ]; then
             test_out_of_tree
           fi
           ;;
         in-tree)
           setup_venv "$python_version" "$TM_TORCH_VERSION"
-          build_in_tree "$TM_USE_PYTORCH_BINARY" "$python_version"
+          build_in_tree "$TM_USE_PYTORCH_BINARY" "$python_version" "$TM_TORCH_VERSION"
           if [ "${TM_UPDATE_ODS_AND_ABSTRACT_INTERP_LIB}" == "ON" ]; then
             pushd /main_checkout/torch-mlir
             ./build_tools/update_torch_ods.sh
@@ -208,6 +208,14 @@ function run_in_docker() {
 function build_in_tree() {
   local torch_from_bin="$1"
   local python_version="$2"
+
+  local torch_version="$3"
+  local enable_ltc="ON"
+  if [[ "${torch_version}" == "stable" ]]
+  then
+    enable_ltc="OFF"
+  fi
+
   echo ":::: Build in-tree Torch from binary: $torch_from_bin with Python: $python_version"
   cmake -GNinja -B/main_checkout/torch-mlir/build \
       -DCMAKE_BUILD_TYPE=Release \
@@ -225,7 +233,7 @@ function build_in_tree() {
       -DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR="/main_checkout/torch-mlir/externals/llvm-external-projects/torch-mlir-dialects" \
       -DLLVM_TARGETS_TO_BUILD=host \
       -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
-      -DTORCH_MLIR_ENABLE_LTC=ON \
+      -DTORCH_MLIR_ENABLE_LTC=${enable_ltc} \
       -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
       -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
       -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
@@ -288,12 +296,16 @@ function test_in_tree() {
 
       echo ":::: Run Lazy Tensor Core e2e integration tests"
       python -m e2e_testing.main --config=lazy_tensor_core -v
+
+      echo ":::: Run Linalg e2e integration tests"
+      python -m e2e_testing.main --config=linalg -v
       ;;
     stable)
       echo ":::: Test with stable torch"
 
-      echo ":::: Run Lazy Tensor Core e2e integration tests in experimental mode"
-      python -m e2e_testing.main --config=lazy_tensor_core -v --ignore_failures
+      # Disabled until the next stable PyTorch release (v2.1) is available
+      # echo ":::: Run Lazy Tensor Core e2e integration tests in experimental mode"
+      # python -m e2e_testing.main --config=lazy_tensor_core -v --ignore_failures
       ;;
     *)
       echo "Unrecognized torch version '$torch_version'"
@@ -307,9 +319,6 @@ function test_in_tree() {
   echo ":::: Run TorchDynamo e2e integration tests"
   python -m e2e_testing.main --config=torchdynamo -v
 
-  echo ":::: Run Linalg e2e integration tests"
-  python -m e2e_testing.main --config=linalg -v
-
   echo ":::: Run StableHLO e2e integration tests"
   python -m e2e_testing.main --config=stablehlo -v
 
@@ -353,6 +362,13 @@ function build_out_of_tree() {
   local python_version="$2"
   echo ":::: Build out-of-tree Torch from binary: $torch_from_bin with Python: $python_version"
 
+  local torch_version="$3"
+  local enable_ltc="ON"
+  if [[ "${torch_version}" == "stable" ]]
+  then
+    enable_ltc="OFF"
+  fi
+
   if [ ! -d "/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" ]
   then
   echo ":::: LLVM / MLIR is not built so building it first.."
@@ -386,7 +402,7 @@ function build_out_of_tree() {
       -DLLVM_DIR="/main_checkout/torch-mlir/llvm-build/lib/cmake/llvm/" \
       -DMLIR_DIR="/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" \
       -DMLIR_ENABLE_BINDINGS_PYTHON=OFF \
-      -DTORCH_MLIR_ENABLE_LTC=ON \
+      -DTORCH_MLIR_ENABLE_LTC=${enable_ltc} \
       -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
       -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
       -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \