diff --git a/build_tools/python_deploy/build_linux_packages.sh b/build_tools/python_deploy/build_linux_packages.sh
index c0367fca1..36e1e8761 100755
--- a/build_tools/python_deploy/build_linux_packages.sh
+++ b/build_tools/python_deploy/build_linux_packages.sh
@@ -187,7 +187,17 @@ function run_in_docker() {
 function build_in_tree() {
   local torch_from_bin="$1"
   local python_version="$2"
-  echo ":::: Build in-tree Torch from binary: $torch_from_bin with Python: $python_version"
+
+  if [ "${torch_from_bin}" == "OFF" ]
+  then
+    echo ":::: Installing PyTorch from source"
+    echo ":::: repo: ${TORCH_MLIR_SRC_PYTORCH_REPO}"
+    echo ":::: branch / commit: ${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
+    echo ":::: install without rebuild: ${TM_PYTORCH_INSTALL_WITHOUT_REBUILD}"
+    /main_checkout/torch-mlir/build_tools/build_libtorch.sh
+  fi
+
+  echo ":::: Build in-tree Torch with Python: $python_version"
   cmake -GNinja -B/main_checkout/torch-mlir/build \
       -DCMAKE_BUILD_TYPE=Release \
       -DCMAKE_C_COMPILER=clang \
@@ -205,10 +215,7 @@ function build_in_tree() {
       -DLLVM_TARGETS_TO_BUILD=host \
       -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
       -DTORCH_MLIR_ENABLE_LTC=ON \
-      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
-      -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
-      -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
-      -DTM_PYTORCH_INSTALL_WITHOUT_REBUILD=${TM_PYTORCH_INSTALL_WITHOUT_REBUILD} \
+      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="ON" \
       -DPython3_EXECUTABLE="$(which python3)" \
       /main_checkout/torch-mlir/externals/llvm-project/llvm
   cmake --build /main_checkout/torch-mlir/build
@@ -289,6 +296,15 @@ function build_out_of_tree() {
   local python_version="$2"
   echo ":::: Build out-of-tree Torch from binary: $torch_from_bin with Python: $python_version"
 
+  if [ "${torch_from_bin}" == "OFF" ]
+  then
+    echo ":::: Installing PyTorch from source"
+    echo ":::: repo: ${TORCH_MLIR_SRC_PYTORCH_REPO}"
+    echo ":::: branch / commit: ${TORCH_MLIR_SRC_PYTORCH_BRANCH}"
+    echo ":::: install without rebuild: ${TM_PYTORCH_INSTALL_WITHOUT_REBUILD}"
+    /main_checkout/torch-mlir/build_tools/build_libtorch.sh
+  fi
+
   if [ ! -d "/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" ]
   then
   echo ":::: LLVM / MLIR is not built so building it first.."
@@ -323,10 +339,7 @@ function build_out_of_tree() {
       -DMLIR_DIR="/main_checkout/torch-mlir/llvm-build/lib/cmake/mlir/" \
       -DMLIR_ENABLE_BINDINGS_PYTHON=OFF \
       -DTORCH_MLIR_ENABLE_LTC=ON \
-      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="$torch_from_bin" \
-      -DTORCH_MLIR_SRC_PYTORCH_REPO=${TORCH_MLIR_SRC_PYTORCH_REPO} \
-      -DTORCH_MLIR_SRC_PYTORCH_BRANCH=${TORCH_MLIR_SRC_PYTORCH_BRANCH} \
-      -DTM_PYTORCH_INSTALL_WITHOUT_REBUILD=${TM_PYTORCH_INSTALL_WITHOUT_REBUILD} \
+      -DTORCH_MLIR_USE_INSTALLED_PYTORCH="ON" \
       -DPython3_EXECUTABLE="$(which python3)" \
       /main_checkout/torch-mlir
   cmake --build /main_checkout/torch-mlir/build_oot