From 874fdb7e429175b701602e08df027f756bdf6ba9 Mon Sep 17 00:00:00 2001 From: Ashay Rane Date: Wed, 6 Jul 2022 14:39:30 -0700 Subject: [PATCH] build: improve robustness of cmake and shell scripts (#1018) On my local machine, `unzip` didn't exist (producing a "command not found" error), but CMake ignored the error. Although the build did succeed (because it found a previously-built version of libtorch), it seems better to abort builds on such failures, so this patch checks the return code of all external process invocations. Along similar lines, this patch also updates the shell scripts in `build_tools` to extensively use double-quoting to prevent unintentional word splitting or globbing. Since some of the scripts execute `rm` while using shell variables, this patch also adds the preamble `set -u` to abort execution if an undefined variable is referenced, so that we reduce the chances of executing `rm -rf /` if the path expression happens to refer to an undefined variable. --- build_tools/build_libtorch.sh | 29 +++++------ build_tools/build_python_wheels.sh | 8 ++-- build_tools/build_standalone.sh | 4 +- .../python_deploy/build_linux_packages.sh | 20 ++++---- .../python_deploy/build_macos_packages.sh | 48 +++++++++---------- .../python_deploy/build_upload_m1_snapshot.sh | 12 ++--- .../python_deploy/install_macos_deps.sh | 8 ++-- .../generate_serialized_tests.sh | 12 ++--- build_tools/update_shape_lib.sh | 4 +- build_tools/update_torch_ods.sh | 4 +- build_tools/write_env_file.sh | 6 ++- python/CMakeLists.txt | 4 ++ .../cmake/modules/TorchMLIRPyTorch.cmake | 16 +++++++ tools/torchscript_e2e_test.sh | 2 +- 14 files changed, 97 insertions(+), 80 deletions(-) diff --git a/build_tools/build_libtorch.sh b/build_tools/build_libtorch.sh index cc1d10c57..57aa04fb9 100755 --- a/build_tools/build_libtorch.sh +++ b/build_tools/build_libtorch.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -xe pipefail +set -xeu -o pipefail SRC_ROOT="$( cd "$(dirname "$0")" ; pwd -P)/.." PYTORCH_ROOT=${PYTORCH_ROOT:-$SRC_ROOT/externals/pytorch} @@ -14,7 +14,6 @@ WHEELHOUSE="${WHEELHOUSE:-$SRC_ROOT/build_tools/python_deploy/wheelhouse}" Red='\033[0;31m' Green='\033[0;32m' Yellow='\033[1;33m' -White='\033[1;37m' NC='\033[0m' echo "SRC_ROOT=${SRC_ROOT}" @@ -30,20 +29,18 @@ if [[ "$LIBTORCH_VARIANT" == *"cxx11-abi"* ]]; then echo _GLIBCXX_USE_CXX11_ABI=1 export _GLIBCXX_USE_CXX11_ABI=1 CXX_ABI=1 - LIBTORCH_ABI="cxx11-abi-" else echo _GLIBCXX_USE_CXX11_ABI=0 export _GLIBCXX_USE_CXX11_ABI=0 CXX_ABI=0 - LIBTORCH_ABI= fi retry () { - $* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) + "$@" || (sleep 1 && "$@") || (sleep 2 && "$@") || (sleep 4 && "$@") || (sleep 8 && "$@") } install_requirements() { - pip install -qr $PYTORCH_ROOT/requirements.txt + pip install -qr "$PYTORCH_ROOT/requirements.txt" pip list } @@ -71,7 +68,7 @@ MACOS_X86_URL="https://download.pytorch.org/libtorch/nightly/cpu/libtorch-macos- LINUX_X86_URL="https://download.pytorch.org/libtorch/nightly/cpu/libtorch-static-without-deps-latest.zip" download_libtorch() { - cd $SRC_ROOT + cd "$SRC_ROOT" if [[ $(uname -s) = 'Darwin' ]]; then echo "Apple macOS detected" if [[ $(uname -m) == 'arm64' ]]; then @@ -82,7 +79,7 @@ download_libtorch() { DOWNLOAD_URL=${MACOS_X86_URL} fi elif [[ $(uname -s) = 'Linux' ]]; then - echo "$Linux detected" + echo "Linux detected" DOWNLOAD_URL=${LINUX_X86_URL} else echo "OS not detected. Pray and Play" @@ -105,16 +102,16 @@ fi checkout_pytorch() { if [[ ! -d "$PYTORCH_ROOT" ]]; then - git clone https://github.com/pytorch/pytorch $PYTORCH_ROOT + git clone https://github.com/pytorch/pytorch "$PYTORCH_ROOT" fi - cd $PYTORCH_ROOT + cd "$PYTORCH_ROOT" git fetch --all - git checkout origin/${PYTORCH_BRANCH} + git checkout origin/"${PYTORCH_BRANCH}" git submodule update --init --recursive } build_pytorch() { - cd $PYTORCH_ROOT + cd "$PYTORCH_ROOT" # Uncomment the next line if you want to iterate on source builds python setup.py clean @@ -130,7 +127,7 @@ build_pytorch() { # STATIC_DISPATCH_BACKEND=ON # BUILD_LITE_INTERPRETER=OFF fi - BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} USE_LIGHTWEIGHT_DISPATCH=${USE_LIGHTWEIGHT_DISPATCH} STATIC_DISPATCH_BACKEND=${STATIC_DISPATCH_BACKEND} BUILD_LITE_INTERPRETER=${BUILD_LITE_INTERPRETER} BUILD_TEST=OFF USE_GLOO=OFF USE_MPS=OFF USE_PYTORCH_QNNPACK=OFF USE_OPENMP=OFF USE_OBSERVERS=OFF USE_KINETO=OFF USE_EIGEN_FOR_BLAS=OFF CMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=${CXX_ABI}" USE_FBGEMM=OFF USE_NCCL=OFF INTERN_DISABLE_ONNX=OFF USE_CUDA=OFF USE_MKL=OFF USE_XNNPACK=OFF USE_DISTRIBUTED=OFF USE_BREAKPAD=OFF USE_MKLDNN=OFF USE_QNNPACK=OFF USE_NNPACK=OFF ONNX_ML=OFF CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} python setup.py bdist_wheel -d $WHEELHOUSE + BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} USE_LIGHTWEIGHT_DISPATCH=${USE_LIGHTWEIGHT_DISPATCH} STATIC_DISPATCH_BACKEND=${STATIC_DISPATCH_BACKEND} BUILD_LITE_INTERPRETER=${BUILD_LITE_INTERPRETER} BUILD_TEST=OFF USE_GLOO=OFF USE_MPS=OFF USE_PYTORCH_QNNPACK=OFF USE_OPENMP=OFF USE_OBSERVERS=OFF USE_KINETO=OFF USE_EIGEN_FOR_BLAS=OFF CMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=${CXX_ABI}" USE_FBGEMM=OFF USE_NCCL=OFF INTERN_DISABLE_ONNX=OFF USE_CUDA=OFF USE_MKL=OFF USE_XNNPACK=OFF USE_DISTRIBUTED=OFF USE_BREAKPAD=OFF USE_MKLDNN=OFF USE_QNNPACK=OFF USE_NNPACK=OFF ONNX_ML=OFF CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} python setup.py bdist_wheel -d "$WHEELHOUSE" } package_pytorch() { @@ -146,11 +143,11 @@ package_pytorch() { mv build/include/* libtorch/include/ echo "${PYTORCH_BUILD_VERSION}" > libtorch/build-version - echo "$(pushd $PYTORCH_ROOT && git rev-parse HEAD)" > libtorch/build-hash + (pushd "$PYTORCH_ROOT" && git rev-parse HEAD) > libtorch/build-hash echo "Installing libtorch in ${PYTORCH_ROOT}/../../" echo "deleteing old ${PYTORCH_ROOT}/../../libtorch" - rm -rf ${PYTORCH_ROOT}/../../libtorch - mv libtorch ${PYTORCH_ROOT}/../../ + rm -rf "${PYTORCH_ROOT}"/../../libtorch + mv libtorch "${PYTORCH_ROOT}"/../../ } #main diff --git a/build_tools/build_python_wheels.sh b/build_tools/build_python_wheels.sh index c766323e6..371bf6263 100755 --- a/build_tools/build_python_wheels.sh +++ b/build_tools/build_python_wheels.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +set -eu -o pipefail if [ -z "$PYTHON" ]; then PYTHON="$(which python)" @@ -7,11 +7,11 @@ fi version="$("$PYTHON" --version)" echo "Using python: $PYTHON (version $version)" -repo_root="$(cd $(dirname $0)/.. && pwd)" +repo_root="$(cd "$(dirname "$0")"/.. && pwd)" wheelhouse="$repo_root/wheelhouse" package_test_venv="$wheelhouse/package-test.venv" -mkdir -p $wheelhouse -cd $wheelhouse +mkdir -p "$wheelhouse" +cd "$wheelhouse" echo "---- BUILDING torch-mlir ----" CMAKE_GENERATOR=Ninja \ diff --git a/build_tools/build_standalone.sh b/build_tools/build_standalone.sh index 1a221423e..a6b13ffc1 100755 --- a/build_tools/build_standalone.sh +++ b/build_tools/build_standalone.sh @@ -11,7 +11,7 @@ set -eu -o errtrace -project_dir="$(cd $(dirname $0)/.. && pwd)" +project_dir="$(cd "$(dirname "$0")"/.. && pwd)" llvm_project_dir="$project_dir/externals/llvm-project" build_dir="$project_dir/build" @@ -21,7 +21,7 @@ cmake -GNinja -B"$build_dir" "$llvm_project_dir/llvm" \ -DLLVM_ENABLE_PROJECTS=mlir \ -DLLVM_EXTERNAL_PROJECTS="torch-mlir;torch-mlir-dialects" \ -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$project_dir" \ - -DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR=${project_dir}/externals/llvm-external-projects/torch-mlir-dialects \ + -DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR="${project_dir}"/externals/llvm-external-projects/torch-mlir-dialects \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DLLVM_TARGETS_TO_BUILD=host diff --git a/build_tools/python_deploy/build_linux_packages.sh b/build_tools/python_deploy/build_linux_packages.sh index 056413c0b..958b63dff 100755 --- a/build_tools/python_deploy/build_linux_packages.sh +++ b/build_tools/python_deploy/build_linux_packages.sh @@ -37,16 +37,14 @@ # to packaging to avoid stomping on development artifacts. set -eu -o errtrace -this_dir="$(cd $(dirname $0) && pwd)" -script_name="$(basename $0)" -repo_root="$(cd $this_dir/../../ && pwd)" -script_name="$(basename $0)" +this_dir="$(cd "$(dirname "$0")" && pwd)" +repo_root="$(cd "$this_dir"/../../ && pwd)" manylinux_docker_image="${manylinux_docker_image:-stellaraccident/manylinux2014_x86_64-bazel-5.1.0:latest}" python_versions="${TM_PYTHON_VERSIONS:-cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310}" output_dir="${output_dir:-${this_dir}/wheelhouse}" packages="${packages:-torch-mlir}" -PKG_VER_FILE=${repo_root}/torch_mlir_package_version ; [ -f $PKG_VER_FILE ] && . $PKG_VER_FILE +PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE" export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}" echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" @@ -63,7 +61,7 @@ function run_on_host() { -e "TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" \ -e "python_versions=${python_versions}" \ -e "packages=${packages}" \ - ${manylinux_docker_image} \ + "${manylinux_docker_image}" \ -- bash /main_checkout/torch-mlir/build_tools/python_deploy/build_linux_packages.sh } @@ -86,9 +84,9 @@ function run_in_docker() { echo ":::: Python version $(python --version)" case "$package" in torch-mlir) - clean_wheels torch_mlir $python_version + clean_wheels torch_mlir "$python_version" build_torch_mlir - #run_audit_wheel torch_mlir $python_version + #run_audit_wheel torch_mlir "$python_version" ;; *) echo "Unrecognized package '$package'" @@ -112,15 +110,15 @@ function run_audit_wheel() { local python_version="$2" generic_wheel="/wheelhouse/${wheel_basename}-*-${python_version}-linux_x86_64.whl" echo ":::: Auditwheel $generic_wheel" - auditwheel repair -w /wheelhouse $generic_wheel - rm $generic_wheel + auditwheel repair -w /wheelhouse "$generic_wheel" + rm "$generic_wheel" } function clean_wheels() { local wheel_basename="$1" local python_version="$2" echo ":::: Clean wheels $wheel_basename $python_version" - rm -f /wheelhouse/${wheel_basename}-*-${python_version}-*.whl + rm -f /wheelhouse/"${wheel_basename}"-*-"${python_version}"-*.whl } # Trampoline to the docker container if running on the host. diff --git a/build_tools/python_deploy/build_macos_packages.sh b/build_tools/python_deploy/build_macos_packages.sh index dd54ec27a..1a4ffd8a0 100755 --- a/build_tools/python_deploy/build_macos_packages.sh +++ b/build_tools/python_deploy/build_macos_packages.sh @@ -18,13 +18,13 @@ set -eu -o errtrace -this_dir="$(cd $(dirname $0) && pwd)" -repo_root="$(cd $this_dir/../../ && pwd)" +this_dir="$(cd "$(dirname "$0")" && pwd)" +repo_root="$(cd "$this_dir"/../../ && pwd)" python_versions="${TORCH_MLIR_PYTHON_VERSIONS:-3.9 3.10}" output_dir="${output_dir:-${this_dir}/wheelhouse}" packages="${packages:-torch-mlir}" -PKG_VER_FILE=${repo_root}/torch_mlir_package_version ; [ -f $PKG_VER_FILE ] && . $PKG_VER_FILE +PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE" export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}" echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" @@ -53,9 +53,9 @@ function run() { echo ":::: Python version $(python3 --version)" case "$package" in torch-mlir) - clean_wheels torch_mlir $python_version - build_torch_mlir torch_mlir $python_version - run_audit_wheel torch_mlir $python_version + clean_wheels torch_mlir "$python_version" + build_torch_mlir torch_mlir "$python_version" + run_audit_wheel torch_mlir "$python_version" ;; *) echo "Unrecognized package '$package'" @@ -69,45 +69,45 @@ function run() { function build_torch_mlir() { local wheel_basename="$1" local python_version="$2" - rm -rf $output_dir/build_venv - python${python_version} -m venv $output_dir/build_venv - source $output_dir/build_venv/bin/activate - python${python_version} -m pip install -U pip - python${python_version} -m pip install -r $repo_root/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu + rm -rf "$output_dir"/build_venv + python"${python_version}" -m venv "$output_dir"/build_venv + source "$output_dir"/build_venv/bin/activate + python"${python_version}" -m pip install -U pip + python"${python_version}" -m pip install -r "$repo_root"/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu CMAKE_GENERATOR=Ninja \ TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \ MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ CMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \ - python${python_version} -m pip wheel -v -w $output_dir $repo_root --extra-index-url https://download.pytorch.org/whl/nightly/cpu + python"${python_version}" -m pip wheel -v -w "$output_dir" "$repo_root" --extra-index-url https://download.pytorch.org/whl/nightly/cpu deactivate - rm -rf $output_dir/build_venv + rm -rf "$output_dir"/build_venv } function clean_wheels() { local wheel_basename="$1" local python_version="$2" echo ":::: Clean wheels $wheel_basename $python_version" - rm -rf $repo_root/build/ - rm -f $output_dir/${wheel_basename}-*-${python_version//./}-*.whl + rm -rf "$repo_root"/build/ + rm -f "$output_dir"/"${wheel_basename}"-*-"${python_version//./}"-*.whl } function run_audit_wheel() { set +x local wheel_basename="$1" local python_version="$2" - generic_wheel=`ls $output_dir/${wheel_basename}-* | grep ${python_version//./}` + generic_wheel=$(ls "$output_dir"/"${wheel_basename}"-* | grep "${python_version//./}") echo "Looking for $generic_wheel" if [ -f "$generic_wheel" ]; then echo "$generic_wheel found. Delocating it.." - rm -rf $output_dir/test_venv - python${python_version} -m venv $output_dir/test_venv - source $output_dir/test_venv/bin/activate - python${python_version} -m pip install -U pip - python${python_version} -m pip install -r $repo_root/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu - python${python_version} -m pip install $generic_wheel --extra-index-url https://download.pytorch.org/whl/nightly/cpu - DYLD_LIBRARY_PATH=$repo_root/libtorch/lib delocate-wheel -v $generic_wheel + rm -rf "$output_dir"/test_venv + python"${python_version}" -m venv "$output_dir"/test_venv + source "$output_dir"/test_venv/bin/activate + python"${python_version}" -m pip install -U pip + python"${python_version}" -m pip install -r "$repo_root"/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu + python"${python_version}" -m pip install "$generic_wheel" --extra-index-url https://download.pytorch.org/whl/nightly/cpu + DYLD_LIBRARY_PATH="$repo_root"/libtorch/lib delocate-wheel -v "$generic_wheel" deactivate - rm -rf $output_dir/test_venv + rm -rf "$output_dir"/test_venv fi } diff --git a/build_tools/python_deploy/build_upload_m1_snapshot.sh b/build_tools/python_deploy/build_upload_m1_snapshot.sh index 9a6a8bf38..b876801c7 100755 --- a/build_tools/python_deploy/build_upload_m1_snapshot.sh +++ b/build_tools/python_deploy/build_upload_m1_snapshot.sh @@ -13,14 +13,14 @@ set -eu -o errtrace -this_dir="$(cd $(dirname $0) && pwd)" -repo_root="$(cd $this_dir/../../ && pwd)" +this_dir="$(cd "$(dirname "$0")" && pwd)" +repo_root="$(cd "$this_dir"/../../ && pwd)" output_dir="${output_dir:-${this_dir}/wheelhouse}" -rm -rf ${output_dir} +rm -rf "${output_dir}" git fetch --all latest_snapshot=$(git for-each-ref --sort=creatordate --format '%(refname:short)' refs/tags | tail -n 1) -git checkout ${latest_snapshot} +git checkout "${latest_snapshot}" git submodule update --init package_version=${latest_snapshot#"snapshot-"} echo "Latest snapshot tag is: ${latest_snapshot}" @@ -35,6 +35,6 @@ TORCH_MLIR_OSX_ARCH=arm64 \ TORCH_MLIR_OSX_TARGET=11.0 \ TORCH_MLIR_PYTHON_PACKAGE_VERSION="${package_version}" \ TORCH_MLIR_PYTHON_VERSIONS="${TORCH_MLIR_PYTHON_VERSIONS}" \ -${repo_root}/build_tools/python_deploy/build_macos_packages.sh +"${repo_root}"/build_tools/python_deploy/build_macos_packages.sh -gh release upload ${latest_snapshot} ${repo_root}/build_tools/python_deploy/wheelhouse/torch*.whl +gh release upload "${latest_snapshot}" "${repo_root}"/build_tools/python_deploy/wheelhouse/torch*.whl diff --git a/build_tools/python_deploy/install_macos_deps.sh b/build_tools/python_deploy/install_macos_deps.sh index 6da876076..335eb813f 100755 --- a/build_tools/python_deploy/install_macos_deps.sh +++ b/build_tools/python_deploy/install_macos_deps.sh @@ -11,7 +11,7 @@ # Usage: # sudo install_macos_deps.sh -set -e -o pipefail +set -eu -o pipefail if [[ "$(whoami)" != "root" ]]; then echo "ERROR: Must setup deps as root" @@ -37,11 +37,11 @@ for python_spec in $PYTHON_SPECS; do # Install Python. if ! [ -x "$python_exe" ]; then - package_basename="$(basename $url)" + package_basename="$(basename "$url")" download_path="/tmp/torch_mlir_python_install/$package_basename" - mkdir -p "$(dirname $download_path)" + mkdir -p "$(dirname "$download_path")" echo "Downloading $url -> $download_path" - curl $url -o "$download_path" + curl "$url" -o "$download_path" echo "Installing $download_path" installer -pkg "$download_path" -target / diff --git a/build_tools/torchscript_e2e_heavydep_tests/generate_serialized_tests.sh b/build_tools/torchscript_e2e_heavydep_tests/generate_serialized_tests.sh index 42920b83a..aa0d42953 100755 --- a/build_tools/torchscript_e2e_heavydep_tests/generate_serialized_tests.sh +++ b/build_tools/torchscript_e2e_heavydep_tests/generate_serialized_tests.sh @@ -13,13 +13,13 @@ fi venv_dir=$1 serialized_test_dir=$2 -here="$(realpath $(dirname $0))" +here="$(realpath "$(dirname "$0")")" torch_mlir_src_root="$here/../../" -mkdir -p $venv_dir -mkdir -p $serialized_test_dir -python3 -m venv $venv_dir -source $venv_dir/bin/activate +mkdir -p "$venv_dir" +mkdir -p "$serialized_test_dir" +python3 -m venv "$venv_dir" +source "$venv_dir"/bin/activate # latest torch-version and torch-vision module is required. python3 -m pip install --upgrade -r "$torch_mlir_src_root/requirements.txt" @@ -38,4 +38,4 @@ cd "$torch_mlir_src_root" export PYTHONPATH=${PYTHONPATH-} source "$torch_mlir_src_root/.env" -python3 -m build_tools.torchscript_e2e_heavydep_tests.main --output_dir=$serialized_test_dir +python3 -m build_tools.torchscript_e2e_heavydep_tests.main --output_dir="$serialized_test_dir" diff --git a/build_tools/update_shape_lib.sh b/build_tools/update_shape_lib.sh index 8912805ed..e0de9f6cb 100755 --- a/build_tools/update_shape_lib.sh +++ b/build_tools/update_shape_lib.sh @@ -9,9 +9,9 @@ # For more information on supporting custom operators, see: # ${TORCH_MLIR}/python/torch_mlir/_torch_mlir_custom_op_example/README.md -set -eo pipefail +set -euo pipefail -src_dir="$(realpath $(dirname $0)/..)" +src_dir="$(realpath "$(dirname "$0")"/..)" build_dir="$(realpath "${TORCH_MLIR_BUILD_DIR:-$src_dir/build}")" torch_transforms_cpp_dir="${src_dir}/lib/Dialect/Torch/Transforms" python_packages_dir="${build_dir}/tools/torch-mlir/python_packages" diff --git a/build_tools/update_torch_ods.sh b/build_tools/update_torch_ods.sh index e7620437f..2e22ab12e 100755 --- a/build_tools/update_torch_ods.sh +++ b/build_tools/update_torch_ods.sh @@ -9,9 +9,9 @@ # For more information on supporting custom operators, see: # ${TORCH_MLIR}/python/torch_mlir/_torch_mlir_custom_op_example/README.md -set -eo pipefail +set -euo pipefail -src_dir="$(realpath $(dirname $0)/..)" +src_dir="$(realpath "$(dirname "$0")"/..)" build_dir="$(realpath "${TORCH_MLIR_BUILD_DIR:-$src_dir/build}")" torch_ir_include_dir="${src_dir}/include/torch-mlir/Dialect/Torch/IR" python_packages_dir="${build_dir}/tools/torch-mlir/python_packages" diff --git a/build_tools/write_env_file.sh b/build_tools/write_env_file.sh index c9e0d0aff..05179c56a 100755 --- a/build_tools/write_env_file.sh +++ b/build_tools/write_env_file.sh @@ -4,12 +4,14 @@ # For arbitrary build/install directories, set the env variables: # - TORCH_MLIR_BUILD_DIR +set -eu -o pipefail + portable_realpath() { # Create the directory if needed so that the `cd` doesn't fail. - mkdir -p $1 && cd $1 && pwd + mkdir -p "$1" && cd "$1" && pwd } -td="$(portable_realpath $(dirname $0)/..)" +td="$(portable_realpath "$(dirname "$0")"/..)" build_dir="$(portable_realpath "${TORCH_MLIR_BUILD_DIR:-$td/build}")" python_packages_dir="$build_dir/tools/torch-mlir/python_packages" diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index cc2b18aae..56de16bba 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -81,7 +81,11 @@ if(TORCH_MLIR_ENABLE_JIT_IR_IMPORTER) LIBTORCH_VARIANT=${LIBTORCH_VARIANT} CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} ${CMAKE_CURRENT_SOURCE_DIR}/../build_tools/build_libtorch.sh + RESULT_VARIABLE _result ) + if(_result) + message(FATAL_ERROR "Failed to run `build_libtorch.sh`") + endif() set(TORCH_INSTALL_PREFIX "libtorch") add_subdirectory(torch_mlir/dialects/torch/importer/jit_ir) add_subdirectory(torch_mlir_e2e_test) diff --git a/python/torch_mlir/dialects/torch/importer/jit_ir/cmake/modules/TorchMLIRPyTorch.cmake b/python/torch_mlir/dialects/torch/importer/jit_ir/cmake/modules/TorchMLIRPyTorch.cmake index 828ad88db..53253c8c7 100644 --- a/python/torch_mlir/dialects/torch/importer/jit_ir/cmake/modules/TorchMLIRPyTorch.cmake +++ b/python/torch_mlir/dialects/torch/importer/jit_ir/cmake/modules/TorchMLIRPyTorch.cmake @@ -59,16 +59,24 @@ function(TorchMLIRConfigurePyTorch) execute_process( COMMAND ${Python3_EXECUTABLE} -c "import torch; import sys; sys.stdout.write('1' if torch.compiled_with_cxx11_abi() else '0')" + RESULT_VARIABLE _result WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE _use_cxx11_abi) + if(_result) + message(FATAL_ERROR "Failed to determine C++ Dual ABI") + endif() message(STATUS "PyTorch C++ Dual ABI setting: \"${_use_cxx11_abi}\"") # Check ABI compatibility version execute_process( COMMAND ${Python3_EXECUTABLE} -c "import torch; import sys; abi=torch._C._PYBIND11_BUILD_ABI; abi.startswith('_cxxabi10') or sys.exit(1); sys.stdout.write(str(abi[-2:]))" + RESULT_VARIABLE _result WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} OUTPUT_VARIABLE _cxx_abi_version) + if(_result) + message(FATAL_ERROR "Failed to determine C++ ABI version") + endif() message(STATUS "PyTorch C++ ABI version: \"${_cxx_abi_version}\"") # Specialize compile flags for compiler @@ -91,15 +99,23 @@ function(TorchMLIRConfigureLibTorch) # Check dual ABI setting first execute_process( COMMAND bash "-c" "cat ${TORCH_INSTALL_PREFIX}/share/cmake/Torch/TorchConfig.cmake | egrep -o '_GLIBCXX_USE_CXX11_ABI=[0-1]' | egrep -o '.$'" + RESULT_VARIABLE _result OUTPUT_VARIABLE _use_cxx11_abi OUTPUT_STRIP_TRAILING_WHITESPACE) + if(_result) + message(FATAL_ERROR "Failed to determine LibTorch C++ Dual ABI") + endif() message(STATUS "LibTorch C++ Dual ABI setting: \"${_use_cxx11_abi}\"") # Check ABI compatibility version execute_process( COMMAND bash "-c" "strings ${TORCH_INSTALL_PREFIX}/lib/libtorch_python.so | egrep '^_cxxabi[0-9]{4}' | egrep -o '..$'" + RESULT_VARIABLE _result OUTPUT_VARIABLE _cxx_abi_version OUTPUT_STRIP_TRAILING_WHITESPACE) + if(_result) + message(FATAL_ERROR "Failed to determine LibTorch C++ ABI version") + endif() message(STATUS "LibTorch C++ ABI version: \"${_cxx_abi_version}\"") # Specialize compile flags for compiler diff --git a/tools/torchscript_e2e_test.sh b/tools/torchscript_e2e_test.sh index adf3dfe7c..152075e63 100755 --- a/tools/torchscript_e2e_test.sh +++ b/tools/torchscript_e2e_test.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -src_dir="$(realpath $(dirname $0)/..)" +src_dir="$(realpath "$(dirname "$0")"/..)" cd "$src_dir"