build: improve robustness of cmake and shell scripts (#1018)

On my local machine, `unzip` didn't exist (producing a "command not
found" error), but CMake ignored the error.  Although the build did
succeed (because it found a previously-built version of libtorch), it
seems better to abort builds on such failures, so this patch checks the
return code of all external process invocations.

Along similar lines, this patch also updates the shell scripts in
`build_tools` to extensively use double-quoting to prevent unintentional
word splitting or globbing.  Since some of the scripts execute `rm`
while using shell variables, this patch also adds the preamble `set -u`
to abort execution if an undefined variable is referenced, so that we
reduce the chances of executing `rm -rf /` if the path expression
happens to refer to an undefined variable.
pull/1019/head
Ashay Rane 2022-07-06 14:39:30 -07:00 committed by GitHub
parent bbb648410e
commit 874fdb7e42
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 97 additions and 80 deletions

View File

@ -1,6 +1,6 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -xe pipefail set -xeu -o pipefail
SRC_ROOT="$( cd "$(dirname "$0")" ; pwd -P)/.." SRC_ROOT="$( cd "$(dirname "$0")" ; pwd -P)/.."
PYTORCH_ROOT=${PYTORCH_ROOT:-$SRC_ROOT/externals/pytorch} PYTORCH_ROOT=${PYTORCH_ROOT:-$SRC_ROOT/externals/pytorch}
@ -14,7 +14,6 @@ WHEELHOUSE="${WHEELHOUSE:-$SRC_ROOT/build_tools/python_deploy/wheelhouse}"
Red='\033[0;31m' Red='\033[0;31m'
Green='\033[0;32m' Green='\033[0;32m'
Yellow='\033[1;33m' Yellow='\033[1;33m'
White='\033[1;37m'
NC='\033[0m' NC='\033[0m'
echo "SRC_ROOT=${SRC_ROOT}" echo "SRC_ROOT=${SRC_ROOT}"
@ -30,20 +29,18 @@ if [[ "$LIBTORCH_VARIANT" == *"cxx11-abi"* ]]; then
echo _GLIBCXX_USE_CXX11_ABI=1 echo _GLIBCXX_USE_CXX11_ABI=1
export _GLIBCXX_USE_CXX11_ABI=1 export _GLIBCXX_USE_CXX11_ABI=1
CXX_ABI=1 CXX_ABI=1
LIBTORCH_ABI="cxx11-abi-"
else else
echo _GLIBCXX_USE_CXX11_ABI=0 echo _GLIBCXX_USE_CXX11_ABI=0
export _GLIBCXX_USE_CXX11_ABI=0 export _GLIBCXX_USE_CXX11_ABI=0
CXX_ABI=0 CXX_ABI=0
LIBTORCH_ABI=
fi fi
retry () { retry () {
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*) "$@" || (sleep 1 && "$@") || (sleep 2 && "$@") || (sleep 4 && "$@") || (sleep 8 && "$@")
} }
install_requirements() { install_requirements() {
pip install -qr $PYTORCH_ROOT/requirements.txt pip install -qr "$PYTORCH_ROOT/requirements.txt"
pip list pip list
} }
@ -71,7 +68,7 @@ MACOS_X86_URL="https://download.pytorch.org/libtorch/nightly/cpu/libtorch-macos-
LINUX_X86_URL="https://download.pytorch.org/libtorch/nightly/cpu/libtorch-static-without-deps-latest.zip" LINUX_X86_URL="https://download.pytorch.org/libtorch/nightly/cpu/libtorch-static-without-deps-latest.zip"
download_libtorch() { download_libtorch() {
cd $SRC_ROOT cd "$SRC_ROOT"
if [[ $(uname -s) = 'Darwin' ]]; then if [[ $(uname -s) = 'Darwin' ]]; then
echo "Apple macOS detected" echo "Apple macOS detected"
if [[ $(uname -m) == 'arm64' ]]; then if [[ $(uname -m) == 'arm64' ]]; then
@ -82,7 +79,7 @@ download_libtorch() {
DOWNLOAD_URL=${MACOS_X86_URL} DOWNLOAD_URL=${MACOS_X86_URL}
fi fi
elif [[ $(uname -s) = 'Linux' ]]; then elif [[ $(uname -s) = 'Linux' ]]; then
echo "$Linux detected" echo "Linux detected"
DOWNLOAD_URL=${LINUX_X86_URL} DOWNLOAD_URL=${LINUX_X86_URL}
else else
echo "OS not detected. Pray and Play" echo "OS not detected. Pray and Play"
@ -105,16 +102,16 @@ fi
checkout_pytorch() { checkout_pytorch() {
if [[ ! -d "$PYTORCH_ROOT" ]]; then if [[ ! -d "$PYTORCH_ROOT" ]]; then
git clone https://github.com/pytorch/pytorch $PYTORCH_ROOT git clone https://github.com/pytorch/pytorch "$PYTORCH_ROOT"
fi fi
cd $PYTORCH_ROOT cd "$PYTORCH_ROOT"
git fetch --all git fetch --all
git checkout origin/${PYTORCH_BRANCH} git checkout origin/"${PYTORCH_BRANCH}"
git submodule update --init --recursive git submodule update --init --recursive
} }
build_pytorch() { build_pytorch() {
cd $PYTORCH_ROOT cd "$PYTORCH_ROOT"
# Uncomment the next line if you want to iterate on source builds # Uncomment the next line if you want to iterate on source builds
python setup.py clean python setup.py clean
@ -130,7 +127,7 @@ build_pytorch() {
# STATIC_DISPATCH_BACKEND=ON # STATIC_DISPATCH_BACKEND=ON
# BUILD_LITE_INTERPRETER=OFF # BUILD_LITE_INTERPRETER=OFF
fi fi
BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} USE_LIGHTWEIGHT_DISPATCH=${USE_LIGHTWEIGHT_DISPATCH} STATIC_DISPATCH_BACKEND=${STATIC_DISPATCH_BACKEND} BUILD_LITE_INTERPRETER=${BUILD_LITE_INTERPRETER} BUILD_TEST=OFF USE_GLOO=OFF USE_MPS=OFF USE_PYTORCH_QNNPACK=OFF USE_OPENMP=OFF USE_OBSERVERS=OFF USE_KINETO=OFF USE_EIGEN_FOR_BLAS=OFF CMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=${CXX_ABI}" USE_FBGEMM=OFF USE_NCCL=OFF INTERN_DISABLE_ONNX=OFF USE_CUDA=OFF USE_MKL=OFF USE_XNNPACK=OFF USE_DISTRIBUTED=OFF USE_BREAKPAD=OFF USE_MKLDNN=OFF USE_QNNPACK=OFF USE_NNPACK=OFF ONNX_ML=OFF CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} python setup.py bdist_wheel -d $WHEELHOUSE BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} USE_LIGHTWEIGHT_DISPATCH=${USE_LIGHTWEIGHT_DISPATCH} STATIC_DISPATCH_BACKEND=${STATIC_DISPATCH_BACKEND} BUILD_LITE_INTERPRETER=${BUILD_LITE_INTERPRETER} BUILD_TEST=OFF USE_GLOO=OFF USE_MPS=OFF USE_PYTORCH_QNNPACK=OFF USE_OPENMP=OFF USE_OBSERVERS=OFF USE_KINETO=OFF USE_EIGEN_FOR_BLAS=OFF CMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=${CXX_ABI}" USE_FBGEMM=OFF USE_NCCL=OFF INTERN_DISABLE_ONNX=OFF USE_CUDA=OFF USE_MKL=OFF USE_XNNPACK=OFF USE_DISTRIBUTED=OFF USE_BREAKPAD=OFF USE_MKLDNN=OFF USE_QNNPACK=OFF USE_NNPACK=OFF ONNX_ML=OFF CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} python setup.py bdist_wheel -d "$WHEELHOUSE"
} }
package_pytorch() { package_pytorch() {
@ -146,11 +143,11 @@ package_pytorch() {
mv build/include/* libtorch/include/ mv build/include/* libtorch/include/
echo "${PYTORCH_BUILD_VERSION}" > libtorch/build-version echo "${PYTORCH_BUILD_VERSION}" > libtorch/build-version
echo "$(pushd $PYTORCH_ROOT && git rev-parse HEAD)" > libtorch/build-hash (pushd "$PYTORCH_ROOT" && git rev-parse HEAD) > libtorch/build-hash
echo "Installing libtorch in ${PYTORCH_ROOT}/../../" echo "Installing libtorch in ${PYTORCH_ROOT}/../../"
echo "deleteing old ${PYTORCH_ROOT}/../../libtorch" echo "deleteing old ${PYTORCH_ROOT}/../../libtorch"
rm -rf ${PYTORCH_ROOT}/../../libtorch rm -rf "${PYTORCH_ROOT}"/../../libtorch
mv libtorch ${PYTORCH_ROOT}/../../ mv libtorch "${PYTORCH_ROOT}"/../../
} }
#main #main

View File

@ -1,5 +1,5 @@
#!/bin/bash #!/bin/bash
set -e set -eu -o pipefail
if [ -z "$PYTHON" ]; then if [ -z "$PYTHON" ]; then
PYTHON="$(which python)" PYTHON="$(which python)"
@ -7,11 +7,11 @@ fi
version="$("$PYTHON" --version)" version="$("$PYTHON" --version)"
echo "Using python: $PYTHON (version $version)" echo "Using python: $PYTHON (version $version)"
repo_root="$(cd $(dirname $0)/.. && pwd)" repo_root="$(cd "$(dirname "$0")"/.. && pwd)"
wheelhouse="$repo_root/wheelhouse" wheelhouse="$repo_root/wheelhouse"
package_test_venv="$wheelhouse/package-test.venv" package_test_venv="$wheelhouse/package-test.venv"
mkdir -p $wheelhouse mkdir -p "$wheelhouse"
cd $wheelhouse cd "$wheelhouse"
echo "---- BUILDING torch-mlir ----" echo "---- BUILDING torch-mlir ----"
CMAKE_GENERATOR=Ninja \ CMAKE_GENERATOR=Ninja \

View File

@ -11,7 +11,7 @@
set -eu -o errtrace set -eu -o errtrace
project_dir="$(cd $(dirname $0)/.. && pwd)" project_dir="$(cd "$(dirname "$0")"/.. && pwd)"
llvm_project_dir="$project_dir/externals/llvm-project" llvm_project_dir="$project_dir/externals/llvm-project"
build_dir="$project_dir/build" build_dir="$project_dir/build"
@ -21,7 +21,7 @@ cmake -GNinja -B"$build_dir" "$llvm_project_dir/llvm" \
-DLLVM_ENABLE_PROJECTS=mlir \ -DLLVM_ENABLE_PROJECTS=mlir \
-DLLVM_EXTERNAL_PROJECTS="torch-mlir;torch-mlir-dialects" \ -DLLVM_EXTERNAL_PROJECTS="torch-mlir;torch-mlir-dialects" \
-DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$project_dir" \ -DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$project_dir" \
-DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR=${project_dir}/externals/llvm-external-projects/torch-mlir-dialects \ -DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR="${project_dir}"/externals/llvm-external-projects/torch-mlir-dialects \
-DMLIR_ENABLE_BINDINGS_PYTHON=ON \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
-DLLVM_ENABLE_ASSERTIONS=ON \ -DLLVM_ENABLE_ASSERTIONS=ON \
-DLLVM_TARGETS_TO_BUILD=host -DLLVM_TARGETS_TO_BUILD=host

View File

@ -37,16 +37,14 @@
# to packaging to avoid stomping on development artifacts. # to packaging to avoid stomping on development artifacts.
set -eu -o errtrace set -eu -o errtrace
this_dir="$(cd $(dirname $0) && pwd)" this_dir="$(cd "$(dirname "$0")" && pwd)"
script_name="$(basename $0)" repo_root="$(cd "$this_dir"/../../ && pwd)"
repo_root="$(cd $this_dir/../../ && pwd)"
script_name="$(basename $0)"
manylinux_docker_image="${manylinux_docker_image:-stellaraccident/manylinux2014_x86_64-bazel-5.1.0:latest}" manylinux_docker_image="${manylinux_docker_image:-stellaraccident/manylinux2014_x86_64-bazel-5.1.0:latest}"
python_versions="${TM_PYTHON_VERSIONS:-cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310}" python_versions="${TM_PYTHON_VERSIONS:-cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310}"
output_dir="${output_dir:-${this_dir}/wheelhouse}" output_dir="${output_dir:-${this_dir}/wheelhouse}"
packages="${packages:-torch-mlir}" packages="${packages:-torch-mlir}"
PKG_VER_FILE=${repo_root}/torch_mlir_package_version ; [ -f $PKG_VER_FILE ] && . $PKG_VER_FILE PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE"
export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}" export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}"
@ -63,7 +61,7 @@ function run_on_host() {
-e "TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" \ -e "TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" \
-e "python_versions=${python_versions}" \ -e "python_versions=${python_versions}" \
-e "packages=${packages}" \ -e "packages=${packages}" \
${manylinux_docker_image} \ "${manylinux_docker_image}" \
-- bash /main_checkout/torch-mlir/build_tools/python_deploy/build_linux_packages.sh -- bash /main_checkout/torch-mlir/build_tools/python_deploy/build_linux_packages.sh
} }
@ -86,9 +84,9 @@ function run_in_docker() {
echo ":::: Python version $(python --version)" echo ":::: Python version $(python --version)"
case "$package" in case "$package" in
torch-mlir) torch-mlir)
clean_wheels torch_mlir $python_version clean_wheels torch_mlir "$python_version"
build_torch_mlir build_torch_mlir
#run_audit_wheel torch_mlir $python_version #run_audit_wheel torch_mlir "$python_version"
;; ;;
*) *)
echo "Unrecognized package '$package'" echo "Unrecognized package '$package'"
@ -112,15 +110,15 @@ function run_audit_wheel() {
local python_version="$2" local python_version="$2"
generic_wheel="/wheelhouse/${wheel_basename}-*-${python_version}-linux_x86_64.whl" generic_wheel="/wheelhouse/${wheel_basename}-*-${python_version}-linux_x86_64.whl"
echo ":::: Auditwheel $generic_wheel" echo ":::: Auditwheel $generic_wheel"
auditwheel repair -w /wheelhouse $generic_wheel auditwheel repair -w /wheelhouse "$generic_wheel"
rm $generic_wheel rm "$generic_wheel"
} }
function clean_wheels() { function clean_wheels() {
local wheel_basename="$1" local wheel_basename="$1"
local python_version="$2" local python_version="$2"
echo ":::: Clean wheels $wheel_basename $python_version" echo ":::: Clean wheels $wheel_basename $python_version"
rm -f /wheelhouse/${wheel_basename}-*-${python_version}-*.whl rm -f /wheelhouse/"${wheel_basename}"-*-"${python_version}"-*.whl
} }
# Trampoline to the docker container if running on the host. # Trampoline to the docker container if running on the host.

View File

@ -18,13 +18,13 @@
set -eu -o errtrace set -eu -o errtrace
this_dir="$(cd $(dirname $0) && pwd)" this_dir="$(cd "$(dirname "$0")" && pwd)"
repo_root="$(cd $this_dir/../../ && pwd)" repo_root="$(cd "$this_dir"/../../ && pwd)"
python_versions="${TORCH_MLIR_PYTHON_VERSIONS:-3.9 3.10}" python_versions="${TORCH_MLIR_PYTHON_VERSIONS:-3.9 3.10}"
output_dir="${output_dir:-${this_dir}/wheelhouse}" output_dir="${output_dir:-${this_dir}/wheelhouse}"
packages="${packages:-torch-mlir}" packages="${packages:-torch-mlir}"
PKG_VER_FILE=${repo_root}/torch_mlir_package_version ; [ -f $PKG_VER_FILE ] && . $PKG_VER_FILE PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE"
export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}" export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}"
@ -53,9 +53,9 @@ function run() {
echo ":::: Python version $(python3 --version)" echo ":::: Python version $(python3 --version)"
case "$package" in case "$package" in
torch-mlir) torch-mlir)
clean_wheels torch_mlir $python_version clean_wheels torch_mlir "$python_version"
build_torch_mlir torch_mlir $python_version build_torch_mlir torch_mlir "$python_version"
run_audit_wheel torch_mlir $python_version run_audit_wheel torch_mlir "$python_version"
;; ;;
*) *)
echo "Unrecognized package '$package'" echo "Unrecognized package '$package'"
@ -69,45 +69,45 @@ function run() {
function build_torch_mlir() { function build_torch_mlir() {
local wheel_basename="$1" local wheel_basename="$1"
local python_version="$2" local python_version="$2"
rm -rf $output_dir/build_venv rm -rf "$output_dir"/build_venv
python${python_version} -m venv $output_dir/build_venv python"${python_version}" -m venv "$output_dir"/build_venv
source $output_dir/build_venv/bin/activate source "$output_dir"/build_venv/bin/activate
python${python_version} -m pip install -U pip python"${python_version}" -m pip install -U pip
python${python_version} -m pip install -r $repo_root/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu python"${python_version}" -m pip install -r "$repo_root"/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
CMAKE_GENERATOR=Ninja \ CMAKE_GENERATOR=Ninja \
TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \ TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \
MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \ MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \
CMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \ CMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \
python${python_version} -m pip wheel -v -w $output_dir $repo_root --extra-index-url https://download.pytorch.org/whl/nightly/cpu python"${python_version}" -m pip wheel -v -w "$output_dir" "$repo_root" --extra-index-url https://download.pytorch.org/whl/nightly/cpu
deactivate deactivate
rm -rf $output_dir/build_venv rm -rf "$output_dir"/build_venv
} }
function clean_wheels() { function clean_wheels() {
local wheel_basename="$1" local wheel_basename="$1"
local python_version="$2" local python_version="$2"
echo ":::: Clean wheels $wheel_basename $python_version" echo ":::: Clean wheels $wheel_basename $python_version"
rm -rf $repo_root/build/ rm -rf "$repo_root"/build/
rm -f $output_dir/${wheel_basename}-*-${python_version//./}-*.whl rm -f "$output_dir"/"${wheel_basename}"-*-"${python_version//./}"-*.whl
} }
function run_audit_wheel() { function run_audit_wheel() {
set +x set +x
local wheel_basename="$1" local wheel_basename="$1"
local python_version="$2" local python_version="$2"
generic_wheel=`ls $output_dir/${wheel_basename}-* | grep ${python_version//./}` generic_wheel=$(ls "$output_dir"/"${wheel_basename}"-* | grep "${python_version//./}")
echo "Looking for $generic_wheel" echo "Looking for $generic_wheel"
if [ -f "$generic_wheel" ]; then if [ -f "$generic_wheel" ]; then
echo "$generic_wheel found. Delocating it.." echo "$generic_wheel found. Delocating it.."
rm -rf $output_dir/test_venv rm -rf "$output_dir"/test_venv
python${python_version} -m venv $output_dir/test_venv python"${python_version}" -m venv "$output_dir"/test_venv
source $output_dir/test_venv/bin/activate source "$output_dir"/test_venv/bin/activate
python${python_version} -m pip install -U pip python"${python_version}" -m pip install -U pip
python${python_version} -m pip install -r $repo_root/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu python"${python_version}" -m pip install -r "$repo_root"/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
python${python_version} -m pip install $generic_wheel --extra-index-url https://download.pytorch.org/whl/nightly/cpu python"${python_version}" -m pip install "$generic_wheel" --extra-index-url https://download.pytorch.org/whl/nightly/cpu
DYLD_LIBRARY_PATH=$repo_root/libtorch/lib delocate-wheel -v $generic_wheel DYLD_LIBRARY_PATH="$repo_root"/libtorch/lib delocate-wheel -v "$generic_wheel"
deactivate deactivate
rm -rf $output_dir/test_venv rm -rf "$output_dir"/test_venv
fi fi
} }

View File

@ -13,14 +13,14 @@
set -eu -o errtrace set -eu -o errtrace
this_dir="$(cd $(dirname $0) && pwd)" this_dir="$(cd "$(dirname "$0")" && pwd)"
repo_root="$(cd $this_dir/../../ && pwd)" repo_root="$(cd "$this_dir"/../../ && pwd)"
output_dir="${output_dir:-${this_dir}/wheelhouse}" output_dir="${output_dir:-${this_dir}/wheelhouse}"
rm -rf ${output_dir} rm -rf "${output_dir}"
git fetch --all git fetch --all
latest_snapshot=$(git for-each-ref --sort=creatordate --format '%(refname:short)' refs/tags | tail -n 1) latest_snapshot=$(git for-each-ref --sort=creatordate --format '%(refname:short)' refs/tags | tail -n 1)
git checkout ${latest_snapshot} git checkout "${latest_snapshot}"
git submodule update --init git submodule update --init
package_version=${latest_snapshot#"snapshot-"} package_version=${latest_snapshot#"snapshot-"}
echo "Latest snapshot tag is: ${latest_snapshot}" echo "Latest snapshot tag is: ${latest_snapshot}"
@ -35,6 +35,6 @@ TORCH_MLIR_OSX_ARCH=arm64 \
TORCH_MLIR_OSX_TARGET=11.0 \ TORCH_MLIR_OSX_TARGET=11.0 \
TORCH_MLIR_PYTHON_PACKAGE_VERSION="${package_version}" \ TORCH_MLIR_PYTHON_PACKAGE_VERSION="${package_version}" \
TORCH_MLIR_PYTHON_VERSIONS="${TORCH_MLIR_PYTHON_VERSIONS}" \ TORCH_MLIR_PYTHON_VERSIONS="${TORCH_MLIR_PYTHON_VERSIONS}" \
${repo_root}/build_tools/python_deploy/build_macos_packages.sh "${repo_root}"/build_tools/python_deploy/build_macos_packages.sh
gh release upload ${latest_snapshot} ${repo_root}/build_tools/python_deploy/wheelhouse/torch*.whl gh release upload "${latest_snapshot}" "${repo_root}"/build_tools/python_deploy/wheelhouse/torch*.whl

View File

@ -11,7 +11,7 @@
# Usage: # Usage:
# sudo install_macos_deps.sh # sudo install_macos_deps.sh
set -e -o pipefail set -eu -o pipefail
if [[ "$(whoami)" != "root" ]]; then if [[ "$(whoami)" != "root" ]]; then
echo "ERROR: Must setup deps as root" echo "ERROR: Must setup deps as root"
@ -37,11 +37,11 @@ for python_spec in $PYTHON_SPECS; do
# Install Python. # Install Python.
if ! [ -x "$python_exe" ]; then if ! [ -x "$python_exe" ]; then
package_basename="$(basename $url)" package_basename="$(basename "$url")"
download_path="/tmp/torch_mlir_python_install/$package_basename" download_path="/tmp/torch_mlir_python_install/$package_basename"
mkdir -p "$(dirname $download_path)" mkdir -p "$(dirname "$download_path")"
echo "Downloading $url -> $download_path" echo "Downloading $url -> $download_path"
curl $url -o "$download_path" curl "$url" -o "$download_path"
echo "Installing $download_path" echo "Installing $download_path"
installer -pkg "$download_path" -target / installer -pkg "$download_path" -target /

View File

@ -13,13 +13,13 @@ fi
venv_dir=$1 venv_dir=$1
serialized_test_dir=$2 serialized_test_dir=$2
here="$(realpath $(dirname $0))" here="$(realpath "$(dirname "$0")")"
torch_mlir_src_root="$here/../../" torch_mlir_src_root="$here/../../"
mkdir -p $venv_dir mkdir -p "$venv_dir"
mkdir -p $serialized_test_dir mkdir -p "$serialized_test_dir"
python3 -m venv $venv_dir python3 -m venv "$venv_dir"
source $venv_dir/bin/activate source "$venv_dir"/bin/activate
# latest torch-version and torch-vision module is required. # latest torch-version and torch-vision module is required.
python3 -m pip install --upgrade -r "$torch_mlir_src_root/requirements.txt" python3 -m pip install --upgrade -r "$torch_mlir_src_root/requirements.txt"
@ -38,4 +38,4 @@ cd "$torch_mlir_src_root"
export PYTHONPATH=${PYTHONPATH-} export PYTHONPATH=${PYTHONPATH-}
source "$torch_mlir_src_root/.env" source "$torch_mlir_src_root/.env"
python3 -m build_tools.torchscript_e2e_heavydep_tests.main --output_dir=$serialized_test_dir python3 -m build_tools.torchscript_e2e_heavydep_tests.main --output_dir="$serialized_test_dir"

View File

@ -9,9 +9,9 @@
# For more information on supporting custom operators, see: # For more information on supporting custom operators, see:
# ${TORCH_MLIR}/python/torch_mlir/_torch_mlir_custom_op_example/README.md # ${TORCH_MLIR}/python/torch_mlir/_torch_mlir_custom_op_example/README.md
set -eo pipefail set -euo pipefail
src_dir="$(realpath $(dirname $0)/..)" src_dir="$(realpath "$(dirname "$0")"/..)"
build_dir="$(realpath "${TORCH_MLIR_BUILD_DIR:-$src_dir/build}")" build_dir="$(realpath "${TORCH_MLIR_BUILD_DIR:-$src_dir/build}")"
torch_transforms_cpp_dir="${src_dir}/lib/Dialect/Torch/Transforms" torch_transforms_cpp_dir="${src_dir}/lib/Dialect/Torch/Transforms"
python_packages_dir="${build_dir}/tools/torch-mlir/python_packages" python_packages_dir="${build_dir}/tools/torch-mlir/python_packages"

View File

@ -9,9 +9,9 @@
# For more information on supporting custom operators, see: # For more information on supporting custom operators, see:
# ${TORCH_MLIR}/python/torch_mlir/_torch_mlir_custom_op_example/README.md # ${TORCH_MLIR}/python/torch_mlir/_torch_mlir_custom_op_example/README.md
set -eo pipefail set -euo pipefail
src_dir="$(realpath $(dirname $0)/..)" src_dir="$(realpath "$(dirname "$0")"/..)"
build_dir="$(realpath "${TORCH_MLIR_BUILD_DIR:-$src_dir/build}")" build_dir="$(realpath "${TORCH_MLIR_BUILD_DIR:-$src_dir/build}")"
torch_ir_include_dir="${src_dir}/include/torch-mlir/Dialect/Torch/IR" torch_ir_include_dir="${src_dir}/include/torch-mlir/Dialect/Torch/IR"
python_packages_dir="${build_dir}/tools/torch-mlir/python_packages" python_packages_dir="${build_dir}/tools/torch-mlir/python_packages"

View File

@ -4,12 +4,14 @@
# For arbitrary build/install directories, set the env variables: # For arbitrary build/install directories, set the env variables:
# - TORCH_MLIR_BUILD_DIR # - TORCH_MLIR_BUILD_DIR
set -eu -o pipefail
portable_realpath() { portable_realpath() {
# Create the directory if needed so that the `cd` doesn't fail. # Create the directory if needed so that the `cd` doesn't fail.
mkdir -p $1 && cd $1 && pwd mkdir -p "$1" && cd "$1" && pwd
} }
td="$(portable_realpath $(dirname $0)/..)" td="$(portable_realpath "$(dirname "$0")"/..)"
build_dir="$(portable_realpath "${TORCH_MLIR_BUILD_DIR:-$td/build}")" build_dir="$(portable_realpath "${TORCH_MLIR_BUILD_DIR:-$td/build}")"
python_packages_dir="$build_dir/tools/torch-mlir/python_packages" python_packages_dir="$build_dir/tools/torch-mlir/python_packages"

View File

@ -81,7 +81,11 @@ if(TORCH_MLIR_ENABLE_JIT_IR_IMPORTER)
LIBTORCH_VARIANT=${LIBTORCH_VARIANT} LIBTORCH_VARIANT=${LIBTORCH_VARIANT}
CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
${CMAKE_CURRENT_SOURCE_DIR}/../build_tools/build_libtorch.sh ${CMAKE_CURRENT_SOURCE_DIR}/../build_tools/build_libtorch.sh
RESULT_VARIABLE _result
) )
if(_result)
message(FATAL_ERROR "Failed to run `build_libtorch.sh`")
endif()
set(TORCH_INSTALL_PREFIX "libtorch") set(TORCH_INSTALL_PREFIX "libtorch")
add_subdirectory(torch_mlir/dialects/torch/importer/jit_ir) add_subdirectory(torch_mlir/dialects/torch/importer/jit_ir)
add_subdirectory(torch_mlir_e2e_test) add_subdirectory(torch_mlir_e2e_test)

View File

@ -59,16 +59,24 @@ function(TorchMLIRConfigurePyTorch)
execute_process( execute_process(
COMMAND ${Python3_EXECUTABLE} COMMAND ${Python3_EXECUTABLE}
-c "import torch; import sys; sys.stdout.write('1' if torch.compiled_with_cxx11_abi() else '0')" -c "import torch; import sys; sys.stdout.write('1' if torch.compiled_with_cxx11_abi() else '0')"
RESULT_VARIABLE _result
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
OUTPUT_VARIABLE _use_cxx11_abi) OUTPUT_VARIABLE _use_cxx11_abi)
if(_result)
message(FATAL_ERROR "Failed to determine C++ Dual ABI")
endif()
message(STATUS "PyTorch C++ Dual ABI setting: \"${_use_cxx11_abi}\"") message(STATUS "PyTorch C++ Dual ABI setting: \"${_use_cxx11_abi}\"")
# Check ABI compatibility version # Check ABI compatibility version
execute_process( execute_process(
COMMAND ${Python3_EXECUTABLE} COMMAND ${Python3_EXECUTABLE}
-c "import torch; import sys; abi=torch._C._PYBIND11_BUILD_ABI; abi.startswith('_cxxabi10') or sys.exit(1); sys.stdout.write(str(abi[-2:]))" -c "import torch; import sys; abi=torch._C._PYBIND11_BUILD_ABI; abi.startswith('_cxxabi10') or sys.exit(1); sys.stdout.write(str(abi[-2:]))"
RESULT_VARIABLE _result
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
OUTPUT_VARIABLE _cxx_abi_version) OUTPUT_VARIABLE _cxx_abi_version)
if(_result)
message(FATAL_ERROR "Failed to determine C++ ABI version")
endif()
message(STATUS "PyTorch C++ ABI version: \"${_cxx_abi_version}\"") message(STATUS "PyTorch C++ ABI version: \"${_cxx_abi_version}\"")
# Specialize compile flags for compiler # Specialize compile flags for compiler
@ -91,15 +99,23 @@ function(TorchMLIRConfigureLibTorch)
# Check dual ABI setting first # Check dual ABI setting first
execute_process( execute_process(
COMMAND bash "-c" "cat ${TORCH_INSTALL_PREFIX}/share/cmake/Torch/TorchConfig.cmake | egrep -o '_GLIBCXX_USE_CXX11_ABI=[0-1]' | egrep -o '.$'" COMMAND bash "-c" "cat ${TORCH_INSTALL_PREFIX}/share/cmake/Torch/TorchConfig.cmake | egrep -o '_GLIBCXX_USE_CXX11_ABI=[0-1]' | egrep -o '.$'"
RESULT_VARIABLE _result
OUTPUT_VARIABLE _use_cxx11_abi OUTPUT_VARIABLE _use_cxx11_abi
OUTPUT_STRIP_TRAILING_WHITESPACE) OUTPUT_STRIP_TRAILING_WHITESPACE)
if(_result)
message(FATAL_ERROR "Failed to determine LibTorch C++ Dual ABI")
endif()
message(STATUS "LibTorch C++ Dual ABI setting: \"${_use_cxx11_abi}\"") message(STATUS "LibTorch C++ Dual ABI setting: \"${_use_cxx11_abi}\"")
# Check ABI compatibility version # Check ABI compatibility version
execute_process( execute_process(
COMMAND bash "-c" "strings ${TORCH_INSTALL_PREFIX}/lib/libtorch_python.so | egrep '^_cxxabi[0-9]{4}' | egrep -o '..$'" COMMAND bash "-c" "strings ${TORCH_INSTALL_PREFIX}/lib/libtorch_python.so | egrep '^_cxxabi[0-9]{4}' | egrep -o '..$'"
RESULT_VARIABLE _result
OUTPUT_VARIABLE _cxx_abi_version OUTPUT_VARIABLE _cxx_abi_version
OUTPUT_STRIP_TRAILING_WHITESPACE) OUTPUT_STRIP_TRAILING_WHITESPACE)
if(_result)
message(FATAL_ERROR "Failed to determine LibTorch C++ ABI version")
endif()
message(STATUS "LibTorch C++ ABI version: \"${_cxx_abi_version}\"") message(STATUS "LibTorch C++ ABI version: \"${_cxx_abi_version}\"")
# Specialize compile flags for compiler # Specialize compile flags for compiler

View File

@ -1,7 +1,7 @@
#!/bin/bash #!/bin/bash
set -euo pipefail set -euo pipefail
src_dir="$(realpath $(dirname $0)/..)" src_dir="$(realpath "$(dirname "$0")"/..)"
cd "$src_dir" cd "$src_dir"