mirror of https://github.com/llvm/torch-mlir
build: improve robustness of cmake and shell scripts (#1018)
On my local machine, `unzip` didn't exist (producing a "command not found" error), but CMake ignored the error. Although the build did succeed (because it found a previously-built version of libtorch), it seems better to abort builds on such failures, so this patch checks the return code of all external process invocations. Along similar lines, this patch also updates the shell scripts in `build_tools` to extensively use double-quoting to prevent unintentional word splitting or globbing. Since some of the scripts execute `rm` while using shell variables, this patch also adds the preamble `set -u` to abort execution if an undefined variable is referenced, so that we reduce the chances of executing `rm -rf /` if the path expression happens to refer to an undefined variable.pull/1019/head
parent
bbb648410e
commit
874fdb7e42
|
@ -1,6 +1,6 @@
|
||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
set -xe pipefail
|
set -xeu -o pipefail
|
||||||
|
|
||||||
SRC_ROOT="$( cd "$(dirname "$0")" ; pwd -P)/.."
|
SRC_ROOT="$( cd "$(dirname "$0")" ; pwd -P)/.."
|
||||||
PYTORCH_ROOT=${PYTORCH_ROOT:-$SRC_ROOT/externals/pytorch}
|
PYTORCH_ROOT=${PYTORCH_ROOT:-$SRC_ROOT/externals/pytorch}
|
||||||
|
@ -14,7 +14,6 @@ WHEELHOUSE="${WHEELHOUSE:-$SRC_ROOT/build_tools/python_deploy/wheelhouse}"
|
||||||
Red='\033[0;31m'
|
Red='\033[0;31m'
|
||||||
Green='\033[0;32m'
|
Green='\033[0;32m'
|
||||||
Yellow='\033[1;33m'
|
Yellow='\033[1;33m'
|
||||||
White='\033[1;37m'
|
|
||||||
NC='\033[0m'
|
NC='\033[0m'
|
||||||
|
|
||||||
echo "SRC_ROOT=${SRC_ROOT}"
|
echo "SRC_ROOT=${SRC_ROOT}"
|
||||||
|
@ -30,20 +29,18 @@ if [[ "$LIBTORCH_VARIANT" == *"cxx11-abi"* ]]; then
|
||||||
echo _GLIBCXX_USE_CXX11_ABI=1
|
echo _GLIBCXX_USE_CXX11_ABI=1
|
||||||
export _GLIBCXX_USE_CXX11_ABI=1
|
export _GLIBCXX_USE_CXX11_ABI=1
|
||||||
CXX_ABI=1
|
CXX_ABI=1
|
||||||
LIBTORCH_ABI="cxx11-abi-"
|
|
||||||
else
|
else
|
||||||
echo _GLIBCXX_USE_CXX11_ABI=0
|
echo _GLIBCXX_USE_CXX11_ABI=0
|
||||||
export _GLIBCXX_USE_CXX11_ABI=0
|
export _GLIBCXX_USE_CXX11_ABI=0
|
||||||
CXX_ABI=0
|
CXX_ABI=0
|
||||||
LIBTORCH_ABI=
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
retry () {
|
retry () {
|
||||||
$* || (sleep 1 && $*) || (sleep 2 && $*) || (sleep 4 && $*) || (sleep 8 && $*)
|
"$@" || (sleep 1 && "$@") || (sleep 2 && "$@") || (sleep 4 && "$@") || (sleep 8 && "$@")
|
||||||
}
|
}
|
||||||
|
|
||||||
install_requirements() {
|
install_requirements() {
|
||||||
pip install -qr $PYTORCH_ROOT/requirements.txt
|
pip install -qr "$PYTORCH_ROOT/requirements.txt"
|
||||||
pip list
|
pip list
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -71,7 +68,7 @@ MACOS_X86_URL="https://download.pytorch.org/libtorch/nightly/cpu/libtorch-macos-
|
||||||
LINUX_X86_URL="https://download.pytorch.org/libtorch/nightly/cpu/libtorch-static-without-deps-latest.zip"
|
LINUX_X86_URL="https://download.pytorch.org/libtorch/nightly/cpu/libtorch-static-without-deps-latest.zip"
|
||||||
|
|
||||||
download_libtorch() {
|
download_libtorch() {
|
||||||
cd $SRC_ROOT
|
cd "$SRC_ROOT"
|
||||||
if [[ $(uname -s) = 'Darwin' ]]; then
|
if [[ $(uname -s) = 'Darwin' ]]; then
|
||||||
echo "Apple macOS detected"
|
echo "Apple macOS detected"
|
||||||
if [[ $(uname -m) == 'arm64' ]]; then
|
if [[ $(uname -m) == 'arm64' ]]; then
|
||||||
|
@ -82,7 +79,7 @@ download_libtorch() {
|
||||||
DOWNLOAD_URL=${MACOS_X86_URL}
|
DOWNLOAD_URL=${MACOS_X86_URL}
|
||||||
fi
|
fi
|
||||||
elif [[ $(uname -s) = 'Linux' ]]; then
|
elif [[ $(uname -s) = 'Linux' ]]; then
|
||||||
echo "$Linux detected"
|
echo "Linux detected"
|
||||||
DOWNLOAD_URL=${LINUX_X86_URL}
|
DOWNLOAD_URL=${LINUX_X86_URL}
|
||||||
else
|
else
|
||||||
echo "OS not detected. Pray and Play"
|
echo "OS not detected. Pray and Play"
|
||||||
|
@ -105,16 +102,16 @@ fi
|
||||||
|
|
||||||
checkout_pytorch() {
|
checkout_pytorch() {
|
||||||
if [[ ! -d "$PYTORCH_ROOT" ]]; then
|
if [[ ! -d "$PYTORCH_ROOT" ]]; then
|
||||||
git clone https://github.com/pytorch/pytorch $PYTORCH_ROOT
|
git clone https://github.com/pytorch/pytorch "$PYTORCH_ROOT"
|
||||||
fi
|
fi
|
||||||
cd $PYTORCH_ROOT
|
cd "$PYTORCH_ROOT"
|
||||||
git fetch --all
|
git fetch --all
|
||||||
git checkout origin/${PYTORCH_BRANCH}
|
git checkout origin/"${PYTORCH_BRANCH}"
|
||||||
git submodule update --init --recursive
|
git submodule update --init --recursive
|
||||||
}
|
}
|
||||||
|
|
||||||
build_pytorch() {
|
build_pytorch() {
|
||||||
cd $PYTORCH_ROOT
|
cd "$PYTORCH_ROOT"
|
||||||
# Uncomment the next line if you want to iterate on source builds
|
# Uncomment the next line if you want to iterate on source builds
|
||||||
python setup.py clean
|
python setup.py clean
|
||||||
|
|
||||||
|
@ -130,7 +127,7 @@ build_pytorch() {
|
||||||
# STATIC_DISPATCH_BACKEND=ON
|
# STATIC_DISPATCH_BACKEND=ON
|
||||||
# BUILD_LITE_INTERPRETER=OFF
|
# BUILD_LITE_INTERPRETER=OFF
|
||||||
fi
|
fi
|
||||||
BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} USE_LIGHTWEIGHT_DISPATCH=${USE_LIGHTWEIGHT_DISPATCH} STATIC_DISPATCH_BACKEND=${STATIC_DISPATCH_BACKEND} BUILD_LITE_INTERPRETER=${BUILD_LITE_INTERPRETER} BUILD_TEST=OFF USE_GLOO=OFF USE_MPS=OFF USE_PYTORCH_QNNPACK=OFF USE_OPENMP=OFF USE_OBSERVERS=OFF USE_KINETO=OFF USE_EIGEN_FOR_BLAS=OFF CMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=${CXX_ABI}" USE_FBGEMM=OFF USE_NCCL=OFF INTERN_DISABLE_ONNX=OFF USE_CUDA=OFF USE_MKL=OFF USE_XNNPACK=OFF USE_DISTRIBUTED=OFF USE_BREAKPAD=OFF USE_MKLDNN=OFF USE_QNNPACK=OFF USE_NNPACK=OFF ONNX_ML=OFF CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} python setup.py bdist_wheel -d $WHEELHOUSE
|
BUILD_SHARED_LIBS=${BUILD_SHARED_LIBS} USE_LIGHTWEIGHT_DISPATCH=${USE_LIGHTWEIGHT_DISPATCH} STATIC_DISPATCH_BACKEND=${STATIC_DISPATCH_BACKEND} BUILD_LITE_INTERPRETER=${BUILD_LITE_INTERPRETER} BUILD_TEST=OFF USE_GLOO=OFF USE_MPS=OFF USE_PYTORCH_QNNPACK=OFF USE_OPENMP=OFF USE_OBSERVERS=OFF USE_KINETO=OFF USE_EIGEN_FOR_BLAS=OFF CMAKE_CXX_FLAGS="-D_GLIBCXX_USE_CXX11_ABI=${CXX_ABI}" USE_FBGEMM=OFF USE_NCCL=OFF INTERN_DISABLE_ONNX=OFF USE_CUDA=OFF USE_MKL=OFF USE_XNNPACK=OFF USE_DISTRIBUTED=OFF USE_BREAKPAD=OFF USE_MKLDNN=OFF USE_QNNPACK=OFF USE_NNPACK=OFF ONNX_ML=OFF CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} python setup.py bdist_wheel -d "$WHEELHOUSE"
|
||||||
}
|
}
|
||||||
|
|
||||||
package_pytorch() {
|
package_pytorch() {
|
||||||
|
@ -146,11 +143,11 @@ package_pytorch() {
|
||||||
mv build/include/* libtorch/include/
|
mv build/include/* libtorch/include/
|
||||||
|
|
||||||
echo "${PYTORCH_BUILD_VERSION}" > libtorch/build-version
|
echo "${PYTORCH_BUILD_VERSION}" > libtorch/build-version
|
||||||
echo "$(pushd $PYTORCH_ROOT && git rev-parse HEAD)" > libtorch/build-hash
|
(pushd "$PYTORCH_ROOT" && git rev-parse HEAD) > libtorch/build-hash
|
||||||
echo "Installing libtorch in ${PYTORCH_ROOT}/../../"
|
echo "Installing libtorch in ${PYTORCH_ROOT}/../../"
|
||||||
echo "deleteing old ${PYTORCH_ROOT}/../../libtorch"
|
echo "deleteing old ${PYTORCH_ROOT}/../../libtorch"
|
||||||
rm -rf ${PYTORCH_ROOT}/../../libtorch
|
rm -rf "${PYTORCH_ROOT}"/../../libtorch
|
||||||
mv libtorch ${PYTORCH_ROOT}/../../
|
mv libtorch "${PYTORCH_ROOT}"/../../
|
||||||
}
|
}
|
||||||
|
|
||||||
#main
|
#main
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -e
|
set -eu -o pipefail
|
||||||
|
|
||||||
if [ -z "$PYTHON" ]; then
|
if [ -z "$PYTHON" ]; then
|
||||||
PYTHON="$(which python)"
|
PYTHON="$(which python)"
|
||||||
|
@ -7,11 +7,11 @@ fi
|
||||||
version="$("$PYTHON" --version)"
|
version="$("$PYTHON" --version)"
|
||||||
echo "Using python: $PYTHON (version $version)"
|
echo "Using python: $PYTHON (version $version)"
|
||||||
|
|
||||||
repo_root="$(cd $(dirname $0)/.. && pwd)"
|
repo_root="$(cd "$(dirname "$0")"/.. && pwd)"
|
||||||
wheelhouse="$repo_root/wheelhouse"
|
wheelhouse="$repo_root/wheelhouse"
|
||||||
package_test_venv="$wheelhouse/package-test.venv"
|
package_test_venv="$wheelhouse/package-test.venv"
|
||||||
mkdir -p $wheelhouse
|
mkdir -p "$wheelhouse"
|
||||||
cd $wheelhouse
|
cd "$wheelhouse"
|
||||||
|
|
||||||
echo "---- BUILDING torch-mlir ----"
|
echo "---- BUILDING torch-mlir ----"
|
||||||
CMAKE_GENERATOR=Ninja \
|
CMAKE_GENERATOR=Ninja \
|
||||||
|
|
|
@ -11,7 +11,7 @@
|
||||||
|
|
||||||
set -eu -o errtrace
|
set -eu -o errtrace
|
||||||
|
|
||||||
project_dir="$(cd $(dirname $0)/.. && pwd)"
|
project_dir="$(cd "$(dirname "$0")"/.. && pwd)"
|
||||||
llvm_project_dir="$project_dir/externals/llvm-project"
|
llvm_project_dir="$project_dir/externals/llvm-project"
|
||||||
build_dir="$project_dir/build"
|
build_dir="$project_dir/build"
|
||||||
|
|
||||||
|
@ -21,7 +21,7 @@ cmake -GNinja -B"$build_dir" "$llvm_project_dir/llvm" \
|
||||||
-DLLVM_ENABLE_PROJECTS=mlir \
|
-DLLVM_ENABLE_PROJECTS=mlir \
|
||||||
-DLLVM_EXTERNAL_PROJECTS="torch-mlir;torch-mlir-dialects" \
|
-DLLVM_EXTERNAL_PROJECTS="torch-mlir;torch-mlir-dialects" \
|
||||||
-DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$project_dir" \
|
-DLLVM_EXTERNAL_TORCH_MLIR_SOURCE_DIR="$project_dir" \
|
||||||
-DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR=${project_dir}/externals/llvm-external-projects/torch-mlir-dialects \
|
-DLLVM_EXTERNAL_TORCH_MLIR_DIALECTS_SOURCE_DIR="${project_dir}"/externals/llvm-external-projects/torch-mlir-dialects \
|
||||||
-DMLIR_ENABLE_BINDINGS_PYTHON=ON \
|
-DMLIR_ENABLE_BINDINGS_PYTHON=ON \
|
||||||
-DLLVM_ENABLE_ASSERTIONS=ON \
|
-DLLVM_ENABLE_ASSERTIONS=ON \
|
||||||
-DLLVM_TARGETS_TO_BUILD=host
|
-DLLVM_TARGETS_TO_BUILD=host
|
||||||
|
|
|
@ -37,16 +37,14 @@
|
||||||
# to packaging to avoid stomping on development artifacts.
|
# to packaging to avoid stomping on development artifacts.
|
||||||
set -eu -o errtrace
|
set -eu -o errtrace
|
||||||
|
|
||||||
this_dir="$(cd $(dirname $0) && pwd)"
|
this_dir="$(cd "$(dirname "$0")" && pwd)"
|
||||||
script_name="$(basename $0)"
|
repo_root="$(cd "$this_dir"/../../ && pwd)"
|
||||||
repo_root="$(cd $this_dir/../../ && pwd)"
|
|
||||||
script_name="$(basename $0)"
|
|
||||||
manylinux_docker_image="${manylinux_docker_image:-stellaraccident/manylinux2014_x86_64-bazel-5.1.0:latest}"
|
manylinux_docker_image="${manylinux_docker_image:-stellaraccident/manylinux2014_x86_64-bazel-5.1.0:latest}"
|
||||||
python_versions="${TM_PYTHON_VERSIONS:-cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310}"
|
python_versions="${TM_PYTHON_VERSIONS:-cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310}"
|
||||||
output_dir="${output_dir:-${this_dir}/wheelhouse}"
|
output_dir="${output_dir:-${this_dir}/wheelhouse}"
|
||||||
packages="${packages:-torch-mlir}"
|
packages="${packages:-torch-mlir}"
|
||||||
|
|
||||||
PKG_VER_FILE=${repo_root}/torch_mlir_package_version ; [ -f $PKG_VER_FILE ] && . $PKG_VER_FILE
|
PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE"
|
||||||
export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
|
export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
|
||||||
echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}"
|
echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}"
|
||||||
|
|
||||||
|
@ -63,7 +61,7 @@ function run_on_host() {
|
||||||
-e "TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" \
|
-e "TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION}" \
|
||||||
-e "python_versions=${python_versions}" \
|
-e "python_versions=${python_versions}" \
|
||||||
-e "packages=${packages}" \
|
-e "packages=${packages}" \
|
||||||
${manylinux_docker_image} \
|
"${manylinux_docker_image}" \
|
||||||
-- bash /main_checkout/torch-mlir/build_tools/python_deploy/build_linux_packages.sh
|
-- bash /main_checkout/torch-mlir/build_tools/python_deploy/build_linux_packages.sh
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -86,9 +84,9 @@ function run_in_docker() {
|
||||||
echo ":::: Python version $(python --version)"
|
echo ":::: Python version $(python --version)"
|
||||||
case "$package" in
|
case "$package" in
|
||||||
torch-mlir)
|
torch-mlir)
|
||||||
clean_wheels torch_mlir $python_version
|
clean_wheels torch_mlir "$python_version"
|
||||||
build_torch_mlir
|
build_torch_mlir
|
||||||
#run_audit_wheel torch_mlir $python_version
|
#run_audit_wheel torch_mlir "$python_version"
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Unrecognized package '$package'"
|
echo "Unrecognized package '$package'"
|
||||||
|
@ -112,15 +110,15 @@ function run_audit_wheel() {
|
||||||
local python_version="$2"
|
local python_version="$2"
|
||||||
generic_wheel="/wheelhouse/${wheel_basename}-*-${python_version}-linux_x86_64.whl"
|
generic_wheel="/wheelhouse/${wheel_basename}-*-${python_version}-linux_x86_64.whl"
|
||||||
echo ":::: Auditwheel $generic_wheel"
|
echo ":::: Auditwheel $generic_wheel"
|
||||||
auditwheel repair -w /wheelhouse $generic_wheel
|
auditwheel repair -w /wheelhouse "$generic_wheel"
|
||||||
rm $generic_wheel
|
rm "$generic_wheel"
|
||||||
}
|
}
|
||||||
|
|
||||||
function clean_wheels() {
|
function clean_wheels() {
|
||||||
local wheel_basename="$1"
|
local wheel_basename="$1"
|
||||||
local python_version="$2"
|
local python_version="$2"
|
||||||
echo ":::: Clean wheels $wheel_basename $python_version"
|
echo ":::: Clean wheels $wheel_basename $python_version"
|
||||||
rm -f /wheelhouse/${wheel_basename}-*-${python_version}-*.whl
|
rm -f /wheelhouse/"${wheel_basename}"-*-"${python_version}"-*.whl
|
||||||
}
|
}
|
||||||
|
|
||||||
# Trampoline to the docker container if running on the host.
|
# Trampoline to the docker container if running on the host.
|
||||||
|
|
|
@ -18,13 +18,13 @@
|
||||||
|
|
||||||
set -eu -o errtrace
|
set -eu -o errtrace
|
||||||
|
|
||||||
this_dir="$(cd $(dirname $0) && pwd)"
|
this_dir="$(cd "$(dirname "$0")" && pwd)"
|
||||||
repo_root="$(cd $this_dir/../../ && pwd)"
|
repo_root="$(cd "$this_dir"/../../ && pwd)"
|
||||||
python_versions="${TORCH_MLIR_PYTHON_VERSIONS:-3.9 3.10}"
|
python_versions="${TORCH_MLIR_PYTHON_VERSIONS:-3.9 3.10}"
|
||||||
output_dir="${output_dir:-${this_dir}/wheelhouse}"
|
output_dir="${output_dir:-${this_dir}/wheelhouse}"
|
||||||
packages="${packages:-torch-mlir}"
|
packages="${packages:-torch-mlir}"
|
||||||
|
|
||||||
PKG_VER_FILE=${repo_root}/torch_mlir_package_version ; [ -f $PKG_VER_FILE ] && . $PKG_VER_FILE
|
PKG_VER_FILE="${repo_root}"/torch_mlir_package_version ; [ -f "$PKG_VER_FILE" ] && . "$PKG_VER_FILE"
|
||||||
export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
|
export TORCH_MLIR_PYTHON_PACKAGE_VERSION="${TORCH_MLIR_PYTHON_PACKAGE_VERSION:-0.0.1}"
|
||||||
echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}"
|
echo "Setting torch-mlir Python Package version to: ${TORCH_MLIR_PYTHON_PACKAGE_VERSION}"
|
||||||
|
|
||||||
|
@ -53,9 +53,9 @@ function run() {
|
||||||
echo ":::: Python version $(python3 --version)"
|
echo ":::: Python version $(python3 --version)"
|
||||||
case "$package" in
|
case "$package" in
|
||||||
torch-mlir)
|
torch-mlir)
|
||||||
clean_wheels torch_mlir $python_version
|
clean_wheels torch_mlir "$python_version"
|
||||||
build_torch_mlir torch_mlir $python_version
|
build_torch_mlir torch_mlir "$python_version"
|
||||||
run_audit_wheel torch_mlir $python_version
|
run_audit_wheel torch_mlir "$python_version"
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Unrecognized package '$package'"
|
echo "Unrecognized package '$package'"
|
||||||
|
@ -69,45 +69,45 @@ function run() {
|
||||||
function build_torch_mlir() {
|
function build_torch_mlir() {
|
||||||
local wheel_basename="$1"
|
local wheel_basename="$1"
|
||||||
local python_version="$2"
|
local python_version="$2"
|
||||||
rm -rf $output_dir/build_venv
|
rm -rf "$output_dir"/build_venv
|
||||||
python${python_version} -m venv $output_dir/build_venv
|
python"${python_version}" -m venv "$output_dir"/build_venv
|
||||||
source $output_dir/build_venv/bin/activate
|
source "$output_dir"/build_venv/bin/activate
|
||||||
python${python_version} -m pip install -U pip
|
python"${python_version}" -m pip install -U pip
|
||||||
python${python_version} -m pip install -r $repo_root/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
|
python"${python_version}" -m pip install -r "$repo_root"/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
|
||||||
CMAKE_GENERATOR=Ninja \
|
CMAKE_GENERATOR=Ninja \
|
||||||
TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \
|
TORCH_MLIR_PYTHON_PACKAGE_VERSION=${TORCH_MLIR_PYTHON_PACKAGE_VERSION} \
|
||||||
MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \
|
MACOSX_DEPLOYMENT_TARGET=$MACOSX_DEPLOYMENT_TARGET \
|
||||||
CMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \
|
CMAKE_OSX_ARCHITECTURES=$CMAKE_OSX_ARCHITECTURES \
|
||||||
python${python_version} -m pip wheel -v -w $output_dir $repo_root --extra-index-url https://download.pytorch.org/whl/nightly/cpu
|
python"${python_version}" -m pip wheel -v -w "$output_dir" "$repo_root" --extra-index-url https://download.pytorch.org/whl/nightly/cpu
|
||||||
deactivate
|
deactivate
|
||||||
rm -rf $output_dir/build_venv
|
rm -rf "$output_dir"/build_venv
|
||||||
}
|
}
|
||||||
|
|
||||||
function clean_wheels() {
|
function clean_wheels() {
|
||||||
local wheel_basename="$1"
|
local wheel_basename="$1"
|
||||||
local python_version="$2"
|
local python_version="$2"
|
||||||
echo ":::: Clean wheels $wheel_basename $python_version"
|
echo ":::: Clean wheels $wheel_basename $python_version"
|
||||||
rm -rf $repo_root/build/
|
rm -rf "$repo_root"/build/
|
||||||
rm -f $output_dir/${wheel_basename}-*-${python_version//./}-*.whl
|
rm -f "$output_dir"/"${wheel_basename}"-*-"${python_version//./}"-*.whl
|
||||||
}
|
}
|
||||||
|
|
||||||
function run_audit_wheel() {
|
function run_audit_wheel() {
|
||||||
set +x
|
set +x
|
||||||
local wheel_basename="$1"
|
local wheel_basename="$1"
|
||||||
local python_version="$2"
|
local python_version="$2"
|
||||||
generic_wheel=`ls $output_dir/${wheel_basename}-* | grep ${python_version//./}`
|
generic_wheel=$(ls "$output_dir"/"${wheel_basename}"-* | grep "${python_version//./}")
|
||||||
echo "Looking for $generic_wheel"
|
echo "Looking for $generic_wheel"
|
||||||
if [ -f "$generic_wheel" ]; then
|
if [ -f "$generic_wheel" ]; then
|
||||||
echo "$generic_wheel found. Delocating it.."
|
echo "$generic_wheel found. Delocating it.."
|
||||||
rm -rf $output_dir/test_venv
|
rm -rf "$output_dir"/test_venv
|
||||||
python${python_version} -m venv $output_dir/test_venv
|
python"${python_version}" -m venv "$output_dir"/test_venv
|
||||||
source $output_dir/test_venv/bin/activate
|
source "$output_dir"/test_venv/bin/activate
|
||||||
python${python_version} -m pip install -U pip
|
python"${python_version}" -m pip install -U pip
|
||||||
python${python_version} -m pip install -r $repo_root/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
|
python"${python_version}" -m pip install -r "$repo_root"/requirements.txt --extra-index-url https://download.pytorch.org/whl/nightly/cpu
|
||||||
python${python_version} -m pip install $generic_wheel --extra-index-url https://download.pytorch.org/whl/nightly/cpu
|
python"${python_version}" -m pip install "$generic_wheel" --extra-index-url https://download.pytorch.org/whl/nightly/cpu
|
||||||
DYLD_LIBRARY_PATH=$repo_root/libtorch/lib delocate-wheel -v $generic_wheel
|
DYLD_LIBRARY_PATH="$repo_root"/libtorch/lib delocate-wheel -v "$generic_wheel"
|
||||||
deactivate
|
deactivate
|
||||||
rm -rf $output_dir/test_venv
|
rm -rf "$output_dir"/test_venv
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -13,14 +13,14 @@
|
||||||
|
|
||||||
set -eu -o errtrace
|
set -eu -o errtrace
|
||||||
|
|
||||||
this_dir="$(cd $(dirname $0) && pwd)"
|
this_dir="$(cd "$(dirname "$0")" && pwd)"
|
||||||
repo_root="$(cd $this_dir/../../ && pwd)"
|
repo_root="$(cd "$this_dir"/../../ && pwd)"
|
||||||
output_dir="${output_dir:-${this_dir}/wheelhouse}"
|
output_dir="${output_dir:-${this_dir}/wheelhouse}"
|
||||||
rm -rf ${output_dir}
|
rm -rf "${output_dir}"
|
||||||
|
|
||||||
git fetch --all
|
git fetch --all
|
||||||
latest_snapshot=$(git for-each-ref --sort=creatordate --format '%(refname:short)' refs/tags | tail -n 1)
|
latest_snapshot=$(git for-each-ref --sort=creatordate --format '%(refname:short)' refs/tags | tail -n 1)
|
||||||
git checkout ${latest_snapshot}
|
git checkout "${latest_snapshot}"
|
||||||
git submodule update --init
|
git submodule update --init
|
||||||
package_version=${latest_snapshot#"snapshot-"}
|
package_version=${latest_snapshot#"snapshot-"}
|
||||||
echo "Latest snapshot tag is: ${latest_snapshot}"
|
echo "Latest snapshot tag is: ${latest_snapshot}"
|
||||||
|
@ -35,6 +35,6 @@ TORCH_MLIR_OSX_ARCH=arm64 \
|
||||||
TORCH_MLIR_OSX_TARGET=11.0 \
|
TORCH_MLIR_OSX_TARGET=11.0 \
|
||||||
TORCH_MLIR_PYTHON_PACKAGE_VERSION="${package_version}" \
|
TORCH_MLIR_PYTHON_PACKAGE_VERSION="${package_version}" \
|
||||||
TORCH_MLIR_PYTHON_VERSIONS="${TORCH_MLIR_PYTHON_VERSIONS}" \
|
TORCH_MLIR_PYTHON_VERSIONS="${TORCH_MLIR_PYTHON_VERSIONS}" \
|
||||||
${repo_root}/build_tools/python_deploy/build_macos_packages.sh
|
"${repo_root}"/build_tools/python_deploy/build_macos_packages.sh
|
||||||
|
|
||||||
gh release upload ${latest_snapshot} ${repo_root}/build_tools/python_deploy/wheelhouse/torch*.whl
|
gh release upload "${latest_snapshot}" "${repo_root}"/build_tools/python_deploy/wheelhouse/torch*.whl
|
||||||
|
|
|
@ -11,7 +11,7 @@
|
||||||
# Usage:
|
# Usage:
|
||||||
# sudo install_macos_deps.sh
|
# sudo install_macos_deps.sh
|
||||||
|
|
||||||
set -e -o pipefail
|
set -eu -o pipefail
|
||||||
|
|
||||||
if [[ "$(whoami)" != "root" ]]; then
|
if [[ "$(whoami)" != "root" ]]; then
|
||||||
echo "ERROR: Must setup deps as root"
|
echo "ERROR: Must setup deps as root"
|
||||||
|
@ -37,11 +37,11 @@ for python_spec in $PYTHON_SPECS; do
|
||||||
|
|
||||||
# Install Python.
|
# Install Python.
|
||||||
if ! [ -x "$python_exe" ]; then
|
if ! [ -x "$python_exe" ]; then
|
||||||
package_basename="$(basename $url)"
|
package_basename="$(basename "$url")"
|
||||||
download_path="/tmp/torch_mlir_python_install/$package_basename"
|
download_path="/tmp/torch_mlir_python_install/$package_basename"
|
||||||
mkdir -p "$(dirname $download_path)"
|
mkdir -p "$(dirname "$download_path")"
|
||||||
echo "Downloading $url -> $download_path"
|
echo "Downloading $url -> $download_path"
|
||||||
curl $url -o "$download_path"
|
curl "$url" -o "$download_path"
|
||||||
|
|
||||||
echo "Installing $download_path"
|
echo "Installing $download_path"
|
||||||
installer -pkg "$download_path" -target /
|
installer -pkg "$download_path" -target /
|
||||||
|
|
|
@ -13,13 +13,13 @@ fi
|
||||||
|
|
||||||
venv_dir=$1
|
venv_dir=$1
|
||||||
serialized_test_dir=$2
|
serialized_test_dir=$2
|
||||||
here="$(realpath $(dirname $0))"
|
here="$(realpath "$(dirname "$0")")"
|
||||||
torch_mlir_src_root="$here/../../"
|
torch_mlir_src_root="$here/../../"
|
||||||
|
|
||||||
mkdir -p $venv_dir
|
mkdir -p "$venv_dir"
|
||||||
mkdir -p $serialized_test_dir
|
mkdir -p "$serialized_test_dir"
|
||||||
python3 -m venv $venv_dir
|
python3 -m venv "$venv_dir"
|
||||||
source $venv_dir/bin/activate
|
source "$venv_dir"/bin/activate
|
||||||
|
|
||||||
# latest torch-version and torch-vision module is required.
|
# latest torch-version and torch-vision module is required.
|
||||||
python3 -m pip install --upgrade -r "$torch_mlir_src_root/requirements.txt"
|
python3 -m pip install --upgrade -r "$torch_mlir_src_root/requirements.txt"
|
||||||
|
@ -38,4 +38,4 @@ cd "$torch_mlir_src_root"
|
||||||
export PYTHONPATH=${PYTHONPATH-}
|
export PYTHONPATH=${PYTHONPATH-}
|
||||||
source "$torch_mlir_src_root/.env"
|
source "$torch_mlir_src_root/.env"
|
||||||
|
|
||||||
python3 -m build_tools.torchscript_e2e_heavydep_tests.main --output_dir=$serialized_test_dir
|
python3 -m build_tools.torchscript_e2e_heavydep_tests.main --output_dir="$serialized_test_dir"
|
||||||
|
|
|
@ -9,9 +9,9 @@
|
||||||
# For more information on supporting custom operators, see:
|
# For more information on supporting custom operators, see:
|
||||||
# ${TORCH_MLIR}/python/torch_mlir/_torch_mlir_custom_op_example/README.md
|
# ${TORCH_MLIR}/python/torch_mlir/_torch_mlir_custom_op_example/README.md
|
||||||
|
|
||||||
set -eo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
src_dir="$(realpath $(dirname $0)/..)"
|
src_dir="$(realpath "$(dirname "$0")"/..)"
|
||||||
build_dir="$(realpath "${TORCH_MLIR_BUILD_DIR:-$src_dir/build}")"
|
build_dir="$(realpath "${TORCH_MLIR_BUILD_DIR:-$src_dir/build}")"
|
||||||
torch_transforms_cpp_dir="${src_dir}/lib/Dialect/Torch/Transforms"
|
torch_transforms_cpp_dir="${src_dir}/lib/Dialect/Torch/Transforms"
|
||||||
python_packages_dir="${build_dir}/tools/torch-mlir/python_packages"
|
python_packages_dir="${build_dir}/tools/torch-mlir/python_packages"
|
||||||
|
|
|
@ -9,9 +9,9 @@
|
||||||
# For more information on supporting custom operators, see:
|
# For more information on supporting custom operators, see:
|
||||||
# ${TORCH_MLIR}/python/torch_mlir/_torch_mlir_custom_op_example/README.md
|
# ${TORCH_MLIR}/python/torch_mlir/_torch_mlir_custom_op_example/README.md
|
||||||
|
|
||||||
set -eo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
src_dir="$(realpath $(dirname $0)/..)"
|
src_dir="$(realpath "$(dirname "$0")"/..)"
|
||||||
build_dir="$(realpath "${TORCH_MLIR_BUILD_DIR:-$src_dir/build}")"
|
build_dir="$(realpath "${TORCH_MLIR_BUILD_DIR:-$src_dir/build}")"
|
||||||
torch_ir_include_dir="${src_dir}/include/torch-mlir/Dialect/Torch/IR"
|
torch_ir_include_dir="${src_dir}/include/torch-mlir/Dialect/Torch/IR"
|
||||||
python_packages_dir="${build_dir}/tools/torch-mlir/python_packages"
|
python_packages_dir="${build_dir}/tools/torch-mlir/python_packages"
|
||||||
|
|
|
@ -4,12 +4,14 @@
|
||||||
# For arbitrary build/install directories, set the env variables:
|
# For arbitrary build/install directories, set the env variables:
|
||||||
# - TORCH_MLIR_BUILD_DIR
|
# - TORCH_MLIR_BUILD_DIR
|
||||||
|
|
||||||
|
set -eu -o pipefail
|
||||||
|
|
||||||
portable_realpath() {
|
portable_realpath() {
|
||||||
# Create the directory if needed so that the `cd` doesn't fail.
|
# Create the directory if needed so that the `cd` doesn't fail.
|
||||||
mkdir -p $1 && cd $1 && pwd
|
mkdir -p "$1" && cd "$1" && pwd
|
||||||
}
|
}
|
||||||
|
|
||||||
td="$(portable_realpath $(dirname $0)/..)"
|
td="$(portable_realpath "$(dirname "$0")"/..)"
|
||||||
build_dir="$(portable_realpath "${TORCH_MLIR_BUILD_DIR:-$td/build}")"
|
build_dir="$(portable_realpath "${TORCH_MLIR_BUILD_DIR:-$td/build}")"
|
||||||
python_packages_dir="$build_dir/tools/torch-mlir/python_packages"
|
python_packages_dir="$build_dir/tools/torch-mlir/python_packages"
|
||||||
|
|
||||||
|
|
|
@ -81,7 +81,11 @@ if(TORCH_MLIR_ENABLE_JIT_IR_IMPORTER)
|
||||||
LIBTORCH_VARIANT=${LIBTORCH_VARIANT}
|
LIBTORCH_VARIANT=${LIBTORCH_VARIANT}
|
||||||
CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
|
CMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES}
|
||||||
${CMAKE_CURRENT_SOURCE_DIR}/../build_tools/build_libtorch.sh
|
${CMAKE_CURRENT_SOURCE_DIR}/../build_tools/build_libtorch.sh
|
||||||
|
RESULT_VARIABLE _result
|
||||||
)
|
)
|
||||||
|
if(_result)
|
||||||
|
message(FATAL_ERROR "Failed to run `build_libtorch.sh`")
|
||||||
|
endif()
|
||||||
set(TORCH_INSTALL_PREFIX "libtorch")
|
set(TORCH_INSTALL_PREFIX "libtorch")
|
||||||
add_subdirectory(torch_mlir/dialects/torch/importer/jit_ir)
|
add_subdirectory(torch_mlir/dialects/torch/importer/jit_ir)
|
||||||
add_subdirectory(torch_mlir_e2e_test)
|
add_subdirectory(torch_mlir_e2e_test)
|
||||||
|
|
|
@ -59,16 +59,24 @@ function(TorchMLIRConfigurePyTorch)
|
||||||
execute_process(
|
execute_process(
|
||||||
COMMAND ${Python3_EXECUTABLE}
|
COMMAND ${Python3_EXECUTABLE}
|
||||||
-c "import torch; import sys; sys.stdout.write('1' if torch.compiled_with_cxx11_abi() else '0')"
|
-c "import torch; import sys; sys.stdout.write('1' if torch.compiled_with_cxx11_abi() else '0')"
|
||||||
|
RESULT_VARIABLE _result
|
||||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
OUTPUT_VARIABLE _use_cxx11_abi)
|
OUTPUT_VARIABLE _use_cxx11_abi)
|
||||||
|
if(_result)
|
||||||
|
message(FATAL_ERROR "Failed to determine C++ Dual ABI")
|
||||||
|
endif()
|
||||||
message(STATUS "PyTorch C++ Dual ABI setting: \"${_use_cxx11_abi}\"")
|
message(STATUS "PyTorch C++ Dual ABI setting: \"${_use_cxx11_abi}\"")
|
||||||
|
|
||||||
# Check ABI compatibility version
|
# Check ABI compatibility version
|
||||||
execute_process(
|
execute_process(
|
||||||
COMMAND ${Python3_EXECUTABLE}
|
COMMAND ${Python3_EXECUTABLE}
|
||||||
-c "import torch; import sys; abi=torch._C._PYBIND11_BUILD_ABI; abi.startswith('_cxxabi10') or sys.exit(1); sys.stdout.write(str(abi[-2:]))"
|
-c "import torch; import sys; abi=torch._C._PYBIND11_BUILD_ABI; abi.startswith('_cxxabi10') or sys.exit(1); sys.stdout.write(str(abi[-2:]))"
|
||||||
|
RESULT_VARIABLE _result
|
||||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
OUTPUT_VARIABLE _cxx_abi_version)
|
OUTPUT_VARIABLE _cxx_abi_version)
|
||||||
|
if(_result)
|
||||||
|
message(FATAL_ERROR "Failed to determine C++ ABI version")
|
||||||
|
endif()
|
||||||
message(STATUS "PyTorch C++ ABI version: \"${_cxx_abi_version}\"")
|
message(STATUS "PyTorch C++ ABI version: \"${_cxx_abi_version}\"")
|
||||||
|
|
||||||
# Specialize compile flags for compiler
|
# Specialize compile flags for compiler
|
||||||
|
@ -91,15 +99,23 @@ function(TorchMLIRConfigureLibTorch)
|
||||||
# Check dual ABI setting first
|
# Check dual ABI setting first
|
||||||
execute_process(
|
execute_process(
|
||||||
COMMAND bash "-c" "cat ${TORCH_INSTALL_PREFIX}/share/cmake/Torch/TorchConfig.cmake | egrep -o '_GLIBCXX_USE_CXX11_ABI=[0-1]' | egrep -o '.$'"
|
COMMAND bash "-c" "cat ${TORCH_INSTALL_PREFIX}/share/cmake/Torch/TorchConfig.cmake | egrep -o '_GLIBCXX_USE_CXX11_ABI=[0-1]' | egrep -o '.$'"
|
||||||
|
RESULT_VARIABLE _result
|
||||||
OUTPUT_VARIABLE _use_cxx11_abi
|
OUTPUT_VARIABLE _use_cxx11_abi
|
||||||
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||||
|
if(_result)
|
||||||
|
message(FATAL_ERROR "Failed to determine LibTorch C++ Dual ABI")
|
||||||
|
endif()
|
||||||
message(STATUS "LibTorch C++ Dual ABI setting: \"${_use_cxx11_abi}\"")
|
message(STATUS "LibTorch C++ Dual ABI setting: \"${_use_cxx11_abi}\"")
|
||||||
|
|
||||||
# Check ABI compatibility version
|
# Check ABI compatibility version
|
||||||
execute_process(
|
execute_process(
|
||||||
COMMAND bash "-c" "strings ${TORCH_INSTALL_PREFIX}/lib/libtorch_python.so | egrep '^_cxxabi[0-9]{4}' | egrep -o '..$'"
|
COMMAND bash "-c" "strings ${TORCH_INSTALL_PREFIX}/lib/libtorch_python.so | egrep '^_cxxabi[0-9]{4}' | egrep -o '..$'"
|
||||||
|
RESULT_VARIABLE _result
|
||||||
OUTPUT_VARIABLE _cxx_abi_version
|
OUTPUT_VARIABLE _cxx_abi_version
|
||||||
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
OUTPUT_STRIP_TRAILING_WHITESPACE)
|
||||||
|
if(_result)
|
||||||
|
message(FATAL_ERROR "Failed to determine LibTorch C++ ABI version")
|
||||||
|
endif()
|
||||||
message(STATUS "LibTorch C++ ABI version: \"${_cxx_abi_version}\"")
|
message(STATUS "LibTorch C++ ABI version: \"${_cxx_abi_version}\"")
|
||||||
|
|
||||||
# Specialize compile flags for compiler
|
# Specialize compile flags for compiler
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
src_dir="$(realpath $(dirname $0)/..)"
|
src_dir="$(realpath "$(dirname "$0")"/..)"
|
||||||
|
|
||||||
cd "$src_dir"
|
cd "$src_dir"
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue