From b0623b779337c2c1711bb630b201b7192d6d8041 Mon Sep 17 00:00:00 2001 From: Stella Laurenzo Date: Tue, 24 Nov 2020 13:10:27 -0800 Subject: [PATCH] Bump LLVM version to 4f5355ee73626f8b8fe6bf0dd6d167fea7628a2c. * Incorporates changes around LLVM StringRef. * Ports fix in upstream pybind11 detection. * Disables CI hack due to broken pybind detection. --- .github/workflows/buildAndTest.yml | 12 -------- cmake/modules/NpcompDetectPythonEnv.cmake | 2 +- external/llvm-project | 2 +- .../pytorch/csrc/builder/acap_dispatch.cpp | 5 ++-- .../pytorch/csrc/builder/func_builder.cpp | 24 ++++++++-------- frontends/pytorch/csrc/builder/func_builder.h | 5 +++- .../pytorch/csrc/builder/graph_importer.cpp | 5 +++- frontends/pytorch/csrc/builder/mlir_utils.h | 28 +++++++++++++++++++ 8 files changed, 53 insertions(+), 30 deletions(-) create mode 100644 frontends/pytorch/csrc/builder/mlir_utils.h diff --git a/.github/workflows/buildAndTest.yml b/.github/workflows/buildAndTest.yml index b87cf8084..1921e493f 100644 --- a/.github/workflows/buildAndTest.yml +++ b/.github/workflows/buildAndTest.yml @@ -34,9 +34,6 @@ jobs: mkdir -p ${LLVM}/build mkdir -p ${LLVM}/install cd ${LLVM}/build - # TODO: This should be auto-detecting. Figure out why it isn't (only seems to happen in the CI). - export CMAKE_PREFIX_PATH="$(python3 -c "import pybind11;print(pybind11.get_cmake_dir(), end='')")" - echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" cmake ../llvm -DLLVM_BUILD_EXAMPLES=OFF -DLLVM_TARGETS_TO_BUILD="host" -DCMAKE_INSTALL_PREFIX=../install -DLLVM_ENABLE_PROJECTS='mlir' -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_ENABLE_OCAMLDOC=OFF -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_INSTALL_UTILS=ON -DLLVM_BUILD_TOOLS=ON -DLLVM_INCLUDE_TESTS=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_ENABLE_LLD=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DMLIR_BINDINGS_PYTHON_ENABLED=ON cmake --build . --target install -- -j$(nproc) build: @@ -69,26 +66,17 @@ jobs: mkdir ${LLVM}/build mkdir ${LLVM}/install cd ${LLVM}/build - # TODO: This should be auto-detecting. Figure out why it isn't (only seems to happen in the CI). - export CMAKE_PREFIX_PATH="$(python3 -c "import pybind11;print(pybind11.get_cmake_dir(), end='')")" - echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" cmake ../llvm -DLLVM_BUILD_EXAMPLES=OFF -DLLVM_TARGETS_TO_BUILD="host" -DCMAKE_INSTALL_PREFIX=../install -DLLVM_ENABLE_PROJECTS='mlir' -DLLVM_OPTIMIZED_TABLEGEN=ON -DLLVM_ENABLE_OCAMLDOC=OFF -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_INSTALL_UTILS=ON -DLLVM_INCLUDE_TESTS=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_ENABLE_LLD=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_RTTI=ON -DLLVM_BUILD_LLVM_DYLIB=ON -DMLIR_BINDINGS_PYTHON_ENABLED=ON cmake --build . --target install -- -j$(nproc) - name: Build and Test npcomp (Assert) run: | mkdir build_assert cd build_assert - # TODO: This should be auto-detecting. Figure out why it isn't (only seems to happen in the CI). - export CMAKE_PREFIX_PATH="$(python3 -c "import pybind11;print(pybind11.get_cmake_dir(), end='')")" - echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" cmake .. -DCMAKE_BUILD_TYPE=Debug -DLLVM_ENABLE_ASSERTIONS=ON -DMLIR_DIR=../${LLVM}/install/lib/cmake/mlir/ -DLLVM_DIR=../${LLVM}/install/lib/cmake/llvm/ -DCMAKE_LINKER=lld -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DPYTHON_EXECUTABLE=/usr/bin/python3 -DLLVM_EXTERNAL_LIT=`pwd`/../${LLVM}/build/bin/llvm-lit make check-npcomp -j$(nproc) - name: Build and Test npcomp (Release) run: | mkdir build_release cd build_release - # TODO: This should be auto-detecting. Figure out why it isn't (only seems to happen in the CI). - export CMAKE_PREFIX_PATH="$(python3 -c "import pybind11;print(pybind11.get_cmake_dir(), end='')")" - echo "CMAKE_PREFIX_PATH=$CMAKE_PREFIX_PATH" cmake .. -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=OFF -DMLIR_DIR=../${LLVM}/install/lib/cmake/mlir/ -DLLVM_DIR=../${LLVM}/install/lib/cmake/llvm/ -DCMAKE_LINKER=lld -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DPYTHON_EXECUTABLE=/usr/bin/python3 -DLLVM_EXTERNAL_LIT=`pwd`/../${LLVM}/build/bin/llvm-lit make check-npcomp -j$(nproc) diff --git a/cmake/modules/NpcompDetectPythonEnv.cmake b/cmake/modules/NpcompDetectPythonEnv.cmake index 3bdc975a8..9367d0de5 100644 --- a/cmake/modules/NpcompDetectPythonEnv.cmake +++ b/cmake/modules/NpcompDetectPythonEnv.cmake @@ -14,7 +14,7 @@ function(npcomp_detect_pybind11_install) else() message(CHECK_START "Checking for pybind11 in python path...") execute_process( - COMMAND "${PYTHON_EXECUTABLE}" + COMMAND "${Python3_EXECUTABLE}" -c "import pybind11;print(pybind11.get_cmake_dir(), end='')" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} RESULT_VARIABLE STATUS diff --git a/external/llvm-project b/external/llvm-project index f4f8a67aa..4f5355ee7 160000 --- a/external/llvm-project +++ b/external/llvm-project @@ -1 +1 @@ -Subproject commit f4f8a67aaf13bc66a2b7d55561b14a3724a5e0de +Subproject commit 4f5355ee73626f8b8fe6bf0dd6d167fea7628a2c diff --git a/frontends/pytorch/csrc/builder/acap_dispatch.cpp b/frontends/pytorch/csrc/builder/acap_dispatch.cpp index 9bb09c3ca..c5cbaadd6 100644 --- a/frontends/pytorch/csrc/builder/acap_dispatch.cpp +++ b/frontends/pytorch/csrc/builder/acap_dispatch.cpp @@ -7,6 +7,7 @@ #include "acap_dispatch.h" #include "debug.h" +#include "mlir_utils.h" #include "mlir-c/StandardAttributes.h" #include "mlir-c/StandardTypes.h" @@ -603,8 +604,8 @@ MlirValue AcapController::importTensorByValue(at::Tensor tensor) { // Create an array from the tensor constant via the // numpy.create_array_from_tensor op. MlirType constArrayType = npcompNdArrayTypeGetFromShaped(shapedType); - MlirOperationState state = - mlirOperationStateGet("numpy.create_array_from_tensor", loc); + MlirOperationState state = mlirOperationStateGet( + toMlirStringRef("numpy.create_array_from_tensor"), loc); mlirOperationStateAddOperands(&state, 1, &constTensorValue); mlirOperationStateAddResults(&state, 1, &constArrayType); MlirOperation constArrayOp = mlirOperationCreate(&state); diff --git a/frontends/pytorch/csrc/builder/func_builder.cpp b/frontends/pytorch/csrc/builder/func_builder.cpp index f8a79fadd..a7967e31d 100644 --- a/frontends/pytorch/csrc/builder/func_builder.cpp +++ b/frontends/pytorch/csrc/builder/func_builder.cpp @@ -17,7 +17,7 @@ using namespace torch_mlir; static MlirOperation createStandardConstant(MlirLocation loc, MlirType type, MlirAttribute value) { OperationStateHolder s("std.constant", loc); - MlirNamedAttribute valueAttr = mlirNamedAttributeGet("value", value); + MlirNamedAttribute valueAttr = mlirNamedAttributeGet(toMlirStringRef("value"), value); mlirOperationStateAddResults(s, 1, &type); mlirOperationStateAddAttributes(s, 1, &valueAttr); return s.createOperation(); @@ -30,7 +30,7 @@ KernelCallBuilder::KernelCallBuilder(MlirContext context, MlirLocation loc, kernelName(kernelName), schema(schema) { (void)this->context; // Preserve for future. MlirNamedAttribute kernelNameAttr = mlirNamedAttributeGet( - "kernelName", + toMlirStringRef("kernelName"), mlirStringAttrGet(context, kernelName.size(), kernelName.data())); mlirOperationStateAddAttributes(state, 1, &kernelNameAttr); addSchemaAttrs(); @@ -45,11 +45,11 @@ void KernelCallBuilder::addSchemaAttrs() { // sigIsMutable llvm::SmallVector attrs; attrs.push_back(mlirNamedAttributeGet( - "sigIsMutable", mlirBoolAttrGet(context, schema.is_mutable()))); + toMlirStringRef("sigIsMutable"), mlirBoolAttrGet(context, schema.is_mutable()))); attrs.push_back(mlirNamedAttributeGet( - "sigIsVararg", mlirBoolAttrGet(context, schema.is_vararg()))); + toMlirStringRef("sigIsVararg"), mlirBoolAttrGet(context, schema.is_vararg()))); attrs.push_back(mlirNamedAttributeGet( - "sigIsVarret", mlirBoolAttrGet(context, schema.is_varret()))); + toMlirStringRef("sigIsVarret"), mlirBoolAttrGet(context, schema.is_varret()))); // Arg types. llvm::SmallVector args; @@ -58,7 +58,7 @@ void KernelCallBuilder::addSchemaAttrs() { args.push_back(mlirStringAttrGet(context, typeStr.size(), typeStr.data())); } attrs.push_back(mlirNamedAttributeGet( - "sigArgTypes", mlirArrayAttrGet(context, args.size(), args.data()))); + toMlirStringRef("sigArgTypes"), mlirArrayAttrGet(context, args.size(), args.data()))); // Return types. llvm::SmallVector returns; @@ -68,7 +68,7 @@ void KernelCallBuilder::addSchemaAttrs() { mlirStringAttrGet(context, typeStr.size(), typeStr.data())); } attrs.push_back(mlirNamedAttributeGet( - "sigRetTypes", + toMlirStringRef("sigRetTypes"), mlirArrayAttrGet(context, returns.size(), returns.data()))); // Add attrs to op. @@ -204,13 +204,13 @@ FuncBuilder::createFunction(FuncBuilder::Inserter &inserter, // (this is fragile and reveals details that are not guaranteed). llvm::SmallVector funcAttrs; funcAttrs.push_back(mlirNamedAttributeGet( - "type", mlirTypeAttrGet(mlirFunctionTypeGet( + toMlirStringRef("type"), mlirTypeAttrGet(mlirFunctionTypeGet( context, inputTypes.size(), inputTypes.data(), /*numResults=*/0, /*results=*/nullptr)))); funcAttrs.push_back(mlirNamedAttributeGet( - "sym_name", mlirStringAttrGet(context, name.size(), name.data()))); + toMlirStringRef("sym_name"), mlirStringAttrGet(context, name.size(), name.data()))); - MlirOperationState state = mlirOperationStateGet("func", location); + MlirOperationState state = mlirOperationStateGet(toMlirStringRef("func"), location); mlirOperationStateAddAttributes(&state, funcAttrs.size(), funcAttrs.data()); { // Don't access these once ownership transferred. @@ -234,7 +234,7 @@ FuncBuilder::createFunction(FuncBuilder::Inserter &inserter, void FuncBuilder::rewriteFuncReturnTypes( llvm::SmallVectorImpl &resultTypes) { // Get inputs from current function type. - MlirAttribute funcTypeAttr = mlirOperationGetAttributeByName(funcOp, "type"); + MlirAttribute funcTypeAttr = mlirOperationGetAttributeByName(funcOp, toMlirStringRef("type")); assert(!mlirAttributeIsNull(funcTypeAttr) && "function missing 'type' attribute"); assert(mlirAttributeIsAType(funcTypeAttr) && @@ -250,7 +250,7 @@ void FuncBuilder::rewriteFuncReturnTypes( mlirFunctionTypeGet(context, inputTypes.size(), inputTypes.data(), resultTypes.size(), resultTypes.data()); MlirAttribute newFuncTypeAttr = mlirTypeAttrGet(newFuncType); - mlirOperationSetAttributeByName(funcOp, "type", newFuncTypeAttr); + mlirOperationSetAttributeByName(funcOp, toMlirStringRef("type"), newFuncTypeAttr); (void)newFuncTypeAttr; } diff --git a/frontends/pytorch/csrc/builder/func_builder.h b/frontends/pytorch/csrc/builder/func_builder.h index dc3dc8a47..fc38591d0 100644 --- a/frontends/pytorch/csrc/builder/func_builder.h +++ b/frontends/pytorch/csrc/builder/func_builder.h @@ -8,6 +8,8 @@ #ifndef NPCOMP_FRONTENDS_PYTORCH_CSRC_BUILDER_FUNC_BUILDER_H #define NPCOMP_FRONTENDS_PYTORCH_CSRC_BUILDER_FUNC_BUILDER_H +#include "mlir_utils.h" + #include "mlir-c/IR.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/StringRef.h" @@ -22,7 +24,8 @@ namespace torch_mlir { class OperationStateHolder { public: OperationStateHolder(const char *name, MlirLocation loc) - : state(mlirOperationStateGet(name, loc)) {} + : state( + mlirOperationStateGet(toMlirStringRef(name), loc)) {} OperationStateHolder(const OperationStateHolder &) = delete; OperationStateHolder(OperationStateHolder &&other) = delete; ~OperationStateHolder() { diff --git a/frontends/pytorch/csrc/builder/graph_importer.cpp b/frontends/pytorch/csrc/builder/graph_importer.cpp index 94dc47b97..6ff7abdb9 100644 --- a/frontends/pytorch/csrc/builder/graph_importer.cpp +++ b/frontends/pytorch/csrc/builder/graph_importer.cpp @@ -7,6 +7,8 @@ #include "graph_importer.h" +#include "mlir_utils.h" + #include "mlir-c/Diagnostics.h" #include "mlir-c/StandardAttributes.h" #include "mlir-c/StandardTypes.h" @@ -296,7 +298,8 @@ MlirLocation GraphImporter::extractCallstackLoc(torch::jit::Node *node, const std::string &file = std::get<0>(*flc); int line = std::get<1>(*flc); int col = std::get<2>(*flc); - return mlirLocationFileLineColGet(context(), file.c_str(), line, col); + return mlirLocationFileLineColGet(context(), toMlirStringRef(file), line, + col); } return useDefault ? defaultLoc : MlirLocation{nullptr}; diff --git a/frontends/pytorch/csrc/builder/mlir_utils.h b/frontends/pytorch/csrc/builder/mlir_utils.h new file mode 100644 index 000000000..49e73ccba --- /dev/null +++ b/frontends/pytorch/csrc/builder/mlir_utils.h @@ -0,0 +1,28 @@ +//===- mlir_utils.h ---------------------------------------------*- C++ -*-===// +// +// This file is licensed under a pytorch-style license +// See frontends/pytorch/LICENSE for license information. +// +//===----------------------------------------------------------------------===// + +#ifndef NPCOMP_FRONTENDS_PYTORCH_CSRC_MLIR_UTILS_H +#define NPCOMP_FRONTENDS_PYTORCH_CSRC_MLIR_UTILS_H + +#include +#include + +#include "mlir-c/Support.h" + +namespace torch_mlir { + +inline MlirStringRef toMlirStringRef(const std::string &s) { + return mlirStringRefCreate(s.data(), s.size()); +} + +inline MlirStringRef toMlirStringRef(const char *s) { + return mlirStringRefCreate(s, std::strlen(s)); +} + +} // namespace torch_mlir + +#endif // NPCOMP_FRONTENDS_PYTORCH_CSRC_MLIR_UTILS_H