Start reworking towards a shared library build.

* Need to have a dag of shared library deps in order to interop across python extensions (as presented in ODM).
* Introduced add_npcomp_library and friends to mirror the MLIR setup.
* Adds a libNPCOMP.so shared library.
* Redirects tools and extensions to link against libNPCOMP.so (instead of static libs).
* Moves all libraries to lib/, all binaries to bin/ and all python extensions to python/. The invariant is that the rpaths are setup to have a one level directory structure.
* Reworks the _torch_mlir extension to build like the others (still need to come up with a consolidated rule to do this instead of open coded).
* Includes an upstream version bump to pick up needed changes.

Sizes with dynamic linking (stripped, release, asserts enabled):
  libNPCOMP.so: 43M (includes much of the underlying LLVM codegen deps)
  libMLIR.so: 31M
  _npcomp.so: 1.6M (python extension)
  _torch_mlir.so: 670K (python extension)
  npcomp-capi-ir-test: 6.3K
  npcomp-opt: 351K
  npcomp-run-mlir: 461K
  mnist-playground: 530K

Still more can be done to normalize and optimize but this gets us structurally to the starting point.

CMakeLists.txt

@@ -28,6 +28,7 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules")
 
 option(NPCOMP_ENABLE_IREE "Enables the IREE backend (must configure location via IREE_DIR)." OFF)
 option(NPCOMP_ENABLE_REFJIT "Enables the reference JIT backend." ON)
+option(NPCOMP_BUILD_NPCOMP_DYLIB "Enables shared build of NPCOMP dylib (depends on LLVM/MLIR dylib support)" ON)
 set(NPCOMP_IREE_SRCDIR "" CACHE STRING "If building IREE, then setting this elects to build from a source directory (versus installed package)")
 
 #-------------------------------------------------------------------------------
@@ -66,6 +67,7 @@ list(APPEND CMAKE_MODULE_PATH "${LLVM_CMAKE_DIR}")
 include(TableGen)
 include(AddLLVM)
 include(AddMLIR)
+include(AddNPCOMP)
 include(HandleLLVMOptions)
 include_directories(${LLVM_INCLUDE_DIRS})
 include_directories(${MLIR_INCLUDE_DIRS})
@@ -148,7 +150,10 @@ add_dependencies(check-all check-npcomp)
 
 add_subdirectory(include/npcomp)
 add_subdirectory(lib)
-add_subdirectory(tools)
 add_subdirectory(python)
 add_subdirectory(test)
 add_subdirectory(frontends)
+
+# Tools needs to come late to ensure that NPCOMP_ALL_LIBS is populated.
+# Generally things after this point may depend on NPCOMP_ALL_LIBS or libNPCOMP.so.
+add_subdirectory(tools)

build_tools/install_mlir.sh

@@ -32,6 +32,7 @@ cmake -GNinja \
   "-H$LLVM_SRC_DIR/llvm" \
   "-B$build_mlir" \
   -DCMAKE_EXPORT_COMPILE_COMMANDS=TRUE \
+  -DLLVM_BUILD_LLVM_DYLIB=ON \
   -DLLVM_INSTALL_UTILS=ON \
   -DLLVM_ENABLE_PROJECTS=mlir \
   -DLLVM_TARGETS_TO_BUILD="X86" \

cmake/modules/AddNPCOMP.cmake

@@ -0,0 +1,71 @@
+# Declare an npcomp library which can be compiled in libNPCOMP.so.
+# This is adapted from add_mlir_library.
+function(add_npcomp_library name)
+  cmake_parse_arguments(ARG
+    "SHARED;EXCLUDE_FROM_LIBNPCOMP"
+    ""
+    "ADDITIONAL_HEADERS;DEPENDS;LINK_COMPONENTS;LINK_LIBS"
+  ${ARGN})
+  set(srcs)
+  # TODO: Port the source description logic for IDEs from add_mlir_library.
+
+
+  if(ARG_SHARED)
+    # Rule explicitly requested a shared library.
+    set(LIBTYPE SHARED)
+  else()
+    if(NOT ARG_EXCLUDE_FROM_LIBNPCOMP)
+      set_property(GLOBAL APPEND PROPERTY NPCOMP_STATIC_LIBS ${name})
+    endif()
+  endif()
+
+  # TODO: Enable npcomp header export.
+  # list(APPEND ARG_DEPENDS npcomp-generic-headers)
+  llvm_add_library(
+    ${name} ${LIBTYPE} ${ARG_UNPARSED_ARGUMENTS} ${srcs}
+    OBJECT
+    DEPENDS ${ARG_DEPENDS}
+    LINK_COMPONENTS ${ARG_LINK_COMPONENTS}
+    LINK_LIBS ${ARG_LINK_LIBS})
+
+  set_target_properties(${name} PROPERTIES FOLDER "NPCOMP libraries")
+endfunction()
+
+# Declare the library associated with a dialect.
+function(add_npcomp_dialect_library name)
+  set_property(GLOBAL APPEND PROPERTY NPCOMP_DIALECT_LIBS ${name})
+  # TODO: Add DEPENDS npcomp-headers
+  add_npcomp_library(${ARGV})
+endfunction()
+
+# Declare the library associated with a conversion.
+function(add_npcomp_conversion_library name)
+  set_property(GLOBAL APPEND PROPERTY NPCOMP_CONVERSION_LIBS ${name})
+  # TODO: Add DEPENDS npcomp-headers
+  add_npcomp_library(${ARGV})
+endfunction()
+
+function(add_npcomp_executable name)
+  add_executable(${ARGV})
+  llvm_update_compile_flags(${name})
+  add_link_opts( ${name} )
+  set_output_directory(${name}
+    BINARY_DIR ${PROJECT_BINARY_DIR}/bin
+    LIBRARY_DIR ${PROJECT_BINARY_DIR}/lib)
+  if (LLVM_PTHREAD_LIB)
+    # libpthreads overrides some standard library symbols, so main
+    # executable must be linked with it in order to provide consistent
+    # API for all shared libaries loaded by this executable.
+    target_link_libraries(${name} PRIVATE ${LLVM_PTHREAD_LIB})
+  endif()
+endfunction()
+
+function(npcomp_enable_exceptions name)
+  target_compile_options(${name} PRIVATE
+  $<$<OR:$<CXX_COMPILER_ID:Clang>,$<CXX_COMPILER_ID:AppleClang>,$<CXX_COMPILER_ID:GNU>>:
+    -fexceptions
+  >
+  $<$<CXX_COMPILER_ID:MSVC>:
+  /EHsc>
+  )
+endfunction()

external/llvm-project

@@ -1 +1 @@
-Subproject commit 4aa217160e5f06a96c6effc4950c3b402374de58
+Subproject commit e20792795065b2fb41128537314044f0a8f0a912

frontends/pytorch/csrc/CMakeLists.txt

@@ -1,3 +1,11 @@
+if(NPCOMP_ENABLE_TORCH_TYPE_DISPATCH)
+  add_subdirectory(type_dispatch)
+else()
+  add_subdirectory(c10_dispatch)
+endif()
+
+include(NpcompPython)
+
 include_directories(
   ${TORCH_INCLUDE_DIRS}
   ${TORCH_INSTALL_PREFIX}/include/TH
@@ -19,22 +27,26 @@ else()
   )
 endif()
 
-add_library(_torch_mlir SHARED
+add_library(NPCOMPTorchMLIRExt SHARED
   init_python_bindings.cpp
   )
-set_target_properties(_torch_mlir PROPERTIES PREFIX "")
-get_property(mlir_libs GLOBAL PROPERTY MLIR_ALL_LIBS)
-target_link_libraries(_torch_mlir
-  NPCOMPATenDialect
+target_link_libraries(NPCOMPTorchMLIRExt
   ${TORCH_LIBRARIES}
-  ${mlir_libs}
   ${PYTHON_LIBRARIES}
   ${torch_mlir_optional_libraries}
   torch_python
+
+  # NPCOMP shared library.
+  NPCOMP
+)
+
+set_target_properties(NPCOMPTorchMLIRExt PROPERTIES
+  LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/python
+  OUTPUT_NAME _torch_mlir
+  PREFIX "${PYTHON_MODULE_PREFIX}"
+  SUFFIX "${PYTHON_MODULE_EXTENSION}"
+  CXX_VISIBILITY_PRESET "hidden"
   )
 
-if(NPCOMP_ENABLE_TORCH_TYPE_DISPATCH)
-  add_subdirectory(type_dispatch)
-else()
-  add_subdirectory(c10_dispatch)
-endif()
+npcomp_python_target_compile_options(NPCOMPTorchMLIRExt)
+mlir_check_all_link_libraries(NPCOMPTorchMLIRExt)

frontends/pytorch/csrc/c10_dispatch/CMakeLists.txt

@@ -14,11 +14,8 @@ add_library(npcomp_torch_c10_dispatch_bindings
   python_bindings.cpp
 )
 
-get_property(mlir_libs GLOBAL PROPERTY MLIR_ALL_LIBS)
 target_link_libraries(npcomp_torch_c10_dispatch_bindings
-  NPCOMPCAPIIR
   ${TORCH_LIBRARIES}
   ${PYTHON_LIBRARIES}
-  ${mlir_libs}
   torch_python
   )

frontends/pytorch/test/CMakeLists.txt

@@ -11,7 +11,7 @@ configure_lit_site_cfg(
 
 set(TEST_DEPENDS
   FileCheck count not
-  _torch_mlir
+  NPCOMPTorchMLIRExt
   )
 
 

frontends/pytorch/test/lit.cfg.py

@@ -21,11 +21,7 @@ from lit.llvm.subst import FindTool
 config.name = 'FRONTENDS_PYTORCH'
 
 config.test_format = lit.formats.ShTest(not llvm_config.use_lit_shell)
-config.environment['PYTHONPATH'] = "{}:{}".format(
-   os.path.join(config.npcomp_obj_root, "python"),
-   # path to our python hooks
-   os.path.join(config.npcomp_obj_root, "frontends", "pytorch", "csrc"))
-
+config.environment['PYTHONPATH'] = os.path.join(config.npcomp_obj_root, "python")
 if 'TEST_SRC_PATH' in os.environ:
    config.environment['TEST_SRC_PATH'] = os.environ['TEST_SRC_PATH']
 

include/npcomp-c/InitLLVM.h

@@ -0,0 +1,25 @@
+/*===-- npcomp-c/InitLLVM.h - C API for initializing LLVM  --------*- C -*-===*\
+|*                                                                            *|
+|* Part of the LLVM Project, under the Apache License v2.0 with LLVM          *|
+|* Exceptions.                                                                *|
+|* See https://llvm.org/LICENSE.txt for license information.                  *|
+|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception                    *|
+|*                                                                            *|
+\*===----------------------------------------------------------------------===*/
+
+#ifndef NPCOMP_C_INITLLVM_H
+#define NPCOMP_C_INITLLVM_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/** Initializes LLVM codegen infrastructure and related MLIR bridge components.
+ */
+void npcompInitializeLLVMCodegen();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // NPCOMP_C_INITLLVM_H

lib/Backend/RefJIT/CMakeLists.txt

@@ -12,6 +12,7 @@ add_library(NPCOMPBackendRefJITPythonModule
 )
 
 target_link_libraries(NPCOMPBackendRefJITPythonModule
+  pybind11::module
   MLIRExecutionEngine
   MLIRTargetLLVMIR
 

lib/CAPI/CMakeLists.txt

@@ -1,9 +1,18 @@
-add_mlir_library(NPCOMPCAPIIR
+set(LLVM_LINK_COMPONENTS
+  Core
+  Support
+  nativecodegen
+  )
+
+add_npcomp_library(NPCOMPCAPI
+  InitLLVM.cpp
   Registration.cpp
   Types.cpp
-  EXCLUDE_FROM_LIBMLIR
 
   LINK_LIBS PUBLIC
+  MLIRExecutionEngine
+  MLIRLLVMIR
+  MLIRTargetLLVMIR
   NPCOMPInitAll
   NPCOMPBasicpyDialect
   NPCOMPNumpyDialect

lib/CAPI/InitLLVM.cpp

@@ -0,0 +1,18 @@
+//===- InitLLVM.cpp - C API for initializing LLVM -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "npcomp-c/InitLLVM.h"
+
+#include "mlir/ExecutionEngine/OptUtils.h"
+#include "llvm/Support/TargetSelect.h"
+
+void npcompInitializeLLVMCodegen() {
+  llvm::InitializeNativeTarget();
+  llvm::InitializeNativeTargetAsmPrinter();
+  mlir::initializeLLVMPasses();
+}

lib/CMakeLists.txt

@@ -28,10 +28,16 @@ if(NPCOMP_ENABLE_IREE)
   )
 endif()
 
-get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
-get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
+get_property(mlir_dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
+get_property(mlir_conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
 
-add_mlir_library(NPCOMPInitAll
+get_property(npcomp_dialect_libs GLOBAL PROPERTY NPCOMP_DIALECT_LIBS)
+get_property(npcomp_conversion_libs GLOBAL PROPERTY NPCOMP_CONVERSION_LIBS)
+
+message(STATUS "NPCOMP Dialect libs: ${npcomp_dialect_libs}")
+message(STATUS "NPCOMP Conversion libs: ${npcomp_conversion_libs}")
+
+add_npcomp_library(NPCOMPInitAll
   InitAll.cpp
 
   LINK_LIBS
@@ -53,7 +59,9 @@ add_mlir_library(NPCOMPInitAll
   NPCOMPTCFPasses
   NPCOMPTypingPasses
 
-  ${dialect_libs}
-  ${conversion_libs}
+  ${npcomp_dialect_libs}
+  ${npcomp_conversion_libs}
+  ${mlir_dialect_libs}
+  ${mlir_conversion_libs}
   ${ALL_DEPENDS}
 )

lib/Conversion/BasicpyToIREEVM/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_conversion_library(BasicpyToIREEVM
+add_npcomp_conversion_library(BasicpyToIREEVM
   Passes.cpp
 
   DEPENDS

lib/Conversion/BasicpyToStd/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_conversion_library(NPCOMPBasicpyToSTD
+add_npcomp_conversion_library(NPCOMPBasicpyToSTD
   Passes.cpp
   PrimitiveOpsConversion.cpp
 

lib/Conversion/CMakeLists.txt

@@ -6,7 +6,7 @@ if(NPCOMP_ENABLE_IREE)
 add_subdirectory(BasicpyToIREEVM)
 endif()
 
-add_mlir_library(NPCOMPConversionPasses
+add_npcomp_library(NPCOMPConversionPasses
   Passes.cpp
 
   DEPENDS

lib/Conversion/NumpyToTCF/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_conversion_library(NPCOMPNumpyToTCF
+add_npcomp_conversion_library(NPCOMPNumpyToTCF
   Passes.cpp
 
   DEPENDS

lib/Conversion/TCFToTCP/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_conversion_library(NPCOMPTCFToTCP
+add_npcomp_conversion_library(NPCOMPTCFToTCP
   TCFToTCP.cpp
 
   ADDITIONAL_HEADER_DIRS

lib/Dialect/ATen/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_dialect_library(NPCOMPATenDialect
+add_npcomp_dialect_library(NPCOMPATenDialect
   ATenDialect.cpp
   ATenDialectOpStats.cpp
   ATenPasses.cpp

lib/Dialect/Basicpy/IR/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_dialect_library(NPCOMPBasicpyDialect
+add_npcomp_dialect_library(NPCOMPBasicpyDialect
   BasicpyDialect.cpp
   BasicpyOps.cpp
 

lib/Dialect/Basicpy/Transforms/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_conversion_library(NPCOMPBasicpyPasses
+add_npcomp_conversion_library(NPCOMPBasicpyPasses
   Passes.cpp
   TypeInference.cpp
 

lib/Dialect/Numpy/IR/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_dialect_library(NPCOMPNumpyDialect
+add_npcomp_dialect_library(NPCOMPNumpyDialect
   NumpyDialect.cpp
   NumpyOps.cpp
 

lib/Dialect/Numpy/Transforms/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_conversion_library(NPCOMPNumpyPasses
+add_npcomp_conversion_library(NPCOMPNumpyPasses
   Passes.cpp
   PublicFunctionToTensor.cpp
 

lib/Dialect/Refback/IR/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_dialect_library(NPCOMPRefbackDialect
+add_npcomp_dialect_library(NPCOMPRefbackDialect
   RefbackDialect.cpp
   RefbackOps.cpp
 

lib/Dialect/Refbackrt/IR/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_dialect_library(NPCOMPRefbackrtDialect
+add_npcomp_dialect_library(NPCOMPRefbackrtDialect
   RefbackrtDialect.cpp
   RefbackrtOps.cpp
 

lib/Dialect/TCF/IR/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_dialect_library(NPCOMPTCFDialect
+add_npcomp_dialect_library(NPCOMPTCFDialect
   TCFDialect.cpp
   TCFOps.cpp
 

lib/Dialect/TCF/Transforms/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_conversion_library(NPCOMPTCFPasses
+add_npcomp_conversion_library(NPCOMPTCFPasses
   Passes.cpp
   ShapeRefinement.cpp
 

lib/Dialect/TCP/IR/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_dialect_library(NPCOMPTCPDialect
+add_npcomp_dialect_library(NPCOMPTCPDialect
   TCPDialect.cpp
   TCPOps.cpp
 

lib/Dialect/Torch/IR/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_dialect_library(NPCOMPTorchDialect
+add_npcomp_dialect_library(NPCOMPTorchDialect
   TorchDialect.cpp
   TorchOps.cpp
 

lib/Python/CMakeLists.txt

@@ -8,15 +8,19 @@ include(NpcompPython)
 # TODO: This should not be wired in at such a low/unconditional level.
 # It is done here to be kept with the other LLVM initialization until a better
 # place can be found for it.
-set(ExtraInit_LIBADD)
-if(NPCOMP_ENABLE_REFJIT)
-  llvm_map_components_to_libnames(refjit_llvm_libs
-    nativecodegen
-  )
-  message(STATUS "Including LLVM libs for RefJit: ${refjit_llvm_libs}")
-  list(APPEND ExtraInit_LIBADD
-    ${refjit_llvm_libs})
-endif()
+# set(ExtraInit_LIBADD)
+# if(NPCOMP_ENABLE_REFJIT)
+#   llvm_map_components_to_libnames(refjit_llvm_libs
+#     nativecodegen
+#   )
+#   message(STATUS "Including LLVM libs for RefJit: ${refjit_llvm_libs}")
+#   list(APPEND ExtraInit_LIBADD
+#     ${refjit_llvm_libs})
+# endif()
+
+# include_directories(
+#   ${PYTHON_INCLUDE_DIRS}
+#   )
 
 set(PYBIND_SOURCES
   MlirInit.cpp
@@ -32,7 +36,6 @@ add_library(NPCOMPPythonCommon
 
 target_link_libraries(NPCOMPPythonCommon
   pybind11::module
-
   NPCOMPInitAll
 
   # Core dialects

lib/Python/MlirInit.cpp

@@ -14,6 +14,7 @@
 #include "mlir/InitAllPasses.h"
 #include "mlir/Pass/PassManager.h"
 #include "mlir/Pass/PassRegistry.h"
+#include "npcomp-c/InitLLVM.h"
 #include "npcomp/Dialect/Basicpy/IR/BasicpyDialect.h"
 #include "npcomp/Dialect/Numpy/IR/NumpyDialect.h"
 #include "npcomp/InitAll.h"
@@ -41,10 +42,8 @@ bool mlir::npcomp::python::npcompMlirInitialize() {
   ::mlir::registerAllPasses();
   ::mlir::NPCOMP::registerAllPasses();
 
-  // LLVM codegen initialization.
-  llvm::InitializeNativeTarget();
-  llvm::InitializeNativeTargetAsmPrinter();
-  mlir::initializeLLVMPasses();
+  // Initialize code generation.
+  npcompInitializeLLVMCodegen();
 
   return true;
 }

lib/RefBackend/CMakeLists.txt

@@ -1,7 +1,7 @@
 add_subdirectory(Runtime)
 add_subdirectory(JITHelpers)
 
-add_mlir_library(NPCOMPRefBackend
+add_npcomp_library(NPCOMPRefBackend
   BypassShapes.cpp
   RefBackend.cpp
   LowerToLLVM.cpp

lib/RefBackend/JITHelpers/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_library(NPCOMPRefBackendJITHelpers
+add_npcomp_library(NPCOMPRefBackendJITHelpers
   JITModule.cpp
 
   ADDITIONAL_HEADER_DIRS

lib/RefBackend/JITHelpers/JITModule.cpp

@@ -10,8 +10,6 @@
 #include "mlir/ExecutionEngine/CRunnerUtils.h"
 #include "mlir/ExecutionEngine/OptUtils.h"
 #include "npcomp/RefBackend/RefBackend.h"
-#include "llvm/Support/InitLLVM.h"
-#include "llvm/Support/TargetSelect.h"
 
 using namespace refback;
 using namespace mlir;

lib/RefBackend/Runtime/CMakeLists.txt

@@ -8,10 +8,8 @@ set(LLVM_OPTIONAL_SOURCES
 
 # The library that users link against, defining basic interactions with an
 # refbackrt module and the relevant data structures.
-add_mlir_library(NPCOMPRuntime
+add_npcomp_library(NPCOMPRuntime
   Runtime.cpp
-
-  EXCLUDE_FROM_LIBMLIR
 )
 
 mlir_check_all_link_libraries(NPCOMPRuntime)
@@ -25,11 +23,11 @@ mlir_check_all_link_libraries(NPCOMPRuntime)
 # This is currently done as a shared library to make it suitable for being
 # loaded by mlir::ExecutionEngine. In e.g. an embedded scenario, we would
 # need to create a static library and link that into the binary.
-add_mlir_library(NPCOMPCompilerRuntimeShlib
+add_npcomp_library(NPCOMPCompilerRuntimeShlib
   SHARED
   CompilerRuntime.cpp
 
-  EXCLUDE_FROM_LIBMLIR
+  EXCLUDE_FROM_LIBNPCOMP
 )
 target_link_libraries(NPCOMPCompilerRuntimeShlib PRIVATE NPCOMPRuntime)
 set_target_properties(NPCOMPCompilerRuntimeShlib PROPERTIES LINK_FLAGS

lib/Typing/Analysis/CPA/CMakeLists.txt

@@ -1,13 +1,12 @@
-add_library(NPCOMPTypingCPA
+add_npcomp_library(NPCOMPTypingCPA
   Algorithm.cpp
   Interfaces.cpp
   Types.cpp
-)
 
-target_link_libraries(NPCOMPTypingCPA
+  DEPENDS
+  NPCOMPTypingCPAInterfacesIncGen
+
+  LINK_LIBS
+  PUBLIC
   MLIRIR
 )
-
-add_dependencies(NPCOMPTypingCPA
-  NPCOMPTypingCPAInterfacesIncGen
-)

lib/Typing/Support/CMakeLists.txt

@@ -1,8 +1,8 @@
-add_library(NPCOMPTypingCPASupport
+add_npcomp_library(NPCOMPTypingCPASupport
   CPAIrHelpers.cpp
-)
 
-target_link_libraries(NPCOMPTypingCPASupport
+  LINK_LIBS
+  PUBLIC
   MLIRIR
   NPCOMPTypingCPA
   NPCOMPBasicpyDialect

lib/Typing/Transforms/CMakeLists.txt

@@ -1,4 +1,4 @@
-add_mlir_conversion_library(NPCOMPTypingPasses
+add_npcomp_conversion_library(NPCOMPTypingPasses
   Passes.cpp
   CPATypeInference.cpp
 

python/CMakeLists.txt

@@ -92,64 +92,36 @@ if(NPCOMP_ENABLE_REFJIT)
   )
 endif()
 
-# When building the extension, distinguish between those sources that use
-# pybind (and need rtti/exceptions) and those that only use LLVM/MLIR.
-# Some of the low-level components do not support mixing RTTI modes and are
-# compiled separately for now.
-set(extension_target NPCOMPNativePyExt)
-set(extension_llvm_sources
-)
-set(extension_pybind_sources
+add_library(NPCOMPNativePyExt ${NPCOMP_PYEXT_LINK_MODE}
   NpcompModule.cpp
 )
-add_library(${extension_target} ${NPCOMP_PYEXT_LINK_MODE}
-  ${extension_pybind_sources}
-  ${extension_llvm_sources}
-)
 
-set_target_properties(${extension_target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
+set_target_properties(NPCOMPNativePyExt PROPERTIES LIBRARY_OUTPUT_DIRECTORY
                       "${CMAKE_CURRENT_BINARY_DIR}")
-set_target_properties(${extension_target} PROPERTIES OUTPUT_NAME _npcomp)
-set_target_properties(${extension_target} PROPERTIES PREFIX
+set_target_properties(NPCOMPNativePyExt PROPERTIES OUTPUT_NAME _npcomp)
+set_target_properties(NPCOMPNativePyExt PROPERTIES PREFIX
                       "${PYTHON_MODULE_PREFIX}")
-set_target_properties(${extension_target} PROPERTIES SUFFIX
+set_target_properties(NPCOMPNativePyExt PROPERTIES SUFFIX
                       "${PYTHON_MODULE_EXTENSION}")
 
 # pybind requires binding code to be compiled with -fvisibility=hidden
 # Better code can be generated if the entire project compiles that way, but
-# that is not enforced here. Instead, include a linker script that explicitly
-# hides anything but the PyInit_* symbols, allowing gc to take place.
-# TODO(laurenzo): Windows needs a .def file and different flags.
-set_target_properties(${extension_target} PROPERTIES CXX_VISIBILITY_PRESET "hidden")
-set_target_properties(${extension_target} PROPERTIES LINK_FLAGS
-  "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/unix_version.script")
+# that is not enforced here.
+set_target_properties(NPCOMPNativePyExt PROPERTIES CXX_VISIBILITY_PRESET "hidden")
 
 get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
 
-target_link_libraries(${extension_target}
+target_link_libraries(NPCOMPNativePyExt
   PRIVATE
-    NPCOMPPythonCommon
+    # Transitive dep on the shared library links most things from there.
+    NPCOMP
+
     NPCOMPInitAll
-
-    ${conversion_libs}
-    ${dialect_libs}
-
-    # Upstream depends
-    MLIRAffineToStandard
-    MLIRAffineTransforms
-    MLIRDialect
-    MLIREDSC
-    MLIRIR
-    MLIRSCFToStandard
-    MLIRLLVMIR
-    MLIRPass
-    MLIRTargetLLVMIR
-    MLIRTransforms
-
+    NPCOMPPythonCommon
     ${NPCOMP_PYEXT_LIBADD}
 )
 
-npcomp_python_target_compile_options(${extension_target})
+npcomp_python_target_compile_options(NPCOMPNativePyExt)
 
-mlir_check_all_link_libraries(${extension_target})
+mlir_check_all_link_libraries(NPCOMPNativePyExt)

python/unix_version.script

@@ -1,4 +0,0 @@
-{
-  global: PyInit__npcomp;
-  local: *;
-};

test/CAPI/CMakeLists.txt

@@ -1,16 +1,6 @@
-set(LLVM_LINK_COMPONENTS
-  Core
-  Support
-  )
-
-add_llvm_executable(npcomp-capi-ir-test
-  ir.c
-  )
+add_npcomp_executable(npcomp-capi-ir-test ir.c)
 llvm_update_compile_flags(npcomp-capi-ir-test)
 
-get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 target_link_libraries(npcomp-capi-ir-test
   PRIVATE
-  NPCOMPCAPIIR
-  MLIRCAPIRegistration
-  ${dialect_libs})
+  NPCOMP)

test/lit.cfg.py

@@ -48,7 +48,7 @@ config.test_source_root = os.path.dirname(__file__)
 
 # test_exec_root: The root path where tests should be run.
 config.test_exec_root = os.path.join(config.npcomp_obj_root, 'test')
-config.npcomp_tools_dir = os.path.join(config.npcomp_obj_root, 'tools')
+config.npcomp_bin_dir = os.path.join(config.npcomp_obj_root, 'bin')
 config.npcomp_runtime_shlib = os.path.join(
     config.npcomp_obj_root, 'lib',
     'libNPCOMPCompilerRuntimeShlib' + config.llvm_shlib_ext)
@@ -60,8 +60,7 @@ llvm_config.with_environment('PYTHONPATH',
                              append_path=True)
 
 tool_dirs = [
-    os.path.join(config.npcomp_tools_dir, 'npcomp-opt'),
-    os.path.join(config.npcomp_tools_dir, 'npcomp-run-mlir'),
+    os.path.join(config.npcomp_bin_dir),
     os.path.join(config.test_exec_root, 'CAPI'),
     config.llvm_tools_dir,
 ]

tools/CMakeLists.txt

@@ -1,3 +1,4 @@
 add_subdirectory(npcomp-opt)
 add_subdirectory(mnist-playground)
 add_subdirectory(npcomp-run-mlir)
+add_subdirectory(npcomp-shlib)

tools/mnist-playground/CMakeLists.txt

@@ -17,13 +17,17 @@ set(LLVM_LINK_COMPONENTS
   nativecodegen
   )
 
-add_llvm_tool(mnist-playground
+add_npcomp_executable(mnist-playground
   mnist-playground.cpp
   )
 llvm_update_compile_flags(mnist-playground)
 get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
 get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
 target_link_libraries(mnist-playground PRIVATE
+  # Shared library deps first ensure we get most of what we need from libraries.
+  NPCOMP
+  MLIR
+
   MLIRAnalysis
   MLIREDSC
   MLIRExecutionEngine

tools/npcomp-opt/CMakeLists.txt

@@ -1,14 +1,11 @@
-get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
-get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
-set(LIBS
-        MLIROptLib
-        NPCOMPInitAll
-        ${conversion_libs}
-        ${dialect_libs}
-)
-add_llvm_executable(npcomp-opt npcomp-opt.cpp)
+# npcomp-opt is always linked dynamically as we want to distribute the
+# binaries with the python packages for hacking/debugging.
+add_npcomp_executable(npcomp-opt npcomp-opt.cpp)
 
-llvm_update_compile_flags(npcomp-opt)
-target_link_libraries(npcomp-opt PRIVATE ${LIBS})
+target_link_libraries(npcomp-opt PRIVATE
+  # Shared library deps first ensure we get most of what we need from libraries.
+  NPCOMP
+  MLIR
+)
 
 mlir_check_all_link_libraries(npcomp-opt)

tools/npcomp-run-mlir/CMakeLists.txt

@@ -1,30 +1,31 @@
-set(LLVM_LINK_COMPONENTS
-  Core
-  Support
-  nativecodegen
-  )
+# npcomp-run-mlir is always linked dynamically as we want to distribute the
+# binaries with the python packages for hacking/debugging.
 
-add_llvm_tool(npcomp-run-mlir
+get_property(dialect_libs GLOBAL PROPERTY NPCOMP_DIALECT_LIBS)
+get_property(conversion_libs GLOBAL PROPERTY NPCOMP_CONVERSION_LIBS)
+
+add_npcomp_executable(npcomp-run-mlir
   npcomp-run-mlir.cpp
   )
+
 llvm_update_compile_flags(npcomp-run-mlir)
-get_property(dialect_libs GLOBAL PROPERTY MLIR_DIALECT_LIBS)
-get_property(conversion_libs GLOBAL PROPERTY MLIR_CONVERSION_LIBS)
 target_link_libraries(npcomp-run-mlir PRIVATE
+  # Shared library deps first ensure we get most of what we need from libraries.
+  NPCOMP
+  MLIR
+
+  NPCOMPCAPI
   MLIRAnalysis
   MLIREDSC
-  MLIRExecutionEngine
   MLIRIR
   MLIRJitRunner
-  MLIRLLVMIR
   MLIRParser
-  MLIRTargetLLVMIR
   MLIRSupport
   NPCOMPInitAll
   NPCOMPRefBackendJITHelpers
   ${conversion_libs}
   ${dialect_libs}
-  )
+)
 add_dependencies(npcomp-run-mlir
   NPCOMPCompilerRuntimeShlib
   )

tools/npcomp-run-mlir/npcomp-run-mlir.cpp

@@ -11,17 +11,16 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/Support/InitLLVM.h"
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
-#include "mlir/ExecutionEngine/OptUtils.h"
 #include "mlir/IR/AsmState.h"
 #include "mlir/InitAllDialects.h"
 #include "mlir/InitAllPasses.h"
 #include "mlir/Parser.h"
 #include "mlir/Pass/PassManager.h"
+#include "npcomp-c/InitLLVM.h"
 #include "npcomp/InitAll.h"
 #include "npcomp/RefBackend/JITHelpers/JITModule.h"
-#include "llvm/Support/InitLLVM.h"
-#include "llvm/Support/TargetSelect.h"
 
 using namespace mlir;
 using llvm::Error;
@@ -186,9 +185,7 @@ int main(int argc, char **argv) {
   mlir::NPCOMP::registerAllPasses();
 
   llvm::InitLLVM y(argc, argv);
-  llvm::InitializeNativeTarget();
-  llvm::InitializeNativeTargetAsmPrinter();
-  mlir::initializeLLVMPasses();
+  npcompInitializeLLVMCodegen();
 
   mlir::registerAsmPrinterCLOptions();
   mlir::registerPassManagerCLOptions();

tools/npcomp-shlib/CMakeLists.txt

@@ -0,0 +1,48 @@
+# Building libmlir-cpp.so fails if LLVM_ENABLE_PIC=Off
+if (NOT LLVM_ENABLE_PIC)
+  message(WARNING "Not building NPCOMP dylib because PIC is disabled")
+  return()
+endif()
+
+# Building libmlir-cpp.so may not work on MSVC
+if (MSVC)
+  message(WARNING "Not building NPCOMP dylib because not yet supported on MSVC")
+  return()
+endif()
+
+if(NOT NPCOMP_BUILD_NPCOMP_DYLIB)
+  message(WARNING "Not building NPCOMP dylib (not NPCOMP_BUILD_NPCOMP_DYLIB): Fully static builds not yet supported")
+  return()
+endif()
+
+get_property(npcomp_libs GLOBAL PROPERTY NPCOMP_STATIC_LIBS)
+list(REMOVE_DUPLICATES npcomp_libs)
+
+# Populate _OBJECTS and _DEPS as necessary per platform.
+foreach (lib ${npcomp_libs})
+  if(XCODE)
+    # Xcode doesn't support object libraries, so we have to trick it into
+    # linking the static libraries instead.
+    list(APPEND _DEPS "-force_load" ${lib})
+  else()
+    list(APPEND _OBJECTS $<TARGET_OBJECTS:obj.${lib}>)
+  endif()
+  # Add transitive deps explcitly since otherwise, there would just be
+  # objects.
+  list(APPEND _DEPS  $<TARGET_PROPERTY:${lib},LINK_LIBRARIES>)
+endforeach()
+
+# Note: Does not use add_npcomp_library, which is used for things that go
+# *into* the libNPCOMP.so. This is building the shared library, so use
+# a higher-level rule.
+llvm_add_library(
+  NPCOMP
+  SHARED
+  npcomp-shlib.cpp
+  ${_OBJECTS}
+  LINK_LIBS PUBLIC
+  # Public dependencies on the MLIR shared lib.
+  MLIR
+  ${_DEPS}
+)
+target_link_libraries(NPCOMP PRIVATE ${LLVM_PTHREAD_LIB})

tools/npcomp-shlib/npcomp-shlib.cpp

@@ -0,0 +1 @@
+// Intentionally empty source file to make CMake happy