Move python native library to python_native/_npcomp...so.

This allows binary and source packages to exist at different physical paths.

CMakeLists.txt

@@ -48,5 +48,6 @@ message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}")
 add_subdirectory(include/npcomp)
 add_subdirectory(lib)
 add_subdirectory(python)
+add_subdirectory(python_native)
 add_subdirectory(tools)
 add_subdirectory(test)

README.md

@@ -38,31 +38,26 @@ ninja check-npcomp-opt
 ./python/run_tests.py
 ```
 
+## Interactive Use
+
+The cmake configuration populates symlinks in the `build/python` directory
+mirroring the source layout. This allows edit-run without rebuilding (unless
+if files are added/removed).
+
+Configuring the `PYTHONPATH` should be sufficient to run any interactive
+tooling (`python3`, Jupyter/Colab, etc).
+
+```shell
+export PYTHONPATH="$(realpath build/python):$(realpath build/python_native)"
+```
+
+The `run_tests.py` script is special in that it sets up the PYTHONPATH
+correctly when run.
+
 ### Things to look at:
 
 * `python/npcomp/tracing/mlir_trace_test.py` : Simple test case of tracing a function to an MLIR module.
 
-### Installing pybind11
-
-The native extension relies on pybind11. In a perfect world, this could just
-be installed with your system package manager. However, at least on
-some tested versions of Ubuntu, the system package installed with broken cmake 
-files.
-
-If this happens, you must install pybind11 from source.
-
-### Building the python native library
-
-```shell
-# From the build directory
-ninja NPCOMPNativePyExt
-ninja check-npcomp
-python3 ./python/run_tests.py
-
-# Setup PYTHONPATH for interactive use.
-export PYTHONPATH=$(pwd)/tools/npcomp/python
-```
-
 Notes:
 
 * Python sources are symlinked to the output directory at configure time.

python/CMakeLists.txt

@@ -1,5 +1,3 @@
-add_subdirectory(npcomp)
-
 ################################################################################
 # Manage python source files
 ################################################################################

python/npcomp/dialect/Basicpy.py

@@ -2,8 +2,8 @@
 #  See https://llvm.org/LICENSE.txt for license information.
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-from ..native.dialect import BasicpyDialectHelper as _BaseDialectHelper
+from _npcomp.dialect import BasicpyDialectHelper as _BaseDialectHelper
-from ..native.mlir import ir
+from _npcomp.mlir import ir
 
 __all__ = [
     "DialectHelper",

python/npcomp/dialect/Numpy.py

@@ -3,7 +3,7 @@
 #  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
 from npcomp.dialect import Basicpy
-from npcomp.native.mlir import ir
+from _npcomp.mlir import ir
 
 __all__ = [
   "load_builtin_module",

python/npcomp/mlir_ir_test.py

@@ -4,7 +4,7 @@
 
 """Test for the MLIR IR Python bindings"""
 
-from npcomp.native.mlir import ir
+from _npcomp.mlir import ir
 from npcomp.utils import test_utils
 
 test_utils.start_filecheck_test()

python/npcomp/smoketest.py

@@ -1,2 +1,2 @@
-from . import native
+import _npcomp
-print(native.__doc__)
+print(_npcomp.__doc__)

python/npcomp/tracing/mlir_trace.py

@@ -6,9 +6,9 @@ import re
 from typing import Iterable
 import numpy as np
 
-from ..dialect import Numpy
+from _npcomp.mlir import ir
-from ..native.mlir import ir
 
+from ..dialect import Numpy
 from .context import *
 from .emitters import *
 from ..exporter import *

python/npcomp/unix_version.script

@@ -1,4 +0,0 @@
-{
-  global: PyInit_native;
-  local: *;
-};

python/run_tests.py

@@ -17,8 +17,14 @@ TEST_MODULES = (
 )
 
 # Compute PYTHONPATH for sub processes.
-DIRSEP = ":" if os.path.sep == "/" else ";"
+DIRSEP = os.path.pathsep
-PYTHONPATH = os.path.abspath(os.path.dirname(__file__))
+LOCAL_PYTHONPATH_COMPONENTS = [
+  # This directory.
+  os.path.abspath(os.path.dirname(__file__)),
+  # The parallel python_native directory (assuming in the build tree).
+  os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "python_native"))
+]
+PYTHONPATH = DIRSEP.join(LOCAL_PYTHONPATH_COMPONENTS)
 if "PYTHONPATH" in os.environ:
   PYTHONPATH = PYTHONPATH + DIRSEP + os.environ["PYTHONPATH"]
 CHILD_ENVIRON = dict(os.environ)

python_native/CMakeLists.txt

@@ -25,11 +25,11 @@ set(NPCOMP_PYEXT_LIBADD ${PYTHON_LIBRARIES})
 # compiled separately for now.
 set(extension_target NPCOMPNativePyExt)
 set(extension_pybind_sources
-  native.cpp
+  MlirInit.cpp
-  mlir_init.cpp
+  MlirIr.cpp
-  mlir_ir.cpp
+  NpcompDialect.cpp
-  npcomp_dialect.cpp
+  NpcompModule.cpp
-  pybind_utils.cpp
+  PybindUtils.cpp
 )
 set_source_files_properties(
   ${extension_pybind_sources}
@@ -41,7 +41,7 @@ add_library(${extension_target} ${NPCOMP_PYEXT_LINK_MODE}
 
 set_target_properties(${extension_target} PROPERTIES LIBRARY_OUTPUT_DIRECTORY
                       "${CMAKE_CURRENT_BINARY_DIR}")
-set_target_properties(${extension_target} PROPERTIES OUTPUT_NAME native)
+set_target_properties(${extension_target} PROPERTIES OUTPUT_NAME _npcomp)
 set_target_properties(${extension_target} PROPERTIES PREFIX 
                       "${PYTHON_MODULE_PREFIX}")
 set_target_properties(${extension_target} PROPERTIES SUFFIX 

python_native/MlirInit.cpp

@@ -1,4 +1,4 @@
-//===- mlir_init.cpp ------------------------------------------------------===//
+//===- MlirInit.cpp -------------------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.

python_native/MlirIr.cpp

@@ -1,4 +1,4 @@
-//===- mlir_ir.cpp - MLIR IR Bindings -------------------------------------===//
+//===- MlirIr.cpp - MLIR IR Bindings --------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir_ir.h"
+#include "MlirIr.h"
-#include "native.h"
+#include "NpcompModule.h"
 
 #include "mlir/Dialect/StandardOps/IR/Ops.h"
 #include "mlir/IR/Attributes.h"

python_native/MlirIr.h

@@ -1,4 +1,4 @@
-//===- mlir_ir.h - MLIR IR Bindings -------------------------------------===//
+//===- MlirIr.h - MLIR IR Bindings ----------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef NPCOMP_PYTHON_MLIR_IR_H
+#ifndef NPCOMP_PYTHON_NATIVE_MLIR_IR_H
-#define NPCOMP_PYTHON_MLIR_IR_H
+#define NPCOMP_PYTHON_NATIVE_MLIR_IR_H
 
-#include "pybind_utils.h"
+#include "PybindUtils.h"
 
 #include "mlir/IR/Block.h"
 #include "mlir/IR/Builders.h"
@@ -174,4 +174,4 @@ protected:
 
 } // namespace mlir
 
-#endif // NPCOMP_PYTHON_MLIR_IR_H
+#endif // NPCOMP_PYTHON_NATIVE_MLIR_IR_H

python_native/NpcompDialect.cpp

@@ -1,4 +1,4 @@
-//===- npcomp_dialect.cpp - Custom dialect classes ------------------------===//
+//===- NpcompDialect.cpp - Custom dialect classes -------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "mlir_ir.h"
+#include "MlirIr.h"
-#include "native.h"
+#include "NpcompModule.h"
 
 #include "npcomp/Dialect/Basicpy/BasicpyDialect.h"
 #include "npcomp/Dialect/Basicpy/BasicpyOps.h"

python_native/NpcompModule.cpp

@@ -9,8 +9,8 @@
 #include <cstddef>
 #include <unordered_map>
 
-#include "native.h"
+#include "NpcompModule.h"
-#include "pybind_utils.h"
+#include "PybindUtils.h"
 
 #include "llvm/Support/CommandLine.h"
 
@@ -48,7 +48,7 @@ void defineLLVMModule(pybind11::module m) {
         py::arg("name"));
 }
 
-PYBIND11_MODULE(native, m) {
+PYBIND11_MODULE(_npcomp, m) {
   // Guard the once init to happen once per process (vs module, which in
   // mondo builds can happen multiple times).
   static bool llvm_init_baton = ([]() { return npcompMlirInitialize(); })();

python_native/NpcompModule.h

@@ -1,4 +1,4 @@
-//===- dialect.h - Module registrations -----------------------------------===//
+//===- NpcompModule.h - Module registrations ------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,10 +6,10 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef NPCOMP_PYTHON_NATIVE_H
+#ifndef NPCOMP_PYTHON_NATIVE_NPCOMP_MODULE_H
-#define NPCOMP_PYTHON_NATIVE_H
+#define NPCOMP_PYTHON_NATIVE_NPCOMP_MODULE_H
 
-#include "pybind_utils.h"
+#include "PybindUtils.h"
 
 namespace mlir {
 void defineMlirIrModule(py::module m);
@@ -24,4 +24,4 @@ void defineNpcompDialect(py::module m);
 } // namespace npcomp
 } // namespace mlir
 
-#endif // NPCOMP_PYTHON_NATIVE_H
+#endif // NPCOMP_PYTHON_NATIVE_NPCOMP_MODULE_H

python_native/PybindUtils.cpp

@@ -1,4 +1,4 @@
-//===- pybind_utils.cpp - Utilities for interop with python ---------------===//
+//===- PybindUtils.cpp - Utilities for interop with python ----------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "pybind_utils.h"
+#include "PybindUtils.h"
 
 namespace pybind11 {
 

python_native/PybindUtils.h

@@ -1,4 +1,4 @@
-//===- pybind_utils.h - Utilities for interop with python -----------------===//
+//===- PybindUtils.h - Utilities for interop with python ------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef NPCOMP_PYTHON_PYBIND_UTILS_H
+#ifndef NPCOMP_PYTHON_NATIVE_PYBIND_UTILS_H
-#define NPCOMP_PYTHON_PYBIND_UTILS_H
+#define NPCOMP_PYTHON_NATIVE_PYBIND_UTILS_H
 
 #include <string>
 
@@ -52,4 +52,4 @@ inline pybind11::error_already_set raiseValueError(const std::string &message) {
 
 } // namespace pybind11
 
-#endif // NPCOMP_PYTHON_PYBIND_UTILS_H
+#endif // NPCOMP_PYTHON_NATIVE_PYBIND_UTILS_H

python_native/unix_version.script

@@ -0,0 +1,4 @@
+{
+  global: PyInit__npcomp;
+  local: *;
+};