diff --git a/.github/workflows/buildAndTest.yml b/.github/workflows/buildAndTest.yml index 3b5ae5fc9..aa00cbe69 100644 --- a/.github/workflows/buildAndTest.yml +++ b/.github/workflows/buildAndTest.yml @@ -48,8 +48,8 @@ jobs: - name: Refbackend integration tests run: | cd $GITHUB_WORKSPACE - export PYTHONPATH="$GITHUB_WORKSPACE/build/python_packages/npcomp_core:$GITHUB_WORKSPACE/build/python_packages/npcomp_torch:$GITHUB_WORKSPACE/build/python_packages/torch_mlir:$GITHUB_WORKSPACE/build/python_packages/torch_mlir_dialects" - python -m frontends.pytorch.e2e_testing.torchscript.main --config=refbackend + export PYTHONPATH="$GITHUB_WORKSPACE/build/python_packages/npcomp_core:$GITHUB_WORKSPACE/build/python_packages/torch_mlir" + python -m e2e_testing.torchscript.main --config=refbackend -v # build_wheels: # name: Build Wheels and Smoketest diff --git a/CMakeLists.txt b/CMakeLists.txt index 95dfac50b..53f430335 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -245,13 +245,15 @@ endif() add_custom_target(check-npcomp) add_custom_target(check-npcomp-all) -add_dependencies(check-npcomp-all check-npcomp check-frontends-pytorch check-torch-mlir check-torch-mlir-plugin) +add_dependencies(check-npcomp-all + check-npcomp + check-npcomp-python + check-torch-mlir + check-torch-mlir-plugin + ) add_subdirectory(include/npcomp) add_subdirectory(lib) add_subdirectory(python) add_subdirectory(test) add_subdirectory(tools) - -message(STATUS "Adding PyTorch frontend support...") -add_subdirectory(frontends/pytorch) diff --git a/build_tools/test_all.sh b/build_tools/test_all.sh index 5fd7df4c3..7a71a24b4 100755 --- a/build_tools/test_all.sh +++ b/build_tools/test_all.sh @@ -1,18 +1,8 @@ #!/bin/bash -# Runs all the tests we are aware of in mlir-npcomp. -# TODO: This should eventually all be folded into a `ninja check-npcomp`. -# Unfortunately, that seems to be a level of build engineering that we -# haven't had enough bandwidth for. -# -# The basic idea seems to be to teach `lit` how to run our Python tests. -# -# There is precendent for doing this, e.g. -# - For GoogleTest tests (i.e. "unittests"): https://github.com/llvm/llvm-project/blob/c490c5e81ac90cbf079c7cee18cd56171f1e27af/llvm/test/Unit/lit.cfg.py#L25 -# - LLDB tests: https://github.com/llvm/llvm-project/blob/eb7d32e46fe184fdfcb52e0a25973e713047e305/lldb/test/API/lldbtest.py#L32 -# - libcxx tests: https://github.com/llvm/llvm-project/blob/eb7d32e46fe184fdfcb52e0a25973e713047e305/libcxx/utils/libcxx/test/newformat.py#L192 -# -# The `getTestsInDirectory` keyword seems to be what to search for. +# Script for invoking all test tests. It also prints out a banner in case +# of success. +# TODO: I think we can just remove this. set -euo pipefail td="$(realpath $(dirname $0)/..)" @@ -20,13 +10,9 @@ td="$(realpath $(dirname $0)/..)" cd $td/build ninja -ninja check-npcomp -ninja check-torch-mlir -ninja check-frontends-pytorch +ninja check-npcomp-all echo echo "========" echo "ALL PASS" echo "========" - - diff --git a/build_tools/torchscript_e2e_heavydep_tests/basic_mt.py b/build_tools/torchscript_e2e_heavydep_tests/basic_mt.py index efc3862ad..3aa3400e9 100644 --- a/build_tools/torchscript_e2e_heavydep_tests/basic_mt.py +++ b/build_tools/torchscript_e2e_heavydep_tests/basic_mt.py @@ -27,9 +27,9 @@ from fairseq.sequence_generator import SequenceGenerator from fairseq.tasks.fairseq_task import LegacyFairseqTask from fairseq import utils -from torch_mlir_torchscript.e2e_test.framework import TestUtils -from torch_mlir_torchscript.e2e_test.registry import register_test_case -from torch_mlir_torchscript.annotations import annotate_args, export +from npcomp_torchscript.e2e_test.framework import TestUtils +from npcomp_torchscript.e2e_test.registry import register_test_case +from npcomp_torchscript.annotations import annotate_args, export DEFAULT_TEST_VOCAB_SIZE = 100 diff --git a/build_tools/torchscript_e2e_heavydep_tests/main.py b/build_tools/torchscript_e2e_heavydep_tests/main.py index 7c6137d04..ada150274 100644 --- a/build_tools/torchscript_e2e_heavydep_tests/main.py +++ b/build_tools/torchscript_e2e_heavydep_tests/main.py @@ -8,9 +8,9 @@ import pickle import torch -from torch_mlir_torchscript.e2e_test.registry import GLOBAL_TEST_REGISTRY -from torch_mlir_torchscript.e2e_test.framework import SerializableTest, generate_golden_trace -from torch_mlir_torchscript.annotations import extract_serializable_annotations +from npcomp_torchscript.e2e_test.registry import GLOBAL_TEST_REGISTRY +from npcomp_torchscript.e2e_test.framework import SerializableTest, generate_golden_trace +from npcomp_torchscript.annotations import extract_serializable_annotations from . import basic_mt diff --git a/build_tools/write_env_file.sh b/build_tools/write_env_file.sh index e358a58f2..a5f17d90d 100755 --- a/build_tools/write_env_file.sh +++ b/build_tools/write_env_file.sh @@ -15,7 +15,7 @@ python_packages_dir="$build_dir/python_packages" write_env_file() { echo "Updating $build_dir/.env file" - echo "PYTHONPATH=\"$(portable_realpath "$python_packages_dir/npcomp_core"):$(portable_realpath "$python_packages_dir/npcomp_torch"):$(portable_realpath "$python_packages_dir/torch_mlir"):$(portable_realpath "$python_packages_dir/torch_mlir_dialects")\"" > "$build_dir/.env" + echo "PYTHONPATH=\"$(portable_realpath "$python_packages_dir/npcomp_core"):$(portable_realpath "$python_packages_dir/torch_mlir"):$(portable_realpath "$python_packages_dir/torch_mlir_dialects")\"" > "$build_dir/.env" if ! cp "$build_dir/.env" "$td/.env"; then echo "WARNING: Failed to write $td/.env" fi diff --git a/frontends/pytorch/e2e_testing/torchscript/basic.py b/e2e_testing/torchscript/basic.py similarity index 95% rename from frontends/pytorch/e2e_testing/torchscript/basic.py rename to e2e_testing/torchscript/basic.py index dfe7e08ad..a97fa4d7e 100644 --- a/frontends/pytorch/e2e_testing/torchscript/basic.py +++ b/e2e_testing/torchscript/basic.py @@ -4,9 +4,9 @@ import torch -from torch_mlir_torchscript.e2e_test.framework import TestUtils -from torch_mlir_torchscript.e2e_test.registry import register_test_case -from torch_mlir_torchscript.annotations import annotate_args, export +from npcomp_torchscript.e2e_test.framework import TestUtils +from npcomp_torchscript.e2e_test.registry import register_test_case +from npcomp_torchscript.annotations import annotate_args, export # ============================================================================== diff --git a/frontends/pytorch/e2e_testing/torchscript/batchnorm.py b/e2e_testing/torchscript/batchnorm.py similarity index 93% rename from frontends/pytorch/e2e_testing/torchscript/batchnorm.py rename to e2e_testing/torchscript/batchnorm.py index f6e62cad8..7957f6062 100644 --- a/frontends/pytorch/e2e_testing/torchscript/batchnorm.py +++ b/e2e_testing/torchscript/batchnorm.py @@ -4,9 +4,9 @@ import torch -from torch_mlir_torchscript.e2e_test.framework import TestUtils -from torch_mlir_torchscript.e2e_test.registry import register_test_case -from torch_mlir_torchscript.annotations import annotate_args, export +from npcomp_torchscript.e2e_test.framework import TestUtils +from npcomp_torchscript.e2e_test.registry import register_test_case +from npcomp_torchscript.annotations import annotate_args, export # ============================================================================== class BatchNorm1DModule(torch.nn.Module): diff --git a/frontends/pytorch/e2e_testing/torchscript/conv.py b/e2e_testing/torchscript/conv.py similarity index 91% rename from frontends/pytorch/e2e_testing/torchscript/conv.py rename to e2e_testing/torchscript/conv.py index a1bbb7f78..fc9c2197b 100644 --- a/frontends/pytorch/e2e_testing/torchscript/conv.py +++ b/e2e_testing/torchscript/conv.py @@ -3,9 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception import torch -from torch_mlir_torchscript.e2e_test.framework import TestUtils -from torch_mlir_torchscript.e2e_test.registry import register_test_case -from torch_mlir_torchscript.annotations import annotate_args, export +from npcomp_torchscript.e2e_test.framework import TestUtils +from npcomp_torchscript.e2e_test.registry import register_test_case +from npcomp_torchscript.annotations import annotate_args, export # ============================================================================== diff --git a/frontends/pytorch/e2e_testing/torchscript/elementwise.py b/e2e_testing/torchscript/elementwise.py similarity index 96% rename from frontends/pytorch/e2e_testing/torchscript/elementwise.py rename to e2e_testing/torchscript/elementwise.py index 7da9a12c5..8b05ac8d7 100644 --- a/frontends/pytorch/e2e_testing/torchscript/elementwise.py +++ b/e2e_testing/torchscript/elementwise.py @@ -4,9 +4,9 @@ import torch -from torch_mlir_torchscript.e2e_test.framework import TestUtils -from torch_mlir_torchscript.e2e_test.registry import register_test_case -from torch_mlir_torchscript.annotations import annotate_args, export +from npcomp_torchscript.e2e_test.framework import TestUtils +from npcomp_torchscript.e2e_test.registry import register_test_case +from npcomp_torchscript.annotations import annotate_args, export # TODO: Support scalar !torch.int/!torch.float variants. Add support to # ReduceOpVariants to implement them in terms of the tensor-only variants + @@ -190,4 +190,3 @@ def ElementwiseSigmoidModule_basic(module, tu: TestUtils): module.forward(tu.rand(3, 5)) # ============================================================================== - diff --git a/frontends/pytorch/e2e_testing/torchscript/list_programs.py b/e2e_testing/torchscript/list_programs.py similarity index 74% rename from frontends/pytorch/e2e_testing/torchscript/list_programs.py rename to e2e_testing/torchscript/list_programs.py index ea6448450..75e2c31e1 100644 --- a/frontends/pytorch/e2e_testing/torchscript/list_programs.py +++ b/e2e_testing/torchscript/list_programs.py @@ -4,9 +4,9 @@ import torch -from torch_mlir_torchscript.e2e_test.framework import TestUtils -from torch_mlir_torchscript.e2e_test.registry import register_test_case -from torch_mlir_torchscript.annotations import annotate_args, export +from npcomp_torchscript.e2e_test.framework import TestUtils +from npcomp_torchscript.e2e_test.registry import register_test_case +from npcomp_torchscript.annotations import annotate_args, export # ============================================================================== diff --git a/frontends/pytorch/e2e_testing/torchscript/main.py b/e2e_testing/torchscript/main.py similarity index 93% rename from frontends/pytorch/e2e_testing/torchscript/main.py rename to e2e_testing/torchscript/main.py index bc2403370..95ea3b655 100644 --- a/frontends/pytorch/e2e_testing/torchscript/main.py +++ b/e2e_testing/torchscript/main.py @@ -8,12 +8,12 @@ import pickle import re import sys -from torch_mlir_torchscript.e2e_test.framework import run_tests -from torch_mlir_torchscript.e2e_test.reporting import report_results -from torch_mlir_torchscript.e2e_test.registry import GLOBAL_TEST_REGISTRY +from npcomp_torchscript.e2e_test.framework import run_tests +from npcomp_torchscript.e2e_test.reporting import report_results +from npcomp_torchscript.e2e_test.registry import GLOBAL_TEST_REGISTRY # Available test configs. -from torch_mlir_torchscript_e2e_test_configs import ( +from npcomp_torchscript_e2e_test_configs import ( NpcompBackendTestConfig, NativeTorchTestConfig, TorchScriptTestConfig ) diff --git a/frontends/pytorch/e2e_testing/torchscript/mlp.py b/e2e_testing/torchscript/mlp.py similarity index 88% rename from frontends/pytorch/e2e_testing/torchscript/mlp.py rename to e2e_testing/torchscript/mlp.py index bc2b27695..15b5f65dc 100644 --- a/frontends/pytorch/e2e_testing/torchscript/mlp.py +++ b/e2e_testing/torchscript/mlp.py @@ -5,9 +5,9 @@ import torch import torch.nn as nn -from torch_mlir_torchscript.e2e_test.framework import TestUtils -from torch_mlir_torchscript.e2e_test.registry import register_test_case -from torch_mlir_torchscript.annotations import annotate_args, export +from npcomp_torchscript.e2e_test.framework import TestUtils +from npcomp_torchscript.e2e_test.registry import register_test_case +from npcomp_torchscript.annotations import annotate_args, export # ============================================================================== diff --git a/frontends/pytorch/e2e_testing/torchscript/quantized_models.py b/e2e_testing/torchscript/quantized_models.py similarity index 87% rename from frontends/pytorch/e2e_testing/torchscript/quantized_models.py rename to e2e_testing/torchscript/quantized_models.py index 81d908d9a..5a482eb18 100644 --- a/frontends/pytorch/e2e_testing/torchscript/quantized_models.py +++ b/e2e_testing/torchscript/quantized_models.py @@ -5,9 +5,9 @@ import torch from torch import nn -from torch_mlir_torchscript.e2e_test.framework import TestUtils -from torch_mlir_torchscript.e2e_test.registry import register_test_case -from torch_mlir_torchscript.annotations import annotate_args, export +from npcomp_torchscript.e2e_test.framework import TestUtils +from npcomp_torchscript.e2e_test.registry import register_test_case +from npcomp_torchscript.annotations import annotate_args, export # ============================================================================== diff --git a/frontends/pytorch/e2e_testing/torchscript/reduction.py b/e2e_testing/torchscript/reduction.py similarity index 89% rename from frontends/pytorch/e2e_testing/torchscript/reduction.py rename to e2e_testing/torchscript/reduction.py index 1675db6f2..027c87e0e 100644 --- a/frontends/pytorch/e2e_testing/torchscript/reduction.py +++ b/e2e_testing/torchscript/reduction.py @@ -4,9 +4,9 @@ import torch -from torch_mlir_torchscript.e2e_test.framework import TestUtils -from torch_mlir_torchscript.e2e_test.registry import register_test_case -from torch_mlir_torchscript.annotations import annotate_args, export +from npcomp_torchscript.e2e_test.framework import TestUtils +from npcomp_torchscript.e2e_test.registry import register_test_case +from npcomp_torchscript.annotations import annotate_args, export # ============================================================================== diff --git a/frontends/pytorch/e2e_testing/torchscript/vision_models.py b/e2e_testing/torchscript/vision_models.py similarity index 81% rename from frontends/pytorch/e2e_testing/torchscript/vision_models.py rename to e2e_testing/torchscript/vision_models.py index 0294d0c10..35ad73a6b 100644 --- a/frontends/pytorch/e2e_testing/torchscript/vision_models.py +++ b/e2e_testing/torchscript/vision_models.py @@ -5,9 +5,9 @@ import torch import torchvision.models as models -from torch_mlir_torchscript.e2e_test.framework import TestUtils -from torch_mlir_torchscript.e2e_test.registry import register_test_case -from torch_mlir_torchscript.annotations import annotate_args, export +from npcomp_torchscript.e2e_test.framework import TestUtils +from npcomp_torchscript.e2e_test.registry import register_test_case +from npcomp_torchscript.annotations import annotate_args, export # ============================================================================== diff --git a/frontends/pytorch/e2e_testing/torchscript/xfail_sets.py b/e2e_testing/torchscript/xfail_sets.py similarity index 100% rename from frontends/pytorch/e2e_testing/torchscript/xfail_sets.py rename to e2e_testing/torchscript/xfail_sets.py diff --git a/frontends/pytorch/examples/__init__.py b/examples/__init__.py similarity index 100% rename from frontends/pytorch/examples/__init__.py rename to examples/__init__.py diff --git a/frontends/pytorch/examples/resnet_inference.ipynb b/examples/resnet_inference.ipynb similarity index 99% rename from frontends/pytorch/examples/resnet_inference.ipynb rename to examples/resnet_inference.ipynb index 988bf37c7..2a31c403b 100644 --- a/frontends/pytorch/examples/resnet_inference.ipynb +++ b/examples/resnet_inference.ipynb @@ -93,7 +93,7 @@ "\n", "from npcomp.passmanager import PassManager\n", "\n", - "from torch_mlir_torchscript.annotations import annotate_args, export\n", + "from npcomp_torchscript.annotations import annotate_args, export\n", "from torch_mlir.torchscript_annotations import extract_annotations" ] }, @@ -227,7 +227,7 @@ " # NOTE: THis is different from `torch.jit.export`. The `torch.jit.export`\n", " # decorator controls which methods of the original torch.nn.Module get\n", " # compiled into TorchScript. This decorator\n", - " # (`torch_mlir_torchscript.annotations.export`) controls which TorchScript\n", + " # (`npcomp_torchscript.annotations.export`) controls which TorchScript\n", " # methods are compiled by npcomp. \n", " @export\n", " # The `annotate_args` annotation provides metadata to the npcomp compiler\n", diff --git a/frontends/pytorch/examples/torchscript_resnet18_e2e.py b/examples/torchscript_resnet18_e2e.py similarity index 94% rename from frontends/pytorch/examples/torchscript_resnet18_e2e.py rename to examples/torchscript_resnet18_e2e.py index c764b1da2..580b74bde 100644 --- a/frontends/pytorch/examples/torchscript_resnet18_e2e.py +++ b/examples/torchscript_resnet18_e2e.py @@ -1,6 +1,6 @@ -# -*- Python -*- -# This file is licensed under a pytorch-style license -# See frontends/pytorch/LICENSE for license information. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception from PIL import Image import requests diff --git a/external/torch-mlir/TorchPlugin/python/torch_mlir/torchscript_annotations.py b/external/torch-mlir/TorchPlugin/python/torch_mlir/torchscript_annotations.py index 526191ac1..82d6b744a 100644 --- a/external/torch-mlir/TorchPlugin/python/torch_mlir/torchscript_annotations.py +++ b/external/torch-mlir/TorchPlugin/python/torch_mlir/torchscript_annotations.py @@ -16,7 +16,7 @@ import torch_mlir # to be expressed conveniently and gives clearer error reports when # the annotations aren't acceptable. -# This module is kept separate from torch_mlir_torchscript.annotations so that +# This module is kept separate from npcomp_torchscript.annotations so that # we can use that module from code without C++ dependencies, which prevent us # from interfacing the test framework across environments. diff --git a/external/torch-mlir/TorchPlugin/setup.py b/external/torch-mlir/TorchPlugin/setup.py index 404f4b715..b308d6532 100644 --- a/external/torch-mlir/TorchPlugin/setup.py +++ b/external/torch-mlir/TorchPlugin/setup.py @@ -45,9 +45,5 @@ setup( packages=find_packages("./python", include=[ "torch_mlir", "torch_mlir.*", - "torch_mlir_torchscript", - "torch_mlir_torchscript.*", - "torch_mlir_torchscript_e2e_test_configs", - "torch_mlir_torchscript_e2e_test_configs.*", ]), ) diff --git a/frontends/pytorch/.gitignore b/frontends/pytorch/.gitignore deleted file mode 100644 index 2247d5f95..000000000 --- a/frontends/pytorch/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/build -/dist diff --git a/frontends/pytorch/CMakeLists.txt b/frontends/pytorch/CMakeLists.txt deleted file mode 100644 index e7c8bae4c..000000000 --- a/frontends/pytorch/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -#------------------------------------------------------------------------------- -# Sub project setup -#------------------------------------------------------------------------------- - -cmake_minimum_required(VERSION 3.13.4) - -if(POLICY CMP0068) - cmake_policy(SET CMP0068 NEW) - set(CMAKE_BUILD_WITH_INSTALL_NAME_DIR ON) -endif() - -if(POLICY CMP0075) - cmake_policy(SET CMP0075 NEW) -endif() - -if(POLICY CMP0077) - cmake_policy(SET CMP0077 NEW) -endif() - -project(npcomp_pytorch LANGUAGES CXX C) -set(CMAKE_C_STANDARD 11) -set(CMAKE_CXX_STANDARD 14) - -#------------------------------------------------------------------------------- -# Output paths -#------------------------------------------------------------------------------- - -if(NOT MLIR_NPCOMP_PYTHON_PACKAGES_DIR) - set(MLIR_NPCOMP_PYTHON_PACKAGES_DIR "${CMAKE_CURRENT_BINARY_DIR}/python_packages") -endif() - -#------------------------------------------------------------------------------- -# Subdirectories -#------------------------------------------------------------------------------- - -add_subdirectory(python) -add_subdirectory(test) diff --git a/frontends/pytorch/LICENSE b/frontends/pytorch/LICENSE deleted file mode 100644 index b6c3eaad6..000000000 --- a/frontends/pytorch/LICENSE +++ /dev/null @@ -1,65 +0,0 @@ -In order to facilitate future incorporation in pytorch, the code in this -directory (frontends/pytorch) is provided under the below license. - -Copyright (c) 2020 LLVM Foundation. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America - and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. - -The design of this code is highly inspired by the design of the xla device for -pytorch (git@github.com:pytorch/xla.git). The license for pytorch/xla is: - -Copyright (c) 2018 Google Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America - and IDIAP Research Institute nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE -LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -POSSIBILITY OF SUCH DAMAGE. diff --git a/frontends/pytorch/python/CMakeLists.txt b/frontends/pytorch/python/CMakeLists.txt deleted file mode 100644 index 11152557b..000000000 --- a/frontends/pytorch/python/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -################################################################################ -# Manage python source files -# Collapse all local python sources to the project level python/ directory. -################################################################################ - -include(NpcompPython) - -npcomp_python_create_symlinks( - ${MLIR_NPCOMP_PYTHON_PACKAGES_DIR}/npcomp_torch ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/frontends/pytorch/test/CMakeLists.txt b/frontends/pytorch/test/CMakeLists.txt deleted file mode 100644 index 97eeb7469..000000000 --- a/frontends/pytorch/test/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -configure_lit_site_cfg( - ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in - ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py - MAIN_CONFIG - ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py -) - -set(TEST_DEPENDS - FileCheck count not - npcomp-opt - TorchMLIRTorchPlugin - TorchMLIRPluginPythonModules - NPCOMPPythonModules - ) - -add_lit_testsuite(check-frontends-pytorch "Running the frontends-pytorch regression tests" - ${CMAKE_CURRENT_BINARY_DIR} - DEPENDS ${TEST_DEPENDS} - ) -set_target_properties(check-frontends-pytorch PROPERTIES FOLDER "Tests") - -add_lit_testsuites(FRONTENDS_PYTORCH ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${TEST_DEPENDS}) -add_dependencies(check-npcomp-all check-frontends-pytorch) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 7ac233336..73a13db6e 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -29,14 +29,10 @@ add_dependencies(NPCOMPPythonResources declare_mlir_python_sources(NPCOMPPythonSources) declare_mlir_python_sources(NPCOMPPythonSources.Core - ADD_TO_PARENT NPCOMPPythonSources ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/npcomp" - SOURCES - build.py - __init__.py + ADD_TO_PARENT NPCOMPPythonSources SOURCES_GLOB - compiler/*.py - utils/*.py + *.py ) declare_mlir_python_sources(NPCOMPPythonSources.Dialects @@ -113,5 +109,39 @@ add_mlir_python_modules(NPCOMPPythonModules COMMON_CAPI_LINK_LIBS NPCOMPPythonCAPI ) +add_dependencies(NPCOMPPythonModules + NPCOMPPythonResources + TorchMLIRPluginPythonModules + ) -add_dependencies(NPCOMPPythonModules NPCOMPPythonResources) +################################################################################ +# Torch support libraries. +################################################################################ + +# Torch-related modules that are too broad of a scope/opinion for torch-mlir +# belong there. +declare_mlir_python_sources(NPCOMPTorchSupportPythonSources + ADD_TO_PARENT NPCOMPPythonSources + ROOT_DIR "${CMAKE_CURRENT_SOURCE_DIR}/torch_support" + SOURCES_GLOB + npcomp_torchscript/*.py + npcomp_torchscript_e2e_test_configs/*.py +) + +add_mlir_python_modules(NPCOMPTorchSupportPythonModules + ROOT_PREFIX "${MLIR_NPCOMP_PYTHON_PACKAGES_DIR}/npcomp_core" + INSTALL_PREFIX "python_packages/npcomp_core" + DECLARED_SOURCES + NPCOMPTorchSupportPythonSources +) +add_dependencies(NPCOMPTorchSupportPythonModules + NPCOMPPythonModules + ) + + + +################################################################################ +# Recurse into the tests. +################################################################################ + +add_subdirectory(test) diff --git a/python/test/CMakeLists.txt b/python/test/CMakeLists.txt new file mode 100644 index 000000000..f6a19151d --- /dev/null +++ b/python/test/CMakeLists.txt @@ -0,0 +1,21 @@ +configure_lit_site_cfg( + ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in + ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py + MAIN_CONFIG + ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py +) + +set(TEST_DEPENDS + FileCheck count not + npcomp-opt + NPCOMPTorchSupportPythonModules + ) + +add_lit_testsuite(check-npcomp-python "Running the npcomp-python regression tests" + ${CMAKE_CURRENT_BINARY_DIR} + DEPENDS ${TEST_DEPENDS} + ) +set_target_properties(check-npcomp-python PROPERTIES FOLDER "Tests") + +add_lit_testsuites(NPCOMP_PYTHON ${CMAKE_CURRENT_SOURCE_DIR} DEPENDS ${TEST_DEPENDS}) +add_dependencies(check-npcomp-all check-npcomp-python) diff --git a/frontends/pytorch/test/annotations-sugar.py b/python/test/annotations-sugar.py similarity index 82% rename from frontends/pytorch/test/annotations-sugar.py rename to python/test/annotations-sugar.py index 40dbd0df1..16fa20dc1 100644 --- a/frontends/pytorch/test/annotations-sugar.py +++ b/python/test/annotations-sugar.py @@ -1,13 +1,13 @@ -# -*- Python -*- -# This file is licensed under a pytorch-style license -# See frontends/pytorch/LICENSE for license information. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # RUN: %PYTHON %s | FileCheck %s import torch import torch_mlir -from torch_mlir_torchscript.annotations import annotate_args, export +from npcomp_torchscript.annotations import annotate_args, export from torch_mlir.torchscript_annotations import extract_annotations class MmModule(torch.nn.Module): diff --git a/frontends/pytorch/test/lit.cfg.py b/python/test/lit.cfg.py similarity index 92% rename from frontends/pytorch/test/lit.cfg.py rename to python/test/lit.cfg.py index 8cf5f77e2..fcd25f897 100644 --- a/frontends/pytorch/test/lit.cfg.py +++ b/python/test/lit.cfg.py @@ -1,6 +1,6 @@ -# -*- Python -*- -# This file is licensed under a pytorch-style license -# See frontends/pytorch/LICENSE for license information. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception import os import platform @@ -62,7 +62,6 @@ npcomp_python_dir = "python" if config.npcomp_built_standalone else "tools/npcom llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True) llvm_config.with_environment('PYTHONPATH', [ os.path.join(config.npcomp_python_packages_dir, 'npcomp_core'), - os.path.join(config.npcomp_python_packages_dir, 'npcomp_torch'), os.path.join(config.torch_mlir_python_packages_dir, 'torch_mlir'), os.path.join(config.torch_mlir_python_packages_dir, 'torch_mlir_dialects'), ], diff --git a/frontends/pytorch/test/lit.site.cfg.py.in b/python/test/lit.site.cfg.py.in similarity index 91% rename from frontends/pytorch/test/lit.site.cfg.py.in rename to python/test/lit.site.cfg.py.in index 684dba38c..20c721fb1 100644 --- a/frontends/pytorch/test/lit.site.cfg.py.in +++ b/python/test/lit.site.cfg.py.in @@ -1,6 +1,7 @@ # -*- Python -*- -# This file is licensed under a pytorch-style license -# See frontends/pytorch/LICENSE for license information. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @LIT_SITE_CFG_IN_HEADER@ diff --git a/frontends/pytorch/test/torchscript_e2e_test/README.md b/python/test/torchscript_e2e_test/README.md similarity index 100% rename from frontends/pytorch/test/torchscript_e2e_test/README.md rename to python/test/torchscript_e2e_test/README.md diff --git a/frontends/pytorch/test/torchscript_e2e_test/basic.py b/python/test/torchscript_e2e_test/basic.py similarity index 61% rename from frontends/pytorch/test/torchscript_e2e_test/basic.py rename to python/test/torchscript_e2e_test/basic.py index 2002148c4..0b0b96467 100644 --- a/frontends/pytorch/test/torchscript_e2e_test/basic.py +++ b/python/test/torchscript_e2e_test/basic.py @@ -1,15 +1,15 @@ -# -*- Python -*- -# This file is licensed under a pytorch-style license -# See frontends/pytorch/LICENSE for license information. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # RUN: %PYTHON %s | FileCheck %s import torch -from torch_mlir_torchscript.e2e_test.framework import run_tests, TestUtils -from torch_mlir_torchscript.e2e_test.reporting import report_results -from torch_mlir_torchscript.e2e_test.registry import register_test_case, GLOBAL_TEST_REGISTRY -from torch_mlir_torchscript_e2e_test_configs import TorchScriptTestConfig +from npcomp_torchscript.e2e_test.framework import run_tests, TestUtils +from npcomp_torchscript.e2e_test.reporting import report_results +from npcomp_torchscript.e2e_test.registry import register_test_case, GLOBAL_TEST_REGISTRY +from npcomp_torchscript_e2e_test_configs import TorchScriptTestConfig class MmModule(torch.nn.Module): diff --git a/frontends/pytorch/test/torchscript_e2e_test/compilation_failure.py b/python/test/torchscript_e2e_test/compilation_failure.py similarity index 65% rename from frontends/pytorch/test/torchscript_e2e_test/compilation_failure.py rename to python/test/torchscript_e2e_test/compilation_failure.py index 28b02086a..558b3985d 100644 --- a/frontends/pytorch/test/torchscript_e2e_test/compilation_failure.py +++ b/python/test/torchscript_e2e_test/compilation_failure.py @@ -1,15 +1,15 @@ -# -*- Python -*- -# This file is licensed under a pytorch-style license -# See frontends/pytorch/LICENSE for license information. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # RUN: %PYTHON %s | FileCheck %s import torch -from torch_mlir_torchscript.e2e_test.framework import run_tests, TestUtils -from torch_mlir_torchscript.e2e_test.reporting import report_results -from torch_mlir_torchscript.e2e_test.registry import register_test_case, GLOBAL_TEST_REGISTRY -from torch_mlir_torchscript_e2e_test_configs import TorchScriptTestConfig +from npcomp_torchscript.e2e_test.framework import run_tests, TestUtils +from npcomp_torchscript.e2e_test.reporting import report_results +from npcomp_torchscript.e2e_test.registry import register_test_case, GLOBAL_TEST_REGISTRY +from npcomp_torchscript_e2e_test_configs import TorchScriptTestConfig class MmModule(torch.nn.Module): diff --git a/frontends/pytorch/test/torchscript_e2e_test/error_reports.py b/python/test/torchscript_e2e_test/error_reports.py similarity index 91% rename from frontends/pytorch/test/torchscript_e2e_test/error_reports.py rename to python/test/torchscript_e2e_test/error_reports.py index 00e4f5295..7f97eb466 100644 --- a/frontends/pytorch/test/torchscript_e2e_test/error_reports.py +++ b/python/test/torchscript_e2e_test/error_reports.py @@ -1,6 +1,6 @@ -# -*- Python -*- -# This file is licensed under a pytorch-style license -# See frontends/pytorch/LICENSE for license information. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # RUN: %PYTHON %s | FileCheck %s @@ -8,10 +8,10 @@ from typing import List, Tuple, Dict import torch -from torch_mlir_torchscript.e2e_test.framework import run_tests, TestUtils -from torch_mlir_torchscript.e2e_test.reporting import report_results -from torch_mlir_torchscript.e2e_test.registry import register_test_case, GLOBAL_TEST_REGISTRY -from torch_mlir_torchscript_e2e_test_configs import TorchScriptTestConfig +from npcomp_torchscript.e2e_test.framework import run_tests, TestUtils +from npcomp_torchscript.e2e_test.reporting import report_results +from npcomp_torchscript.e2e_test.registry import register_test_case, GLOBAL_TEST_REGISTRY +from npcomp_torchscript_e2e_test_configs import TorchScriptTestConfig # CHECK: FAIL - "ErroneousModule_basic" diff --git a/frontends/pytorch/test/torchscript_e2e_test/non_tensor_values.py b/python/test/torchscript_e2e_test/non_tensor_values.py similarity index 67% rename from frontends/pytorch/test/torchscript_e2e_test/non_tensor_values.py rename to python/test/torchscript_e2e_test/non_tensor_values.py index 1c48cdbe5..c532bc6dd 100644 --- a/frontends/pytorch/test/torchscript_e2e_test/non_tensor_values.py +++ b/python/test/torchscript_e2e_test/non_tensor_values.py @@ -1,6 +1,6 @@ -# -*- Python -*- -# This file is licensed under a pytorch-style license -# See frontends/pytorch/LICENSE for license information. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # RUN: %PYTHON %s | FileCheck %s @@ -8,10 +8,10 @@ from typing import List, Tuple, Dict import torch -from torch_mlir_torchscript.e2e_test.framework import run_tests, TestUtils -from torch_mlir_torchscript.e2e_test.reporting import report_results -from torch_mlir_torchscript.e2e_test.registry import register_test_case, GLOBAL_TEST_REGISTRY -from torch_mlir_torchscript_e2e_test_configs import TorchScriptTestConfig +from npcomp_torchscript.e2e_test.framework import run_tests, TestUtils +from npcomp_torchscript.e2e_test.reporting import report_results +from npcomp_torchscript.e2e_test.registry import register_test_case, GLOBAL_TEST_REGISTRY +from npcomp_torchscript_e2e_test_configs import TorchScriptTestConfig class NonTensorValuesModule(torch.nn.Module): diff --git a/frontends/pytorch/test/torchscript_e2e_test/submodule.py b/python/test/torchscript_e2e_test/submodule.py similarity index 63% rename from frontends/pytorch/test/torchscript_e2e_test/submodule.py rename to python/test/torchscript_e2e_test/submodule.py index 84b5673dc..191c6b8ac 100644 --- a/frontends/pytorch/test/torchscript_e2e_test/submodule.py +++ b/python/test/torchscript_e2e_test/submodule.py @@ -1,15 +1,15 @@ -# -*- Python -*- -# This file is licensed under a pytorch-style license -# See frontends/pytorch/LICENSE for license information. +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # RUN: %PYTHON %s | FileCheck %s import torch -from torch_mlir_torchscript.e2e_test.framework import run_tests, TestUtils -from torch_mlir_torchscript.e2e_test.reporting import report_results -from torch_mlir_torchscript.e2e_test.registry import register_test_case, GLOBAL_TEST_REGISTRY -from torch_mlir_torchscript_e2e_test_configs import TorchScriptTestConfig +from npcomp_torchscript.e2e_test.framework import run_tests, TestUtils +from npcomp_torchscript.e2e_test.reporting import report_results +from npcomp_torchscript.e2e_test.registry import register_test_case, GLOBAL_TEST_REGISTRY +from npcomp_torchscript_e2e_test_configs import TorchScriptTestConfig class Submodule2(torch.nn.Module): def __init__(self): diff --git a/frontends/pytorch/python/torch_mlir_torchscript/__init__.py b/python/torch_support/npcomp_torchscript/__init__.py similarity index 100% rename from frontends/pytorch/python/torch_mlir_torchscript/__init__.py rename to python/torch_support/npcomp_torchscript/__init__.py diff --git a/frontends/pytorch/python/torch_mlir_torchscript/annotations.py b/python/torch_support/npcomp_torchscript/annotations.py similarity index 100% rename from frontends/pytorch/python/torch_mlir_torchscript/annotations.py rename to python/torch_support/npcomp_torchscript/annotations.py diff --git a/frontends/pytorch/python/torch_mlir_torchscript/e2e_test/__init__.py b/python/torch_support/npcomp_torchscript/e2e_test/__init__.py similarity index 100% rename from frontends/pytorch/python/torch_mlir_torchscript/e2e_test/__init__.py rename to python/torch_support/npcomp_torchscript/e2e_test/__init__.py diff --git a/frontends/pytorch/python/torch_mlir_torchscript/e2e_test/framework.py b/python/torch_support/npcomp_torchscript/e2e_test/framework.py similarity index 100% rename from frontends/pytorch/python/torch_mlir_torchscript/e2e_test/framework.py rename to python/torch_support/npcomp_torchscript/e2e_test/framework.py diff --git a/frontends/pytorch/python/torch_mlir_torchscript/e2e_test/registry.py b/python/torch_support/npcomp_torchscript/e2e_test/registry.py similarity index 100% rename from frontends/pytorch/python/torch_mlir_torchscript/e2e_test/registry.py rename to python/torch_support/npcomp_torchscript/e2e_test/registry.py diff --git a/frontends/pytorch/python/torch_mlir_torchscript/e2e_test/reporting.py b/python/torch_support/npcomp_torchscript/e2e_test/reporting.py similarity index 100% rename from frontends/pytorch/python/torch_mlir_torchscript/e2e_test/reporting.py rename to python/torch_support/npcomp_torchscript/e2e_test/reporting.py diff --git a/frontends/pytorch/python/torch_mlir_torchscript_e2e_test_configs/__init__.py b/python/torch_support/npcomp_torchscript_e2e_test_configs/__init__.py similarity index 100% rename from frontends/pytorch/python/torch_mlir_torchscript_e2e_test_configs/__init__.py rename to python/torch_support/npcomp_torchscript_e2e_test_configs/__init__.py diff --git a/frontends/pytorch/python/torch_mlir_torchscript_e2e_test_configs/native_torch.py b/python/torch_support/npcomp_torchscript_e2e_test_configs/native_torch.py similarity index 92% rename from frontends/pytorch/python/torch_mlir_torchscript_e2e_test_configs/native_torch.py rename to python/torch_support/npcomp_torchscript_e2e_test_configs/native_torch.py index 95ba0c59b..6e8163863 100644 --- a/frontends/pytorch/python/torch_mlir_torchscript_e2e_test_configs/native_torch.py +++ b/python/torch_support/npcomp_torchscript_e2e_test_configs/native_torch.py @@ -7,7 +7,7 @@ from typing import Any import torch -from torch_mlir_torchscript.e2e_test.framework import TestConfig, Trace, TraceItem +from npcomp_torchscript.e2e_test.framework import TestConfig, Trace, TraceItem class NativeTorchTestConfig(TestConfig): diff --git a/frontends/pytorch/python/torch_mlir_torchscript_e2e_test_configs/npcomp_backend.py b/python/torch_support/npcomp_torchscript_e2e_test_configs/npcomp_backend.py similarity index 98% rename from frontends/pytorch/python/torch_mlir_torchscript_e2e_test_configs/npcomp_backend.py rename to python/torch_support/npcomp_torchscript_e2e_test_configs/npcomp_backend.py index 21321c0c9..d6be23a8e 100644 --- a/frontends/pytorch/python/torch_mlir_torchscript_e2e_test_configs/npcomp_backend.py +++ b/python/torch_support/npcomp_torchscript_e2e_test_configs/npcomp_backend.py @@ -16,7 +16,7 @@ import npcomp from npcomp.passmanager import PassManager from npcomp.compiler.pytorch.backend import refjit from npcomp.compiler.pytorch.backend.abc import NpcompBackend -from torch_mlir_torchscript.e2e_test.framework import TestConfig, Trace, TraceItem +from npcomp_torchscript.e2e_test.framework import TestConfig, Trace, TraceItem from torch_mlir.torchscript_annotations import extract_annotations def _recursively_convert_to_numpy(o: Any): diff --git a/frontends/pytorch/python/torch_mlir_torchscript_e2e_test_configs/torchscript.py b/python/torch_support/npcomp_torchscript_e2e_test_configs/torchscript.py similarity index 93% rename from frontends/pytorch/python/torch_mlir_torchscript_e2e_test_configs/torchscript.py rename to python/torch_support/npcomp_torchscript_e2e_test_configs/torchscript.py index ed22f7157..9eb9a579a 100644 --- a/frontends/pytorch/python/torch_mlir_torchscript_e2e_test_configs/torchscript.py +++ b/python/torch_support/npcomp_torchscript_e2e_test_configs/torchscript.py @@ -7,7 +7,7 @@ from typing import Any import torch -from torch_mlir_torchscript.e2e_test.framework import TestConfig, Trace, TraceItem +from npcomp_torchscript.e2e_test.framework import TestConfig, Trace, TraceItem class TorchScriptTestConfig(TestConfig): diff --git a/tools/torchscript_e2e_test.sh b/tools/torchscript_e2e_test.sh index 273b03235..adf3dfe7c 100755 --- a/tools/torchscript_e2e_test.sh +++ b/tools/torchscript_e2e_test.sh @@ -9,4 +9,4 @@ cd "$src_dir" export PYTHONPATH=${PYTHONPATH-} source .env -python -m frontends.pytorch.e2e_testing.torchscript.main "$@" +python -m e2e_testing.torchscript.main "$@"