2021-09-30 00:03:40 +08:00
|
|
|
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
|
|
# See https://llvm.org/LICENSE.txt for license information.
|
|
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
# Also available under a BSD-style license. See LICENSE.
|
2021-04-20 06:12:29 +08:00
|
|
|
|
|
|
|
import argparse
|
2021-07-10 03:22:45 +08:00
|
|
|
import os
|
|
|
|
import pickle
|
2021-04-22 06:07:15 +08:00
|
|
|
import re
|
|
|
|
import sys
|
2021-04-20 06:12:29 +08:00
|
|
|
|
2021-10-05 02:24:12 +08:00
|
|
|
from torch_mlir_e2e_test.torchscript.framework import TestConfig, run_tests
|
2021-09-28 02:36:44 +08:00
|
|
|
from torch_mlir_e2e_test.torchscript.reporting import report_results
|
|
|
|
from torch_mlir_e2e_test.torchscript.registry import GLOBAL_TEST_REGISTRY
|
2021-04-20 06:12:29 +08:00
|
|
|
|
|
|
|
# Available test configs.
|
2021-09-28 02:36:44 +08:00
|
|
|
from torch_mlir_e2e_test.torchscript.configs import (
|
2021-10-08 10:07:03 +08:00
|
|
|
LinalgOnTensorsBackendTestConfig, NativeTorchTestConfig, TorchScriptTestConfig, TosaBackendTestConfig
|
2021-04-20 06:12:29 +08:00
|
|
|
)
|
|
|
|
|
2021-09-28 02:36:44 +08:00
|
|
|
from torch_mlir_e2e_test.linalg_on_tensors_backends.refbackend import RefBackendLinalgOnTensorsBackend
|
2021-10-08 10:07:03 +08:00
|
|
|
from torch_mlir_e2e_test.tosa_backends.linalg_on_tensors import LinalgOnTensorsTosaBackend
|
2021-07-01 05:13:21 +08:00
|
|
|
|
2021-10-08 10:07:03 +08:00
|
|
|
from .xfail_sets import REFBACKEND_XFAIL_SET, TOSA_PASS_SET, COMMON_TORCH_MLIR_LOWERING_XFAILS
|
2021-07-01 05:13:21 +08:00
|
|
|
|
2021-04-20 06:12:29 +08:00
|
|
|
# Import tests to register them in the global registry.
|
2021-07-01 05:13:21 +08:00
|
|
|
# Make sure to use `tools/torchscript_e2e_test.sh` wrapper for invoking
|
|
|
|
# this script.
|
|
|
|
from . import basic
|
|
|
|
from . import vision_models
|
|
|
|
from . import mlp
|
2021-06-29 08:01:12 +08:00
|
|
|
from . import conv
|
2021-07-01 05:13:21 +08:00
|
|
|
from . import batchnorm
|
|
|
|
from . import quantized_models
|
|
|
|
from . import elementwise
|
2021-10-16 06:23:40 +08:00
|
|
|
from . import type_promotion
|
|
|
|
from . import type_conversion
|
2021-11-08 23:56:40 +08:00
|
|
|
from . import backprop
|
2021-09-02 02:58:31 +08:00
|
|
|
from . import reduction
|
2021-09-08 01:18:10 +08:00
|
|
|
from . import argmax
|
2021-10-21 13:15:10 +08:00
|
|
|
from . import matmul
|
2021-10-27 18:18:06 +08:00
|
|
|
from . import view
|
2021-04-20 06:12:29 +08:00
|
|
|
|
2021-05-20 08:36:00 +08:00
|
|
|
def _get_argparse():
|
2021-10-08 10:07:03 +08:00
|
|
|
config_choices = ['native_torch', 'torchscript', 'refbackend', 'tosa', 'external']
|
2021-04-20 06:12:29 +08:00
|
|
|
parser = argparse.ArgumentParser(description='Run torchscript e2e tests.')
|
2021-08-06 03:00:38 +08:00
|
|
|
parser.add_argument('-c', '--config',
|
2021-07-01 05:13:21 +08:00
|
|
|
choices=config_choices,
|
2021-04-20 06:12:29 +08:00
|
|
|
default='refbackend',
|
2021-07-01 05:13:21 +08:00
|
|
|
help=f'''
|
2021-04-20 06:12:29 +08:00
|
|
|
Meaning of options:
|
2021-09-28 02:36:44 +08:00
|
|
|
"refbackend": run through torch-mlir's RefBackend.
|
2021-10-08 10:07:03 +08:00
|
|
|
"tosa": run through torch-mlir's default TOSA backend.
|
2021-04-21 05:45:43 +08:00
|
|
|
"native_torch": run the torch.nn.Module as-is without compiling (useful for verifying model is deterministic; ALL tests should pass in this configuration).
|
2021-04-20 06:12:29 +08:00
|
|
|
"torchscript": compile the model to a torch.jit.ScriptModule, and then run that as-is (useful for verifying TorchScript is modeling the program correctly).
|
2021-10-05 02:24:12 +08:00
|
|
|
"external": use an external backend, specified by the `--external-backend` option.
|
|
|
|
''')
|
|
|
|
parser.add_argument('--external-config',
|
|
|
|
help=f'''
|
|
|
|
Specifies a path to a Python file, which will be `exec`'ed.
|
|
|
|
The file has the following contract:
|
|
|
|
- The global variable `config` should be set to an instance of `TestConfig`.
|
|
|
|
- `xfail_set` should be set to a set of test unique identifiers that are
|
|
|
|
expected to fail. The global `COMMON_TORCH_MLIR_LOWERING_XFAILS` provides
|
|
|
|
a common set of xfails that won't work on backends because torch-mlir
|
|
|
|
itself does not handle them.
|
2021-04-22 06:07:15 +08:00
|
|
|
''')
|
2021-08-10 04:55:20 +08:00
|
|
|
parser.add_argument('-f', '--filter', default='.*', help='''
|
2021-04-22 06:07:15 +08:00
|
|
|
Regular expression specifying which tests to include in this run.
|
2021-04-20 06:12:29 +08:00
|
|
|
''')
|
2021-05-20 08:36:00 +08:00
|
|
|
parser.add_argument('-v', '--verbose',
|
|
|
|
default=False,
|
|
|
|
action='store_true',
|
|
|
|
help='report test results with additional detail')
|
2021-07-10 03:22:45 +08:00
|
|
|
parser.add_argument('--serialized-test-dir', default=None, type=str, help='''
|
|
|
|
The directory containing serialized pre-built tests.
|
|
|
|
Right now, these are additional tests which require heavy Python dependencies
|
|
|
|
to generate (or cannot even be generated with the version of PyTorch used by
|
2021-09-28 02:36:44 +08:00
|
|
|
torch-mlir).
|
2021-07-10 03:22:45 +08:00
|
|
|
See `build_tools/torchscript_e2e_heavydep_tests/generate_serialized_tests.sh`
|
|
|
|
for more information on building these artifacts.
|
|
|
|
''')
|
2021-05-20 08:36:00 +08:00
|
|
|
return parser
|
|
|
|
|
|
|
|
def main():
|
|
|
|
args = _get_argparse().parse_args()
|
|
|
|
|
2021-10-08 10:07:03 +08:00
|
|
|
all_tests = list(GLOBAL_TEST_REGISTRY)
|
|
|
|
if args.serialized_test_dir:
|
|
|
|
for root, dirs, files in os.walk(args.serialized_test_dir):
|
|
|
|
for filename in files:
|
|
|
|
with open(os.path.join(root, filename), 'rb') as f:
|
|
|
|
all_tests.append(pickle.load(f).as_test())
|
|
|
|
all_test_unique_names = set(test.unique_name for test in all_tests)
|
|
|
|
|
2021-05-20 08:36:00 +08:00
|
|
|
# Find the selected config.
|
2021-04-20 06:12:29 +08:00
|
|
|
if args.config == 'refbackend':
|
2021-09-28 02:36:44 +08:00
|
|
|
config = LinalgOnTensorsBackendTestConfig(RefBackendLinalgOnTensorsBackend())
|
2021-10-08 10:07:03 +08:00
|
|
|
xfail_set = REFBACKEND_XFAIL_SET
|
|
|
|
if args.config == 'tosa':
|
|
|
|
config = TosaBackendTestConfig(LinalgOnTensorsTosaBackend())
|
|
|
|
xfail_set = all_test_unique_names - TOSA_PASS_SET
|
2021-04-20 06:12:29 +08:00
|
|
|
elif args.config == 'native_torch':
|
|
|
|
config = NativeTorchTestConfig()
|
2021-10-08 10:07:03 +08:00
|
|
|
xfail_set = {}
|
2021-04-20 06:12:29 +08:00
|
|
|
elif args.config == 'torchscript':
|
|
|
|
config = TorchScriptTestConfig()
|
2021-10-08 10:07:03 +08:00
|
|
|
xfail_set = {}
|
2021-10-05 02:24:12 +08:00
|
|
|
elif args.config == 'external':
|
|
|
|
with open(args.external_config, 'r') as f:
|
|
|
|
code = compile(f.read(), args.external_config, 'exec')
|
|
|
|
exec_globals = {
|
|
|
|
'COMMON_TORCH_MLIR_LOWERING_XFAILS': COMMON_TORCH_MLIR_LOWERING_XFAILS}
|
|
|
|
exec(code, exec_globals)
|
|
|
|
config = exec_globals.get('config')
|
|
|
|
xfail_set = exec_globals.get('xfail_set')
|
|
|
|
if config is None or not isinstance(config, TestConfig):
|
|
|
|
print(
|
|
|
|
f'ERROR: the script {args.external_config} did not set a global variable `config`'
|
|
|
|
)
|
|
|
|
sys.exit(1)
|
|
|
|
if xfail_set is None:
|
|
|
|
print(
|
|
|
|
f'ERROR: the script {args.external_config} did not set a global variable `xfail_set`'
|
|
|
|
)
|
|
|
|
sys.exit(1)
|
2021-04-22 06:07:15 +08:00
|
|
|
|
2021-05-20 08:36:00 +08:00
|
|
|
# Find the selected tests, and emit a diagnostic if none are found.
|
2021-04-22 06:07:15 +08:00
|
|
|
tests = [
|
2021-07-10 03:22:45 +08:00
|
|
|
test for test in all_tests
|
2021-04-22 06:07:15 +08:00
|
|
|
if re.match(args.filter, test.unique_name)
|
|
|
|
]
|
|
|
|
if len(tests) == 0:
|
|
|
|
print(
|
|
|
|
f'ERROR: the provided filter {args.filter!r} does not match any tests'
|
|
|
|
)
|
|
|
|
print('The available tests are:')
|
2021-07-10 03:22:45 +08:00
|
|
|
for test in all_tests:
|
2021-04-22 06:07:15 +08:00
|
|
|
print(test.unique_name)
|
|
|
|
sys.exit(1)
|
2021-05-20 08:36:00 +08:00
|
|
|
|
|
|
|
# Run the tests.
|
2021-08-28 02:01:08 +08:00
|
|
|
results = run_tests(tests, config)
|
2021-05-20 08:36:00 +08:00
|
|
|
|
|
|
|
# Report the test results.
|
2021-10-05 02:24:12 +08:00
|
|
|
failed = report_results(results, xfail_set, args.verbose)
|
2021-07-01 05:13:21 +08:00
|
|
|
sys.exit(1 if failed else 0)
|
2021-04-20 06:12:29 +08:00
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
main()
|