| #!/usr/bin/env python2 | |
| # | |
| # Copyright (c) 2016 ARM Limited | |
| # All rights reserved | |
| # | |
| # The license below extends only to copyright in the software and shall | |
| # not be construed as granting a license to any other intellectual | |
| # property including but not limited to intellectual property relating | |
| # to a hardware implementation of the functionality of the software | |
| # licensed hereunder. You may use the software subject to the license | |
| # terms below provided that you ensure that this notice is replicated | |
| # unmodified and in its entirety in all distributions of the software, | |
| # modified or unmodified, in source code or in binary form. | |
| # | |
| # Redistribution and use in source and binary forms, with or without | |
| # modification, are permitted provided that the following conditions are | |
| # met: redistributions of source code must retain the above copyright | |
| # notice, this list of conditions and the following disclaimer; | |
| # redistributions in binary form must reproduce the above copyright | |
| # notice, this list of conditions and the following disclaimer in the | |
| # documentation and/or other materials provided with the distribution; | |
| # neither the name of the copyright holders nor the names of its | |
| # contributors may be used to endorse or promote products derived from | |
| # this software without specific prior written permission. | |
| # | |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| # | |
| # Authors: Andreas Sandberg | |
| from __future__ import print_function | |
| import argparse | |
| import sys | |
| import os | |
| import pickle | |
| from testing.tests import * | |
| import testing.results | |
| class ParagraphHelpFormatter(argparse.HelpFormatter): | |
| def _fill_text(self, text, width, indent): | |
| return "\n\n".join([ | |
| super(ParagraphHelpFormatter, self)._fill_text(p, width, indent) \ | |
| for p in text.split("\n\n") ]) | |
| formatters = { | |
| "junit" : testing.results.JUnit, | |
| "text" : testing.results.Text, | |
| "summary" : testing.results.TextSummary, | |
| "pickle" : testing.results.Pickle, | |
| } | |
| def _add_format_args(parser): | |
| parser.add_argument("--format", choices=formatters, default="text", | |
| help="Output format") | |
| parser.add_argument("--no-junit-xlate-names", action="store_true", | |
| help="Don't translate test names to " \ | |
| "package-like names") | |
| parser.add_argument("--output", "-o", | |
| type=argparse.FileType('w'), default=sys.stdout, | |
| help="Test result output file") | |
| def _create_formatter(args): | |
| formatter = formatters[args.format] | |
| kwargs = { | |
| "fout" : args.output, | |
| "verbose" : args.verbose | |
| } | |
| if issubclass(formatter, testing.results.JUnit): | |
| kwargs.update({ | |
| "translate_names" : not args.no_junit_xlate_names, | |
| }) | |
| return formatter(**kwargs) | |
| def _list_tests_args(subparsers): | |
| parser = subparsers.add_parser( | |
| "list", | |
| formatter_class=ParagraphHelpFormatter, | |
| help="List available tests", | |
| description="List available tests", | |
| epilog=""" | |
| Generate a list of available tests using a list filter. | |
| The filter is a string consisting of the target ISA optionally | |
| followed by the test category and mode separated by | |
| slashes. The test names emitted by this command can be fed | |
| into the run command. | |
| For example, to list all quick arm tests, run the following: | |
| tests.py list arm/quick | |
| Non-mandatory parts of the filter string (anything other than | |
| the ISA) can be left out or replaced with the wildcard | |
| character. For example, all full-system tests can be listed | |
| with this command: tests.py list arm/*/fs""") | |
| parser.add_argument("--ruby-protocol", type=str, default=None, | |
| help="Ruby protocol") | |
| parser.add_argument("--gpu-isa", type=str, default=None, | |
| help="GPU ISA") | |
| parser.add_argument("list_filter", metavar="ISA[/category/mode]", | |
| action="append", type=str, | |
| help="List available test cases") | |
| def _list_tests(args): | |
| for isa, categories, modes in \ | |
| ( parse_test_filter(f) for f in args.list_filter ): | |
| for test in get_tests(isa, categories=categories, modes=modes, | |
| ruby_protocol=args.ruby_protocol, | |
| gpu_isa=args.gpu_isa): | |
| print("/".join(test)) | |
| sys.exit(0) | |
| def _run_tests_args(subparsers): | |
| parser = subparsers.add_parser( | |
| "run", | |
| formatter_class=ParagraphHelpFormatter, | |
| help='Run one or more tests', | |
| description="Run one or more tests.", | |
| epilog=""" | |
| Run one or more tests described by a gem5 test tuple. | |
| The test tuple consists of a test category (quick or long), a | |
| test mode (fs or se), a workload name, an isa, an operating | |
| system, and a config name separate by slashes. For example: | |
| quick/se/00.hello/arm/linux/simple-timing | |
| Available tests can be listed using the 'list' sub-command | |
| (e.g., "tests.py list arm/quick" or one of the scons test list | |
| targets (e.g., "scons build/ARM/tests/opt/quick.list"). | |
| The test results can be stored in multiple different output | |
| formats. See the help for the show command for more details | |
| about output formatting.""") | |
| parser.add_argument("gem5", type=str, | |
| help="gem5 binary") | |
| parser.add_argument("test", type=str, nargs="*", | |
| help="List of tests to execute") | |
| parser.add_argument("--directory", "-d", | |
| type=str, default="m5tests", | |
| help="Test work directory") | |
| parser.add_argument("--timeout", "-t", | |
| type=int, default="0", metavar="MINUTES", | |
| help="Timeout, 0 to disable") | |
| parser.add_argument("--skip-diff-out", action="store_true", | |
| help="Skip output diffing stage") | |
| parser.add_argument("--skip-diff-stat", action="store_true", | |
| help="Skip stat diffing stage") | |
| _add_format_args(parser) | |
| def _run_tests(args): | |
| if not os.path.isfile(args.gem5) or not os.access(args.gem5, os.X_OK): | |
| print("gem5 binary '%s' not an executable file" % args.gem5, | |
| file=sys.stderr) | |
| sys.exit(2) | |
| formatter = _create_formatter(args) | |
| out_base = os.path.abspath(args.directory) | |
| if not os.path.exists(out_base): | |
| os.mkdir(out_base) | |
| tests = [] | |
| for test_name in args.test: | |
| config = ClassicConfig(*test_name.split("/")) | |
| out_dir = os.path.join(out_base, "/".join(config)) | |
| tests.append( | |
| ClassicTest(args.gem5, out_dir, config, | |
| timeout=args.timeout, | |
| skip_diff_stat=args.skip_diff_stat, | |
| skip_diff_out=args.skip_diff_out)) | |
| all_results = [] | |
| print("Running %i tests" % len(tests)) | |
| for testno, test in enumerate(tests): | |
| print("%i: Running '%s'..." % (testno, test)) | |
| all_results.append(test.run()) | |
| formatter.dump_suites(all_results) | |
| def _show_args(subparsers): | |
| parser = subparsers.add_parser( | |
| "show", | |
| formatter_class=ParagraphHelpFormatter, | |
| help='Display pickled test results', | |
| description='Display pickled test results', | |
| epilog=""" | |
| Reformat the pickled output from one or more test runs. This | |
| command is typically used with the output from a single test | |
| run, but it can also be used to merge the outputs from | |
| multiple runs. | |
| The 'text' format is a verbose output format that provides | |
| information about individual test units and the output from | |
| failed tests. It's mainly useful for debugging test failures. | |
| The 'summary' format provides outputs the results of one test | |
| per line with the test's overall status (OK, SKIPPED, or | |
| FAILED). | |
| The 'junit' format is primarily intended for use with CI | |
| systems. It provides an XML representation of test | |
| status. Similar to the text format, it includes detailed | |
| information about test failures. Since many JUnit parser make | |
| assume that test names look like Java packet strings, the | |
| JUnit formatter automatically to something the looks like a | |
| Java class path ('.'->'-', '/'->'.'). | |
| The 'pickle' format stores the raw results in a format that | |
| can be reformatted using this command. It's typically used | |
| with the show command to merge multiple test results into one | |
| pickle file.""") | |
| _add_format_args(parser) | |
| parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", | |
| help="Pickled test results") | |
| def _show(args): | |
| def _load(f): | |
| # Load the pickled status file, sometimes e.g., when a | |
| # regression is still running the status file might be | |
| # incomplete. | |
| try: | |
| return pickle.load(f) | |
| except EOFError: | |
| print('Could not read file %s' % f.name, file=sys.stderr) | |
| return [] | |
| formatter = _create_formatter(args) | |
| suites = sum([ _load(f) for f in args.result ], []) | |
| formatter.dump_suites(suites) | |
| def _test_args(subparsers): | |
| parser = subparsers.add_parser( | |
| "test", | |
| formatter_class=ParagraphHelpFormatter, | |
| help='Probe test results and set exit code', | |
| epilog=""" | |
| Load one or more pickled test file and return an exit code | |
| corresponding to the test outcome. The following exit codes | |
| can be returned: | |
| 0: All tests were successful or skipped. | |
| 1: General fault in the script such as incorrect parameters or | |
| failing to parse a pickle file. | |
| 2: At least one test failed to run. This is what the summary | |
| formatter usually shows as a 'FAILED'. | |
| 3: All tests ran correctly, but at least one failed to | |
| verify its output. When displaying test output using the | |
| summary formatter, such a test would show up as 'CHANGED'. | |
| """) | |
| parser.add_argument("result", type=argparse.FileType("rb"), nargs="*", | |
| help="Pickled test results") | |
| def _test(args): | |
| try: | |
| suites = sum([ pickle.load(f) for f in args.result ], []) | |
| except EOFError: | |
| print('Could not read all files', file=sys.stderr) | |
| sys.exit(2) | |
| if all(s for s in suites): | |
| sys.exit(0) | |
| elif any([ s.failed_run() for s in suites ]): | |
| sys.exit(2) | |
| elif any([ s.changed() for s in suites ]): | |
| sys.exit(3) | |
| else: | |
| assert False, "Unexpected return status from test" | |
| _commands = { | |
| "list" : (_list_tests, _list_tests_args), | |
| "run" : (_run_tests, _run_tests_args), | |
| "show" : (_show, _show_args), | |
| "test" : (_test, _test_args), | |
| } | |
| def main(): | |
| parser = argparse.ArgumentParser( | |
| formatter_class=ParagraphHelpFormatter, | |
| description="""gem5 testing multi tool.""", | |
| epilog=""" | |
| This tool provides an interface to gem5's test framework that | |
| doesn't depend on gem5's build system. It supports test | |
| listing, running, and output formatting. | |
| The list sub-command (e.g., "test.py list arm/quick") produces | |
| a list of tests tuples that can be used by the run command | |
| (e.g., "tests.py run gem5.opt | |
| quick/se/00.hello/arm/linux/simple-timing"). | |
| The run command supports several output formats. One of them, | |
| pickle, contains the raw output from the tests and can be | |
| re-formatted using the show command (e.g., "tests.py show | |
| --format summary *.pickle"). Such pickle files are also | |
| generated by the build system when scons is used to run | |
| regressions. | |
| See the usage strings for the individual sub-commands for | |
| details.""") | |
| parser.add_argument("--verbose", action="store_true", | |
| help="Produce more verbose output") | |
| subparsers = parser.add_subparsers(dest="command") | |
| for key, (impl, cmd_parser) in _commands.items(): | |
| cmd_parser(subparsers) | |
| args = parser.parse_args() | |
| impl, cmd_parser = _commands[args.command] | |
| impl(args) | |
| if __name__ == "__main__": | |
| main() |