| #!/usr/bin/env python2 | |
| # | |
| # Copyright (c) 2016 ARM Limited | |
| # All rights reserved | |
| # | |
| # The license below extends only to copyright in the software and shall | |
| # not be construed as granting a license to any other intellectual | |
| # property including but not limited to intellectual property relating | |
| # to a hardware implementation of the functionality of the software | |
| # licensed hereunder. You may use the software subject to the license | |
| # terms below provided that you ensure that this notice is replicated | |
| # unmodified and in its entirety in all distributions of the software, | |
| # modified or unmodified, in source code or in binary form. | |
| # | |
| # Redistribution and use in source and binary forms, with or without | |
| # modification, are permitted provided that the following conditions are | |
| # met: redistributions of source code must retain the above copyright | |
| # notice, this list of conditions and the following disclaimer; | |
| # redistributions in binary form must reproduce the above copyright | |
| # notice, this list of conditions and the following disclaimer in the | |
| # documentation and/or other materials provided with the distribution; | |
| # neither the name of the copyright holders nor the names of its | |
| # contributors may be used to endorse or promote products derived from | |
| # this software without specific prior written permission. | |
| # | |
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| # | |
| # Authors: Andreas Sandberg | |
| from __future__ import print_function | |
| from abc import ABCMeta, abstractmethod | |
| import inspect | |
| import pickle | |
| import string | |
| import sys | |
| import xml.etree.cElementTree as ET | |
| class UnitResult(object): | |
| """Results of a single test unit. | |
| A test result can be one of: | |
| - STATE_OK: Test ran successfully. | |
| - STATE_SKIPPED: The test was skipped. | |
| - STATE_ERROR: The test failed to run. | |
| - STATE_FAILED: Test ran, but failed. | |
| The difference between STATE_ERROR and STATE_FAILED is very | |
| subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed | |
| to start or crashed, while STATE_FAILED would mean that a test | |
| failed (e.g., statistics mismatch). | |
| """ | |
| STATE_OK = 0 | |
| STATE_SKIPPED = 1 | |
| STATE_ERROR = 2 | |
| STATE_FAILURE = 3 | |
| state_names = { | |
| STATE_OK : "OK", | |
| STATE_SKIPPED : "SKIPPED", | |
| STATE_ERROR : "ERROR", | |
| STATE_FAILURE : "FAILURE", | |
| } | |
| def __init__(self, name, state, message="", stderr="", stdout="", | |
| runtime=0.0): | |
| self.name = name | |
| self.state = state | |
| self.message = message | |
| self.stdout = stdout | |
| self.stderr = stderr | |
| self.runtime = runtime | |
| def skipped(self): | |
| return self.state == UnitResult.STATE_SKIPPED | |
| def success(self): | |
| return self.state == UnitResult.STATE_OK | |
| def state_name(self): | |
| return UnitResult.state_names[self.state] | |
| def __nonzero__(self): | |
| return self.success() or self.skipped() | |
| def __str__(self): | |
| state_name = self.state_name() | |
| status = "%s: %s" % (state_name, self.message) if self.message else \ | |
| state_name | |
| return "%s: %s" % (self.name, status) | |
| class TestResult(object): | |
| """Results for from a single test consisting of one or more units.""" | |
| def __init__(self, name, run_results=[], verify_results=[]): | |
| self.name = name | |
| self.results = run_results + verify_results | |
| self.run_results = run_results | |
| self.verify_results = verify_results | |
| def success(self): | |
| return self.success_run() and self.success_verify() | |
| def success_run(self): | |
| return all([ r.success() for r in self.run_results ]) | |
| def success_verify(self): | |
| return all([ r.success() for r in self.verify_results ]) | |
| def failed(self): | |
| return self.failed_run() or self.failed_verify() | |
| def failed_run(self): | |
| return any([ not r for r in self.run_results ]) | |
| def failed_verify(self): | |
| return any([ not r for r in self.verify_results ]) | |
| def skipped(self): | |
| return all([ r.skipped() for r in self.run_results ]) | |
| def changed(self): | |
| return self.success_run() and self.failed_verify() | |
| def runtime(self): | |
| return sum([ r.runtime for r in self.results ]) | |
| def __nonzero__(self): | |
| return all([ r for r in self.results ]) | |
| class ResultFormatter(object): | |
| __metaclass__ = ABCMeta | |
| def __init__(self, fout=sys.stdout, verbose=False): | |
| self.verbose = verbose | |
| self.fout = fout | |
| @abstractmethod | |
| def dump_suites(self, suites): | |
| pass | |
| class Pickle(ResultFormatter): | |
| """Save test results as a binary using Python's pickle | |
| functionality. | |
| """ | |
| def __init__(self, **kwargs): | |
| super(Pickle, self).__init__(**kwargs) | |
| def dump_suites(self, suites): | |
| pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL) | |
| class Text(ResultFormatter): | |
| """Output test results as text.""" | |
| def __init__(self, **kwargs): | |
| super(Text, self).__init__(**kwargs) | |
| def dump_suites(self, suites): | |
| fout = self.fout | |
| for suite in suites: | |
| print("--- %s ---" % suite.name, file=fout) | |
| for t in suite.results: | |
| print("*** %s" % t, file=fout) | |
| if t and not self.verbose: | |
| continue | |
| if t.message: | |
| print(t.message, file=fout) | |
| if t.stderr: | |
| print(t.stderr, file=fout) | |
| if t.stdout: | |
| print(t.stdout, file=fout) | |
| class TextSummary(ResultFormatter): | |
| """Output test results as a text summary""" | |
| def __init__(self, **kwargs): | |
| super(TextSummary, self).__init__(**kwargs) | |
| def test_status(self, suite): | |
| if suite.skipped(): | |
| return "SKIPPED" | |
| elif suite.changed(): | |
| return "CHANGED" | |
| elif suite: | |
| return "OK" | |
| else: | |
| return "FAILED" | |
| def dump_suites(self, suites): | |
| fout = self.fout | |
| for suite in suites: | |
| status = self.test_status(suite) | |
| print("%s: %s" % (suite.name, status), file=fout) | |
| class JUnit(ResultFormatter): | |
| """Output test results as JUnit XML""" | |
| def __init__(self, translate_names=True, **kwargs): | |
| super(JUnit, self).__init__(**kwargs) | |
| if translate_names: | |
| self.name_table = string.maketrans( | |
| "/.", | |
| ".-", | |
| ) | |
| else: | |
| self.name_table = string.maketrans("", "") | |
| def convert_unit(self, x_suite, test): | |
| x_test = ET.SubElement(x_suite, "testcase", | |
| name=test.name, | |
| time="%f" % test.runtime) | |
| x_state = None | |
| if test.state == UnitResult.STATE_OK: | |
| pass | |
| elif test.state == UnitResult.STATE_SKIPPED: | |
| x_state = ET.SubElement(x_test, "skipped") | |
| elif test.state == UnitResult.STATE_FAILURE: | |
| x_state = ET.SubElement(x_test, "failure") | |
| elif test.state == UnitResult.STATE_ERROR: | |
| x_state = ET.SubElement(x_test, "error") | |
| else: | |
| assert False, "Unknown test state" | |
| if x_state is not None: | |
| if test.message: | |
| x_state.set("message", test.message) | |
| msg = [] | |
| if test.stderr: | |
| msg.append("*** Standard Errror: ***") | |
| msg.append(test.stderr) | |
| if test.stdout: | |
| msg.append("*** Standard Out: ***") | |
| msg.append(test.stdout) | |
| x_state.text = "\n".join(msg) | |
| return x_test | |
| def convert_suite(self, x_suites, suite): | |
| x_suite = ET.SubElement(x_suites, "testsuite", | |
| name=suite.name.translate(self.name_table), | |
| time="%f" % suite.runtime()) | |
| errors = 0 | |
| failures = 0 | |
| skipped = 0 | |
| for test in suite.results: | |
| if test.state != UnitResult.STATE_OK: | |
| if test.state == UnitResult.STATE_SKIPPED: | |
| skipped += 1 | |
| elif test.state == UnitResult.STATE_ERROR: | |
| errors += 1 | |
| elif test.state == UnitResult.STATE_FAILURE: | |
| failures += 1 | |
| x_test = self.convert_unit(x_suite, test) | |
| x_suite.set("errors", str(errors)) | |
| x_suite.set("failures", str(failures)) | |
| x_suite.set("skipped", str(skipped)) | |
| x_suite.set("tests", str(len(suite.results))) | |
| return x_suite | |
| def convert_suites(self, suites): | |
| x_root = ET.Element("testsuites") | |
| for suite in suites: | |
| self.convert_suite(x_root, suite) | |
| return x_root | |
| def dump_suites(self, suites): | |
| et = ET.ElementTree(self.convert_suites(suites)) | |
| et.write(self.fout, encoding="UTF-8") |