tests/testing/results.py

#!/usr/bin/env python2.7
#
# Copyright (c) 2016 ARM Limited
# All rights reserved
#
# The license below extends only to copyright in the software and shall
# not be construed as granting a license to any other intellectual
# property including but not limited to intellectual property relating
# to a hardware implementation of the functionality of the software
# licensed hereunder.  You may use the software subject to the license
# terms below provided that you ensure that this notice is replicated
# unmodified and in its entirety in all distributions of the software,
# modified or unmodified, in source code or in binary form.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met: redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer;
# redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution;
# neither the name of the copyright holders nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Andreas Sandberg

from __future__ import print_function

from abc import ABCMeta, abstractmethod
import inspect
import pickle
import string
import sys

import xml.etree.cElementTree as ET

class UnitResult(object):
    """Results of a single test unit.

    A test result can be one of:
        - STATE_OK: Test ran successfully.
        - STATE_SKIPPED: The test was skipped.
        - STATE_ERROR: The test failed to run.
        - STATE_FAILED: Test ran, but failed.

    The difference between STATE_ERROR and STATE_FAILED is very
    subtle. In a gem5 context, STATE_ERROR would mean that gem5 failed
    to start or crashed, while STATE_FAILED would mean that a test
    failed (e.g., statistics mismatch).

    """

    STATE_OK = 0
    STATE_SKIPPED = 1
    STATE_ERROR = 2
    STATE_FAILURE = 3

    state_names = {
        STATE_OK : "OK",
        STATE_SKIPPED : "SKIPPED",
        STATE_ERROR : "ERROR",
        STATE_FAILURE : "FAILURE",
    }

    def __init__(self, name, state, message="", stderr="", stdout="",
                 runtime=0.0):
        self.name = name
        self.state = state
        self.message = message
        self.stdout = stdout
        self.stderr = stderr
        self.runtime = runtime

    def skipped(self):
        return self.state == UnitResult.STATE_SKIPPED

    def success(self):
        return self.state == UnitResult.STATE_OK

    def state_name(self):
        return UnitResult.state_names[self.state]

    def __nonzero__(self):
        return self.success() or self.skipped()

    def __str__(self):
        state_name = self.state_name()

        status = "%s: %s" % (state_name, self.message) if self.message else \
                 state_name

        return "%s: %s" % (self.name, status)

class TestResult(object):
    """Results for from a single test consisting of one or more units."""

    def __init__(self, name, run_results=[], verify_results=[]):
        self.name = name
        self.results = run_results + verify_results
        self.run_results = run_results
        self.verify_results = verify_results

    def success(self):
        return self.success_run() and self.success_verify()

    def success_run(self):
        return all([ r.success() for r in self.run_results ])

    def success_verify(self):
        return all([ r.success() for r in self.verify_results ])

    def failed(self):
        return self.failed_run() or self.failed_verify()

    def failed_run(self):
        return any([ not r for r in self.run_results ])

    def failed_verify(self):
        return any([ not r for r in self.verify_results ])

    def skipped(self):
        return all([ r.skipped() for r in self.run_results ])

    def changed(self):
        return self.success_run() and self.failed_verify()

    def runtime(self):
        return sum([ r.runtime for r in self.results ])

    def __nonzero__(self):
        return all([ r for r in self.results ])

class ResultFormatter(object):
    __metaclass__ = ABCMeta

    def __init__(self, fout=sys.stdout, verbose=False):
        self.verbose = verbose
        self.fout = fout

    @abstractmethod
    def dump_suites(self, suites):
        pass

class Pickle(ResultFormatter):
    """Save test results as a binary using Python's pickle
    functionality.

    """

    def __init__(self, **kwargs):
        super(Pickle, self).__init__(**kwargs)

    def dump_suites(self, suites):
        pickle.dump(suites, self.fout, pickle.HIGHEST_PROTOCOL)

class Text(ResultFormatter):
    """Output test results as text."""

    def __init__(self, **kwargs):
        super(Text, self).__init__(**kwargs)

    def dump_suites(self, suites):
        fout = self.fout
        for suite in suites:
            print("--- %s ---" % suite.name, file=fout)

            for t in suite.results:
                print("*** %s" % t, file=fout)

                if t and not self.verbose:
                    continue

                if t.message:
                    print(t.message, file=fout)

                if t.stderr:
                    print(t.stderr, file=fout)
                if t.stdout:
                    print(t.stdout, file=fout)

class TextSummary(ResultFormatter):
    """Output test results as a text summary"""

    def __init__(self, **kwargs):
        super(TextSummary, self).__init__(**kwargs)

    def test_status(self, suite):
        if suite.skipped():
            return "SKIPPED"
        elif suite.changed():
            return "CHANGED"
        elif suite:
            return "OK"
        else:
            return "FAILED"

    def dump_suites(self, suites):
        fout = self.fout
        for suite in suites:
            status = self.test_status(suite)
            print("%s: %s" % (suite.name, status), file=fout)

class JUnit(ResultFormatter):
    """Output test results as JUnit XML"""

    def __init__(self, translate_names=True, **kwargs):
        super(JUnit, self).__init__(**kwargs)

        if translate_names:
            self.name_table = string.maketrans(
                "/.",
                ".-",
            )
        else:
            self.name_table = string.maketrans("", "")

    def convert_unit(self, x_suite, test):
        x_test = ET.SubElement(x_suite, "testcase",
                               name=test.name,
                               time="%f" % test.runtime)

        x_state = None
        if test.state == UnitResult.STATE_OK:
            pass
        elif test.state == UnitResult.STATE_SKIPPED:
            x_state = ET.SubElement(x_test, "skipped")
        elif test.state == UnitResult.STATE_FAILURE:
            x_state = ET.SubElement(x_test, "failure")
        elif test.state == UnitResult.STATE_ERROR:
            x_state = ET.SubElement(x_test, "error")
        else:
            assert False, "Unknown test state"

        if x_state is not None:
            if test.message:
                x_state.set("message", test.message)

            msg = []
            if test.stderr:
                msg.append("*** Standard Errror: ***")
                msg.append(test.stderr)
            if test.stdout:
                msg.append("*** Standard Out: ***")
                msg.append(test.stdout)

            x_state.text = "\n".join(msg)

        return x_test

    def convert_suite(self, x_suites, suite):
        x_suite = ET.SubElement(x_suites, "testsuite",
                                name=suite.name.translate(self.name_table),
                                time="%f" % suite.runtime())
        errors = 0
        failures = 0
        skipped = 0

        for test in suite.results:
            if test.state != UnitResult.STATE_OK:
                if test.state == UnitResult.STATE_SKIPPED:
                    skipped += 1
                elif test.state == UnitResult.STATE_ERROR:
                    errors += 1
                elif test.state == UnitResult.STATE_FAILURE:
                    failures += 1

            x_test = self.convert_unit(x_suite, test)

        x_suite.set("errors", str(errors))
        x_suite.set("failures", str(failures))
        x_suite.set("skipped", str(skipped))
        x_suite.set("tests", str(len(suite.results)))

        return x_suite

    def convert_suites(self, suites):
        x_root = ET.Element("testsuites")

        for suite in suites:
            self.convert_suite(x_root, suite)

        return x_root

    def dump_suites(self, suites):
        et = ET.ElementTree(self.convert_suites(suites))
        et.write(self.fout, encoding="UTF-8")