ext/testlib/result.py - public/gem5 - Git at Google

 # Copyright (c) 2020 ARM Limited
 # All rights reserved
 #
 # The license below extends only to copyright in the software and shall
 # not be construed as granting a license to any other intellectual
 # property including but not limited to intellectual property relating
 # to a hardware implementation of the functionality of the software
 # licensed hereunder.  You may use the software subject to the license
 # terms below provided that you ensure that this notice is replicated
 # unmodified and in its entirety in all distributions of the software,
 # modified or unmodified, in source code or in binary form.
 #
 # Copyright (c) 2017 Mark D. Hill and David A. Wood
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met: redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer;
 # redistributions in binary form must reproduce the above copyright
 # notice, this list of conditions and the following disclaimer in the
 # documentation and/or other materials provided with the distribution;
 # neither the name of the copyright holders nor the names of its
 # contributors may be used to endorse or promote products derived from
 # this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 # Authors: Sean Wilson

 import os
 import pickle
 import xml.sax.saxutils

 from testlib.configuration import config
 import testlib.helper as helper
 import testlib.state as state

 def _create_uid_index(iterable):
     index = {}
     for item in iterable:
         assert item.uid not in index
         index[item.uid] = item
     return index


 class _CommonMetadataMixin:
     @property
     def name(self):
         return self._metadata.name
     @property
     def uid(self):
         return self._metadata.uid
     @property
     def result(self):
         return self._metadata.result
     @result.setter
     def result(self, result):
         self._metadata.result = result

     @property
     def unsuccessful(self):
         return self._metadata.result.value != state.Result.Passed

     @property
     def time(self):
         return self._metadata.time


 class InternalTestResult(_CommonMetadataMixin):
     def __init__(self, obj, suite, directory):
         self._metadata = obj.metadata
         self.suite = suite

         self.stderr = os.path.join(
             InternalSavedResults.output_path(self.uid, suite.uid),
             'stderr'
         )
         self.stdout = os.path.join(
             InternalSavedResults.output_path(self.uid, suite.uid),
             'stdout'
         )


 class InternalSuiteResult(_CommonMetadataMixin):
     def __init__(self, obj, directory):
         self._metadata = obj.metadata
         self.directory = directory
         self._wrap_tests(obj)

     def _wrap_tests(self, obj):
         self._tests = [InternalTestResult(test, self, self.directory)
                        for test in obj]
         self._tests_index = _create_uid_index(self._tests)

     def get_test(self, uid):
         return self._tests_index[uid]

     def __iter__(self):
         return iter(self._tests)

     def get_test_result(self, uid):
         return self.get_test(uid)

     def aggregate_test_results(self):
         results = {}
         for test in self:
             helper.append_dictlist(results, test.result.value, test)
         return results


 class InternalLibraryResults(_CommonMetadataMixin):
     def __init__(self, obj, directory):
         self.directory = directory
         self._metadata = obj.metadata
         self._wrap_suites(obj)

     def __iter__(self):
         return iter(self._suites)

     def _wrap_suites(self, obj):
         self._suites = [InternalSuiteResult(suite, self.directory)
                         for suite in obj]
         self._suites_index = _create_uid_index(self._suites)

     def add_suite(self, suite):
         if suite.uid in self._suites:
             raise ValueError('Cannot have duplicate suite UIDs.')
         self._suites[suite.uid] = suite

     def get_suite_result(self, suite_uid):
         return self._suites_index[suite_uid]

     def get_test_result(self, test_uid, suite_uid):
         return self.get_suite_result(suite_uid).get_test_result(test_uid)

     def aggregate_test_results(self):
         results = {}
         for suite in self._suites:
             for test in suite:
                 helper.append_dictlist(results, test.result.value, test)
         return results

 class InternalSavedResults:
     @staticmethod
     def output_path(test_uid, suite_uid, base=None):
         '''
         Return the path which results for a specific test case should be
         stored.
         '''
         if base is None:
             base = config.result_path
         return os.path.join(
                 base,
                 str(suite_uid).replace(os.path.sep, '-'),
                 str(test_uid).replace(os.path.sep, '-'))

     @staticmethod
     def save(results, path, protocol=pickle.HIGHEST_PROTOCOL):
         if not os.path.exists(os.path.dirname(path)):
            try:
                os.makedirs(os.path.dirname(path))
            except OSError as exc: # Guard against race condition
                if exc.errno != errno.EEXIST:
                    raise

         with open(path, 'wb') as f:
             pickle.dump(results, f, protocol)

     @staticmethod
     def load(path):
         with open(path, 'rb') as f:
             return pickle.load(f)


 class XMLElement(object):
     def write(self, file_):
         self.begin(file_)
         self.end(file_)

     def begin(self, file_):
         file_.write('<')
         file_.write(self.name)
         for attr in self.attributes:
             file_.write(' ')
             attr.write(file_)
         file_.write('>')

         self.body(file_)

     def body(self, file_):
         for elem in self.elements:
             file_.write('\n')
             elem.write(file_)
         file_.write('\n')

     def end(self, file_):
         file_.write('</%s>' % self.name)

 class XMLAttribute(object):
     def __init__(self, name, value):
         self.name = name
         self.value = value

     def write(self, file_):
         file_.write('%s=%s' % (self.name,
                 xml.sax.saxutils.quoteattr(self.value)))


 class JUnitTestSuites(XMLElement):
     name = 'testsuites'
     result_map = {
         state.Result.Errored: 'errors',
         state.Result.Failed: 'failures',
         state.Result.Passed: 'tests'
     }

     def __init__(self, internal_results):
         results = internal_results.aggregate_test_results()

         self.attributes = []
         for result, tests in results.items():
             self.attributes.append(self.result_attribute(result,
                     str(len(tests))))

         self.elements = []
         for suite in internal_results:
             self.elements.append(JUnitTestSuite(suite))

     def result_attribute(self, result, count):
         return XMLAttribute(self.result_map[result], count)

 class JUnitTestSuite(JUnitTestSuites):
     name = 'testsuite'
     result_map = {
         state.Result.Errored: 'errors',
         state.Result.Failed: 'failures',
         state.Result.Passed: 'tests',
         state.Result.Skipped: 'skipped'
     }

     def __init__(self, suite_result):
         results = suite_result.aggregate_test_results()

         self.attributes = [
             XMLAttribute('name', suite_result.name)
         ]
         for result, tests in results.items():
             self.attributes.append(self.result_attribute(result,
                     str(len(tests))))

         self.elements = []
         for test in suite_result:
             self.elements.append(JUnitTestCase(test))

     def result_attribute(self, result, count):
         return XMLAttribute(self.result_map[result], count)

 class JUnitTestCase(XMLElement):
     name = 'testcase'
     def __init__(self, test_result):
         self.attributes = [
             XMLAttribute('name', test_result.name),
              # TODO JUnit expects class of test.. add as test metadata.
             XMLAttribute('classname', str(test_result.uid)),
             XMLAttribute('status', str(test_result.result)),
             XMLAttribute('time', str(test_result.time["user_time"])),
         ]

         # TODO JUnit expects a message for the reason a test was
         # skipped or errored, save this with the test metadata.
         # http://llg.cubic.org/docs/junit/
         self.elements = [
             LargeFileElement('system-err', test_result.stderr),
             LargeFileElement('system-out', test_result.stdout),
         ]

         if str(test_result.result) == 'Failed':
             self.elements.append(JUnitFailure('Test failed', 'ERROR'))


 class JUnitFailure(XMLElement):
     name = 'failure'
     def __init__(self, message, fail_type):
         self.attributes = [
             XMLAttribute('message', message),
             XMLAttribute('type', fail_type),
         ]
         self.elements = []


 class LargeFileElement(XMLElement):
     def __init__(self, name, filename):
         self.name = name
         self.filename = filename
         self.attributes = []

     def body(self, file_):
         try:
             with open(self.filename, 'r') as f:
                 for line in f:
                     file_.write(xml.sax.saxutils.escape(line))
         except IOError:
             # TODO Better error logic, this is sometimes O.K.
             # if there was no stdout/stderr captured for the test
             #
             # TODO If that was the case, the file should still be made and it
             # should just be empty instead of not existing.
             pass


 class JUnitSavedResults:
     @staticmethod
     def save(results, path):
         '''
         Compile the internal results into JUnit format writting it to the
         given file.
         '''
         results = JUnitTestSuites(results)
         with open(path, 'w') as f:
             results.write(f)
	# Copyright (c) 2020 ARM Limited
	# All rights reserved
	#
	# The license below extends only to copyright in the software and shall
	# not be construed as granting a license to any other intellectual
	# property including but not limited to intellectual property relating
	# to a hardware implementation of the functionality of the software
	# licensed hereunder. You may use the software subject to the license
	# terms below provided that you ensure that this notice is replicated
	# unmodified and in its entirety in all distributions of the software,
	# modified or unmodified, in source code or in binary form.
	#
	# Copyright (c) 2017 Mark D. Hill and David A. Wood
	# All rights reserved.
	#
	# Redistribution and use in source and binary forms, with or without
	# modification, are permitted provided that the following conditions are
	# met: redistributions of source code must retain the above copyright
	# notice, this list of conditions and the following disclaimer;
	# redistributions in binary form must reproduce the above copyright
	# notice, this list of conditions and the following disclaimer in the
	# documentation and/or other materials provided with the distribution;
	# neither the name of the copyright holders nor the names of its
	# contributors may be used to endorse or promote products derived from
	# this software without specific prior written permission.
	#
	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	#
	# Authors: Sean Wilson

	import os
	import pickle
	import xml.sax.saxutils

	from testlib.configuration import config
	import testlib.helper as helper
	import testlib.state as state

	def _create_uid_index(iterable):
	index = {}
	for item in iterable:
	assert item.uid not in index
	index[item.uid] = item
	return index


	class _CommonMetadataMixin:
	@property
	def name(self):
	return self._metadata.name
	@property
	def uid(self):
	return self._metadata.uid
	@property
	def result(self):
	return self._metadata.result
	@result.setter
	def result(self, result):
	self._metadata.result = result

	@property
	def unsuccessful(self):
	return self._metadata.result.value != state.Result.Passed

	@property
	def time(self):
	return self._metadata.time


	class InternalTestResult(_CommonMetadataMixin):
	def __init__(self, obj, suite, directory):
	self._metadata = obj.metadata
	self.suite = suite

	self.stderr = os.path.join(
	InternalSavedResults.output_path(self.uid, suite.uid),
	'stderr'
	)
	self.stdout = os.path.join(
	InternalSavedResults.output_path(self.uid, suite.uid),
	'stdout'
	)


	class InternalSuiteResult(_CommonMetadataMixin):
	def __init__(self, obj, directory):
	self._metadata = obj.metadata
	self.directory = directory
	self._wrap_tests(obj)

	def _wrap_tests(self, obj):
	self._tests = [InternalTestResult(test, self, self.directory)
	for test in obj]
	self._tests_index = _create_uid_index(self._tests)

	def get_test(self, uid):
	return self._tests_index[uid]

	def __iter__(self):
	return iter(self._tests)

	def get_test_result(self, uid):
	return self.get_test(uid)

	def aggregate_test_results(self):
	results = {}
	for test in self:
	helper.append_dictlist(results, test.result.value, test)
	return results


	class InternalLibraryResults(_CommonMetadataMixin):
	def __init__(self, obj, directory):
	self.directory = directory
	self._metadata = obj.metadata
	self._wrap_suites(obj)

	def __iter__(self):
	return iter(self._suites)

	def _wrap_suites(self, obj):
	self._suites = [InternalSuiteResult(suite, self.directory)
	for suite in obj]
	self._suites_index = _create_uid_index(self._suites)

	def add_suite(self, suite):
	if suite.uid in self._suites:
	raise ValueError('Cannot have duplicate suite UIDs.')
	self._suites[suite.uid] = suite

	def get_suite_result(self, suite_uid):
	return self._suites_index[suite_uid]

	def get_test_result(self, test_uid, suite_uid):
	return self.get_suite_result(suite_uid).get_test_result(test_uid)

	def aggregate_test_results(self):
	results = {}
	for suite in self._suites:
	for test in suite:
	helper.append_dictlist(results, test.result.value, test)
	return results

	class InternalSavedResults:
	@staticmethod
	def output_path(test_uid, suite_uid, base=None):
	'''
	Return the path which results for a specific test case should be
	stored.
	'''
	if base is None:
	base = config.result_path
	return os.path.join(
	base,
	str(suite_uid).replace(os.path.sep, '-'),
	str(test_uid).replace(os.path.sep, '-'))

	@staticmethod
	def save(results, path, protocol=pickle.HIGHEST_PROTOCOL):
	if not os.path.exists(os.path.dirname(path)):
	try:
	os.makedirs(os.path.dirname(path))
	except OSError as exc: # Guard against race condition
	if exc.errno != errno.EEXIST:
	raise

	with open(path, 'wb') as f:
	pickle.dump(results, f, protocol)

	@staticmethod
	def load(path):
	with open(path, 'rb') as f:
	return pickle.load(f)


	class XMLElement(object):
	def write(self, file_):
	self.begin(file_)
	self.end(file_)

	def begin(self, file_):
	file_.write('<')
	file_.write(self.name)
	for attr in self.attributes:
	file_.write(' ')
	attr.write(file_)
	file_.write('>')

	self.body(file_)

	def body(self, file_):
	for elem in self.elements:
	file_.write('\n')
	elem.write(file_)
	file_.write('\n')

	def end(self, file_):
	file_.write('</%s>' % self.name)

	class XMLAttribute(object):
	def __init__(self, name, value):
	self.name = name
	self.value = value

	def write(self, file_):
	file_.write('%s=%s' % (self.name,
	xml.sax.saxutils.quoteattr(self.value)))


	class JUnitTestSuites(XMLElement):
	name = 'testsuites'
	result_map = {
	state.Result.Errored: 'errors',
	state.Result.Failed: 'failures',
	state.Result.Passed: 'tests'
	}

	def __init__(self, internal_results):
	results = internal_results.aggregate_test_results()

	self.attributes = []
	for result, tests in results.items():
	self.attributes.append(self.result_attribute(result,
	str(len(tests))))

	self.elements = []
	for suite in internal_results:
	self.elements.append(JUnitTestSuite(suite))

	def result_attribute(self, result, count):
	return XMLAttribute(self.result_map[result], count)

	class JUnitTestSuite(JUnitTestSuites):
	name = 'testsuite'
	result_map = {
	state.Result.Errored: 'errors',
	state.Result.Failed: 'failures',
	state.Result.Passed: 'tests',
	state.Result.Skipped: 'skipped'
	}

	def __init__(self, suite_result):
	results = suite_result.aggregate_test_results()

	self.attributes = [
	XMLAttribute('name', suite_result.name)
	]
	for result, tests in results.items():
	self.attributes.append(self.result_attribute(result,
	str(len(tests))))

	self.elements = []
	for test in suite_result:
	self.elements.append(JUnitTestCase(test))

	def result_attribute(self, result, count):
	return XMLAttribute(self.result_map[result], count)

	class JUnitTestCase(XMLElement):
	name = 'testcase'
	def __init__(self, test_result):
	self.attributes = [
	XMLAttribute('name', test_result.name),
	# TODO JUnit expects class of test.. add as test metadata.
	XMLAttribute('classname', str(test_result.uid)),
	XMLAttribute('status', str(test_result.result)),
	XMLAttribute('time', str(test_result.time["user_time"])),
	]

	# TODO JUnit expects a message for the reason a test was
	# skipped or errored, save this with the test metadata.
	# http://llg.cubic.org/docs/junit/
	self.elements = [
	LargeFileElement('system-err', test_result.stderr),
	LargeFileElement('system-out', test_result.stdout),
	]

	if str(test_result.result) == 'Failed':
	self.elements.append(JUnitFailure('Test failed', 'ERROR'))


	class JUnitFailure(XMLElement):
	name = 'failure'
	def __init__(self, message, fail_type):
	self.attributes = [
	XMLAttribute('message', message),
	XMLAttribute('type', fail_type),
	]
	self.elements = []


	class LargeFileElement(XMLElement):
	def __init__(self, name, filename):
	self.name = name
	self.filename = filename
	self.attributes = []

	def body(self, file_):
	try:
	with open(self.filename, 'r') as f:
	for line in f:
	file_.write(xml.sax.saxutils.escape(line))
	except IOError:
	# TODO Better error logic, this is sometimes O.K.
	# if there was no stdout/stderr captured for the test
	#
	# TODO If that was the case, the file should still be made and it
	# should just be empty instead of not existing.
	pass



	class JUnitSavedResults:
	@staticmethod
	def save(results, path):
	'''
	Compile the internal results into JUnit format writting it to the
	given file.
	'''
	results = JUnitTestSuites(results)
	with open(path, 'w') as f:
	results.write(f)