tests: Add support for downloaded archive fixtures
This changes add support for specifying fixtures that download
archives and uncompress them to the desired directory.
Change-Id: Ib3f6ee111b8d6130200507cbd170ecaf9fb39445
Signed-off-by: Nikos Nikoleris <nikos.nikoleris@arm.com>
Reviewed-on: https://gem5-review.googlesource.com/c/public/gem5/+/18988
Tested-by: kokoro <noreply+kokoro@google.com>
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Maintainer: Jason Lowe-Power <jason@lowepower.com>
diff --git a/tests/gem5/cpu_tests/test.py b/tests/gem5/cpu_tests/test.py
index f34b23d..58925de 100644
--- a/tests/gem5/cpu_tests/test.py
+++ b/tests/gem5/cpu_tests/test.py
@@ -42,23 +42,26 @@
}
+base_path = joinpath(absdirpath(__file__), 'benchmarks', 'bin')
+base_url = 'http://gem5.org/dist/current/gem5/cpu_tests/benchmarks/bin/'
for isa in valid_isas:
- bm_dir = joinpath('gem5/cpu_tests/benchmarks/bin/', isa)
+ path = joinpath(base_path, isa)
for workload in workloads:
ref_path = joinpath(getcwd(), 'ref', workload)
verifiers = (
verifier.MatchStdout(ref_path),
)
- workload_binary = DownloadedProgram(bm_dir, workload)
- workload_path = workload_binary.path
+ url = base_url + isa + '/' + workload
+ workload_binary = DownloadedProgram(url, path, workload)
+ binary = joinpath(workload_binary.path, workload)
for cpu in valid_isas[isa]:
gem5_verify_config(
name='cpu_test_{}_{}'.format(cpu,workload),
verifiers=verifiers,
config=joinpath(getcwd(), 'run.py'),
- config_args=['--cpu={}'.format(cpu), workload_path],
+ config_args=['--cpu={}'.format(cpu), binary],
valid_isas=(isa.upper(),),
fixtures=[workload_binary]
)
diff --git a/tests/gem5/fixture.py b/tests/gem5/fixture.py
index e84b89f..d39296e 100644
--- a/tests/gem5/fixture.py
+++ b/tests/gem5/fixture.py
@@ -37,11 +37,14 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# Authors: Sean Wilson
+# Nikos Nikoleris
import os
import tempfile
import shutil
import threading
+import urllib
+import urllib2
from testlib.fixture import Fixture
from testlib.config import config, constants
@@ -239,44 +242,43 @@
elif not os.path.exists(self.path):
super(MakeTarget, self).setup()
-class DownloadedProgram(Fixture):
+class DownloadedProgram(UniqueFixture):
""" Like TestProgram, but checks the version in the gem5 binary repository
and downloads an updated version if it is needed.
"""
- urlbase = "http://gem5.org/dist/current/"
- def __init__(self, path, program, **kwargs):
+ def __new__(cls, url, path, filename):
+ target = joinpath(path, filename)
+ return super(DownloadedProgram, cls).__new__(cls, target)
+
+ def _init(self, url, path, filename, **kwargs):
"""
+ url: string
+ The url of the archive
path: string
- The path to the directory containing the binary relative to
- $GEM5_BASE/tests
- program: string
- The name of the binary file
+ The absolute path of the directory containing the archive
+ filename: string
+ The name of the archive
"""
- super(DownloadedProgram, self).__init__("download-" + program,
- build_once=True, **kwargs)
- self.program_dir = path
- relative_path = joinpath(self.program_dir, program)
- self.url = self.urlbase + relative_path
- self.path = os.path.realpath(
- joinpath(absdirpath(__file__), '../', relative_path)
- )
+ self.url = url
+ self.path = path
+ self.filename = joinpath(path, filename)
+ self.name = "Downloaded:" + self.filename
def _download(self):
- import urllib
import errno
log.test_log.debug("Downloading " + self.url + " to " + self.path)
- if not os.path.exists(self.program_dir):
+ if not os.path.exists(self.path):
try:
- os.makedirs(self.program_dir)
+ os.makedirs(self.path)
except OSError as e:
if e.errno != errno.EEXIST:
raise
- urllib.urlretrieve(self.url, self.path)
+ urllib.urlretrieve(self.url, self.filename)
def _getremotetime(self):
- import urllib2, datetime, time
+ import datetime, time
import _strptime # Needed for python threading bug
u = urllib2.urlopen(self.url)
@@ -284,18 +286,44 @@
u.info().getheaders("Last-Modified")[0],
"%a, %d %b %Y %X GMT").timetuple())
- def setup(self, testitem):
- import urllib2
+ def _setup(self, testitem):
# Check to see if there is a file downloaded
- if not os.path.exists(self.path):
+ if not os.path.exists(self.filename):
self._download()
else:
try:
t = self._getremotetime()
except urllib2.URLError:
# Problem checking the server, use the old files.
- log.debug("Could not contact server. Binaries may be old.")
+ log.test_log.debug("Could not contact server. Binaries may be old.")
return
# If the server version is more recent, download it
- if t > os.path.getmtime(self.path):
+ if t > os.path.getmtime(self.filename):
self._download()
+
+class DownloadedArchive(DownloadedProgram):
+ """ Like TestProgram, but checks the version in the gem5 binary repository
+ and downloads an updated version if it is needed.
+ """
+
+ def _extract(self):
+ import tarfile
+ with tarfile.open(self.filename) as tf:
+ tf.extractall(self.path)
+
+ def _setup(self, testitem):
+ # Check to see if there is a file downloaded
+ if not os.path.exists(self.filename):
+ self._download()
+ self._extract()
+ else:
+ try:
+ t = self._getremotetime()
+ except urllib2.URLError:
+ # Problem checking the server, use the old files.
+ log.test_log.debug("Could not contact server. Binaries may be old.")
+ return
+ # If the server version is more recent, download it
+ if t > os.path.getmtime(self.filename):
+ self._download()
+ self._extract()
diff --git a/tests/gem5/hello_se/test_hello_se.py b/tests/gem5/hello_se/test_hello_se.py
index 5017962..cc62efb 100644
--- a/tests/gem5/hello_se/test_hello_se.py
+++ b/tests/gem5/hello_se/test_hello_se.py
@@ -36,11 +36,14 @@
'arm': ('hello64-static', 'hello32-static'),
}
+urlbase = 'http://gem5.org/dist/current/test-progs/hello/bin/'
for isa in test_progs:
for binary in test_progs[isa]:
import os
- path = os.path.join('test-progs', 'hello', 'bin', isa, 'linux')
- hello_program = DownloadedProgram(path, binary)
+ url = urlbase + isa + '/linux/' + binary
+ path = joinpath(absdirpath(__file__), '..', 'test-progs', 'hello',
+ 'bin', isa, 'linux')
+ hello_program = DownloadedProgram(url, path, binary)
ref_path = joinpath(getcwd(), 'ref')
@@ -53,6 +56,6 @@
fixtures=(hello_program,),
verifiers=verifiers,
config=joinpath(config.base_dir, 'configs', 'example','se.py'),
- config_args=['--cmd', hello_program.path],
+ config_args=['--cmd', joinpath(path, binary)],
valid_isas=(isa.upper(),),
)
diff --git a/tests/gem5/m5_util/test_exit.py b/tests/gem5/m5_util/test_exit.py
index a766db4..3c99512 100644
--- a/tests/gem5/m5_util/test_exit.py
+++ b/tests/gem5/m5_util/test_exit.py
@@ -37,8 +37,11 @@
r'Exiting @ tick \d* because m5_exit instruction encountered'
)
-test_program = DownloadedProgram('test-progs/m5-exit/bin/x86/linux/',\
- 'm5_exit')
+path = joinpath(absdirpath(__file__), '..',
+ 'test-progs', 'hello', 'bin', 'x86', 'linux')
+filename = 'm5_exit'
+url = 'http://gem5.org/dist/current/test-progs/m5-exit/bin/x86/linux/m5_exit'
+test_program = DownloadedProgram(url, path, filename)
a = verifier.MatchRegex(m5_exit_regex)
gem5_verify_config(
@@ -46,6 +49,6 @@
verifiers=[a],
fixtures=(test_program,),
config=os.path.join(config.base_dir, 'configs', 'example','se.py'),
- config_args=['--cmd', test_program.path],
+ config_args=['--cmd', joinpath(test_program.path, filename)],
valid_isas=('X86',)
)
