#!/usr/bin/env python
# encoding: utf-8
from glob import glob
import argparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
import unittest
import platform
#Python2/3 compat code for iterating items
try:
dict.iteritems
except AttributeError:
# Python 3
def itervalues(d):
return iter(d.values())
def iteritems(d):
return iter(d.items())
def bytes_to_str(bytes):
return str(bytes, 'utf-8')
else:
# Python 2
def itervalues(d):
return d.itervalues()
def iteritems(d):
return d.iteritems()
def bytes_to_str(bytes):
return str(bytes)
def datadir():
return os.path.join(os.path.dirname(__file__), "data")
def datadir_for_testing():
return os.path.relpath(".", os.path.dirname(__file__))
def out(string):
sys.stdout.write(string)
sys.stdout.flush()
class WidelandsTestCase(unittest.TestCase):
do_use_random_directory = True
path_to_widelands_binary = None
keep_output_around = False
ignore_error_code = False
def __init__(self, test_script, **wlargs):
unittest.TestCase.__init__(self)
self._test_script = test_script
self._wlargs = wlargs
def __str__(self):
return self._test_script
def setUp(self):
if self.do_use_random_directory:
self.run_dir = tempfile.mkdtemp(prefix="widelands_regression_test")
else:
self.run_dir = os.path.join(tempfile.gettempdir(), "widelands_regression_test", self.__class__.__name__)
if os.path.exists(self.run_dir):
if not self.keep_output_around:
shutil.rmtree(self.run_dir)
os.makedirs(self.run_dir)
else:
os.makedirs(self.run_dir)
self.widelands_returncode = 0
def run(self, result=None):
self.currentResult = result # remember result for use in tearDown
unittest.TestCase.run(self, result)
def tearDown(self):
if self.currentResult.wasSuccessful() and not self.keep_output_around:
shutil.rmtree(self.run_dir)
def run_widelands(self, wlargs, which_time):
"""Run Widelands with the given 'wlargs'. 'which_time' is an integer
defining the number of times Widelands has been run this test case
(i.e. because we might load a saved game from an earlier run. This will
impact the filenames for stdout.txt.
Returns the stdout filename."""
stdout_filename = os.path.join(self.run_dir, "stdout_{:02d}.txt".format(which_time))
if (os.path.exists(stdout_filename)):
os.unlink(stdout_filename)
with open(stdout_filename, 'a') as stdout_file:
args = [self.path_to_widelands_binary,
'--verbose=true',
'--datadir={}'.format(datadir()),
'--datadir_for_testing={}'.format(datadir_for_testing()),
'--homedir={}'.format(self.run_dir),
'--nosound',
'--fail-on-lua-error',
'--language=en_US' ]
args += [ "--{}={}".format(key, value) for key, value in iteritems(wlargs) ]
stdout_file.write("Running widelands binary: ")
for anarg in args:
stdout_file.write(anarg)
stdout_file.write(" ")
stdout_file.write("\n")
widelands = subprocess.Popen(
args, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
while 1:
line = widelands.stdout.readline()
if not line:
break
stdout_file.write(bytes_to_str(line))
stdout_file.flush()
widelands.communicate()
stdout_file.write("\nReturned from Widelands, return code is %d\n" % widelands.returncode)
self.widelands_returncode = widelands.returncode
return stdout_filename
def runTest(self):
out("\n Running Widelands ... ")
stdout_filename = self.run_widelands(self._wlargs, 0)
stdout = open(stdout_filename, "r").read()
self.verify_success(stdout, stdout_filename)
find_saves = lambda stdout: re.findall("Script requests save to: (\w+)$", stdout, re.M)
savegame_done = { fn: False for fn in find_saves(stdout) }
which_time = 1
while not all(savegame_done.values()):
for savegame in sorted(savegame_done):
if not savegame_done[savegame]: break
out(" Loading savegame: {} ... ".format(savegame))
stdout_filename = self.run_widelands({ "loadgame": os.path.join(
self.run_dir, "save", "{}.wgf".format(savegame))}, which_time)
which_time += 1
stdout = open(stdout_filename, "r").read()
for new_save in find_saves(stdout):
if new_save not in savegame_done:
savegame_done[new_save] = False
savegame_done[savegame] = True
self.verify_success(stdout, stdout_filename)
def verify_success(self, stdout, stdout_filename):
# Catch instabilities with SDL in CI environment
if self.widelands_returncode == 2:
print("SDL initialization failed. TEST SKIPPED.")
with open(stdout_filename, 'r') as stdout_file:
for line in stdout_file.readlines():
print(line.strip())
out("SKIPPED.\n")
else:
common_msg = "Analyze the files in {} to see why this test case failed. Stdout is\n {}\n\nstdout:\n{}".format(
self.run_dir, stdout_filename, stdout)
if self.widelands_returncode == 1 and self.ignore_error_code:
out("IGNORING error code 1\n")
else:
self.assertTrue(self.widelands_returncode == 0,
"Widelands exited abnormally. {}".format(common_msg)
)
self.assertTrue("All Tests passed" in stdout,
"Not all tests pass. {}.".format(common_msg)
)
self.assertFalse("lua_errors.cc" in stdout,
"Not all tests pass. {}.".format(common_msg)
)
out("done.\n")
if self.keep_output_around:
out(" stdout: {}\n".format(stdout_filename))
def parse_args():
p = argparse.ArgumentParser(description=
"Run the regression tests suite."
)
p.add_argument("-r", "--regexp", type=str,
help = "Run only the tests from the files which filename matches."
)
p.add_argument("-n", "--nonrandom", action="store_true", default = False,
help = "Do not randomize the directories for the tests. This is useful "
"if you want to run a test more often than once and not reopen stdout.txt "
"in your editor."
)
p.add_argument("-k", "--keep-around", action="store_true", default = False,
help = "Keep the output files around even when a test terminates successfully."
)
p.add_argument("-b", "--binary", type=str,
help = "Run this binary as Widelands. Otherwise some default paths are searched."
)
p.add_argument("-i", "--ignore-error-code", action="store_true", default = False,
help = "Assume success on return code 1, to allow running the tests "
"without ASan reporting false positives."
)
args = p.parse_args()
if args.binary is None:
potential_binaries = (
glob("widelands") +
glob("src/widelands") +
glob("../*/src/widelands")
)
if not potential_binaries:
p.error("No widelands binary found. Please specify with -b.")
args.binary = potential_binaries[0]
return args
def discover_loadgame_tests(regexp, suite):
"""Add all tests using --loadgame to the 'suite'."""
for fixture in glob(os.path.join("test", "save", "*")):
if not os.path.isdir(fixture):
continue
savegame = glob(os.path.join(fixture, "*.wgf"))[0]
for test_script in glob(os.path.join(fixture, "test*.lua")):
if regexp is not None and not re.search(regexp, test_script):
continue
suite.addTest(
WidelandsTestCase(test_script,
loadgame=savegame, script=test_script))
def discover_scenario_tests(regexp, suite):
"""Add all tests using --scenario to the 'suite'."""
for wlmap in glob(os.path.join("test", "maps", "*")):
if not os.path.isdir(wlmap):
continue
for test_script in glob(os.path.join(wlmap, "scripting", "test*.lua")):
if regexp is not None and not re.search(regexp, test_script):
continue
suite.addTest(
WidelandsTestCase(test_script,
scenario=wlmap, script=test_script))
def discover_editor_tests(regexp, suite):
"""Add all tests needing --editor to the 'suite'."""
for wlmap in glob(os.path.join("test", "maps", "*")):
if not os.path.isdir(wlmap):
continue
for test_script in glob(os.path.join(wlmap, "scripting", "editor_test*.lua")):
if regexp is not None and not re.search(regexp, test_script):
continue
suite.addTest(
WidelandsTestCase(test_script,
editor=wlmap, script=test_script))
def main():
args = parse_args()
WidelandsTestCase.path_to_widelands_binary = args.binary
print("Using '{}' binary.".format(args.binary))
WidelandsTestCase.do_use_random_directory = not args.nonrandom
WidelandsTestCase.keep_output_around = args.keep_around
WidelandsTestCase.ignore_error_code = args.ignore_error_code
all_files = [os.path.basename(filename) for filename in glob("test/test_*.py") ]
if args.regexp:
all_files = [filename for filename in all_files if re.search(args.regexp, filename) ]
all_modules = [ "test.{}".format(filename[:-3]) for filename in all_files ]
suite = unittest.TestSuite()
discover_loadgame_tests(args.regexp, suite)
discover_scenario_tests(args.regexp, suite)
discover_editor_tests(args.regexp, suite)
return unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful()
if __name__ == '__main__':
sys.exit(0 if main() else 1)