Add a --with-regression-traces configure option, to allow using externally supplied regression traces instead of fetching them from the network.
import os
import sys
import Params
import shutil
import pproc as subprocess
import urllib
import wutils
#
# The directory in which the tarball of the reference traces lives. This is
# used if Mercurial is not on the system.
#
REGRESSION_TRACES_URL = "http://www.nsnam.org/releases/"
#
# The path to the Mercurial repository used to find the reference traces if
# we find "hg" on the system. We expect that the repository will be named
# identically to refDirName below
#
REGRESSION_TRACES_REPO = "http://code.nsnam.org/"
#
# Name of the local directory where the regression code lives.
#
REGRESSION_DIR = "regression"
#
# The last part of the path name to use to find the regression traces. The
# path will be APPNAME + '-' + VERSION + REGRESSION_SUFFIX, e.g.,
# ns-3-dev-ref-traces
#
REGRESSION_SUFFIX = "-ref-traces"
def dev_null():
if sys.platform == 'win32':
return open("NUL:", "w")
else:
return open("/dev/null", "w")
### Regression testing
class Regression(object):
def __init__(self, testdir, reference_traces):
self.testdir = testdir
self.reference_traces = reference_traces
self.env = Params.g_build.env_of_name('default')
def run_test(self, verbose, generate, testName, arguments=[], pyscript=None, refTestName=None):
"""
@param verbose: enable verbose execution
@param generate: generate new traces instead of comparing with the reference
@param testName: name of the test
@arguments: list of extra parameters to pass to the program to be tested
@pyscript: if not None, the test is written in Python and this
parameter contains the path to the python script, relative to
the project root dir
@param refTestName: if not None, this is the name of the directory under refDirName
that contains the reference traces. Otherwise "refDirname/testName + .ref" is used.
"""
if not isinstance(arguments, list):
raise TypeError
if refTestName is None:
refTestDirName = os.path.join(self.reference_traces, (testName + ".ref"))
else:
refTestDirName = os.path.join(self.reference_traces, refTestName)
if not os.path.exists(self.reference_traces):
print"No reference trace repository"
return 1
if generate:
if not os.path.exists(refTestDirName):
print "creating new " + refTestDirName
os.mkdir(refTestDirName)
if pyscript is None:
Params.g_options.cwd_launch = refTestDirName
tmpl = "%s"
for arg in arguments:
tmpl = tmpl + " " + arg
wutils.run_program(testName, tmpl)
else:
argv = [self.env['PYTHON'], os.path.join(Params.g_cwd_launch, *os.path.split(pyscript))] + arguments
before = os.getcwd()
os.chdir(refTestDirName)
try:
wutils.run_argv(argv)
finally:
os.chdir(before)
print "Remember to commit " + refTestDirName
return 0
else:
if not os.path.exists(refTestDirName):
print "Cannot locate reference traces in " + refTestDirName
return 1
if refTestName is None:
traceDirName = testName + ".ref"
else:
traceDirName = refTestName
traceDirName = os.path.join ('traces', traceDirName)
try:
shutil.rmtree(traceDirName)
except OSError:
pass
os.mkdir(traceDirName)
#os.system("./waf --cwd regression/traces --run " +
# testName + " > /dev/null 2>&1")
if pyscript is None:
Params.g_options.cwd_launch = traceDirName
wutils.run_program(testName, command_template=wutils.get_command_template(*arguments))
else:
argv = [self.env['PYTHON'], os.path.join('..', '..', '..', *os.path.split(pyscript))] + arguments
before = os.getcwd()
os.chdir(traceDirName)
try:
wutils.run_argv(argv)
finally:
os.chdir(before)
if verbose:
#diffCmd = "diff traces " + refTestDirName + " | head"
diffCmd = subprocess.Popen([self.env['DIFF'], traceDirName, refTestDirName],
stdout=subprocess.PIPE)
headCmd = subprocess.Popen("head", stdin=diffCmd.stdout)
rc2 = headCmd.wait()
diffCmd.stdout.close()
rc1 = diffCmd.wait()
rc = rc1 or rc2
else:
rc = subprocess.Popen([self.env['DIFF'], traceDirName, refTestDirName], stdout=dev_null()).wait()
if rc:
print "----------"
print "Traces differ in test: test-" + testName
print "Reference traces in directory: regression/" + refTestDirName
print "Traces in directory: traces"
print "Rerun regression test as: " + \
"\"./waf --regression --regression-tests=test-" + testName + "\""
print "Then do \"diff -u regression/" + refTestDirName + " regression/" + traceDirName +\
"\" for details"
print "----------"
return rc
def _find_tests(testdir):
"""Return a list of test modules in the test directory
Arguments:
testdir -- the directory to look in for tests
"""
names = os.listdir(testdir)
tests = []
for name in names:
if name[:5] == "test-" and name[-3:] == ".py":
testname = name[:-3]
tests.append(testname)
tests.sort()
return tests
def run_regression(reference_traces):
"""Execute regression tests. Called with cwd set to the 'regression' subdir of ns-3.
@param reference_traces: reference traces directory, or None for default.
"""
testdir = "tests"
if not os.path.exists(testdir):
print "Tests directory does not exist"
sys.exit(3)
dir_name = (wutils.APPNAME + '-' + wutils.VERSION + REGRESSION_SUFFIX)
if reference_traces is None:
reference_traces = dir_name
no_net = False
else:
no_net = True
sys.path.append(testdir)
sys.modules['tracediff'] = Regression(testdir, reference_traces)
if Params.g_options.regression_tests:
tests = Params.g_options.regression_tests.split(',')
else:
tests = _find_tests(testdir)
print "========== Running Regression Tests =========="
env = Params.g_build.env_of_name('default')
if not no_net:
if env['MERCURIAL']:
print "Synchronizing reference traces using Mercurial."
if not os.path.exists(reference_traces):
print "Cloning " + REGRESSION_TRACES_REPO + dir_name + " from repo."
argv = ["hg", "clone", REGRESSION_TRACES_REPO + dir_name, reference_traces]
rv = subprocess.Popen(argv).wait()
else:
_dir = os.getcwd()
os.chdir(reference_traces)
try:
print "Pulling " + REGRESSION_TRACES_REPO + dir_name + " from repo."
result = subprocess.Popen(["hg", "-q", "pull", REGRESSION_TRACES_REPO + dir_name]).wait()
if not result:
result = subprocess.Popen(["hg", "-q", "update"]).wait()
finally:
os.chdir("..")
if result:
Params.fatal("Synchronizing reference traces using Mercurial failed.")
else:
if not os.path.exists(reference_traces):
traceball = dir_name + wutils.TRACEBALL_SUFFIX
print "Retrieving " + traceball + " from web."
urllib.urlretrieve(REGRESSION_TRACES_URL + traceball, traceball)
os.system("tar -xjf %s -C .." % (traceball))
print "Done."
if not os.path.exists(reference_traces):
print "Reference traces directory (%s) does not exist" % reference_traces
return 3
bad = []
for test in tests:
try:
result = _run_regression_test(test)
if result == 0:
if Params.g_options.regression_generate:
print "GENERATE " + test
else:
print "PASS " + test
else:
bad.append(test)
print "FAIL " + test
except NotImplementedError:
print "SKIP " + test
return len(bad) > 0
def _run_regression_test(test):
"""Run a single test.
Arguments:
test -- the name of the test
"""
if os.path.exists("traces"):
files = os.listdir("traces")
for file in files:
if file == '.' or file == '..':
continue
shutil.rmtree(os.path.join("traces", file), ignore_errors=True)
else:
os.mkdir("traces")
mod = __import__(test, globals(), locals(), [])
return mod.run(verbose=(Params.g_options.verbose > 0),
generate=Params.g_options.regression_generate)