1.1 --- a/regression/regression.py Fri Apr 04 19:02:34 2008 +0200
1.2 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000
1.3 @@ -1,175 +0,0 @@
1.4 -#! /usr/bin/env python
1.5 -# regression.py adapted from python language regression scripts.
1.6 -
1.7 -"""Regression test.
1.8 -
1.9 -This will find all modules whose name is "test-*" in the tests
1.10 -directory, and run them.
1.11 -
1.12 -Command line options:
1.13 -
1.14 --v: verbose -- run tests in verbose mode (print diff output)
1.15 --g: generate -- write the output file for a test instead of comparing it
1.16 -
1.17 -If non-option arguments are present, they are names for tests to run.
1.18 -If no test names are given, all tests are run.
1.19 -
1.20 -"""
1.21 -
1.22 -import getopt
1.23 -import sys
1.24 -import os
1.25 -import urllib
1.26 -import subprocess
1.27 -
1.28 -verbose = 0
1.29 -generate = 0
1.30 -
1.31 -#
1.32 -# The directory in which the tarball of the reference traces lives. This is
1.33 -# used if Mercurial is not on the system.
1.34 -#
1.35 -refUrl = "http://www.nsnam.org/releases/"
1.36 -
1.37 -#
1.38 -# The name of the tarball to find the reference traces in if there is no
1.39 -# mercurial on the system. It is expected to be created using tar -cjf and
1.40 -# will be extracted using tar -xjf
1.41 -#
1.42 -refTarName = "ns-3.0.12-ref-traces.tar.bz2"
1.43 -
1.44 -#
1.45 -# The path to the Mercurial repository used to find the reference traces if
1.46 -# we find "hg" on the system. We expect that the repository will be named
1.47 -# identically to refDirName below
1.48 -#
1.49 -refRepo = "http://code.nsnam.org/"
1.50 -
1.51 -#
1.52 -# The local directory name into which the reference traces will go in either
1.53 -# case (net or hg).
1.54 -#
1.55 -refDirName = "ns-3-dev-ref-traces"
1.56 -
1.57 -def main(tests = None, testdir = None):
1.58 - """Execute regression tests.
1.59 -
1.60 - Arguments:
1.61 - tests -- a list of strings containing test names (optional)
1.62 - testdir -- the directory in which to look for tests (optional)
1.63 - """
1.64 -
1.65 - global verbose
1.66 - global generate
1.67 - global refUrl
1.68 - global refTarName
1.69 - global refRepo
1.70 - global refRepoName
1.71 - global refDirName
1.72 -
1.73 - try:
1.74 - opts, args = getopt.getopt(sys.argv[1:], 'vg')
1.75 - except getopt.error, msg:
1.76 - print msg
1.77 - print __doc__
1.78 - return 2
1.79 -
1.80 - for o, a in opts:
1.81 - if o == '-v': verbose = 1
1.82 - if o == '-g': generate = 1
1.83 -
1.84 - if not testdir:
1.85 - testdir = os.path.join(os.curdir, "tests")
1.86 -
1.87 - if not os.path.exists(testdir):
1.88 - print "Tests directory does not exist"
1.89 - return 3
1.90 -
1.91 - sys.path.append(testdir)
1.92 -
1.93 - for i in range(len(args)):
1.94 - if args[i][-3:] == '.py':
1.95 - args[i] = args[i][:-3]
1.96 -
1.97 - if not tests:
1.98 - tests = args
1.99 -
1.100 - if not tests:
1.101 - tests = findtests(testdir)
1.102 - if not generate:
1.103 - print "========== Running Unit Tests =========="
1.104 - os.system("./waf check")
1.105 -
1.106 - print "========== Running Regression Tests =========="
1.107 -
1.108 - if os.system("hg version > /dev/null 2>&1") == 0:
1.109 - print "Synchronizing reference traces using Mercurial."
1.110 - if not os.path.exists(refDirName):
1.111 - os.system("hg clone " + refRepo + refDirName + " > /dev/null 2>&1")
1.112 - else:
1.113 - os.chdir(refDirName)
1.114 - os.system("hg pull " + refRepo + refDirName + " > /dev/null 2>&1")
1.115 - os.chdir("..")
1.116 - else:
1.117 - print "Synchronizing reference traces from web."
1.118 - urllib.urlretrieve(refUrl + refTarName, refTarName)
1.119 - os.system("tar -xjf " + refTarName)
1.120 -
1.121 - print "Done."
1.122 -
1.123 - if not os.path.exists(refDirName):
1.124 - print "Reference traces directory does not exist"
1.125 - return 3
1.126 -
1.127 - bad = []
1.128 -
1.129 -
1.130 - for test in tests:
1.131 - result = run_test(test)
1.132 - if result == 0:
1.133 - if generate:
1.134 - print "GENERATE " + test
1.135 - else:
1.136 - print "PASS " + test
1.137 - else:
1.138 - bad.append(test)
1.139 - print "FAIL " + test
1.140 -
1.141 - return len(bad) > 0
1.142 -
1.143 -def findtests(testdir):
1.144 - """Return a list of test modules in the test directory
1.145 -
1.146 - Arguments:
1.147 - testdir -- the directory to look in for tests
1.148 - """
1.149 - names = os.listdir(testdir)
1.150 - tests = []
1.151 - for name in names:
1.152 - if name[:5] == "test-" and name[-3:] == ".py":
1.153 - testname = name[:-3]
1.154 - tests.append(testname)
1.155 - tests.sort()
1.156 - return tests
1.157 -
1.158 -def run_test(test):
1.159 - """Run a single test.
1.160 -
1.161 - Arguments:
1.162 - test -- the name of the test
1.163 - """
1.164 - if os.path.exists("traces"):
1.165 - files = os.listdir("traces")
1.166 - for file in files:
1.167 - if file == '.' or file == '..':
1.168 - continue
1.169 - path = "traces" + os.sep + file
1.170 - os.remove(path)
1.171 - else:
1.172 - os.mkdir("traces")
1.173 -
1.174 - mod = __import__(test, globals(), locals(), [])
1.175 - return mod.run(verbose, generate, refDirName)
1.176 -
1.177 -if __name__ == '__main__':
1.178 - sys.exit(main())
2.1 --- a/regression/tracediff.py Fri Apr 04 19:02:34 2008 +0200
2.2 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000
2.3 @@ -1,57 +0,0 @@
2.4 -#! /usr/bin/env python
2.5 -
2.6 -"""Generic trace-comparison-type regression test."""
2.7 -
2.8 -import sys
2.9 -import os
2.10 -import shutil
2.11 -
2.12 -def run_test(verbose, generate, refDirName, testName):
2.13 - """Execute a test."""
2.14 -
2.15 - refTestDirName = refDirName + "/" + testName + ".ref"
2.16 -
2.17 - if not os.path.exists(refDirName):
2.18 - print"No reference trace repository"
2.19 - return 1
2.20 -
2.21 - if generate:
2.22 - if not os.path.exists(refTestDirName):
2.23 - print "creating new " + refTestDirName
2.24 - os.mkdir(refTestDirName)
2.25 -
2.26 - os.system("./waf --cwd regression/" + refTestDirName +
2.27 - " --run " + testName + " > /dev/null 2>&1")
2.28 -
2.29 - print "Remember to commit " + refTestDirName
2.30 - return 0
2.31 - else:
2.32 - if not os.path.exists(refTestDirName):
2.33 - print "Cannot locate reference traces"
2.34 - return 1
2.35 -
2.36 - shutil.rmtree("traces");
2.37 - os.mkdir("traces")
2.38 -
2.39 - os.system("./waf --cwd regression/traces --run " +
2.40 - testName + " > /dev/null 2>&1")
2.41 -
2.42 - if verbose:
2.43 - diffCmd = "diff traces " + refTestDirName + " | head"
2.44 - else:
2.45 - diffCmd = "diff traces " + refTestDirName + \
2.46 - " > /dev/null 2>&1"
2.47 -
2.48 - rc = os.system(diffCmd)
2.49 - if rc:
2.50 - print "----------"
2.51 - print "Traces differ in test: test-" + testName
2.52 - print "Reference traces in directory: " + refTestDirName
2.53 - print "Traces in directory: traces"
2.54 - print "Rerun regression test as: " + \
2.55 - "\"python regression.py test-" + testName + "\""
2.56 - print "Then do \"diff -u traces " + refTestDirName + \
2.57 - "\" for details"
2.58 - print "----------"
2.59 - return rc
2.60 -
3.1 --- a/wscript Fri Apr 04 19:02:34 2008 +0200
3.2 +++ b/wscript Fri Apr 04 19:33:39 2008 +0100
3.3 @@ -12,6 +12,7 @@
3.4
3.5 Params.g_autoconfig = 1
3.6
3.7 +
3.8 # the following two variables are used by the target "waf dist"
3.9 VERSION = file("VERSION").read().strip()
3.10 APPNAME = 'ns'
3.11 @@ -20,6 +21,33 @@
3.12 srcdir = '.'
3.13 blddir = 'build'
3.14
3.15 +#
3.16 +# The directory in which the tarball of the reference traces lives. This is
3.17 +# used if Mercurial is not on the system.
3.18 +#
3.19 +REGRESSION_TRACES_URL = "http://www.nsnam.org/releases/"
3.20 +
3.21 +#
3.22 +# The name of the tarball to find the reference traces in if there is no
3.23 +# mercurial on the system. It is expected to be created using tar -cjf and
3.24 +# will be extracted using tar -xjf
3.25 +#
3.26 +REGRESSION_TRACES_TAR_NAME = "ns-3.0.12-ref-traces.tar.bz2"
3.27 +
3.28 +#
3.29 +# The path to the Mercurial repository used to find the reference traces if
3.30 +# we find "hg" on the system. We expect that the repository will be named
3.31 +# identically to refDirName below
3.32 +#
3.33 +REGRESSION_TRACES_REPO = "http://code.nsnam.org/"
3.34 +
3.35 +#
3.36 +# The local directory name (relative to the 'regression' dir) into
3.37 +# which the reference traces will go in either case (net or hg).
3.38 +#
3.39 +REGRESSION_TRACES_DIR_NAME = "ns-3-dev-ref-traces"
3.40 +
3.41 +
3.42 def dist_hook():
3.43 shutil.rmtree("doc/html", True)
3.44 shutil.rmtree("doc/latex", True)
3.45 @@ -95,6 +123,17 @@
3.46 action="store_true", default=False,
3.47 dest='shell')
3.48
3.49 + opt.add_option('--regression',
3.50 + help=("Enable regression testing; only used for the 'check' target"),
3.51 + default=False, dest='regression', action="store_true")
3.52 + opt.add_option('--regression-generate',
3.53 + help=("Generate new regression test traces; only used for the 'check' target"),
3.54 + default=False, dest='regression_generate', action="store_true")
3.55 + opt.add_option('--regression-tests',
3.56 + help=('For regression testing, only run/generate the indicated regression tests, '
3.57 + 'specified as a comma separated list of test names'),
3.58 + dest='regression_tests', type="string")
3.59 +
3.60 # options provided in a script in a subdirectory named "src"
3.61 opt.sub_options('src')
3.62
3.63 @@ -256,7 +295,7 @@
3.64 if Params.g_options.valgrind:
3.65 if Params.g_options.command_template:
3.66 Params.fatal("Options --command-template and --valgrind are conflicting")
3.67 - return "valgrind %s"
3.68 + return "valgrind --leak-check=full %s"
3.69 else:
3.70 return (Params.g_options.command_template or '%s')
3.71
3.72 @@ -273,6 +312,14 @@
3.73 if Params.g_commands['check']:
3.74 _run_waf_check()
3.75
3.76 + if Params.g_options.regression or Params.g_options.regression_generate:
3.77 + _dir = os.getcwd()
3.78 + os.chdir("regression")
3.79 + try:
3.80 + run_regression()
3.81 + finally:
3.82 + os.chdir(_dir)
3.83 +
3.84 if Params.g_options.lcov_report:
3.85 lcov_report()
3.86
3.87 @@ -588,3 +635,154 @@
3.88 return TMPFOLDER
3.89
3.90 Scripting.DistDir = DistDir
3.91 +
3.92 +
3.93 +
3.94 +
3.95 +### Regression testing
3.96 +class Regression(object):
3.97 + def __init__(self, testdir):
3.98 + self.testdir = testdir
3.99 +
3.100 + def run_test(self, verbose, generate, refDirName, testName):
3.101 + refTestDirName = os.path.join(refDirName, (testName + ".ref"))
3.102 +
3.103 + if not os.path.exists(refDirName):
3.104 + print"No reference trace repository"
3.105 + return 1
3.106 +
3.107 + if generate:
3.108 + if not os.path.exists(refTestDirName):
3.109 + print "creating new " + refTestDirName
3.110 + os.mkdir(refTestDirName)
3.111 +
3.112 + #os.system("./waf --cwd regression/" + refTestDirName +
3.113 + # " --run " + testName + " > /dev/null 2>&1")
3.114 + Params.g_options.cwd_launch = refTestDirName
3.115 + run_program(testName)
3.116 +
3.117 + print "Remember to commit " + refTestDirName
3.118 + return 0
3.119 + else:
3.120 + if not os.path.exists(refTestDirName):
3.121 + print "Cannot locate reference traces"
3.122 + return 1
3.123 +
3.124 + shutil.rmtree("traces");
3.125 + os.mkdir("traces")
3.126 +
3.127 + #os.system("./waf --cwd regression/traces --run " +
3.128 + # testName + " > /dev/null 2>&1")
3.129 + Params.g_options.cwd_launch = "traces"
3.130 + run_program(testName, command_template=get_command_template())
3.131 +
3.132 + if verbose:
3.133 + diffCmd = "diff traces " + refTestDirName + " | head"
3.134 + else:
3.135 + diffCmd = "diff traces " + refTestDirName + \
3.136 + " > /dev/null 2>&1"
3.137 +
3.138 + rc = os.system(diffCmd)
3.139 + if rc:
3.140 + print "----------"
3.141 + print "Traces differ in test: test-" + testName
3.142 + print "Reference traces in directory: " + refTestDirName
3.143 + print "Traces in directory: traces"
3.144 + print "Rerun regression test as: " + \
3.145 + "\"python regression.py test-" + testName + "\""
3.146 + print "Then do \"diff -u traces " + refTestDirName + \
3.147 + "\" for details"
3.148 + print "----------"
3.149 + return rc
3.150 +
3.151 +def _find_tests(testdir):
3.152 + """Return a list of test modules in the test directory
3.153 +
3.154 + Arguments:
3.155 + testdir -- the directory to look in for tests
3.156 + """
3.157 + names = os.listdir(testdir)
3.158 + tests = []
3.159 + for name in names:
3.160 + if name[:5] == "test-" and name[-3:] == ".py":
3.161 + testname = name[:-3]
3.162 + tests.append(testname)
3.163 + tests.sort()
3.164 + return tests
3.165 +
3.166 +def run_regression():
3.167 + """Execute regression tests."""
3.168 +
3.169 + testdir = "tests"
3.170 + if not os.path.exists(testdir):
3.171 + print "Tests directory does not exist"
3.172 + sys.exit(3)
3.173 +
3.174 + sys.path.append(testdir)
3.175 + sys.modules['tracediff'] = Regression(testdir)
3.176 +
3.177 + if Params.g_options.regression_tests:
3.178 + tests = Params.g_options.regression_tests.split(',')
3.179 + else:
3.180 + tests = _find_tests(testdir)
3.181 +
3.182 + print "========== Running Regression Tests =========="
3.183 +
3.184 + if os.system("hg version > /dev/null 2>&1") == 0:
3.185 + print "Synchronizing reference traces using Mercurial."
3.186 + if not os.path.exists(REGRESSION_TRACES_DIR_NAME):
3.187 + os.system("hg clone " + REGRESSION_TRACES_REPO + REGRESSION_TRACES_DIR_NAME + " > /dev/null 2>&1")
3.188 + else:
3.189 + _dir = os.getcwd()
3.190 + os.chdir(REGRESSION_TRACES_DIR_NAME)
3.191 + try:
3.192 + os.system("hg pull " + REGRESSION_TRACES_REPO + REGRESSION_TRACES_DIR_NAME + " > /dev/null 2>&1")
3.193 + finally:
3.194 + os.chdir("..")
3.195 + else:
3.196 + print "Synchronizing reference traces from web."
3.197 + urllib.urlretrieve(REGRESSION_TRACES_URL + REGRESSION_TRACES_TAR_NAME, REGRESSION_TRACES_TAR_NAME)
3.198 + os.system("tar -xjf %s" % (REGRESSION_TRACES_TAR_NAME,))
3.199 +
3.200 + print "Done."
3.201 +
3.202 + if not os.path.exists(REGRESSION_TRACES_DIR_NAME):
3.203 + print "Reference traces directory does not exist"
3.204 + return 3
3.205 +
3.206 + bad = []
3.207 +
3.208 + for test in tests:
3.209 + result = _run_regression_test(test)
3.210 + if result == 0:
3.211 + if Params.g_options.regression_generate:
3.212 + print "GENERATE " + test
3.213 + else:
3.214 + print "PASS " + test
3.215 + else:
3.216 + bad.append(test)
3.217 + print "FAIL " + test
3.218 +
3.219 + return len(bad) > 0
3.220 +
3.221 +
3.222 +def _run_regression_test(test):
3.223 + """Run a single test.
3.224 +
3.225 + Arguments:
3.226 + test -- the name of the test
3.227 + """
3.228 + if os.path.exists("traces"):
3.229 + files = os.listdir("traces")
3.230 + for file in files:
3.231 + if file == '.' or file == '..':
3.232 + continue
3.233 + path = "traces" + os.sep + file
3.234 + os.remove(path)
3.235 + else:
3.236 + os.mkdir("traces")
3.237 +
3.238 + mod = __import__(test, globals(), locals(), [])
3.239 + return mod.run(verbose=(Params.g_options.verbose > 0),
3.240 + generate=Params.g_options.regression_generate,
3.241 + refDirName=REGRESSION_TRACES_DIR_NAME)