regression.py
author Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
Tue, 18 Nov 2008 13:48:26 +0000
changeset 3866 9e946fee902c
child 3872 8e757a83fb36
permissions -rw-r--r--
Refactor wscript code to move regression testing code to a separate python module (bug 325)
gjc@3866
     1
import os
gjc@3866
     2
import sys
gjc@3866
     3
import Params
gjc@3866
     4
import shutil
gjc@3866
     5
import pproc as subprocess
gjc@3866
     6
import urllib
gjc@3866
     7
gjc@3866
     8
import wutils
gjc@3866
     9
gjc@3866
    10
#
gjc@3866
    11
# The directory in which the tarball of the reference traces lives.  This is
gjc@3866
    12
# used if Mercurial is not on the system.
gjc@3866
    13
#
gjc@3866
    14
REGRESSION_TRACES_URL = "http://www.nsnam.org/releases/"
gjc@3866
    15
gjc@3866
    16
#
gjc@3866
    17
# The path to the Mercurial repository used to find the reference traces if
gjc@3866
    18
# we find "hg" on the system.  We expect that the repository will be named
gjc@3866
    19
# identically to refDirName below
gjc@3866
    20
#
gjc@3866
    21
REGRESSION_TRACES_REPO = "http://code.nsnam.org/"
gjc@3866
    22
gjc@3866
    23
#
gjc@3866
    24
# Name of the local directory where the regression code lives.
gjc@3866
    25
#
gjc@3866
    26
REGRESSION_DIR = "regression"
gjc@3866
    27
gjc@3866
    28
#
gjc@3866
    29
# The last part of the path name to use to find the regression traces.  The
gjc@3866
    30
# path will be APPNAME + '-' + VERSION + REGRESSION_SUFFIX, e.g.,
gjc@3866
    31
# ns-3-dev-ref-traces
gjc@3866
    32
#
gjc@3866
    33
REGRESSION_SUFFIX = "-ref-traces"
gjc@3866
    34
gjc@3866
    35
gjc@3866
    36
def dev_null():
gjc@3866
    37
    if sys.platform == 'win32':
gjc@3866
    38
        return open("NUL:", "w")
gjc@3866
    39
    else:
gjc@3866
    40
        return open("/dev/null", "w")
gjc@3866
    41
gjc@3866
    42
gjc@3866
    43
### Regression testing
gjc@3866
    44
class Regression(object):
gjc@3866
    45
    def __init__(self, testdir):
gjc@3866
    46
        self.testdir = testdir
gjc@3866
    47
        self.env = Params.g_build.env_of_name('default')
gjc@3866
    48
gjc@3866
    49
    def run_test(self, verbose, generate, refDirName, testName, arguments=[], pyscript=None, refTestName=None):
gjc@3866
    50
        """
gjc@3866
    51
        @param verbose: enable verbose execution
gjc@3866
    52
gjc@3866
    53
        @param generate: generate new traces instead of comparing with the reference
gjc@3866
    54
gjc@3866
    55
        @param refDirName: name of the base directory containing reference traces
gjc@3866
    56
gjc@3866
    57
        @param testName: name of the test
gjc@3866
    58
gjc@3866
    59
        @arguments: list of extra parameters to pass to the program to be tested
gjc@3866
    60
gjc@3866
    61
        @pyscript: if not None, the test is written in Python and this
gjc@3866
    62
        parameter contains the path to the python script, relative to
gjc@3866
    63
        the project root dir
gjc@3866
    64
gjc@3866
    65
        @param refTestName: if not None, this is the name of the directory under refDirName
gjc@3866
    66
        that contains the reference traces. Otherwise "refDirname/testName + .ref" is used.
gjc@3866
    67
gjc@3866
    68
        """
gjc@3866
    69
        if not isinstance(arguments, list):
gjc@3866
    70
            raise TypeError
gjc@3866
    71
        
gjc@3866
    72
        if refTestName is None:
gjc@3866
    73
            refTestDirName = os.path.join(refDirName, (testName + ".ref"))
gjc@3866
    74
        else:
gjc@3866
    75
            refTestDirName = os.path.join(refDirName, refTestName)
gjc@3866
    76
gjc@3866
    77
        if not os.path.exists(refDirName):
gjc@3866
    78
            print"No reference trace repository"
gjc@3866
    79
            return 1
gjc@3866
    80
gjc@3866
    81
        if generate:
gjc@3866
    82
            if not os.path.exists(refTestDirName):
gjc@3866
    83
                print "creating new " + refTestDirName
gjc@3866
    84
                os.mkdir(refTestDirName)
gjc@3866
    85
gjc@3866
    86
            if pyscript is None:
gjc@3866
    87
                Params.g_options.cwd_launch = refTestDirName
gjc@3866
    88
                tmpl = "%s"
gjc@3866
    89
                for arg in arguments:
gjc@3866
    90
                    tmpl = tmpl + " " + arg
gjc@3866
    91
                wutils.run_program(testName, tmpl)
gjc@3866
    92
            else:
gjc@3866
    93
                argv = [self.env['PYTHON'], os.path.join('..', '..', '..', *os.path.split(pyscript))] + arguments
gjc@3866
    94
                before = os.getcwd()
gjc@3866
    95
                os.chdir(refTestDirName)
gjc@3866
    96
                try:
gjc@3866
    97
                    wutils.run_argv(argv)
gjc@3866
    98
                finally:
gjc@3866
    99
                    os.chdir(before)
gjc@3866
   100
            print "Remember to commit " + refTestDirName
gjc@3866
   101
            return 0
gjc@3866
   102
        else:
gjc@3866
   103
            if not os.path.exists(refTestDirName):
gjc@3866
   104
                print "Cannot locate reference traces in " + refTestDirName
gjc@3866
   105
                return 1
gjc@3866
   106
gjc@3866
   107
            
gjc@3866
   108
            if refTestName is None:
gjc@3866
   109
                traceDirName = testName + ".ref"
gjc@3866
   110
            else:
gjc@3866
   111
                traceDirName = refTestName
gjc@3866
   112
            traceDirName = os.path.join ('traces', traceDirName)
gjc@3866
   113
gjc@3866
   114
            try:
gjc@3866
   115
                shutil.rmtree(traceDirName)
gjc@3866
   116
            except OSError:
gjc@3866
   117
                pass
gjc@3866
   118
            os.mkdir(traceDirName)
gjc@3866
   119
gjc@3866
   120
            #os.system("./waf --cwd regression/traces --run " +
gjc@3866
   121
            #  testName + " > /dev/null 2>&1")
gjc@3866
   122
gjc@3866
   123
            if pyscript is None:
gjc@3866
   124
                Params.g_options.cwd_launch = traceDirName
gjc@3866
   125
                wutils.run_program(testName, command_template=wutils.get_command_template(*arguments))
gjc@3866
   126
            else:
gjc@3866
   127
                argv = [self.env['PYTHON'], os.path.join('..', '..', '..', *os.path.split(pyscript))] + arguments
gjc@3866
   128
                before = os.getcwd()
gjc@3866
   129
                os.chdir(traceDirName)
gjc@3866
   130
                try:
gjc@3866
   131
                    wutils.run_argv(argv)
gjc@3866
   132
                finally:
gjc@3866
   133
                    os.chdir(before)
gjc@3866
   134
gjc@3866
   135
            if verbose:
gjc@3866
   136
                #diffCmd = "diff traces " + refTestDirName + " | head"
gjc@3866
   137
                diffCmd = subprocess.Popen([self.env['DIFF'], traceDirName, refTestDirName],
gjc@3866
   138
                                           stdout=subprocess.PIPE)
gjc@3866
   139
                headCmd = subprocess.Popen("head", stdin=diffCmd.stdout)
gjc@3866
   140
                rc2 = headCmd.wait()
gjc@3866
   141
                diffCmd.stdout.close()
gjc@3866
   142
                rc1 = diffCmd.wait()
gjc@3866
   143
                rc = rc1 or rc2
gjc@3866
   144
            else:
gjc@3866
   145
                rc = subprocess.Popen([self.env['DIFF'], traceDirName, refTestDirName], stdout=dev_null()).wait()
gjc@3866
   146
            if rc:
gjc@3866
   147
                print "----------"
gjc@3866
   148
                print "Traces differ in test: test-" + testName
gjc@3866
   149
                print "Reference traces in directory: regression/" + refTestDirName
gjc@3866
   150
                print "Traces in directory: traces"
gjc@3866
   151
                print "Rerun regression test as: " + \
gjc@3866
   152
                    "\"./waf --regression --regression-tests=test-" + testName + "\""
gjc@3866
   153
                print "Then do \"diff -u regression/" + refTestDirName + " regression/" + traceDirName +\
gjc@3866
   154
                    "\" for details"
gjc@3866
   155
                print "----------"
gjc@3866
   156
            return rc
gjc@3866
   157
gjc@3866
   158
def _find_tests(testdir):
gjc@3866
   159
    """Return a list of test modules in the test directory
gjc@3866
   160
gjc@3866
   161
    Arguments:
gjc@3866
   162
    testdir -- the directory to look in for tests
gjc@3866
   163
    """
gjc@3866
   164
    names = os.listdir(testdir)
gjc@3866
   165
    tests = []
gjc@3866
   166
    for name in names:
gjc@3866
   167
        if name[:5] == "test-" and name[-3:] == ".py":
gjc@3866
   168
            testname = name[:-3]
gjc@3866
   169
            tests.append(testname)
gjc@3866
   170
    tests.sort()
gjc@3866
   171
    return tests
gjc@3866
   172
gjc@3866
   173
def run_regression():
gjc@3866
   174
    """Execute regression tests."""
gjc@3866
   175
gjc@3866
   176
    testdir = "tests"
gjc@3866
   177
    if not os.path.exists(testdir):
gjc@3866
   178
        print "Tests directory does not exist"
gjc@3866
   179
        sys.exit(3)
gjc@3866
   180
    
gjc@3866
   181
    sys.path.append(testdir)
gjc@3866
   182
    sys.modules['tracediff'] = Regression(testdir)
gjc@3866
   183
gjc@3866
   184
    if Params.g_options.regression_tests:
gjc@3866
   185
        tests = Params.g_options.regression_tests.split(',')
gjc@3866
   186
    else:
gjc@3866
   187
        tests = _find_tests(testdir)
gjc@3866
   188
gjc@3866
   189
    print "========== Running Regression Tests =========="
gjc@3866
   190
    dir_name = wutils.APPNAME + '-' + wutils.VERSION + REGRESSION_SUFFIX
gjc@3866
   191
    env = Params.g_build.env_of_name('default')
gjc@3866
   192
    if env['MERCURIAL']:
gjc@3866
   193
        print "Synchronizing reference traces using Mercurial."
gjc@3866
   194
        if not os.path.exists(dir_name):
gjc@3866
   195
            print "Cloning " + REGRESSION_TRACES_REPO + dir_name + " from repo."
gjc@3866
   196
            argv = ["hg", "clone", REGRESSION_TRACES_REPO + dir_name, dir_name]
gjc@3866
   197
            rv = subprocess.Popen(argv).wait()
gjc@3866
   198
        else:
gjc@3866
   199
            _dir = os.getcwd()
gjc@3866
   200
            os.chdir(dir_name)
gjc@3866
   201
            try:
gjc@3866
   202
                print "Pulling " + REGRESSION_TRACES_REPO + dir_name + " from repo."
gjc@3866
   203
                result = subprocess.Popen(["hg", "-q", "pull", REGRESSION_TRACES_REPO + dir_name]).wait()
gjc@3866
   204
                if not result:
gjc@3866
   205
                    result = subprocess.Popen(["hg", "-q", "update"]).wait()
gjc@3866
   206
            finally:
gjc@3866
   207
                os.chdir("..")
gjc@3866
   208
            if result:
gjc@3866
   209
                Params.fatal("Synchronizing reference traces using Mercurial failed.")
gjc@3866
   210
    else:
gjc@3866
   211
        if not os.path.exists(dir_name):
gjc@3866
   212
            traceball = dir_name + wutils.TRACEBALL_SUFFIX
gjc@3866
   213
            print "Retrieving " + traceball + " from web."
gjc@3866
   214
            urllib.urlretrieve(REGRESSION_TRACES_URL + traceball, traceball)
gjc@3866
   215
            os.system("tar -xjf %s -C .." % (traceball))
gjc@3866
   216
            print "Done."
gjc@3866
   217
gjc@3866
   218
    if not os.path.exists(dir_name):
gjc@3866
   219
        print "Reference traces directory (%s) does not exist" % dir_name
gjc@3866
   220
        return 3
gjc@3866
   221
    
gjc@3866
   222
    bad = []
gjc@3866
   223
gjc@3866
   224
    for test in tests:
gjc@3866
   225
        try:
gjc@3866
   226
            result = _run_regression_test(test)
gjc@3866
   227
            if result == 0:
gjc@3866
   228
                if Params.g_options.regression_generate:
gjc@3866
   229
                    print "GENERATE " + test
gjc@3866
   230
                else:
gjc@3866
   231
                    print "PASS " + test
gjc@3866
   232
            else:
gjc@3866
   233
                bad.append(test)
gjc@3866
   234
                print "FAIL " + test
gjc@3866
   235
        except NotImplementedError:
gjc@3866
   236
                print "SKIP " + test            
gjc@3866
   237
gjc@3866
   238
    return len(bad) > 0
gjc@3866
   239
gjc@3866
   240
gjc@3866
   241
def _run_regression_test(test):
gjc@3866
   242
    """Run a single test.
gjc@3866
   243
gjc@3866
   244
    Arguments:
gjc@3866
   245
    test -- the name of the test
gjc@3866
   246
    """
gjc@3866
   247
gjc@3866
   248
    if os.path.exists("traces"):
gjc@3866
   249
        files = os.listdir("traces")
gjc@3866
   250
        for file in files:
gjc@3866
   251
            if file == '.' or file == '..':
gjc@3866
   252
                continue
gjc@3866
   253
            shutil.rmtree(os.path.join("traces", file), ignore_errors=True)
gjc@3866
   254
    else:
gjc@3866
   255
        os.mkdir("traces")
gjc@3866
   256
    
gjc@3866
   257
    dir_name = wutils.APPNAME + '-' + wutils.VERSION + REGRESSION_SUFFIX
gjc@3866
   258
gjc@3866
   259
    mod = __import__(test, globals(), locals(), [])
gjc@3866
   260
    return mod.run(verbose=(Params.g_options.verbose > 0),
gjc@3866
   261
                   generate=Params.g_options.regression_generate,
gjc@3866
   262
                   refDirName=dir_name)