regression.py
author Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
Sun, 18 Jan 2009 22:47:25 +0000
changeset 4110 04170734fa8b
parent 4109 1a251c8ad317
child 4114 a6c5ccfa3451
permissions -rw-r--r--
Don't abuse os.chdir so much (not thread safe)
gjc@4064
     1
# python lib modules
gjc@3866
     2
import os
gjc@3866
     3
import sys
gjc@3866
     4
import shutil
gjc@3866
     5
import pproc as subprocess
gjc@3866
     6
import urllib
gjc@3866
     7
gjc@4064
     8
# WAF modules
gjc@4064
     9
import Build
gjc@4064
    10
import Options
gjc@4064
    11
import Utils
gjc@4064
    12
gjc@4064
    13
# local modules
gjc@3866
    14
import wutils
gjc@3866
    15
gjc@4064
    16
gjc@3866
    17
def dev_null():
gjc@3866
    18
    if sys.platform == 'win32':
gjc@3866
    19
        return open("NUL:", "w")
gjc@3866
    20
    else:
gjc@3866
    21
        return open("/dev/null", "w")
gjc@3866
    22
gjc@3866
    23
gjc@3866
    24
### Regression testing
gjc@3866
    25
class Regression(object):
gjc@3872
    26
    def __init__(self, testdir, reference_traces):
gjc@3866
    27
        self.testdir = testdir
gjc@3872
    28
        self.reference_traces = reference_traces
gjc@4064
    29
        self.env = Build.bld.env
gjc@3866
    30
gjc@3872
    31
    def run_test(self, verbose, generate, testName, arguments=[], pyscript=None, refTestName=None):
gjc@3866
    32
        """
gjc@3866
    33
        @param verbose: enable verbose execution
gjc@3866
    34
gjc@3866
    35
        @param generate: generate new traces instead of comparing with the reference
gjc@3866
    36
gjc@3866
    37
        @param testName: name of the test
gjc@3866
    38
gjc@3866
    39
        @arguments: list of extra parameters to pass to the program to be tested
gjc@3866
    40
gjc@3866
    41
        @pyscript: if not None, the test is written in Python and this
gjc@3866
    42
        parameter contains the path to the python script, relative to
gjc@3866
    43
        the project root dir
gjc@3866
    44
gjc@3866
    45
        @param refTestName: if not None, this is the name of the directory under refDirName
gjc@3866
    46
        that contains the reference traces. Otherwise "refDirname/testName + .ref" is used.
gjc@3866
    47
gjc@3866
    48
        """
gjc@3866
    49
        if not isinstance(arguments, list):
gjc@3866
    50
            raise TypeError
gjc@3866
    51
        
gjc@3866
    52
        if refTestName is None:
gjc@3872
    53
            refTestDirName = os.path.join(self.reference_traces, (testName + ".ref"))
gjc@3866
    54
        else:
gjc@3872
    55
            refTestDirName = os.path.join(self.reference_traces, refTestName)
gjc@3866
    56
gjc@3872
    57
        if not os.path.exists(self.reference_traces):
gjc@3866
    58
            print"No reference trace repository"
gjc@3866
    59
            return 1
gjc@3866
    60
gjc@3866
    61
        if generate:
gjc@3866
    62
            if not os.path.exists(refTestDirName):
gjc@3866
    63
                print "creating new " + refTestDirName
gjc@3866
    64
                os.mkdir(refTestDirName)
gjc@3866
    65
gjc@3866
    66
            if pyscript is None:
gjc@3866
    67
                tmpl = "%s"
gjc@3866
    68
                for arg in arguments:
gjc@3866
    69
                    tmpl = tmpl + " " + arg
gjc@4110
    70
                wutils.run_program(testName, tmpl, cwd=refTestDirName)
gjc@3866
    71
            else:
gjc@4064
    72
                argv = [self.env['PYTHON'], os.path.join(Options.cwd_launch, *os.path.split(pyscript))] + arguments
gjc@4110
    73
                wutils.run_argv(argv, cwd=refTestDirName)
gjc@3866
    74
            print "Remember to commit " + refTestDirName
gjc@3866
    75
            return 0
gjc@3866
    76
        else:
gjc@3866
    77
            if not os.path.exists(refTestDirName):
gjc@3866
    78
                print "Cannot locate reference traces in " + refTestDirName
gjc@3866
    79
                return 1
gjc@3866
    80
gjc@3866
    81
            if refTestName is None:
gjc@3866
    82
                traceDirName = testName + ".ref"
gjc@3866
    83
            else:
gjc@3866
    84
                traceDirName = refTestName
gjc@4110
    85
            traceDirName = os.path.join('regression', 'traces', traceDirName)
gjc@3866
    86
gjc@3866
    87
            try:
gjc@3866
    88
                shutil.rmtree(traceDirName)
gjc@3866
    89
            except OSError:
gjc@3866
    90
                pass
gjc@3866
    91
            os.mkdir(traceDirName)
gjc@3866
    92
gjc@3866
    93
            if pyscript is None:
gjc@4110
    94
                wutils.run_program(testName,
gjc@4110
    95
                                   command_template=wutils.get_command_template(*arguments),
gjc@4110
    96
                                   cwd=traceDirName)
gjc@3866
    97
            else:
gjc@3866
    98
                argv = [self.env['PYTHON'], os.path.join('..', '..', '..', *os.path.split(pyscript))] + arguments
gjc@4110
    99
                wutils.run_argv(argv, cwd=traceDirName)
gjc@3866
   100
gjc@3866
   101
            if verbose:
gjc@3866
   102
                #diffCmd = "diff traces " + refTestDirName + " | head"
gjc@3866
   103
                diffCmd = subprocess.Popen([self.env['DIFF'], traceDirName, refTestDirName],
gjc@3866
   104
                                           stdout=subprocess.PIPE)
gjc@3866
   105
                headCmd = subprocess.Popen("head", stdin=diffCmd.stdout)
gjc@3866
   106
                rc2 = headCmd.wait()
gjc@3866
   107
                diffCmd.stdout.close()
gjc@3866
   108
                rc1 = diffCmd.wait()
gjc@3866
   109
                rc = rc1 or rc2
gjc@3866
   110
            else:
gjc@3866
   111
                rc = subprocess.Popen([self.env['DIFF'], traceDirName, refTestDirName], stdout=dev_null()).wait()
gjc@3866
   112
            if rc:
gjc@3866
   113
                print "----------"
gjc@3866
   114
                print "Traces differ in test: test-" + testName
gjc@3866
   115
                print "Reference traces in directory: regression/" + refTestDirName
gjc@3866
   116
                print "Traces in directory: traces"
gjc@3866
   117
                print "Rerun regression test as: " + \
gjc@3866
   118
                    "\"./waf --regression --regression-tests=test-" + testName + "\""
gjc@3866
   119
                print "Then do \"diff -u regression/" + refTestDirName + " regression/" + traceDirName +\
gjc@3866
   120
                    "\" for details"
gjc@3866
   121
                print "----------"
gjc@3866
   122
            return rc
gjc@3866
   123
gjc@3866
   124
def _find_tests(testdir):
gjc@3866
   125
    """Return a list of test modules in the test directory
gjc@3866
   126
gjc@3866
   127
    Arguments:
gjc@3866
   128
    testdir -- the directory to look in for tests
gjc@3866
   129
    """
gjc@3866
   130
    names = os.listdir(testdir)
gjc@3866
   131
    tests = []
gjc@3866
   132
    for name in names:
gjc@3866
   133
        if name[:5] == "test-" and name[-3:] == ".py":
gjc@3866
   134
            testname = name[:-3]
gjc@3866
   135
            tests.append(testname)
gjc@3866
   136
    tests.sort()
gjc@3866
   137
    return tests
gjc@3866
   138
gjc@3872
   139
def run_regression(reference_traces):
gjc@3872
   140
    """Execute regression tests.  Called with cwd set to the 'regression' subdir of ns-3.
gjc@3872
   141
gjc@4109
   142
    @param reference_traces: reference traces directory.
gjc@3872
   143
gjc@3872
   144
    """
gjc@3866
   145
gjc@4110
   146
    testdir = os.path.join("regression", "tests")
gjc@3866
   147
    if not os.path.exists(testdir):
gjc@3866
   148
        print "Tests directory does not exist"
gjc@3866
   149
        sys.exit(3)
gjc@3872
   150
gjc@3866
   151
    sys.path.append(testdir)
gjc@3872
   152
    sys.modules['tracediff'] = Regression(testdir, reference_traces)
gjc@3866
   153
gjc@4064
   154
    if Options.options.regression_tests:
gjc@4064
   155
        tests = Options.options.regression_tests.split(',')
gjc@3866
   156
    else:
gjc@3866
   157
        tests = _find_tests(testdir)
gjc@3866
   158
gjc@3866
   159
    print "========== Running Regression Tests =========="
gjc@4064
   160
    env = Build.bld.env
gjc@3866
   161
gjc@3872
   162
    if not os.path.exists(reference_traces):
gjc@3872
   163
        print "Reference traces directory (%s) does not exist" % reference_traces
gjc@3866
   164
        return 3
gjc@3866
   165
    
gjc@3866
   166
    bad = []
gjc@3866
   167
gjc@3866
   168
    for test in tests:
gjc@3866
   169
        try:
gjc@3866
   170
            result = _run_regression_test(test)
gjc@3866
   171
            if result == 0:
gjc@4064
   172
                if Options.options.regression_generate:
gjc@3866
   173
                    print "GENERATE " + test
gjc@3866
   174
                else:
gjc@3866
   175
                    print "PASS " + test
gjc@3866
   176
            else:
gjc@3866
   177
                bad.append(test)
gjc@3866
   178
                print "FAIL " + test
gjc@3866
   179
        except NotImplementedError:
gjc@4064
   180
            print "SKIP " + test            
gjc@3866
   181
gjc@4110
   182
    return (len(bad) > 0)
gjc@3866
   183
gjc@3866
   184
gjc@3866
   185
def _run_regression_test(test):
gjc@3866
   186
    """Run a single test.
gjc@3866
   187
gjc@3866
   188
    Arguments:
gjc@3866
   189
    test -- the name of the test
gjc@3866
   190
    """
gjc@4110
   191
    traces_dir = os.path.join("regression", "traces")
gjc@4110
   192
    if os.path.exists(traces_dir):
gjc@4110
   193
        files = os.listdir(traces_dir)
gjc@3866
   194
        for file in files:
gjc@3866
   195
            if file == '.' or file == '..':
gjc@3866
   196
                continue
gjc@3866
   197
            shutil.rmtree(os.path.join("traces", file), ignore_errors=True)
gjc@3866
   198
    else:
gjc@4110
   199
        os.mkdir(traces_dir)
gjc@3866
   200
    
gjc@3866
   201
    mod = __import__(test, globals(), locals(), [])
gjc@4064
   202
    return mod.run(verbose=(Options.options.verbose > 0),
gjc@4064
   203
                   generate=Options.options.regression_generate)
gjc@3872
   204