regression.py
author Mathieu Lacage <mathieu.lacage@sophia.inria.fr>
Sat, 04 Jul 2009 08:15:48 +0200
changeset 4654 2eaebe77d66b
parent 4586 4b9b5f1692d3
permissions -rw-r--r--
Added tag ns-3.5 for changeset c975274c9707
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
4064
10222f483860 Upgrade to new WAF, work in progress
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3872
diff changeset
     1
# python lib modules
3866
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
     2
import os
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
     3
import sys
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
     4
import shutil
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
     5
import pproc as subprocess
4308
b8528d30dfb3 Less error hiding anti-pattern when running regression tests.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4202
diff changeset
     6
import errno
3866
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
     7
4064
10222f483860 Upgrade to new WAF, work in progress
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3872
diff changeset
     8
# WAF modules
10222f483860 Upgrade to new WAF, work in progress
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3872
diff changeset
     9
import Options
10222f483860 Upgrade to new WAF, work in progress
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3872
diff changeset
    10
import Utils
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    11
import Task
4064
10222f483860 Upgrade to new WAF, work in progress
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3872
diff changeset
    12
10222f483860 Upgrade to new WAF, work in progress
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3872
diff changeset
    13
# local modules
3866
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    14
import wutils
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    15
4064
10222f483860 Upgrade to new WAF, work in progress
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3872
diff changeset
    16
3866
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    17
def dev_null():
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    18
    if sys.platform == 'win32':
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    19
        return open("NUL:", "w")
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    20
    else:
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    21
        return open("/dev/null", "w")
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    22
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    23
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    24
def _find_tests(testdir):
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    25
    """Return a list of test modules in the test directory
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    26
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    27
    Arguments:
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    28
    testdir -- the directory to look in for tests
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    29
    """
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    30
    names = os.listdir(testdir)
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    31
    tests = []
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    32
    for name in names:
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    33
        if name[:5] == "test-" and name[-3:] == ".py":
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    34
            testname = name[:-3]
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    35
            tests.append(testname)
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    36
    tests.sort()
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    37
    return tests
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
    38
4202
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    39
def diff(dir1, dir2, verbose):
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    40
    import filecmp
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    41
    comp = filecmp.dircmp(dir1, dir2)
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    42
    differ = (comp.left_only or comp.right_only or comp.diff_files)
4586
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    43
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    44
    if differ:
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    45
        # ok, stupid binary comparison reports differences, but maybe
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    46
        # only text files differ, in which case we should compare
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    47
        # again while ignoring newline differences between
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    48
        # windows/mac/unix.
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    49
        if not comp.left_only and not comp.right_only:
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    50
            for diff_fname in comp.diff_files:
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    51
                if not (diff_fname.endswith(".tr") or diff_fname.endswith(".mob")):
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    52
                    # doesn't look like a text file; it has to differ
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    53
                    break
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    54
                diff_file1 = open(os.path.join(dir1, diff_fname), "rtU").readlines()
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    55
                diff_file2 = open(os.path.join(dir2, diff_fname), "rtU").readlines()
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    56
                if diff_file1 != diff_file2:
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    57
                    break
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    58
                #else:
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    59
                #    print ">>>>>>>> %s file does not really differ!" % (diff_fname)
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    60
            else:
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    61
                differ = False
4b9b5f1692d3 When comparing regression traces, ignore newline differences for text files (.tr and .mob extensions); this makes regression testing in win32/mingw pass.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4584
diff changeset
    62
4202
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    63
    if differ:
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    64
        if verbose:
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    65
            comp.report()
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    66
            import difflib
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    67
            for diff_fname in comp.diff_files:
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    68
                if not (diff_fname.endswith(".tr") or diff_fname.endswith(".mob")):
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    69
                    print "The different file %r does not sound like a text file, not compared." % (diff_fname,)
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    70
                diff_file1 = open(os.path.join(dir1, diff_fname), "rt").readlines()
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    71
                diff_file2 = open(os.path.join(dir2, diff_fname), "rt").readlines()
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    72
                diff = difflib.unified_diff(diff_file1, diff_file2)
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    73
                count = 0
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    74
                print "Differences in file %r" % (diff_fname,)
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    75
                for line in diff:
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    76
                    print line
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    77
                    count += 1
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    78
                    if count > 100:
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    79
                        break
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    80
        return 1
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    81
    else:
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
    82
        return 0
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    83
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    84
class regression_test_task(Task.TaskBase):
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    85
    after = 'cc cxx cc_link cxx_link'
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    86
    color = 'BLUE'
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    87
4326
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
    88
    def __init__(self, bld, env, test_name, test_scripts_dir, build_traces_dir, reference_traces):
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
    89
        self.bld = bld
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
    90
        self.generator = self
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    91
        self.env = env
4326
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
    92
        super(regression_test_task, self).__init__(generator=self, env=env)
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    93
        self.test_name = test_name
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    94
        self.test_scripts_dir = test_scripts_dir
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    95
        self.build_traces_dir = build_traces_dir
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    96
        self.reference_traces_dir = reference_traces
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    97
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    98
    def __str__(self):
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
    99
        return 'regression-test (%s)\n' % self.test_name
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   100
4326
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   101
    def runnable_status(self):
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   102
        return Task.RUN_ME
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   103
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   104
    def run(self):
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   105
        """Run a single test"""
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   106
        sys.path.insert(0, self.test_scripts_dir)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   107
        try:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   108
            mod = __import__(self.test_name, globals(), locals(), [])
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   109
        finally:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   110
            sys.path.remove(self.test_scripts_dir)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   111
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   112
        assert self.test_name.startswith('test-')
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   113
        short_name = self.test_name[len('test-'):]
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   115
        trace_dir_name = getattr(mod, "trace_dir_name", None)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   116
        if trace_dir_name is None:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   117
            trace_dir_name = "%s.ref" % short_name
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   118
        trace_output_path = os.path.join(self.build_traces_dir, trace_dir_name)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   119
        reference_traces_path = os.path.join(self.reference_traces_dir, trace_dir_name)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   120
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   121
        if hasattr(mod, 'get_arguments'):
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   122
            arguments = mod.get_arguments(self.env, '..')
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   123
        else:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   124
            arguments = getattr(mod, "arguments", [])
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   125
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   126
        pyscript = getattr(mod, "pyscript", None)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   127
        if pyscript:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   128
            is_pyscript = True
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   129
            program = pyscript
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   130
        else:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   131
            is_pyscript = False
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   132
            program = getattr(mod, "program", short_name)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   133
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   134
        if hasattr(mod, 'may_run'):
4130
9bddd87225a9 Skip the NSC test when valgrind is enabled.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4129
diff changeset
   135
            reason_cannot_run = mod.may_run(self.env, Options.options)
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   136
        else:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   137
            reason_cannot_run = None
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   138
        if reason_cannot_run:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   139
            print "SKIP %s (%s)" % (self.test_name, reason_cannot_run)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   140
            self.result = None
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   141
            return 0
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   142
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   143
        if Options.options.regression_generate:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   144
            # clean the target dir
4308
b8528d30dfb3 Less error hiding anti-pattern when running regression tests.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4202
diff changeset
   145
            try:
4326
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   146
                shutil.rmtree(reference_traces_path)
4308
b8528d30dfb3 Less error hiding anti-pattern when running regression tests.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4202
diff changeset
   147
            except OSError, ex:
b8528d30dfb3 Less error hiding anti-pattern when running regression tests.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4202
diff changeset
   148
                if ex.errno not in [errno.ENOENT]:
b8528d30dfb3 Less error hiding anti-pattern when running regression tests.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4202
diff changeset
   149
                    raise
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   150
            os.makedirs(reference_traces_path)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   151
            result = self.run_reference_generate(reference_traces_path, program, arguments, is_pyscript)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   152
            if result == 0:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   153
                print "GENERATE " + self.test_name
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   154
            else:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   155
                print "GENERATE FAIL " + self.test_name
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   156
        else:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   157
            # clean the target dir
4308
b8528d30dfb3 Less error hiding anti-pattern when running regression tests.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4202
diff changeset
   158
            try:
b8528d30dfb3 Less error hiding anti-pattern when running regression tests.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4202
diff changeset
   159
                shutil.rmtree(trace_output_path)
b8528d30dfb3 Less error hiding anti-pattern when running regression tests.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4202
diff changeset
   160
            except OSError, ex:
b8528d30dfb3 Less error hiding anti-pattern when running regression tests.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4202
diff changeset
   161
                if ex.errno not in [errno.ENOENT]:
b8528d30dfb3 Less error hiding anti-pattern when running regression tests.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4202
diff changeset
   162
                    raise
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   163
            os.makedirs(trace_output_path)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   164
            # run it
4584
7f4ee7f84b19 Fix win32/mingw bug running commands with command-template: needed to escape backslashes before calling shlex.split().
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4388
diff changeset
   165
            #print "self.run_reference_test:(%r, %r, %r, %r, %r)" \
7f4ee7f84b19 Fix win32/mingw bug running commands with command-template: needed to escape backslashes before calling shlex.split().
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4388
diff changeset
   166
            #    % (reference_traces_path, trace_output_path, program, arguments, is_pyscript)
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   167
            result = self.run_reference_test(reference_traces_path, trace_output_path, program, arguments, is_pyscript)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   168
            if result == 0:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   169
                print "PASS " + self.test_name
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   170
            else:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   171
                print "FAIL " + self.test_name
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   172
        self.result = result
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   173
        return 0
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   174
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   175
    def run_reference_test(self, reference_traces_path, trace_output_path, program, arguments, is_pyscript):
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   176
        if not os.path.exists(reference_traces_path):
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   177
            print "Cannot locate reference traces in " + reference_traces_path
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   178
            return 1
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   179
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   180
        if is_pyscript:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   181
            script = os.path.abspath(os.path.join('..', *os.path.split(program)))
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   182
            argv = [self.env['PYTHON'], script] + arguments
4129
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   183
            try:
4388
fd90dd412e67 Force valgrind to not be used for Python tests
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4332
diff changeset
   184
                wutils.run_argv(argv, self.env, cwd=trace_output_path, force_no_valgrind=True)
4129
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   185
            except Utils.WafError, ex:
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   186
                print >> sys.stderr, ex
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   187
                return 1
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   188
        else:
4129
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   189
            try:
4332
a1c7bc503a0c Detect valgrind memory leak reports; based on patch by Mathieu Lacage; closes bug #532
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4326
diff changeset
   190
                wutils.run_program(program, self.env,
4129
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   191
                                   command_template=wutils.get_command_template(self.env, arguments),
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   192
                                   cwd=trace_output_path)
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   193
            except Utils.WafError, ex:
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   194
                print >> sys.stderr, ex
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   195
                return 1
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   196
4202
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
   197
        rc = diff(trace_output_path, reference_traces_path, Options.options.verbose)
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   198
        if rc:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   199
            print "----------"
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   200
            print "Traces differ in test: ", self.test_name
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   201
            print "Reference traces in directory: " + reference_traces_path
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   202
            print "Traces in directory: " + trace_output_path
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   203
            print "Run the following command for details:"
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   204
            print "\tdiff -u %s %s" % (reference_traces_path, trace_output_path)
4202
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
   205
            if not Options.options.verbose:
272cabf60878 For regression testing, use python modules filecmp and difflib instead of an external diff command, to improve portability (e.g. mingw). Closes #330.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4130
diff changeset
   206
                print "Or re-run regression testing with option -v"
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   207
            print "----------"
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   208
        return rc
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   209
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   210
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   211
    def run_reference_generate(self, trace_output_path, program, arguments, is_pyscript):
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   212
        if is_pyscript:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   213
            script = os.path.abspath(os.path.join('..', *os.path.split(program)))
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   214
            argv = [self.env['PYTHON'], script] + arguments
4129
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   215
            try:
4388
fd90dd412e67 Force valgrind to not be used for Python tests
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4332
diff changeset
   216
                retval = wutils.run_argv(argv, self.env, cwd=trace_output_path, force_no_valgrind=True)
4129
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   217
            except Utils.WafError, ex:
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   218
                print >> sys.stderr, ex
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   219
                return 1
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   220
        else:
4129
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   221
            try:
4332
a1c7bc503a0c Detect valgrind memory leak reports; based on patch by Mathieu Lacage; closes bug #532
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4326
diff changeset
   222
                retval = wutils.run_program(program, self.env,
4129
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   223
                                            command_template=wutils.get_command_template(self.env, arguments),
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   224
                                            cwd=trace_output_path)
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   225
            except Utils.WafError, ex:
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   226
                print >> sys.stderr, ex
9b1f5d58dfb6 Interpret regression program non-zero exit as a test failure, instead of showing a traceback.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4119
diff changeset
   227
                return 1
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   228
        return retval
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   229
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   230
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   231
class regression_test_collector_task(Task.TaskBase):
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   232
    after = 'regression_test_task'
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   233
    color = 'BLUE'
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   234
4326
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   235
    def __init__(self, bld, test_tasks):
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   236
        self.bld = bld
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   237
        super(regression_test_collector_task, self).__init__(generator=self)
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   238
        self.test_tasks = test_tasks
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   239
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   240
    def __str__(self):
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   241
        return 'regression-test-collector\n'
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   242
4326
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   243
    def runnable_status(self):
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   244
        return Task.RUN_ME
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   245
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   246
    def run(self):
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   247
        failed_tests = [test for test in self.test_tasks if test.result is not None and test.result != 0]
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   248
        skipped_tests = [test for test in self.test_tasks if test.result is None]
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   249
        print "Regression testing summary:"
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   250
        if skipped_tests:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   251
            print "SKIP: %i of %i tests have been skipped (%s)" % (
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   252
                len(skipped_tests), len(self.test_tasks),
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   253
                ', '.join([test.test_name for test in skipped_tests]))
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   254
        if failed_tests:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   255
            print "FAIL: %i of %i tests have failed (%s)" % (
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   256
                len(failed_tests), len(self.test_tasks),
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   257
                ', '.join([test.test_name for test in failed_tests]))
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   258
            return 1
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   259
        else:
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   260
            print "PASS: %i of %i tests passed" % (len(self.test_tasks) - len(skipped_tests),
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   261
                                                   len(self.test_tasks))
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   262
            return 0
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   263
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   264
def run_regression(bld, reference_traces):
3872
8e757a83fb36 Add a --with-regression-traces configure option, to allow using externally supplied regression traces instead of fetching them from the network.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3866
diff changeset
   265
    """Execute regression tests.  Called with cwd set to the 'regression' subdir of ns-3.
8e757a83fb36 Add a --with-regression-traces configure option, to allow using externally supplied regression traces instead of fetching them from the network.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3866
diff changeset
   266
4109
1a251c8ad317 Cleanup: remove regression reference traces downloading code (moved to allinone)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4064
diff changeset
   267
    @param reference_traces: reference traces directory.
3872
8e757a83fb36 Add a --with-regression-traces configure option, to allow using externally supplied regression traces instead of fetching them from the network.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3866
diff changeset
   268
8e757a83fb36 Add a --with-regression-traces configure option, to allow using externally supplied regression traces instead of fetching them from the network.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3866
diff changeset
   269
    """
3866
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
   270
4110
04170734fa8b Don't abuse os.chdir so much (not thread safe)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4109
diff changeset
   271
    testdir = os.path.join("regression", "tests")
3866
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
   272
    if not os.path.exists(testdir):
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
   273
        print "Tests directory does not exist"
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
   274
        sys.exit(3)
3872
8e757a83fb36 Add a --with-regression-traces configure option, to allow using externally supplied regression traces instead of fetching them from the network.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3866
diff changeset
   275
4064
10222f483860 Upgrade to new WAF, work in progress
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3872
diff changeset
   276
    if Options.options.regression_tests:
10222f483860 Upgrade to new WAF, work in progress
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3872
diff changeset
   277
        tests = Options.options.regression_tests.split(',')
3866
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
   278
    else:
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
   279
        tests = _find_tests(testdir)
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
   280
3872
8e757a83fb36 Add a --with-regression-traces configure option, to allow using externally supplied regression traces instead of fetching them from the network.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3866
diff changeset
   281
    if not os.path.exists(reference_traces):
8e757a83fb36 Add a --with-regression-traces configure option, to allow using externally supplied regression traces instead of fetching them from the network.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 3866
diff changeset
   282
        print "Reference traces directory (%s) does not exist" % reference_traces
3866
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
   283
        return 3
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
   284
    
4114
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   285
    test_scripts_dir = bld.path.find_dir('regression/tests').abspath()
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   286
    build_traces_dir = bld.path.find_or_declare('regression/traces').abspath(bld.env)
a6c5ccfa3451 Run regression tests as WAF tasks, plus simplify how unit tests are declared. Closes #480.
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4110
diff changeset
   287
    tasks = []
3866
9e946fee902c Refactor wscript code to move regression testing code to a separate python module (bug 325)
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents:
diff changeset
   288
    for test in tests:
4326
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   289
        task = regression_test_task(bld, bld.env, test, test_scripts_dir, build_traces_dir, reference_traces)
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   290
        #bld.task_manager.add_task(task)
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   291
        tasks.append(task)
179f86838e62 Upgrade to WAF 1.5.4
Gustavo J. A. M. Carneiro <gjc@inescporto.pt>
parents: 4308
diff changeset
   292
    regression_test_collector_task(bld, tasks)