author | Craig Dowell <craigdo@ee.washington.edu> |
Mon, 31 Mar 2008 18:20:33 -0700 | |
changeset 2862 | b6ef2bfe89da |
parent 2861 | 18930dcd33e0 |
child 2863 | 0bcad16fc46b |
permissions | -rw-r--r-- |
2846
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1 |
#! /usr/bin/env python |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
2 |
# regression.py adapted from python language regression scripts. |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
3 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
4 |
"""Regression test. |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
5 |
|
2849
fe96c0d98484
some tests and known traces
Craig Dowell <craigdo@ee.washington.edu>
parents:
2847
diff
changeset
|
6 |
This will find all modules whose name is "test-*" in the tests |
2846
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
7 |
directory, and run them. |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
8 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
9 |
Command line options: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
10 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
11 |
-v: verbose -- run tests in verbose mode with output to stdout |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
12 |
-g: generate -- write the output file for a test instead of comparing it |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
13 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
14 |
If non-option arguments are present, they are names for tests to run. |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
15 |
If no test names are given, all tests are run. |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
16 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
17 |
""" |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
18 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
19 |
import sys |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
20 |
import os |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
21 |
import getopt |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
22 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
23 |
verbose = 0 |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
24 |
generate = 0 |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
25 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
26 |
def main(tests = None, testdir = None): |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
27 |
"""Execute regression tests. |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
28 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
29 |
Arguments: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
30 |
tests -- a list of strings containing test names (optional) |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
31 |
testdir -- the directory in which to look for tests (optional) |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
32 |
""" |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
33 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
34 |
global verbose |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
35 |
global generate |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
36 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
37 |
try: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
38 |
opts, args = getopt.getopt(sys.argv[1:], 'vg') |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
39 |
except getopt.error, msg: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
40 |
print msg |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
41 |
print __doc__ |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
42 |
return 2 |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
43 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
44 |
for o, a in opts: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
45 |
if o == '-v': verbose = 1 |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
46 |
if o == '-g': generate = 1 |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
47 |
|
2855
e1063ecd1585
move reference traces out of main repo
Craig Dowell <craigdo@ee.washington.edu>
parents:
2849
diff
changeset
|
48 |
userName = "craigdo/" |
e1063ecd1585
move reference traces out of main repo
Craig Dowell <craigdo@ee.washington.edu>
parents:
2849
diff
changeset
|
49 |
repoName = "ns-3-ref-traces/" |
e1063ecd1585
move reference traces out of main repo
Craig Dowell <craigdo@ee.washington.edu>
parents:
2849
diff
changeset
|
50 |
|
2862 | 51 |
print "========== Running Unit Tests ==========" |
52 |
os.system("./waf check") |
|
53 |
||
54 |
print "========== Running Regression Tests ==========" |
|
2861 | 55 |
print "Synchronizing reference traces." |
56 |
||
2855
e1063ecd1585
move reference traces out of main repo
Craig Dowell <craigdo@ee.washington.edu>
parents:
2849
diff
changeset
|
57 |
if not os.path.exists(repoName): |
2857
2c9890f41feb
quiet clone and pull
Craig Dowell <craigdo@ee.washington.edu>
parents:
2855
diff
changeset
|
58 |
cloneCmd = "hg clone http://code.nsnam.org/" + userName + repoName + " >& /dev/null" |
2855
e1063ecd1585
move reference traces out of main repo
Craig Dowell <craigdo@ee.washington.edu>
parents:
2849
diff
changeset
|
59 |
os.system(cloneCmd) |
e1063ecd1585
move reference traces out of main repo
Craig Dowell <craigdo@ee.washington.edu>
parents:
2849
diff
changeset
|
60 |
else: |
e1063ecd1585
move reference traces out of main repo
Craig Dowell <craigdo@ee.washington.edu>
parents:
2849
diff
changeset
|
61 |
os.chdir(repoName) |
2857
2c9890f41feb
quiet clone and pull
Craig Dowell <craigdo@ee.washington.edu>
parents:
2855
diff
changeset
|
62 |
pullCmd = "hg pull http://code.nsnam.org/" + userName + repoName + " >& /dev/null" |
2855
e1063ecd1585
move reference traces out of main repo
Craig Dowell <craigdo@ee.washington.edu>
parents:
2849
diff
changeset
|
63 |
os.system(pullCmd) |
e1063ecd1585
move reference traces out of main repo
Craig Dowell <craigdo@ee.washington.edu>
parents:
2849
diff
changeset
|
64 |
os.chdir("..") |
e1063ecd1585
move reference traces out of main repo
Craig Dowell <craigdo@ee.washington.edu>
parents:
2849
diff
changeset
|
65 |
|
2861 | 66 |
print "Done." |
67 |
||
2846
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
68 |
bad = [] |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
69 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
70 |
if not testdir: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
71 |
testdir = os.path.join(os.curdir, "tests") |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
72 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
73 |
if not os.path.exists(testdir): |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
74 |
print "Tests directory does not exist" |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
75 |
return 3 |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
76 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
77 |
sys.path.append(testdir) |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
78 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
79 |
for i in range(len(args)): |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
80 |
if args[i][-3:] == '.py': |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
81 |
args[i] = args[i][:-3] |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
82 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
83 |
if not tests: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
84 |
tests = args |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
85 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
86 |
if not tests: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
87 |
tests = findtests(testdir) |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
88 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
89 |
for test in tests: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
90 |
if verbose: |
2862 | 91 |
print "Running test " + test |
2846
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
92 |
result = runtest(test) |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
93 |
if result == 0: |
2847
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
94 |
if generate: |
2862 | 95 |
print "GENERATE" + test |
2847
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
96 |
else: |
2862 | 97 |
print "PASS " + test |
2846
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
98 |
else: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
99 |
bad.append(test) |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
100 |
print "FAIL ", test |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
101 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
102 |
return len(bad) > 0 |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
103 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
104 |
def findtests(testdir): |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
105 |
"""Return a list of test modules in the test directory |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
106 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
107 |
Arguments: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
108 |
testdir -- the directory to look in for tests |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
109 |
""" |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
110 |
names = os.listdir(testdir) |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
111 |
if verbose: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
112 |
print "findtests(): found ", names |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
113 |
tests = [] |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
114 |
for name in names: |
2849
fe96c0d98484
some tests and known traces
Craig Dowell <craigdo@ee.washington.edu>
parents:
2847
diff
changeset
|
115 |
if name[:5] == "test-" and name[-3:] == ".py": |
2846
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
116 |
testname = name[:-3] |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
117 |
tests.append(testname) |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
118 |
tests.sort() |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
119 |
return tests |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
120 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
121 |
def runtest(test): |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
122 |
"""Run a single test. |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
123 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
124 |
Arguments: |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
125 |
test -- the name of the test |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
126 |
""" |
2847
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
127 |
if os.path.exists("traces"): |
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
128 |
files = os.listdir("traces") |
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
129 |
for file in files: |
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
130 |
if file == '.' or file == '..': |
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
131 |
continue |
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
132 |
path = "traces" + os.sep + file |
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
133 |
os.remove(path) |
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
134 |
else: |
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
135 |
os.mkdir("traces") |
f0aedbbb037b
trace generation and comparison
Craig Dowell <craigdo@ee.washington.edu>
parents:
2846
diff
changeset
|
136 |
|
2846
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
137 |
mod = __import__(test, globals(), locals(), []) |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
138 |
return mod.run(verbose, generate) |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
139 |
|
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
140 |
if __name__ == '__main__': |
7689461231ac
start of regression tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
141 |
sys.exit(main()) |