author | Craig Dowell <craigdo@ee.washington.edu> |
Tue, 06 Oct 2009 19:34:29 -0700 | |
changeset 5369 | 86beb5869f67 |
parent 5351 | 6abced63cd7f |
child 5370 | a27163124701 |
permissions | -rwxr-xr-x |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1 |
#! /usr/bin/env python |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
2 |
## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*- |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
3 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
4 |
# Copyright (c) 2009 University of Washington |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
5 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
6 |
# This program is free software; you can redistribute it and/or modify |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
7 |
# it under the terms of the GNU General Public License version 2 as |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
8 |
# published by the Free Software Foundation; |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
9 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
10 |
# This program is distributed in the hope that it will be useful, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
11 |
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
12 |
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
13 |
# GNU General Public License for more details. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
14 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
15 |
# You should have received a copy of the GNU General Public License |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
16 |
# along with this program; if not, write to the Free Software |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
17 |
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
18 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
19 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
20 |
import os |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
21 |
import sys |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
22 |
import optparse |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
23 |
import subprocess |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
24 |
import threading |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
25 |
import Queue |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
26 |
import signal |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
27 |
import random |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
28 |
import xml.dom.minidom |
5239
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
29 |
import shutil |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
30 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
31 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
32 |
# XXX This should really be part of a waf command to list the configuration |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
33 |
# items relative to optional ns-3 pieces. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
34 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
35 |
# A list of interesting configuration items in the waf configuration |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
36 |
# cache which we may be interested in when deciding on which examples |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
37 |
# to run and how to run them. These are set by waf during the |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
38 |
# configuration phase and the corresponding assignments are usually |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
39 |
# found in the associated subdirectory wscript files. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
40 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
41 |
interesting_config_items = [ |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
42 |
"NS3_BUILDDIR", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
43 |
"NS3_MODULE_PATH", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
44 |
"ENABLE_NSC", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
45 |
"ENABLE_REAL_TIME", |
5369
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
46 |
"ENABLE_EXAMPLES", |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
47 |
] |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
48 |
|
5295 | 49 |
ENABLE_NSC = False |
50 |
ENABLE_REAL_TIME = False |
|
5369
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
51 |
ENABLE_EXAMPLES = True |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
52 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
53 |
# |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
54 |
# If the user has constrained us to run certain kinds of tests, we can tell waf |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
55 |
# to only build |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
56 |
core_kinds = ["bvt", "core", "system", "unit"] |
5295 | 57 |
|
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
58 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
59 |
# A list of examples to run as smoke tests just to ensure that they remain |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
60 |
# buildable and runnable over time. Also a condition under which to run |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
61 |
# the example (from the waf configuration). |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
62 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
63 |
# XXX Should this not be read from a configuration file somewhere and not |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
64 |
# hardcoded. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
65 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
66 |
example_tests = [ |
5369
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
67 |
("csma/csma-bridge", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
68 |
("csma/csma-bridge-one-hop", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
69 |
("csma/csma-broadcast", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
70 |
("csma/csma-multicast", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
71 |
("csma/csma-one-subnet", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
72 |
("csma/csma-packet-socket", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
73 |
("csma/csma-ping", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
74 |
("csma/csma-raw-ip-socket", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
75 |
("csma/csma-star", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
76 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
77 |
("emulation/emu-ping", "False"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
78 |
("emulation/emu-udp-echo", "False"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
79 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
80 |
("error-model/simple-error-model", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
81 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
82 |
("ipv6/icmpv6-redirect", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
83 |
("ipv6/ping6", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
84 |
("ipv6/radvd", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
85 |
("ipv6/radvd-two-prefix", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
86 |
("ipv6/test-ipv6", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
87 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
88 |
("mesh/mesh", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
89 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
90 |
("naming/object-names", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
91 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
92 |
("realtime/realtime-udp-echo", "ENABLE_REAL_TIME == True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
93 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
94 |
("routing/dynamic-global-routing", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
95 |
("routing/global-injection-slash32", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
96 |
("routing/global-routing-slash32", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
97 |
("routing/mixed-global-routing", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
98 |
("routing/nix-simple", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
99 |
("routing/nms-p2p-nix", "False"), # Takes too long to run |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
100 |
("routing/simple-alternate-routing", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
101 |
("routing/simple-global-routing", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
102 |
("routing/simple-point-to-point-olsr", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
103 |
("routing/simple-routing-ping6", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
104 |
("routing/static-routing-slash32", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
105 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
106 |
("stats/wifi-example-sim", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
107 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
108 |
("tap/tap-wifi-dumbbell", "False"), # Requires manual configuration |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
109 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
110 |
("tcp/star", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
111 |
("tcp/tcp-star-server", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
112 |
("tcp/tcp-large-transfer", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
113 |
("tcp/tcp-nsc-lfn", "ENABLE_NSC == True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
114 |
("tcp/tcp-nsc-zoo", "ENABLE_NSC == True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
115 |
("tcp/tcp-star-server", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
116 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
117 |
("tunneling/virtual-net-device", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
118 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
119 |
("tutorial/first", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
120 |
("tutorial/hello-simulator", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
121 |
("tutorial/second", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
122 |
("tutorial/third", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
123 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
124 |
("udp/udp-echo", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
125 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
126 |
("wireless/mixed-wireless", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
127 |
("wireless/multirate", "False"), # Takes too long to run |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
128 |
("wireless/simple-wifi-frame-aggregation", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
129 |
("wireless/wifi-adhoc", "False"), # Takes too long to run |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
130 |
("wireless/wifi-ap --verbose=0", "True"), # Don't let it spew to stdout |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
131 |
("wireless/wifi-clear-channel-cmu", "False"), # Requires specific hardware |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
132 |
("wireless/wifi-simple-adhoc", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
133 |
("wireless/wifi-simple-adhoc-grid", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
134 |
("wireless/wifi-simple-infra", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
135 |
("wireless/wifi-simple-interference", "True"), |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
136 |
("wireless/wifi-wired-bridging", "True"), |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
137 |
] |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
138 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
139 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
140 |
# Most of the examples produce gangs of trace files, so we want to find |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
141 |
# somewhere to put them that won't pollute the current directory. One |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
142 |
# obvious place is somewhere in /tmp. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
143 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
144 |
TMP_TRACES_DIR = "/tmp/unchecked-traces" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
145 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
146 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
147 |
# The test suites are going to want to output status. They are running |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
148 |
# concurrently. This means that unless we are careful, the output of |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
149 |
# the test suites will be interleaved. Rather than introducing a lock |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
150 |
# file that could unintentionally start serializing execution, we ask |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
151 |
# the tests to write their output to a temporary directory and then |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
152 |
# put together the final output file when we "join" the test tasks back |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
153 |
# to the main thread. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
154 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
155 |
TMP_OUTPUT_DIR = "/tmp/testpy" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
156 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
157 |
def get_node_text(node): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
158 |
for child in node.childNodes: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
159 |
if child.nodeType == child.TEXT_NODE: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
160 |
return child.nodeValue |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
161 |
return "None" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
162 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
163 |
# |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
164 |
# A simple example of writing a text file with a test result summary. It is |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
165 |
# expected that this output will be fine for developers looking for problems. |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
166 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
167 |
def translate_to_text(results_file, text_file): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
168 |
f = open(text_file, 'w') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
169 |
dom = xml.dom.minidom.parse(results_file) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
170 |
for suite in dom.getElementsByTagName("TestSuite"): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
171 |
result = get_node_text(suite.getElementsByTagName("SuiteResult")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
172 |
name = get_node_text(suite.getElementsByTagName("SuiteName")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
173 |
time = get_node_text(suite.getElementsByTagName("SuiteTime")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
174 |
output = "%s: Test Suite \"%s\" (%s)\n" % (result, name, time) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
175 |
f.write(output) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
176 |
if result != "CRASH": |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
177 |
for case in suite.getElementsByTagName("TestCase"): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
178 |
result = get_node_text(case.getElementsByTagName("CaseResult")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
179 |
name = get_node_text(case.getElementsByTagName("CaseName")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
180 |
time = get_node_text(case.getElementsByTagName("CaseTime")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
181 |
output = " %s: Test Case \"%s\" (%s)\n" % (result, name, time) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
182 |
f.write(output) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
183 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
184 |
if result == "FAIL": |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
185 |
for details in case.getElementsByTagName("FailureDetails"): |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
186 |
f.write(" Details:\n") |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
187 |
f.write(" Message: %s\n" % get_node_text(details.getElementsByTagName("Message")[0])) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
188 |
f.write(" Condition: %s\n" % get_node_text(details.getElementsByTagName("Condition")[0])) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
189 |
f.write(" Actual: %s\n" % get_node_text(details.getElementsByTagName("Actual")[0])) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
190 |
f.write(" Limit: %s\n" % get_node_text(details.getElementsByTagName("Limit")[0])) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
191 |
f.write(" File: %s\n" % get_node_text(details.getElementsByTagName("File")[0])) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
192 |
f.write(" Line: %s\n" % get_node_text(details.getElementsByTagName("Line")[0])) |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
193 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
194 |
for example in dom.getElementsByTagName("Example"): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
195 |
result = get_node_text(example.getElementsByTagName("Result")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
196 |
name = get_node_text(example.getElementsByTagName("Name")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
197 |
output = "%s: Example \"%s\"\n" % (result, name) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
198 |
f.write(output) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
199 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
200 |
f.close() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
201 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
202 |
# |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
203 |
# A simple example of writing an HTML file with a test result summary. It is |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
204 |
# expected that this will eventually be made prettier as time progresses and |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
205 |
# we have time to tweak it. This may end up being moved to a separate module |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
206 |
# since it will probably grow over time. |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
207 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
208 |
def translate_to_html(results_file, html_file): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
209 |
f = open(html_file, 'w') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
210 |
f.write("<html>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
211 |
f.write("<body>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
212 |
f.write("<center><h1>ns-3 Test Results</h1></center>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
213 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
214 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
215 |
# Read and parse the whole results file. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
216 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
217 |
dom = xml.dom.minidom.parse(results_file) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
218 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
219 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
220 |
# Iterate through the test suites |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
221 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
222 |
f.write("<h2>Test Suites</h2>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
223 |
for suite in dom.getElementsByTagName("TestSuite"): |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
224 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
225 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
226 |
# For each test suite, get its name, result and execution time info |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
227 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
228 |
name = get_node_text(suite.getElementsByTagName("SuiteName")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
229 |
result = get_node_text(suite.getElementsByTagName("SuiteResult")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
230 |
time = get_node_text(suite.getElementsByTagName("SuiteTime")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
231 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
232 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
233 |
# Print a level three header in green with the result, name and time. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
234 |
# If the test suite passed, the header is printed in green, otherwise |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
235 |
# it is printed in red. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
236 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
237 |
if result == "PASS": |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
238 |
f.write("<h3 style=\"color:green\">%s: %s (%s)</h3>\n" % (result, name, time)) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
239 |
else: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
240 |
f.write("<h3 style=\"color:red\">%s: %s (%s)</h3>\n" % (result, name, time)) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
241 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
242 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
243 |
# The test case information goes in a table. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
244 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
245 |
f.write("<table border=\"1\">\n") |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
246 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
247 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
248 |
# The first column of the table has the heading Result |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
249 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
250 |
f.write("<th> Result </th>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
251 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
252 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
253 |
# If the suite crashed, there is no further information, so just |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
254 |
# delare a new table row with the result (CRASH) in it. Looks like: |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
255 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
256 |
# +--------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
257 |
# | Result | |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
258 |
# +--------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
259 |
# | CRASH | |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
260 |
# +--------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
261 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
262 |
# Then go on to the next test suite |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
263 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
264 |
if result == "CRASH": |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
265 |
f.write("<tr>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
266 |
f.write("<td style=\"color:red\">%s</td>\n" % result) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
267 |
f.write("</tr>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
268 |
f.write("</table>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
269 |
continue |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
270 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
271 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
272 |
# If the suite didn't crash, we expect more information, so fill out |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
273 |
# the table heading row. Like, |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
274 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
275 |
# +--------+----------------+------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
276 |
# | Result | Test Case Name | Time | |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
277 |
# +--------+----------------+------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
278 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
279 |
f.write("<th>Test Case Name</th>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
280 |
f.write("<th> Time </th>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
281 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
282 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
283 |
# If the test case failed, we need to print out some failure details |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
284 |
# so extend the heading row again. Like, |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
285 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
286 |
# +--------+----------------+------+-----------------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
287 |
# | Result | Test Case Name | Time | Failure Details | |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
288 |
# +--------+----------------+------+-----------------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
289 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
290 |
if result == "FAIL": |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
291 |
f.write("<th>Failure Details</th>\n") |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
292 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
293 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
294 |
# Now iterate through all of the test cases. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
295 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
296 |
for case in suite.getElementsByTagName("TestCase"): |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
297 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
298 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
299 |
# Get the name, result and timing information from xml to use in |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
300 |
# printing table below. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
301 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
302 |
name = get_node_text(case.getElementsByTagName("CaseName")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
303 |
result = get_node_text(case.getElementsByTagName("CaseResult")[0]) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
304 |
time = get_node_text(case.getElementsByTagName("CaseTime")[0]) |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
305 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
306 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
307 |
# If the test case failed, we iterate through possibly multiple |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
308 |
# failure details |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
309 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
310 |
if result == "FAIL": |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
311 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
312 |
# There can be multiple failures for each test case. The first |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
313 |
# row always gets the result, name and timing information along |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
314 |
# with the failure details. Remaining failures don't duplicate |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
315 |
# this information but just get blanks for readability. Like, |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
316 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
317 |
# +--------+----------------+------+-----------------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
318 |
# | Result | Test Case Name | Time | Failure Details | |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
319 |
# +--------+----------------+------+-----------------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
320 |
# | FAIL | The name | time | It's busted | |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
321 |
# +--------+----------------+------+-----------------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
322 |
# | | | | Really broken | |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
323 |
# +--------+----------------+------+-----------------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
324 |
# | | | | Busted bad | |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
325 |
# +--------+----------------+------+-----------------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
326 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
327 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
328 |
first_row = True |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
329 |
for details in case.getElementsByTagName("FailureDetails"): |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
330 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
331 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
332 |
# Start a new row in the table for each possible Failure Detail |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
333 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
334 |
f.write("<tr>\n") |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
335 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
336 |
if first_row: |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
337 |
first_row = False |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
338 |
f.write("<td style=\"color:red\">%s</td>\n" % result) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
339 |
f.write("<td>%s</td>\n" % name) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
340 |
f.write("<td>%s</td>\n" % time) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
341 |
else: |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
342 |
f.write("<td></td>\n") |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
343 |
f.write("<td></td>\n") |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
344 |
f.write("<td></td>\n") |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
345 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
346 |
f.write("<td>") |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
347 |
f.write("<b>Message: </b>%s, " % get_node_text(details.getElementsByTagName("Message")[0])) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
348 |
f.write("<b>Condition: </b>%s, " % get_node_text(details.getElementsByTagName("Condition")[0])) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
349 |
f.write("<b>Actual: </b>%s, " % get_node_text(details.getElementsByTagName("Actual")[0])) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
350 |
f.write("<b>Limit: </b>%s, " % get_node_text(details.getElementsByTagName("Limit")[0])) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
351 |
f.write("<b>File: </b>%s, " % get_node_text(details.getElementsByTagName("File")[0])) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
352 |
f.write("<b>Line: </b>%s" % get_node_text(details.getElementsByTagName("Line")[0])) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
353 |
f.write("</td>\n") |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
354 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
355 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
356 |
# End the table row |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
357 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
358 |
f.write("</td>\n") |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
359 |
else: |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
360 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
361 |
# If this particular test case passed, then we just print the PASS |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
362 |
# result in green, followed by the test case name and its execution |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
363 |
# time information. These go off in <td> ... </td> table data. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
364 |
# The details table entry is left blank. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
365 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
366 |
# +--------+----------------+------+---------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
367 |
# | Result | Test Case Name | Time | Details | |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
368 |
# +--------+----------------+------+---------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
369 |
# | PASS | The name | time | | |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
370 |
# +--------+----------------+------+---------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
371 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
372 |
f.write("<tr>\n") |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
373 |
f.write("<td style=\"color:green\">%s</td>\n" % result) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
374 |
f.write("<td>%s</td>\n" % name) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
375 |
f.write("<td>%s</td>\n" % time) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
376 |
f.write("<td></td>\n") |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
377 |
f.write("</tr>\n") |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
378 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
379 |
# All of the rows are written, so we need to end the table. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
380 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
381 |
f.write("</table>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
382 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
383 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
384 |
# That's it for all of the test suites. Now we have to do something about |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
385 |
# our examples. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
386 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
387 |
f.write("<h2>Examples</h2>\n") |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
388 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
389 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
390 |
# Example status is rendered in a table just like the suites. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
391 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
392 |
f.write("<table border=\"1\">\n") |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
393 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
394 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
395 |
# The table headings look like, |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
396 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
397 |
# +--------+--------------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
398 |
# | Result | Example Name | |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
399 |
# +--------+--------------+ |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
400 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
401 |
f.write("<th> Result </th>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
402 |
f.write("<th>Example Name</th>\n") |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
403 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
404 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
405 |
# Now iterate through all of the examples |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
406 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
407 |
for example in dom.getElementsByTagName("Example"): |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
408 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
409 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
410 |
# Start a new row for each example |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
411 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
412 |
f.write("<tr>\n") |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
413 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
414 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
415 |
# Get the result and name of the example in question |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
416 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
417 |
result = get_node_text(example.getElementsByTagName("Result")[0]) |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
418 |
name = get_node_text(example.getElementsByTagName("Name")[0]) |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
419 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
420 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
421 |
# If the example eitehr failed or crashed, print its result status |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
422 |
# in red; otherwise green. This goes in a <td> ... </td> table data |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
423 |
# |
5243
46a3f1348d4d
crashing examples should print red in html
Craig Dowell <craigdo@ee.washington.edu>
parents:
5239
diff
changeset
|
424 |
if result in ["FAIL", "CRASH"]: |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
425 |
f.write("<td style=\"color:red\">%s</td>\n" % result) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
426 |
else: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
427 |
f.write("<td style=\"color:green\">%s</td>\n" % result) |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
428 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
429 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
430 |
# Write the example name as a new tagle data. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
431 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
432 |
f.write("<td>%s</td>\n" % name) |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
433 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
434 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
435 |
# That's it for the current example, so terminate the row. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
436 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
437 |
f.write("</tr>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
438 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
439 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
440 |
# That's it for the table of examples, so terminate the table. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
441 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
442 |
f.write("</table>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
443 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
444 |
# |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
445 |
# And that's it for the report, so finish up. |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
446 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
447 |
f.write("</body>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
448 |
f.write("</html>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
449 |
f.close() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
450 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
451 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
452 |
# Python Control-C handling is broken in the presence of multiple threads. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
453 |
# Signals get delivered to the runnable/running thread by default and if |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
454 |
# it is blocked, the signal is simply ignored. So we hook sigint and set |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
455 |
# a global variable telling the system to shut down gracefully. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
456 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
457 |
thread_exit = False |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
458 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
459 |
def sigint_hook(signal, frame): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
460 |
global thread_exit |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
461 |
thread_exit = True |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
462 |
return 0 |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
463 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
464 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
465 |
# Waf can be configured to compile in debug or optimized modes. In each |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
466 |
# case, the resulting built goes into a different directory. If we want |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
467 |
# test tests to run from the correct code-base, we have to figure out which |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
468 |
# mode waf is running in. This is called its active variant. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
469 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
470 |
# XXX This function pokes around in the waf internal state file. To be a |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
471 |
# little less hacky, we should add a commmand to waf to return this info |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
472 |
# and use that result. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
473 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
474 |
def read_waf_active_variant(): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
475 |
for line in open("build/c4che/default.cache.py").readlines(): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
476 |
if line.startswith("NS3_ACTIVE_VARIANT"): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
477 |
exec(line, globals()) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
478 |
break |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
479 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
480 |
if options.verbose: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
481 |
print "NS3_ACTIVE_VARIANT == %s" % NS3_ACTIVE_VARIANT |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
482 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
483 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
484 |
# In general, the build process itself naturally takes care of figuring out |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
485 |
# which tests are built into the test runner. For example, if waf configure |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
486 |
# determines that ENABLE_EMU is false due to some missing dependency, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
487 |
# the tests for the emu net device simply will not be built and will |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
488 |
# therefore not be included in the built test runner. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
489 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
490 |
# Examples, however, are a different story. In that case, we are just given |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
491 |
# a list of examples that could be run. Instead of just failing, for example, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
492 |
# nsc-tcp-zoo if NSC is not present, we look into the waf saved configuration |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
493 |
# for relevant configuration items. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
494 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
495 |
# XXX This function pokes around in the waf internal state file. To be a |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
496 |
# little less hacky, we should add a commmand to waf to return this info |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
497 |
# and use that result. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
498 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
499 |
def read_waf_config(): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
500 |
for line in open("build/c4che/%s.cache.py" % NS3_ACTIVE_VARIANT).readlines(): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
501 |
for item in interesting_config_items: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
502 |
if line.startswith(item): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
503 |
exec(line, globals()) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
504 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
505 |
if options.verbose: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
506 |
for item in interesting_config_items: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
507 |
print "%s ==" % item, eval(item) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
508 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
509 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
510 |
# It seems pointless to fork a process to run waf to fork a process to run |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
511 |
# the test runner, so we just run the test runner directly. The main thing |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
512 |
# that waf would do for us would be to sort out the shared library path but |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
513 |
# we can deal with that easily and do here. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
514 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
515 |
# There can be many different ns-3 repositories on a system, and each has |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
516 |
# its own shared libraries, so ns-3 doesn't hardcode a shared library search |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
517 |
# path -- it is cooked up dynamically, so we do that too. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
518 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
519 |
def make_library_path(): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
520 |
global LIBRARY_PATH |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
521 |
|
5275
d4008f2981ba
Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents:
5274
diff
changeset
|
522 |
LIBRARY_PATH = "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:'" |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
523 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
524 |
if sys.platform == "darwin": |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
525 |
LIBRARY_PATH = "DYLD_LIBRARY_PATH='" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
526 |
elif sys.platform == "win32": |
5275
d4008f2981ba
Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents:
5274
diff
changeset
|
527 |
LIBRARY_PATH = "PATH=$PATH:'" |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
528 |
elif sys.platform == "cygwin": |
5275
d4008f2981ba
Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents:
5274
diff
changeset
|
529 |
LIBRARY_PATH = "PATH=$PATH:'" |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
530 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
531 |
for path in NS3_MODULE_PATH: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
532 |
LIBRARY_PATH = LIBRARY_PATH + path + ":" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
533 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
534 |
LIBRARY_PATH = LIBRARY_PATH + "'" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
535 |
|
5275
d4008f2981ba
Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents:
5274
diff
changeset
|
536 |
if options.verbose: |
d4008f2981ba
Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents:
5274
diff
changeset
|
537 |
print "LIBRARY_PATH == %s" % LIBRARY_PATH |
d4008f2981ba
Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents:
5274
diff
changeset
|
538 |
|
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
539 |
def run_job_synchronously(shell_command, directory): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
540 |
cmd = "%s %s/%s/%s" % (LIBRARY_PATH, NS3_BUILDDIR, NS3_ACTIVE_VARIANT, shell_command) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
541 |
if options.verbose: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
542 |
print "Synchronously execute %s" % cmd |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
543 |
proc = subprocess.Popen(cmd, shell=True, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
5351
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
544 |
stdout_results, stderr_results = proc.communicate() |
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
545 |
return (proc.returncode, stdout_results, stderr_results) |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
546 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
547 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
548 |
# This class defines a unit of testing work. It will typically refer to |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
549 |
# a test suite to run using the test-runner, or an example to run directly. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
550 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
551 |
class Job(): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
552 |
def __init__(self): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
553 |
self.is_break = False |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
554 |
self.is_example = False |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
555 |
self.shell_command = "" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
556 |
self.display_name = "" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
557 |
self.cwd = "" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
558 |
self.tmp_file_name = "" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
559 |
self.returncode = False |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
560 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
561 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
562 |
# A job is either a standard job or a special job indicating that a worker |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
563 |
# thread should exist. This special job is indicated by setting is_break |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
564 |
# to true. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
565 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
566 |
def set_is_break(self, is_break): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
567 |
self.is_break = is_break |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
568 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
569 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
570 |
# Examples are treated differently than standard test suites. This is |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
571 |
# mostly because they are completely unaware that they are being run as |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
572 |
# tests. So we have to do some special case processing to make them look |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
573 |
# like tests. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
574 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
575 |
def set_is_example(self, is_example): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
576 |
self.is_example = is_example |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
577 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
578 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
579 |
# This is the shell command that will be executed in the job. For example, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
580 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
581 |
# "utils/test-runner --suite=some-test-suite" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
582 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
583 |
def set_shell_command(self, shell_command): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
584 |
self.shell_command = shell_command |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
585 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
586 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
587 |
# This is the dispaly name of the job, typically the test suite or example |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
588 |
# name. For example, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
589 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
590 |
# "some-test-suite" or "udp-echo" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
591 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
592 |
def set_display_name(self, display_name): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
593 |
self.display_name = display_name |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
594 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
595 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
596 |
# This is the base directory of the repository out of which the tests are |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
597 |
# being run. It will be used deep down in the testing framework to determine |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
598 |
# where the source directory of the test was, and therefore where to find |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
599 |
# provided test vectors. For example, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
600 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
601 |
# "/home/user/repos/ns-3-dev" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
602 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
603 |
def set_basedir(self, basedir): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
604 |
self.basedir = basedir |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
605 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
606 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
607 |
# This is the current working directory that will be given to an executing |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
608 |
# test as it is being run. It will be used for examples to tell them where |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
609 |
# to write all of the pcap files that we will be carefully ignoring. For |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
610 |
# example, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
611 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
612 |
# "/tmp/unchecked-traces" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
613 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
614 |
def set_cwd(self, cwd): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
615 |
self.cwd = cwd |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
616 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
617 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
618 |
# This is the temporary results file name that will be given to an executing |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
619 |
# test as it is being run. We will be running all of our tests in parallel |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
620 |
# so there must be multiple temporary output files. These will be collected |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
621 |
# into a single XML file at the end and then be deleted. The file names are |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
622 |
# just giant random numbers, for example |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
623 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
624 |
# "/tmp/testpy/5437925246732857" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
625 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
626 |
def set_tmp_file_name(self, tmp_file_name): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
627 |
self.tmp_file_name = tmp_file_name |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
628 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
629 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
630 |
# The return code received when the job process is executed. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
631 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
632 |
def set_returncode(self, returncode): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
633 |
self.returncode = returncode |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
634 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
635 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
636 |
# The worker thread class that handles the actual running of a given test. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
637 |
# Once spawned, it receives requests for work through its input_queue and |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
638 |
# ships the results back through the output_queue. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
639 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
640 |
class worker_thread(threading.Thread): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
641 |
def __init__(self, input_queue, output_queue): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
642 |
threading.Thread.__init__(self) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
643 |
self.input_queue = input_queue |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
644 |
self.output_queue = output_queue |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
645 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
646 |
def run(self): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
647 |
while True: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
648 |
job = self.input_queue.get() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
649 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
650 |
# Worker threads continue running until explicitly told to stop with |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
651 |
# a special job. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
652 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
653 |
if job.is_break: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
654 |
return |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
655 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
656 |
# If the global interrupt handler sets the thread_exit variable, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
657 |
# we stop doing real work and just report back a "break" in the |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
658 |
# normal command processing has happened. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
659 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
660 |
if thread_exit == True: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
661 |
job.set_is_break(True) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
662 |
self.output_queue.put(job) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
663 |
continue |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
664 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
665 |
# Otherwise go about the business of running tests as normal. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
666 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
667 |
else: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
668 |
if options.verbose: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
669 |
print "Launch %s" % job.shell_command |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
670 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
671 |
if job.is_example: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
672 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
673 |
# If we have an example, the shell command is all we need to |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
674 |
# know. It will be something like "examples/udp-echo" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
675 |
# |
5351
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
676 |
(job.returncode, standard_out, standard_err) = run_job_synchronously(job.shell_command, job.cwd) |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
677 |
else: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
678 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
679 |
# If we're a test suite, we need to provide a little more info |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
680 |
# to the test runner, specifically the base directory and temp |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
681 |
# file name |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
682 |
# |
5351
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
683 |
(job.returncode, standard_out, standard_err) = run_job_synchronously(job.shell_command + |
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
684 |
" --basedir=%s --out=%s" % (job.basedir, job.tmp_file_name), job.cwd) |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
685 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
686 |
if options.verbose: |
5351
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
687 |
print "---------- beign standard out ----------" |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
688 |
print standard_out |
5351
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
689 |
print "---------- begin standard err ----------" |
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
690 |
print standard_err |
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
691 |
print "---------- end standard err ----------" |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
692 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
693 |
self.output_queue.put(job) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
694 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
695 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
696 |
# This is the main function that does the work of interacting with the test-runner |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
697 |
# itself. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
698 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
699 |
def run_tests(): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
700 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
701 |
# Run waf to make sure that everything is built, configured and ready to go |
5369
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
702 |
# unless we are explicitly told not to. We want to be careful about causing |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
703 |
# our users pain while waiting for extraneous stuff to compile and link, so |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
704 |
# we allow users that know what they''re doing to not invoke waf at all. |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
705 |
# |
5369
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
706 |
if not options.nowaf: |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
707 |
|
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
708 |
# |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
709 |
# If the user is running the "kinds" or "list" options, there is an |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
710 |
# implied dependency on the test-runner since we call that program |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
711 |
# if those options are selected. We will exit after processing those |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
712 |
# options, so if we see them, we can safely only build the test-runner. |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
713 |
# |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
714 |
# If the user has constrained us to running only a particular type of |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
715 |
# file, we can only ask waf to build what we know will be necessary. |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
716 |
# For example, if the user only wants to run BVT tests, we only have |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
717 |
# to build the test-runner and can ignore all of the examples. |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
718 |
# |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
719 |
if options.kinds or options.list or (len(options.constrain) and options.constrain in core_kinds): |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
720 |
proc = subprocess.Popen("./waf --target=test-runner", shell=True) |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
721 |
else: |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
722 |
proc = subprocess.Popen("./waf", shell=True) |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
723 |
|
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
724 |
proc.communicate() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
725 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
726 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
727 |
# Pull some interesting configuration information out of waf, primarily |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
728 |
# so we can know where executables can be found, but also to tell us what |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
729 |
# pieces of the system have been built. This will tell us what examples |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
730 |
# are runnable. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
731 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
732 |
read_waf_active_variant() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
733 |
read_waf_config() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
734 |
make_library_path() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
735 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
736 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
737 |
# There are a couple of options that imply we can to exit before starting |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
738 |
# up a bunch of threads and running tests. Let's detect these cases and |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
739 |
# handle them without doing all of the hard work. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
740 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
741 |
if options.kinds: |
5351
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
742 |
(rc, standard_out, standard_err) = run_job_synchronously("utils/test-runner --kinds", os.getcwd()) |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
743 |
print standard_out |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
744 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
745 |
if options.list: |
5351
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
746 |
(rc, standard_out, standard_err) = run_job_synchronously("utils/test-runner --list", os.getcwd()) |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
747 |
print standard_out |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
748 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
749 |
if options.kinds or options.list: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
750 |
return |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
751 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
752 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
753 |
# We communicate results in two ways. First, a simple message relating |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
754 |
# PASS, FAIL, or SKIP is always written to the standard output. It is |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
755 |
# expected that this will be one of the main use cases. A developer can |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
756 |
# just run test.py with no options and see that all of the tests still |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
757 |
# pass. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
758 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
759 |
# The second main use case is when detailed status is requested (with the |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
760 |
# --text or --html options). Typicall this will be text if a developer |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
761 |
# finds a problem, or HTML for nightly builds. In these cases, an |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
762 |
# XML file is written containing the status messages from the test suites. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
763 |
# This file is then read and translated into text or HTML. It is expected |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
764 |
# that nobody will really be interested in the XML, so we write it to |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
765 |
# somewhere in /tmp with a random name to avoid collisions. Just in case |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
766 |
# some strange once-in-a-lifetime error occurs, we always write the info |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
767 |
# so it can be found, we just may not use it. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
768 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
769 |
# When we run examples as smoke tests, they are going to want to create |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
770 |
# lots and lots of trace files. We aren't really interested in the contents |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
771 |
# of the trace files, so we also just stash them off in /tmp somewhere. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
772 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
773 |
if not os.path.exists(TMP_OUTPUT_DIR): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
774 |
os.makedirs(TMP_OUTPUT_DIR) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
775 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
776 |
if not os.path.exists(TMP_TRACES_DIR): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
777 |
os.makedirs(TMP_TRACES_DIR) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
778 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
779 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
780 |
# Create the main output file and start filling it with XML. We need to |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
781 |
# do this since the tests will just append individual results to this file. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
782 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
783 |
xml_results_file = TMP_OUTPUT_DIR + "%d.xml" % random.randint(0, sys.maxint) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
784 |
f = open(xml_results_file, 'w') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
785 |
f.write('<?xml version="1.0"?>\n') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
786 |
f.write('<TestResults>\n') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
787 |
f.close() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
788 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
789 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
790 |
# We need to figure out what test suites to execute. We are either given one |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
791 |
# suite or example explicitly via the --suite or --example option, or we |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
792 |
# need to call into the test runner and ask it to list all of the available |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
793 |
# test suites. Further, we need to provide the constraint information if it |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
794 |
# has been given to us. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
795 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
796 |
# This translates into allowing the following options with respect to the |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
797 |
# suites |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
798 |
# |
5369
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
799 |
# ./test,py: run all of the suites and examples |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
800 |
# ./test.py --constrain=core: run all of the suites of all kinds |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
801 |
# ./test.py --constrain=unit: run all unit suites |
5369
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
802 |
# ./test,py --suite=some-test-suite: run a single suite |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
803 |
# ./test,py --example=udp-echo: run no test suites |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
804 |
# ./test,py --suite=some-suite --example=some-example: run the single suite |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
805 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
806 |
# We can also use the --constrain option to provide an ordering of test |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
807 |
# execution quite easily. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
808 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
809 |
if len(options.suite): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
810 |
suites = options.suite + "\n" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
811 |
elif len(options.example) == 0: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
812 |
if len(options.constrain): |
5351
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
813 |
(rc, suites, standard_err) = run_job_synchronously("utils/test-runner --list --constrain=%s" % |
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
814 |
options.constrain, os.getcwd()) |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
815 |
else: |
5351
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
816 |
(rc, suites, standard_err) = run_job_synchronously("utils/test-runner --list", os.getcwd()) |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
817 |
else: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
818 |
suites = "" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
819 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
820 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
821 |
# suite_list will either a single test suite name that the user has |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
822 |
# indicated she wants to run or a list of test suites provided by |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
823 |
# the test-runner possibly according to user provided constraints. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
824 |
# We go through the trouble of setting up the parallel execution |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
825 |
# even in the case of a single suite to avoid having two process the |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
826 |
# results in two different places. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
827 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
828 |
suite_list = suites.split('\n') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
829 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
830 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
831 |
# We now have a possibly large number of test suites to run, so we want to |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
832 |
# run them in parallel. We're going to spin up a number of worker threads |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
833 |
# that will run our test jobs for us. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
834 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
835 |
input_queue = Queue.Queue(0) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
836 |
output_queue = Queue.Queue(0) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
837 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
838 |
jobs = 0 |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
839 |
threads=[] |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
840 |
|
5273
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
841 |
# |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
842 |
# In Python 2.6 you can just use multiprocessing module, but we don't want |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
843 |
# to introduce that dependency yet; so we jump through a few hoops. |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
844 |
# |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
845 |
processors = 1 |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
846 |
|
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
847 |
if 'SC_NPROCESSORS_ONLN'in os.sysconf_names: |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
848 |
processors = os.sysconf('SC_NPROCESSORS_ONLN') |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
849 |
else: |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
850 |
proc = subprocess.Popen("sysctl -n hw.ncpu", shell = True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
851 |
stdout_results, stderr_results = proc.communicate() |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
852 |
if len(stderr_results) == 0: |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
853 |
processors = int(stdout_results) |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
854 |
|
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
855 |
# |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
856 |
# Now, spin up one thread per processor which will eventually mean one test |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
857 |
# per processor running concurrently. |
c95bd39cc980
Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents:
5257
diff
changeset
|
858 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
859 |
for i in range(processors): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
860 |
thread = worker_thread(input_queue, output_queue) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
861 |
threads.append(thread) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
862 |
thread.start() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
863 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
864 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
865 |
# We now have worker threads spun up, and a list of work to do. So, run |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
866 |
# through the list of test suites and dispatch a job to run each one. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
867 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
868 |
# Dispatching will run with unlimited speed and the worker threads will |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
869 |
# execute as fast as possible from the queue. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
870 |
# |
5279
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
871 |
total_tests = 0 |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
872 |
for test in suite_list: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
873 |
if len(test): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
874 |
job = Job() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
875 |
job.set_is_example(False) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
876 |
job.set_display_name(test) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
877 |
job.set_tmp_file_name(TMP_OUTPUT_DIR + "%d" % random.randint(0, sys.maxint)) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
878 |
job.set_cwd(os.getcwd()) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
879 |
job.set_basedir(os.getcwd()) |
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
880 |
if (options.multiple): |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
881 |
multiple = " --multiple" |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
882 |
else: |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
883 |
multiple = "" |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
884 |
|
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
885 |
job.set_shell_command("utils/test-runner --suite='%s'%s" % (test, multiple)) |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
886 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
887 |
if options.verbose: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
888 |
print "Queue %s" % test |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
889 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
890 |
input_queue.put(job) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
891 |
jobs = jobs + 1 |
5279
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
892 |
total_tests = total_tests + 1 |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
893 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
894 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
895 |
# We've taken care of the discovered or specified test suites. Now we |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
896 |
# have to deal with examples run as smoke tests. We have a list of all of |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
897 |
# the example programs it makes sense to try and run. Each example will |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
898 |
# have a condition associated with it that must evaluate to true for us |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
899 |
# to try and execute it. This is used to determine if the example has |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
900 |
# a dependency that is not satisfied. For example, if an example depends |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
901 |
# on NSC being configured by waf, that example should have a condition |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
902 |
# that evaluates to true if NSC is enabled. For example, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
903 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
904 |
# ("tcp-nsc-zoo", "ENABLE_NSC == True"), |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
905 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
906 |
# In this case, the example "tcp-nsc-zoo" will only be run if we find the |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
907 |
# waf configuration variable "ENABLE_NSC" to be True. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
908 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
909 |
# We don't care at all how the trace files come out, so we just write them |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
910 |
# to a single temporary directory. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
911 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
912 |
# XXX As it stands, all of the trace files have unique names, and so file |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
913 |
# collisions can only happen if two instances of an example are running in |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
914 |
# two versions of the test.py process concurrently. We may want to create |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
915 |
# uniquely named temporary traces directories to avoid this problem. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
916 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
917 |
# We need to figure out what examples to execute. We are either given one |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
918 |
# suite or example explicitly via the --suite or --example option, or we |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
919 |
# need to walk the list of examples looking for available example |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
920 |
# conditions. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
921 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
922 |
# This translates into allowing the following options with respect to the |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
923 |
# suites |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
924 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
925 |
# ./test,py: run all of the examples |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
926 |
# ./test.py --constrain=unit run no examples |
5369
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
927 |
# ./test.py --constrain=example run all of the examples |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
928 |
# ./test,py --suite=some-test-suite: run no examples |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
929 |
# ./test,py --example=some-example: run the single example |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
930 |
# ./test,py --suite=some-suite --example=some-example: run the single example |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
931 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
932 |
# XXX could use constrain to separate out examples used for performance |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
933 |
# testing |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
934 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
935 |
if len(options.suite) == 0 and len(options.example) == 0: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
936 |
if len(options.constrain) == 0 or options.constrain == "example": |
5369
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
937 |
if ENABLE_EXAMPLES: |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
938 |
for test, condition in example_tests: |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
939 |
if eval(condition) == True: |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
940 |
job = Job() |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
941 |
job.set_is_example(True) |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
942 |
job.set_display_name(test) |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
943 |
job.set_tmp_file_name("") |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
944 |
job.set_cwd(TMP_TRACES_DIR) |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
945 |
job.set_basedir(os.getcwd()) |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
946 |
job.set_shell_command("examples/%s" % test) |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
947 |
|
5369
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
948 |
if options.verbose: |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
949 |
print "Queue %s" % test |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
950 |
|
5369
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
951 |
input_queue.put(job) |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
952 |
jobs = jobs + 1 |
86beb5869f67
split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents:
5351
diff
changeset
|
953 |
total_tests = total_tests + 1 |
5279
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
954 |
|
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
955 |
elif len(options.example): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
956 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
957 |
# If you tell me to run an example, I will try and run the example |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
958 |
# irrespective of any condition. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
959 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
960 |
job = Job() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
961 |
job.set_is_example(True) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
962 |
job.set_display_name(options.example) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
963 |
job.set_tmp_file_name("") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
964 |
job.set_cwd(TMP_TRACES_DIR) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
965 |
job.set_basedir(os.getcwd()) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
966 |
job.set_shell_command("examples/%s" % options.example) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
967 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
968 |
if options.verbose: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
969 |
print "Queue %s" % test |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
970 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
971 |
input_queue.put(job) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
972 |
jobs = jobs + 1 |
5279
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
973 |
total_tests = total_tests + 1 |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
974 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
975 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
976 |
# Tell the worker threads to pack up and go home for the day. Each one |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
977 |
# will exit when they see their is_break task. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
978 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
979 |
for i in range(processors): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
980 |
job = Job() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
981 |
job.set_is_break(True) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
982 |
input_queue.put(job) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
983 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
984 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
985 |
# Now all of the tests have been dispatched, so all we have to do here |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
986 |
# in the main thread is to wait for them to complete. Keyboard interrupt |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
987 |
# handling is broken as mentioned above. We use a signal handler to catch |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
988 |
# sigint and set a global variable. When the worker threads sense this |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
989 |
# they stop doing real work and will just start throwing jobs back at us |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
990 |
# with is_break set to True. In this case, there are no real results so we |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
991 |
# ignore them. If there are real results, we always print PASS or FAIL to |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
992 |
# standard out as a quick indication of what happened. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
993 |
# |
5279
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
994 |
passed_tests = 0 |
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
995 |
failed_tests = 0 |
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
996 |
crashed_tests = 0 |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
997 |
for i in range(jobs): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
998 |
job = output_queue.get() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
999 |
if job.is_break: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1000 |
continue |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1001 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1002 |
if job.is_example: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1003 |
kind = "Example" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1004 |
else: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1005 |
kind = "TestSuite" |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1006 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1007 |
if job.returncode == 0: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1008 |
status = "PASS" |
5279
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
1009 |
passed_tests = passed_tests + 1 |
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
1010 |
elif job.returncode == 1: |
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
1011 |
failed_tests = failed_tests + 1 |
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
1012 |
status = "FAIL" |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1013 |
else: |
5279
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
1014 |
crashed_tests = crashed_tests + 1 |
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
1015 |
status = "CRASH" |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1016 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1017 |
print "%s: %s %s" % (status, kind, job.display_name) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1018 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1019 |
if job.is_example == True: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1020 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1021 |
# Examples are the odd man out here. They are written without any |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1022 |
# knowledge that they are going to be run as a test, so we need to |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1023 |
# cook up some kind of output for them. We're writing an xml file, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1024 |
# so we do some simple XML that says we ran the example. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1025 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1026 |
# XXX We could add some timing information to the examples, i.e. run |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1027 |
# them through time and print the results here. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1028 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1029 |
f = open(xml_results_file, 'a') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1030 |
f.write('<Example>\n') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1031 |
example_name = " <Name>%s</Name>\n" % job.display_name |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1032 |
f.write(example_name) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1033 |
if job.returncode == 0: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1034 |
f.write(' <Result>PASS</Result>\n') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1035 |
elif job.returncode == 1: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1036 |
f.write(' <Result>FAIL</Result>\n') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1037 |
else: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1038 |
f.write(' <Result>CRASH</Result>\n') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1039 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1040 |
f.write('</Example>\n') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1041 |
f.close() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1042 |
else: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1043 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1044 |
# If we're not running an example, we're running a test suite. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1045 |
# These puppies are running concurrently and generating output |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1046 |
# that was written to a temporary file to avoid collisions. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1047 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1048 |
# Now that we are executing sequentially in the main thread, we can |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1049 |
# concatenate the contents of the associated temp file to the main |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1050 |
# results file and remove that temp file. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1051 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1052 |
# One thing to consider is that a test suite can crash just as |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1053 |
# well as any other program, so we need to deal with that |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1054 |
# possibility as well. If it ran correctly it will return 0 |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1055 |
# if it passed, or 1 if it failed. In this case, we can count |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1056 |
# on the results file it saved being complete. If it crashed, it |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1057 |
# will return some other code, and the file should be considered |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1058 |
# corrupt and useless. If the suite didn't create any XML, then |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1059 |
# we're going to have to do it ourselves. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1060 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1061 |
if job.returncode == 0 or job.returncode == 1: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1062 |
f_to = open(xml_results_file, 'a') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1063 |
f_from = open(job.tmp_file_name, 'r') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1064 |
f_to.write(f_from.read()) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1065 |
f_to.close() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1066 |
f_from.close() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1067 |
else: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1068 |
f = open(xml_results_file, 'a') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1069 |
f.write("<TestSuite>\n") |
5239
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
1070 |
f.write(" <SuiteName>%s</SuiteName>\n" % job.display_name) |
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
1071 |
f.write(' <SuiteResult>CRASH</SuiteResult>\n') |
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
1072 |
f.write(' <SuiteTime>Execution times not available</SuiteTime>\n') |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1073 |
f.write("</TestSuite>\n") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1074 |
f.close() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1075 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1076 |
os.remove(job.tmp_file_name) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1077 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1078 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1079 |
# We have all of the tests run and the results written out. One final |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1080 |
# bit of housekeeping is to wait for all of the threads to close down |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1081 |
# so we can exit gracefully. |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1082 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1083 |
for thread in threads: |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1084 |
thread.join() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1085 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1086 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1087 |
# Back at the beginning of time, we started the body of an XML document |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1088 |
# since the test suites and examples were going to just write their |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1089 |
# individual pieces. So, we need to finish off and close out the XML |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1090 |
# document |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1091 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1092 |
f = open(xml_results_file, 'a') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1093 |
f.write('</TestResults>\n') |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1094 |
f.close() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1095 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1096 |
# |
5279
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
1097 |
# Print a quick summary of events |
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
1098 |
# |
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
1099 |
print "%d of %d tests passed (%d passed, %d failed, %d crashed)" % (passed_tests, total_tests, passed_tests, |
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
1100 |
failed_tests, crashed_tests) |
1aec8ea805d2
update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents:
5275
diff
changeset
|
1101 |
# |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1102 |
# The last things to do are to translate the XML results file to "human |
5239
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
1103 |
# readable form" if the user asked for it (or make an XML file somewhere) |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1104 |
# |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1105 |
if len(options.html): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1106 |
translate_to_html(xml_results_file, options.html) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1107 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1108 |
if len(options.text): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1109 |
translate_to_text(xml_results_file, options.text) |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1110 |
|
5239
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
1111 |
if len(options.xml): |
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
1112 |
shutil.copyfile(xml_results_file, options.xml) |
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
1113 |
|
5350
9f457bebbcf4
have test.py return status
Craig Dowell <craigdo@ee.washington.edu>
parents:
5324
diff
changeset
|
1114 |
if passed_tests == total_tests: |
5351
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
1115 |
return 0 # success |
5350
9f457bebbcf4
have test.py return status
Craig Dowell <craigdo@ee.washington.edu>
parents:
5324
diff
changeset
|
1116 |
else: |
5351
6abced63cd7f
keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents:
5350
diff
changeset
|
1117 |
return 1 # catchall for general errors |
5350
9f457bebbcf4
have test.py return status
Craig Dowell <craigdo@ee.washington.edu>
parents:
5324
diff
changeset
|
1118 |
|
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1119 |
def main(argv): |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1120 |
random.seed() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1121 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1122 |
parser = optparse.OptionParser() |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1123 |
parser.add_option("-c", "--constrain", action="store", type="string", dest="constrain", default="", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1124 |
metavar="KIND", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1125 |
help="constrain the test-runner by kind of test") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1126 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1127 |
parser.add_option("-e", "--example", action="store", type="string", dest="example", default="", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1128 |
metavar="EXAMPLE", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1129 |
help="specify a single example to run") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1130 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1131 |
parser.add_option("-k", "--kinds", action="store_true", dest="kinds", default=False, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1132 |
help="print the kinds of tests available") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1133 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1134 |
parser.add_option("-l", "--list", action="store_true", dest="list", default=False, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1135 |
help="print the list of known tests") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1136 |
|
5324
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
1137 |
parser.add_option("-m", "--multiple", action="store_true", dest="multiple", default=False, |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
1138 |
help="report multiple failures from test suites and test cases") |
0ba73cdd2a43
Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents:
5295
diff
changeset
|
1139 |
|
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1140 |
parser.add_option("-n", "--nowaf", action="store_true", dest="nowaf", default=False, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1141 |
help="do not run waf before starting testing") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1142 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1143 |
parser.add_option("-s", "--suite", action="store", type="string", dest="suite", default="", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1144 |
metavar="TEST-SUITE", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1145 |
help="specify a single test suite to run") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1146 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1147 |
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1148 |
help="print progress and informational messages") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1149 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1150 |
parser.add_option("-w", "--web", "--html", action="store", type="string", dest="html", default="", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1151 |
metavar="HTML-FILE", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1152 |
help="write detailed test results into HTML-FILE.html") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1153 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1154 |
parser.add_option("-t", "--text", action="store", type="string", dest="text", default="", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1155 |
metavar="TEXT-FILE", |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1156 |
help="write detailed test results into TEXT-FILE.txt") |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1157 |
|
5239
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
1158 |
parser.add_option("-x", "--xml", action="store", type="string", dest="xml", default="", |
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
1159 |
metavar="XML-FILE", |
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
1160 |
help="write detailed test results into XML-FILE.xml") |
2e753de86174
Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents:
4772
diff
changeset
|
1161 |
|
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1162 |
global options |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1163 |
options = parser.parse_args()[0] |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1164 |
signal.signal(signal.SIGINT, sigint_hook) |
5350
9f457bebbcf4
have test.py return status
Craig Dowell <craigdo@ee.washington.edu>
parents:
5324
diff
changeset
|
1165 |
|
9f457bebbcf4
have test.py return status
Craig Dowell <craigdo@ee.washington.edu>
parents:
5324
diff
changeset
|
1166 |
return run_tests() |
4772
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1167 |
|
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1168 |
if __name__ == '__main__': |
7b6ae6bf0055
add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff
changeset
|
1169 |
sys.exit(main(sys.argv)) |