test.py
author Pavel Boyko <boyko@iitp.ru>
Mon, 16 Nov 2009 14:00:51 +0300
changeset 5741 83919caa6208
parent 5483 ad83d869c08c
child 5909 766442c7f240
permissions -rwxr-xr-x
[aodv] examples/routing/aodv added to tested examples
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     1
#! /usr/bin/env python
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     2
## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     3
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     4
# Copyright (c) 2009 University of Washington
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     5
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     6
# This program is free software; you can redistribute it and/or modify
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     7
# it under the terms of the GNU General Public License version 2 as
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     8
# published by the Free Software Foundation;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     9
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    10
# This program is distributed in the hope that it will be useful,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    13
# GNU General Public License for more details.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    14
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    15
# You should have received a copy of the GNU General Public License
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    16
# along with this program; if not, write to the Free Software
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    18
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    19
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    20
import os
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    21
import sys
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
    22
import time
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    23
import optparse
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    24
import subprocess
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    25
import threading
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    26
import Queue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    27
import signal
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    28
import xml.dom.minidom
5239
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
    29
import shutil
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    30
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    31
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    32
# XXX This should really be part of a waf command to list the configuration
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    33
# items relative to optional ns-3 pieces.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    34
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    35
# A list of interesting configuration items in the waf configuration 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    36
# cache which we may be interested in when deciding on which examples
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    37
# to run and how to run them.  These are set by waf during the 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    38
# configuration phase and the corresponding assignments are usually
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    39
# found in the associated subdirectory wscript files.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    40
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    41
interesting_config_items = [
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    42
    "NS3_BUILDDIR",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    43
    "NS3_MODULE_PATH",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    44
    "ENABLE_NSC",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    45
    "ENABLE_REAL_TIME",
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
    46
    "ENABLE_EXAMPLES",
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    47
]
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    48
5295
c1bd4ffb5e47 fix test.py -v
Craig Dowell <craigdo@ee.washington.edu>
parents: 5279
diff changeset
    49
ENABLE_NSC = False
c1bd4ffb5e47 fix test.py -v
Craig Dowell <craigdo@ee.washington.edu>
parents: 5279
diff changeset
    50
ENABLE_REAL_TIME = False
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
    51
ENABLE_EXAMPLES = True
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
    52
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
    53
#
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
    54
# If the user has constrained us to run certain kinds of tests, we can tell waf
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
    55
# to only build
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    56
#
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
    57
core_kinds = ["bvt", "core", "system", "unit"]
5295
c1bd4ffb5e47 fix test.py -v
Craig Dowell <craigdo@ee.washington.edu>
parents: 5279
diff changeset
    58
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    59
#
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    60
# There are some special cases for test suites that kill valgrind.  This is
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    61
# because NSC causes illegal instruction crashes when run under valgrind.
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    62
#
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    63
core_valgrind_skip_tests = [
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    64
    "ns3-tcp-cwnd",
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    65
    "ns3-tcp-interoperability",
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    66
]
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    67
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    68
#
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    69
# A list of examples to run as smoke tests just to ensure that they remain 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    70
# buildable and runnable over time.  Also a condition under which to run
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    71
# the example (from the waf configuration), and a condition under which to
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    72
# run the example under valgrind.  This is because NSC causes illegal 
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    73
# instruction crashes when run under valgrind.
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    74
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    75
# XXX Should this not be read from a configuration file somewhere and not
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    76
# hardcoded.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    77
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    78
example_tests = [
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    79
    ("csma/csma-bridge", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    80
    ("csma/csma-bridge-one-hop", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    81
    ("csma/csma-broadcast", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    82
    ("csma/csma-multicast", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    83
    ("csma/csma-one-subnet", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    84
    ("csma/csma-packet-socket", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    85
    ("csma/csma-ping", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    86
    ("csma/csma-raw-ip-socket", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    87
    ("csma/csma-star", "True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
    88
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    89
    ("emulation/emu-ping", "False", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    90
    ("emulation/emu-udp-echo", "False", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
    91
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    92
    ("error-model/simple-error-model", "True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
    93
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    94
    ("ipv6/icmpv6-redirect", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    95
    ("ipv6/ping6", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    96
    ("ipv6/radvd", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    97
    ("ipv6/radvd-two-prefix", "True", "True"),    
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
    98
    ("ipv6/test-ipv6", "True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
    99
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   100
    ("mesh/mesh", "True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   101
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   102
    ("naming/object-names", "True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   103
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   104
    ("realtime/realtime-udp-echo", "ENABLE_REAL_TIME == True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   105
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   106
    ("routing/dynamic-global-routing", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   107
    ("routing/global-injection-slash32", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   108
    ("routing/global-routing-slash32", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   109
    ("routing/mixed-global-routing", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   110
    ("routing/nix-simple", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   111
    ("routing/nms-p2p-nix", "False", "True"), # Takes too long to run
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   112
    ("routing/simple-alternate-routing", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   113
    ("routing/simple-global-routing", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   114
    ("routing/simple-point-to-point-olsr", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   115
    ("routing/simple-routing-ping6", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   116
    ("routing/static-routing-slash32", "True", "True"),
5741
83919caa6208 [aodv] examples/routing/aodv added to tested examples
Pavel Boyko <boyko@iitp.ru>
parents: 5483
diff changeset
   117
    ("routing/aodv", "True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   118
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   119
    ("stats/wifi-example-sim", "True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   120
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   121
    ("tap/tap-wifi-dumbbell", "False", "True"), # Requires manual configuration
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   122
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   123
    ("tcp/star", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   124
    ("tcp/tcp-large-transfer", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   125
    ("tcp/tcp-nsc-lfn", "ENABLE_NSC == True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   126
    ("tcp/tcp-nsc-zoo", "ENABLE_NSC == True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   127
    ("tcp/tcp-star-server", "True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   128
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   129
    ("tunneling/virtual-net-device", "True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   130
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   131
    ("tutorial/first", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   132
    ("tutorial/hello-simulator", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   133
    ("tutorial/second", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   134
    ("tutorial/third", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   135
    ("tutorial/fourth", "True", "True"),
5483
ad83d869c08c Make tests pass on MinGW, add fifth tutorial example
Craig Dowell <craigdo@ee.washington.edu>
parents: 5481
diff changeset
   136
    ("tutorial/fifth", "True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   137
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   138
    ("udp/udp-echo", "True", "True"),
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   139
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   140
    ("wireless/mixed-wireless", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   141
    ("wireless/multirate", "False", "True"), # Takes too long to run
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   142
    ("wireless/simple-wifi-frame-aggregation", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   143
    ("wireless/wifi-adhoc", "False", "True"), # Takes too long to run
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   144
    ("wireless/wifi-ap --verbose=0", "True", "True"), # Don't let it spew to stdout
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   145
    ("wireless/wifi-clear-channel-cmu", "False", "True"), # Requires specific hardware
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   146
    ("wireless/wifi-simple-adhoc", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   147
    ("wireless/wifi-simple-adhoc-grid", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   148
    ("wireless/wifi-simple-infra", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   149
    ("wireless/wifi-simple-interference", "True", "True"),
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   150
    ("wireless/wifi-wired-bridging", "True", "True"),
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   151
]
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   152
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   153
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   154
# The test suites are going to want to output status.  They are running
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   155
# concurrently.  This means that unless we are careful, the output of
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   156
# the test suites will be interleaved.  Rather than introducing a lock
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   157
# file that could unintentionally start serializing execution, we ask
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   158
# the tests to write their output to a temporary directory and then 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   159
# put together the final output file when we "join" the test tasks back
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   160
# to the main thread.  In addition to this issue, the example programs
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   161
# often write lots and lots of trace files which we will just ignore.
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   162
# We put all of them into the temp directory as well, so they can be
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   163
# easily deleted.
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   164
#
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   165
TMP_OUTPUT_DIR = "testpy-output"
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   166
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   167
def get_node_text(node):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   168
    for child in node.childNodes:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   169
        if child.nodeType == child.TEXT_NODE:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   170
            return child.nodeValue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   171
    return "None"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   172
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   173
#
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   174
# A simple example of writing a text file with a test result summary.  It is 
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   175
# expected that this output will be fine for developers looking for problems.
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   176
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   177
def translate_to_text(results_file, text_file):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   178
    f = open(text_file, 'w')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   179
    dom = xml.dom.minidom.parse(results_file)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   180
    for suite in dom.getElementsByTagName("TestSuite"):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   181
        result = get_node_text(suite.getElementsByTagName("SuiteResult")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   182
        name = get_node_text(suite.getElementsByTagName("SuiteName")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   183
        time = get_node_text(suite.getElementsByTagName("SuiteTime")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   184
        output = "%s: Test Suite \"%s\" (%s)\n" % (result, name, time)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   185
        f.write(output)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   186
        if result != "CRASH":
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   187
            for case in suite.getElementsByTagName("TestCase"):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   188
                result = get_node_text(case.getElementsByTagName("CaseResult")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   189
                name = get_node_text(case.getElementsByTagName("CaseName")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   190
                time = get_node_text(case.getElementsByTagName("CaseTime")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   191
                output =   "  %s: Test Case \"%s\" (%s)\n" % (result, name, time)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   192
                f.write(output)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   193
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   194
                if result == "FAIL":
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   195
                    for details in case.getElementsByTagName("FailureDetails"):
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   196
                        f.write("    Details:\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   197
                        f.write("      Message:   %s\n" % get_node_text(details.getElementsByTagName("Message")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   198
                        f.write("      Condition: %s\n" % get_node_text(details.getElementsByTagName("Condition")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   199
                        f.write("      Actual:    %s\n" % get_node_text(details.getElementsByTagName("Actual")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   200
                        f.write("      Limit:     %s\n" % get_node_text(details.getElementsByTagName("Limit")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   201
                        f.write("      File:      %s\n" % get_node_text(details.getElementsByTagName("File")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   202
                        f.write("      Line:      %s\n" % get_node_text(details.getElementsByTagName("Line")[0]))
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   203
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   204
    for example in dom.getElementsByTagName("Example"):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   205
        result = get_node_text(example.getElementsByTagName("Result")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   206
        name = get_node_text(example.getElementsByTagName("Name")[0])
5460
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   207
        time = get_node_text(example.getElementsByTagName("ElapsedTime")[0])
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   208
        output = "%s: Example \"%s\" (%s)\n" % (result, name, time)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   209
        f.write(output)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   210
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   211
    f.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   212
    
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   213
#
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   214
# A simple example of writing an HTML file with a test result summary.  It is 
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   215
# expected that this will eventually be made prettier as time progresses and
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   216
# we have time to tweak it.  This may end up being moved to a separate module
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   217
# since it will probably grow over time.
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   218
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   219
def translate_to_html(results_file, html_file):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   220
    f = open(html_file, 'w')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   221
    f.write("<html>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   222
    f.write("<body>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   223
    f.write("<center><h1>ns-3 Test Results</h1></center>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   224
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   225
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   226
    # Read and parse the whole results file.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   227
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   228
    dom = xml.dom.minidom.parse(results_file)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   229
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   230
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   231
    # Iterate through the test suites
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   232
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   233
    f.write("<h2>Test Suites</h2>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   234
    for suite in dom.getElementsByTagName("TestSuite"):
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   235
     
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   236
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   237
        # For each test suite, get its name, result and execution time info
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   238
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   239
        name = get_node_text(suite.getElementsByTagName("SuiteName")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   240
        result = get_node_text(suite.getElementsByTagName("SuiteResult")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   241
        time = get_node_text(suite.getElementsByTagName("SuiteTime")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   242
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   243
        # 
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   244
        # Print a level three header with the result, name and time.  If the 
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   245
        # test suite passed, the header is printed in green. If the suite was
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   246
        # skipped, print it in orange, otherwise assume something bad happened
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   247
        # and print in red.
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   248
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   249
        if result == "PASS":
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   250
            f.write("<h3 style=\"color:green\">%s: %s (%s)</h3>\n" % (result, name, time))
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   251
        elif result == "SKIP":
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   252
            f.write("<h3 style=\"color:#ff6600\">%s: %s (%s)</h3>\n" % (result, name, time))
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   253
        else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   254
            f.write("<h3 style=\"color:red\">%s: %s (%s)</h3>\n" % (result, name, time))
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   255
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   256
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   257
        # The test case information goes in a table.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   258
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   259
        f.write("<table border=\"1\">\n")
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   260
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   261
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   262
        # The first column of the table has the heading Result
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   263
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   264
        f.write("<th> Result </th>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   265
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   266
        #
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   267
        # If the suite crashed or is skipped, there is no further information, so just
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   268
        # delare a new table row with the result (CRASH or SKIP) in it.  Looks like:
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   269
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   270
        #   +--------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   271
        #   | Result |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   272
        #   +--------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   273
        #   | CRASH  |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   274
        #   +--------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   275
        #
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   276
        # Then go on to the next test suite.  Valgrind and skipped errors look the same.
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   277
        #
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   278
        if result in ["CRASH", "SKIP", "VALGR"]:
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   279
            f.write("<tr>\n")
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   280
            if result == "SKIP":
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   281
                f.write("<td style=\"color:#ff6600\">%s</td>\n" % result)
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   282
            else:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   283
                f.write("<td style=\"color:red\">%s</td>\n" % result)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   284
            f.write("</tr>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   285
            f.write("</table>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   286
            continue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   287
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   288
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   289
        # If the suite didn't crash, we expect more information, so fill out
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   290
        # the table heading row.  Like,
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   291
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   292
        #   +--------+----------------+------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   293
        #   | Result | Test Case Name | Time |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   294
        #   +--------+----------------+------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   295
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   296
        f.write("<th>Test Case Name</th>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   297
        f.write("<th> Time </th>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   298
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   299
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   300
        # If the test case failed, we need to print out some failure details
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   301
        # so extend the heading row again.  Like,
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   302
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   303
        #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   304
        #   | Result | Test Case Name | Time | Failure Details |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   305
        #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   306
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   307
        if result == "FAIL":
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   308
            f.write("<th>Failure Details</th>\n")
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   309
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   310
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   311
        # Now iterate through all of the test cases.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   312
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   313
        for case in suite.getElementsByTagName("TestCase"):
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   314
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   315
            #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   316
            # Get the name, result and timing information from xml to use in
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   317
            # printing table below.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   318
            #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   319
            name = get_node_text(case.getElementsByTagName("CaseName")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   320
            result = get_node_text(case.getElementsByTagName("CaseResult")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   321
            time = get_node_text(case.getElementsByTagName("CaseTime")[0])
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   322
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   323
            #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   324
            # If the test case failed, we iterate through possibly multiple
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   325
            # failure details
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   326
            #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   327
            if result == "FAIL":
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   328
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   329
                # There can be multiple failures for each test case.  The first
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   330
                # row always gets the result, name and timing information along
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   331
                # with the failure details.  Remaining failures don't duplicate
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   332
                # this information but just get blanks for readability.  Like,
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   333
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   334
                #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   335
                #   | Result | Test Case Name | Time | Failure Details |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   336
                #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   337
                #   |  FAIL  | The name       | time | It's busted     |   
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   338
                #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   339
                #   |        |                |      | Really broken   |   
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   340
                #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   341
                #   |        |                |      | Busted bad      |   
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   342
                #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   343
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   344
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   345
                first_row = True
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   346
                for details in case.getElementsByTagName("FailureDetails"):
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   347
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   348
                    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   349
                    # Start a new row in the table for each possible Failure Detail
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   350
                    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   351
                    f.write("<tr>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   352
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   353
                    if first_row:
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   354
                        first_row = False
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   355
                        f.write("<td style=\"color:red\">%s</td>\n" % result)
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   356
                        f.write("<td>%s</td>\n" % name)
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   357
                        f.write("<td>%s</td>\n" % time)
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   358
                    else:
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   359
                        f.write("<td></td>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   360
                        f.write("<td></td>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   361
                        f.write("<td></td>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   362
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   363
                    f.write("<td>")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   364
                    f.write("<b>Message: </b>%s, " % get_node_text(details.getElementsByTagName("Message")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   365
                    f.write("<b>Condition: </b>%s, " % get_node_text(details.getElementsByTagName("Condition")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   366
                    f.write("<b>Actual: </b>%s, " % get_node_text(details.getElementsByTagName("Actual")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   367
                    f.write("<b>Limit: </b>%s, " % get_node_text(details.getElementsByTagName("Limit")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   368
                    f.write("<b>File: </b>%s, " % get_node_text(details.getElementsByTagName("File")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   369
                    f.write("<b>Line: </b>%s" % get_node_text(details.getElementsByTagName("Line")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   370
                    f.write("</td>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   371
                    
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   372
                    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   373
                    # End the table row
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   374
                    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   375
                    f.write("</td>\n")
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   376
            else:
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   377
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   378
                # If this particular test case passed, then we just print the PASS
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   379
                # result in green, followed by the test case name and its execution
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   380
                # time information.  These go off in <td> ... </td> table data.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   381
                # The details table entry is left blank.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   382
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   383
                #   +--------+----------------+------+---------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   384
                #   | Result | Test Case Name | Time | Details |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   385
                #   +--------+----------------+------+---------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   386
                #   |  PASS  | The name       | time |         |   
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   387
                #   +--------+----------------+------+---------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   388
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   389
                f.write("<tr>\n")
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   390
                f.write("<td style=\"color:green\">%s</td>\n" % result)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   391
                f.write("<td>%s</td>\n" % name)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   392
                f.write("<td>%s</td>\n" % time)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   393
                f.write("<td></td>\n")
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   394
                f.write("</tr>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   395
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   396
        # All of the rows are written, so we need to end the table.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   397
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   398
        f.write("</table>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   399
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   400
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   401
    # That's it for all of the test suites.  Now we have to do something about 
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   402
    # our examples.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   403
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   404
    f.write("<h2>Examples</h2>\n")
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   405
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   406
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   407
    # Example status is rendered in a table just like the suites.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   408
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   409
    f.write("<table border=\"1\">\n")
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   410
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   411
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   412
    # The table headings look like,
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   413
    #
5460
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   414
    #   +--------+--------------+--------------+
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   415
    #   | Result | Example Name | Elapsed Time |
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   416
    #   +--------+--------------+--------------+
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   417
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   418
    f.write("<th> Result </th>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   419
    f.write("<th>Example Name</th>\n")
5460
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   420
    f.write("<th>Elapsed Time</th>\n")
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   421
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   422
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   423
    # Now iterate through all of the examples
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   424
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   425
    for example in dom.getElementsByTagName("Example"):
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   426
        
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   427
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   428
        # Start a new row for each example
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   429
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   430
        f.write("<tr>\n")
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   431
        
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   432
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   433
        # Get the result and name of the example in question
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   434
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   435
        result = get_node_text(example.getElementsByTagName("Result")[0])
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   436
        name =   get_node_text(example.getElementsByTagName("Name")[0])
5460
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   437
        time =   get_node_text(example.getElementsByTagName("ElapsedTime")[0])
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   438
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   439
        #
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
   440
        # If the example either failed or crashed, print its result status
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   441
        # in red; otherwise green.  This goes in a <td> ... </td> table data
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   442
        #
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
   443
        if result == "PASS":
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
   444
            f.write("<td style=\"color:green\">%s</td>\n" % result)
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   445
        elif result == "SKIP":
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   446
            f.write("<td style=\"color:#ff6600\">%s</fd>\n" % result)
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
   447
        else:
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   448
            f.write("<td style=\"color:red\">%s</td>\n" % result)
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   449
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   450
        #
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   451
        # Write the example name as a new tag data.
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   452
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   453
        f.write("<td>%s</td>\n" % name)
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   454
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   455
        #
5460
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   456
        # Write the elapsed time as a new tag data.
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   457
        #
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   458
        f.write("<td>%s</td>\n" % time)
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   459
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   460
        #
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   461
        # That's it for the current example, so terminate the row.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   462
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   463
        f.write("</tr>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   464
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   465
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   466
    # That's it for the table of examples, so terminate the table.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   467
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   468
    f.write("</table>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   469
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   470
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   471
    # And that's it for the report, so finish up.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   472
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   473
    f.write("</body>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   474
    f.write("</html>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   475
    f.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   476
    
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   477
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   478
# Python Control-C handling is broken in the presence of multiple threads.  
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   479
# Signals get delivered to the runnable/running thread by default and if 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   480
# it is blocked, the signal is simply ignored.  So we hook sigint and set 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   481
# a global variable telling the system to shut down gracefully.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   482
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   483
thread_exit = False
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   484
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   485
def sigint_hook(signal, frame):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   486
    global thread_exit
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   487
    thread_exit = True
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   488
    return 0
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   489
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   490
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   491
# Waf can be configured to compile in debug or optimized modes.  In each
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   492
# case, the resulting built goes into a different directory.  If we want
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   493
# test tests to run from the correct code-base, we have to figure out which
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   494
# mode waf is running in.  This is called its active variant.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   495
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   496
# XXX This function pokes around in the waf internal state file.  To be a
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   497
# little less hacky, we should add a commmand to waf to return this info
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   498
# and use that result.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   499
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   500
def read_waf_active_variant():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   501
    for line in open("build/c4che/default.cache.py").readlines():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   502
        if line.startswith("NS3_ACTIVE_VARIANT"):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   503
            exec(line, globals())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   504
            break
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   505
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   506
    if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   507
        print "NS3_ACTIVE_VARIANT == %s" % NS3_ACTIVE_VARIANT
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   508
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   509
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   510
# In general, the build process itself naturally takes care of figuring out
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   511
# which tests are built into the test runner.  For example, if waf configure
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   512
# determines that ENABLE_EMU is false due to some missing dependency,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   513
# the tests for the emu net device simply will not be built and will 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   514
# therefore not be included in the built test runner.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   515
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   516
# Examples, however, are a different story.  In that case, we are just given
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   517
# a list of examples that could be run.  Instead of just failing, for example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   518
# nsc-tcp-zoo if NSC is not present, we look into the waf saved configuration
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   519
# for relevant configuration items.  
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   520
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   521
# XXX This function pokes around in the waf internal state file.  To be a
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   522
# little less hacky, we should add a commmand to waf to return this info
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   523
# and use that result.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   524
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   525
def read_waf_config():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   526
    for line in open("build/c4che/%s.cache.py" % NS3_ACTIVE_VARIANT).readlines():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   527
        for item in interesting_config_items:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   528
            if line.startswith(item):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   529
                exec(line, globals())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   530
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   531
    if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   532
        for item in interesting_config_items:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   533
            print "%s ==" % item, eval(item)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   534
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   535
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   536
# It seems pointless to fork a process to run waf to fork a process to run
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   537
# the test runner, so we just run the test runner directly.  The main thing 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   538
# that waf would do for us would be to sort out the shared library path but
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   539
# we can deal with that easily and do here.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   540
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   541
# There can be many different ns-3 repositories on a system, and each has 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   542
# its own shared libraries, so ns-3 doesn't hardcode a shared library search
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   543
# path -- it is cooked up dynamically, so we do that too.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   544
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   545
def make_library_path():
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   546
    have_DYLD_LIBRARY_PATH = False
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   547
    have_LD_LIBRARY_PATH = False
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   548
    have_PATH = False
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   549
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   550
    keys = os.environ.keys()
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   551
    for key in keys:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   552
        if key == "DYLD_LIBRARY_PATH":
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   553
            have_DYLD_LIBRARY_PATH = True
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   554
        if key == "LD_LIBRARY_PATH":
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   555
            have_LD_LIBRARY_PATH = True
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   556
        if key == "PATH":
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   557
            have_PATH = True
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   558
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   559
    if sys.platform == "darwin":
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   560
        if not have_DYLD_LIBRARY_PATH:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   561
            os.environ["DYLD_LIBRARY_PATH"] = ""
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   562
        for path in NS3_MODULE_PATH:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   563
            os.environ["DYLD_LIBRARY_PATH"] += ":" + path
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   564
        if options.verbose:
5466
c206a8ce5644 Fix testing under Mac OS X
fmoatamr
parents: 5461
diff changeset
   565
            print "os.environ[\"DYLD_LIBRARY_PATH\"] == %s" % os.environ["DYLD_LIBRARY_PATH"]
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   566
    elif sys.platform == "win32":
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   567
        if not have_PATH:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   568
            os.environ["PATH"] = ""
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   569
        for path in NS3_MODULE_PATH:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   570
            os.environ["PATH"] += ';' + path
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   571
        if options.verbose:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   572
            print "os.environ[\"PATH\"] == %s" % os.environ["PATH"]
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   573
    elif sys.platform == "cygwin":
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   574
        if not have_PATH:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   575
            os.environ["PATH"] = ""
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   576
        for path in NS3_MODULE_PATH:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   577
            os.environ["PATH"] += ":" + path
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   578
        if options.verbose:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   579
            print "os.environ[\"PATH\"] == %s" % os.environ["PATH"]
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   580
    else:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   581
        if not have_LD_LIBRARY_PATH:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   582
            os.environ["LD_LIBRARY_PATH"] = ""
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   583
        for path in NS3_MODULE_PATH:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   584
            os.environ["LD_LIBRARY_PATH"] += ":" + path
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   585
        if options.verbose:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   586
            print "os.environ[\"LD_LIBRARY_PATH\"] == %s" % os.environ["LD_LIBRARY_PATH"]
5275
d4008f2981ba Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents: 5274
diff changeset
   587
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
   588
def run_job_synchronously(shell_command, directory, valgrind):
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   589
    path_cmd = os.path.join (NS3_BUILDDIR, NS3_ACTIVE_VARIANT, shell_command)
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
   590
    if valgrind:
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   591
        cmd = "valgrind --leak-check=full --error-exitcode=2 %s" % path_cmd
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
   592
    else:
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   593
        cmd = path_cmd
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
   594
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   595
    if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   596
        print "Synchronously execute %s" % cmd
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
   597
5460
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   598
    start_time = time.time()
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   599
    proc = subprocess.Popen(cmd, shell = True, cwd = directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
5351
6abced63cd7f keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5350
diff changeset
   600
    stdout_results, stderr_results = proc.communicate()
5460
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
   601
    elapsed_time = time.time() - start_time
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   602
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   603
    if options.verbose:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   604
        print "Return code = ", proc.returncode
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   605
        print "stderr = ", stderr_results
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   606
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   607
    return (proc.returncode, stdout_results, stderr_results, elapsed_time)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   608
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   609
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   610
# This class defines a unit of testing work.  It will typically refer to
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   611
# a test suite to run using the test-runner, or an example to run directly.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   612
#
5415
5c2f96b507a9 Stray () in test.py (bug 718)
Craig Dowell <craigdo@ee.washington.edu>
parents: 5414
diff changeset
   613
class Job:
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   614
    def __init__(self):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   615
        self.is_break = False
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   616
        self.is_skip = False
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   617
        self.is_example = False
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   618
        self.shell_command = ""
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   619
        self.display_name = ""
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   620
        self.basedir = ""
5481
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
   621
        self.tempdir = ""
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   622
        self.cwd = ""
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   623
        self.tmp_file_name = ""
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   624
        self.returncode = False
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   625
        self.elapsed_time = 0
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   626
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   627
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   628
    # A job is either a standard job or a special job indicating that a worker
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   629
    # thread should exist.  This special job is indicated by setting is_break 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   630
    # to true.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   631
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   632
    def set_is_break(self, is_break):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   633
        self.is_break = is_break
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   634
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   635
    #
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   636
    # If a job is to be skipped, we actually run it through the worker threads
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   637
    # to keep the PASS, FAIL, CRASH and SKIP processing all in one place.
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   638
    #
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   639
    def set_is_skip(self, is_skip):
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   640
        self.is_skip = is_skip
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   641
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   642
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   643
    # Examples are treated differently than standard test suites.  This is
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   644
    # mostly because they are completely unaware that they are being run as 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   645
    # tests.  So we have to do some special case processing to make them look
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   646
    # like tests.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   647
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   648
    def set_is_example(self, is_example):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   649
        self.is_example = is_example
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   650
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   651
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   652
    # This is the shell command that will be executed in the job.  For example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   653
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   654
    #  "utils/test-runner --suite=some-test-suite"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   655
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   656
    def set_shell_command(self, shell_command):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   657
        self.shell_command = shell_command
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   658
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   659
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   660
    # This is the dispaly name of the job, typically the test suite or example 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   661
    # name.  For example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   662
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   663
    #  "some-test-suite" or "udp-echo"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   664
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   665
    def set_display_name(self, display_name):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   666
        self.display_name = display_name
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   667
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   668
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   669
    # This is the base directory of the repository out of which the tests are
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   670
    # being run.  It will be used deep down in the testing framework to determine
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   671
    # where the source directory of the test was, and therefore where to find 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   672
    # provided test vectors.  For example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   673
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   674
    #  "/home/user/repos/ns-3-dev"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   675
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   676
    def set_basedir(self, basedir):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   677
        self.basedir = basedir
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   678
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   679
    #
5481
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
   680
    # This is the directory to which a running test suite should write any 
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
   681
    # temporary files.
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
   682
    #
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
   683
    def set_tempdir(self, tempdir):
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
   684
        self.tempdir = tempdir
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
   685
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
   686
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   687
    # This is the current working directory that will be given to an executing
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   688
    # test as it is being run.  It will be used for examples to tell them where
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   689
    # to write all of the pcap files that we will be carefully ignoring.  For
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   690
    # example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   691
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   692
    #  "/tmp/unchecked-traces"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   693
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   694
    def set_cwd(self, cwd):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   695
        self.cwd = cwd
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   696
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   697
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   698
    # This is the temporary results file name that will be given to an executing 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   699
    # test as it is being run.  We will be running all of our tests in parallel
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   700
    # so there must be multiple temporary output files.  These will be collected
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   701
    # into a single XML file at the end and then be deleted.  
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   702
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   703
    def set_tmp_file_name(self, tmp_file_name):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   704
        self.tmp_file_name = tmp_file_name
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   705
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   706
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   707
    # The return code received when the job process is executed.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   708
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   709
    def set_returncode(self, returncode):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   710
        self.returncode = returncode
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   711
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   712
    #
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   713
    # The elapsed real time for the job execution.
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   714
    #
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   715
    def set_elapsed_time(self, elapsed_time):
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   716
        self.elapsed_time = elapsed_time
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   717
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   718
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   719
# The worker thread class that handles the actual running of a given test.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   720
# Once spawned, it receives requests for work through its input_queue and
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   721
# ships the results back through the output_queue.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   722
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   723
class worker_thread(threading.Thread):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   724
    def __init__(self, input_queue, output_queue):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   725
        threading.Thread.__init__(self)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   726
        self.input_queue = input_queue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   727
        self.output_queue = output_queue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   728
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   729
    def run(self):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   730
        while True:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   731
            job = self.input_queue.get()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   732
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   733
            # Worker threads continue running until explicitly told to stop with
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   734
            # a special job.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   735
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   736
            if job.is_break:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   737
                return
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   738
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   739
            # If the global interrupt handler sets the thread_exit variable,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   740
            # we stop doing real work and just report back a "break" in the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   741
            # normal command processing has happened.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   742
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   743
            if thread_exit == True:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   744
                job.set_is_break(True)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   745
                self.output_queue.put(job)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   746
                continue
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   747
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   748
            #
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   749
            # If we are actually supposed to skip this job, do so.  Note that
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   750
            # if is_skip is true, returncode is undefined.
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   751
            #
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   752
            if job.is_skip:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   753
                if options.verbose:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   754
                    print "Skip %s" % job.shell_command
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   755
                self.output_queue.put(job)
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   756
                return
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   757
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   758
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   759
            # Otherwise go about the business of running tests as normal.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   760
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   761
            else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   762
                if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   763
                    print "Launch %s" % job.shell_command
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   764
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   765
                if job.is_example:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   766
                    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   767
                    # If we have an example, the shell command is all we need to
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   768
                    # know.  It will be something like "examples/udp-echo"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   769
                    #
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   770
                    (job.returncode, standard_out, standard_err, et) = run_job_synchronously(job.shell_command, 
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   771
                        job.cwd, options.valgrind)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   772
                else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   773
                    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   774
                    # If we're a test suite, we need to provide a little more info
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   775
                    # to the test runner, specifically the base directory and temp
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   776
                    # file name
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   777
                    #
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   778
                    (job.returncode, standard_out, standard_err, et) = run_job_synchronously(job.shell_command + 
5481
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
   779
                        " --basedir=%s --tempdir=%s --out=%s" % (job.basedir, job.tempdir, job.tmp_file_name), 
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
   780
                        job.cwd, options.valgrind)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   781
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   782
                job.set_elapsed_time(et)
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   783
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   784
                if options.verbose:
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
   785
                    print "returncode = %d" % job.returncode
5351
6abced63cd7f keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5350
diff changeset
   786
                    print "---------- beign standard out ----------"
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   787
                    print standard_out
5351
6abced63cd7f keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5350
diff changeset
   788
                    print "---------- begin standard err ----------"
6abced63cd7f keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5350
diff changeset
   789
                    print standard_err
6abced63cd7f keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5350
diff changeset
   790
                    print "---------- end standard err ----------"
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   791
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   792
                self.output_queue.put(job)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   793
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   794
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   795
# This is the main function that does the work of interacting with the test-runner
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   796
# itself.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   797
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   798
def run_tests():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   799
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   800
    # Run waf to make sure that everything is built, configured and ready to go
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   801
    # unless we are explicitly told not to.  We want to be careful about causing
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   802
    # our users pain while waiting for extraneous stuff to compile and link, so
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   803
    # we allow users that know what they''re doing to not invoke waf at all.
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   804
    #
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   805
    if not options.nowaf:
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   806
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   807
        #
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   808
        # If the user is running the "kinds" or "list" options, there is an 
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   809
        # implied dependency on the test-runner since we call that program
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   810
        # if those options are selected.  We will exit after processing those
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   811
        # options, so if we see them, we can safely only build the test-runner.
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   812
        #
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   813
        # If the user has constrained us to running only a particular type of
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   814
        # file, we can only ask waf to build what we know will be necessary.
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   815
        # For example, if the user only wants to run BVT tests, we only have
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   816
        # to build the test-runner and can ignore all of the examples.
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   817
        #
5470
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   818
        # If the user only wants to run a single example, then we can just build
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   819
        # that example.
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   820
        #
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   821
        # If there is no constraint, then we have to build everything since the
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   822
        # user wants to run everything.
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   823
        #
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   824
        if options.kinds or options.list or (len(options.constrain) and options.constrain in core_kinds):
5470
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   825
            if sys.platform == "win32":
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   826
                waf_cmd = "waf --target=test-runner"
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   827
            else:
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   828
                waf_cmd = "./waf --target=test-runner"
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   829
        elif len(options.example):
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   830
            if sys.platform == "win32":
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   831
                waf_cmd = "waf --target=%s" % os.path.basename(options.example)
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   832
            else:
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   833
                waf_cmd = "./waf --target=%s" % os.path.basename(options.example)
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   834
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   835
        else:
5470
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   836
            if sys.platform == "win32":
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   837
                waf_cmd = "waf"
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   838
            else:
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   839
                waf_cmd = "./waf"
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   840
5470
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   841
        if options.verbose:
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   842
            print "Building: %s" % waf_cmd
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   843
d3aabb63dd12 remind test.py how to build on unix and limit build scope when posible
Craig Dowell <craigdo@ee.washington.edu>
parents: 5466
diff changeset
   844
        proc = subprocess.Popen(waf_cmd, shell = True)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   845
        proc.communicate()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   846
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   847
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   848
    # Pull some interesting configuration information out of waf, primarily
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   849
    # so we can know where executables can be found, but also to tell us what
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   850
    # pieces of the system have been built.  This will tell us what examples 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   851
    # are runnable.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   852
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   853
    read_waf_active_variant()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   854
    read_waf_config()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   855
    make_library_path()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   856
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   857
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   858
    # There are a couple of options that imply we can to exit before starting
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   859
    # up a bunch of threads and running tests.  Let's detect these cases and 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   860
    # handle them without doing all of the hard work.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   861
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   862
    if options.kinds:
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   863
        path_cmd = os.path.join("utils", "test-runner --kinds")
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   864
        (rc, standard_out, standard_err, et) = run_job_synchronously(path_cmd, os.getcwd(), False)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   865
        print standard_out
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   866
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   867
    if options.list:
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   868
        path_cmd = os.path.join("utils", "test-runner --list")
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   869
        (rc, standard_out, standard_err, et) = run_job_synchronously(path_cmd, os.getcwd(), False)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   870
        print standard_out
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   871
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   872
    if options.kinds or options.list:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   873
        return
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   874
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   875
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   876
    # We communicate results in two ways.  First, a simple message relating 
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   877
    # PASS, FAIL, CRASH or SKIP is always written to the standard output.  It 
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   878
    # is expected that this will be one of the main use cases.  A developer can
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   879
    # just run test.py with no options and see that all of the tests still 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   880
    # pass.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   881
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   882
    # The second main use case is when detailed status is requested (with the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   883
    # --text or --html options).  Typicall this will be text if a developer
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   884
    # finds a problem, or HTML for nightly builds.  In these cases, an
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   885
    # XML file is written containing the status messages from the test suites.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   886
    # This file is then read and translated into text or HTML.  It is expected
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   887
    # that nobody will really be interested in the XML, so we write it somewhere
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   888
    # with a unique name (time) to avoid collisions.  In case an error happens, we
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   889
    # provide a runtime option to retain the temporary files.
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   890
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   891
    # When we run examples as smoke tests, they are going to want to create
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   892
    # lots and lots of trace files.  We aren't really interested in the contents
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   893
    # of the trace files, so we also just stash them off in the temporary dir.
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   894
    # The retain option also causes these unchecked trace files to be kept.
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   895
    #
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   896
    date_and_time = time.strftime("%Y-%m-%d-%H-%M-%S-CUT", time.gmtime())
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   897
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   898
    if not os.path.exists(TMP_OUTPUT_DIR):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   899
        os.makedirs(TMP_OUTPUT_DIR)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   900
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   901
    testpy_output_dir = os.path.join(TMP_OUTPUT_DIR, date_and_time);
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   902
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   903
    if not os.path.exists(testpy_output_dir):
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   904
        os.makedirs(testpy_output_dir)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   905
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   906
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   907
    # Create the main output file and start filling it with XML.  We need to 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   908
    # do this since the tests will just append individual results to this file.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   909
    #
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
   910
    xml_results_file = os.path.join(testpy_output_dir, "results.xml")
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   911
    f = open(xml_results_file, 'w')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   912
    f.write('<?xml version="1.0"?>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   913
    f.write('<TestResults>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   914
    f.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   915
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   916
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   917
    # We need to figure out what test suites to execute.  We are either given one 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   918
    # suite or example explicitly via the --suite or --example option, or we
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   919
    # need to call into the test runner and ask it to list all of the available
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   920
    # test suites.  Further, we need to provide the constraint information if it
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   921
    # has been given to us.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   922
    # 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   923
    # This translates into allowing the following options with respect to the 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   924
    # suites
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   925
    #
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   926
    #  ./test,py:                                           run all of the suites and examples
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   927
    #  ./test.py --constrain=core:                          run all of the suites of all kinds
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   928
    #  ./test.py --constrain=unit:                          run all unit suites
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
   929
    #  ./test,py --suite=some-test-suite:                   run a single suite
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   930
    #  ./test,py --example=udp-echo:                        run no test suites
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   931
    #  ./test,py --suite=some-suite --example=some-example: run the single suite
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   932
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   933
    # We can also use the --constrain option to provide an ordering of test 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   934
    # execution quite easily.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   935
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   936
    if len(options.suite):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   937
        suites = options.suite + "\n"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   938
    elif len(options.example) == 0:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   939
        if len(options.constrain):
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   940
            path_cmd = os.path.join("utils", "test-runner --list --constrain=%s" % options.constrain)
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   941
            (rc, suites, standard_err, et) = run_job_synchronously(path_cmd, os.getcwd(), False)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   942
        else:
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   943
            path_cmd = os.path.join("utils", "test-runner --list")
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   944
            (rc, suites, standard_err, et) = run_job_synchronously(path_cmd, os.getcwd(), False)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   945
    else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   946
        suites = ""
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   947
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   948
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   949
    # suite_list will either a single test suite name that the user has 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   950
    # indicated she wants to run or a list of test suites provided by
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   951
    # the test-runner possibly according to user provided constraints.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   952
    # We go through the trouble of setting up the parallel execution 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   953
    # even in the case of a single suite to avoid having two process the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   954
    # results in two different places.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   955
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   956
    suite_list = suites.split('\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   957
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   958
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   959
    # We now have a possibly large number of test suites to run, so we want to
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   960
    # run them in parallel.  We're going to spin up a number of worker threads
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   961
    # that will run our test jobs for us.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   962
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   963
    input_queue = Queue.Queue(0)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   964
    output_queue = Queue.Queue(0)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   965
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   966
    jobs = 0
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   967
    threads=[]
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   968
5273
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   969
    #
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   970
    # In Python 2.6 you can just use multiprocessing module, but we don't want
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   971
    # to introduce that dependency yet; so we jump through a few hoops.
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   972
    #
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   973
    processors = 1
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   974
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   975
    if sys.platform != "win32":
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   976
        if 'SC_NPROCESSORS_ONLN'in os.sysconf_names:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   977
            processors = os.sysconf('SC_NPROCESSORS_ONLN')
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   978
        else:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   979
            proc = subprocess.Popen("sysctl -n hw.ncpu", shell = True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   980
            stdout_results, stderr_results = proc.communicate()
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   981
            if len(stderr_results) == 0:
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
   982
                processors = int(stdout_results)
5273
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   983
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   984
    #
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   985
    # Now, spin up one thread per processor which will eventually mean one test
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   986
    # per processor running concurrently.
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   987
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   988
    for i in range(processors):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   989
        thread = worker_thread(input_queue, output_queue)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   990
        threads.append(thread)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   991
        thread.start()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   992
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   993
    #
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   994
    # Keep track of some summary statistics
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   995
    #
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   996
    total_tests = 0
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   997
    skipped_tests = 0
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   998
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
   999
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1000
    # We now have worker threads spun up, and a list of work to do.  So, run 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1001
    # through the list of test suites and dispatch a job to run each one.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1002
    # 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1003
    # Dispatching will run with unlimited speed and the worker threads will 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1004
    # execute as fast as possible from the queue.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1005
    #
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1006
    # Note that we actually dispatch tests to be skipped, so all of the 
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1007
    # PASS, FAIL, CRASH and SKIP processing is done in the same place.
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1008
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1009
    for test in suite_list:
5461
ac47584eca3e Make test.py work on MinGW
Craig Dowell <craigdo@ee.washington.edu>
parents: 5460
diff changeset
  1010
        test = test.strip()
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1011
        if len(test):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1012
            job = Job()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1013
            job.set_is_example(False)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1014
            job.set_display_name(test)
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1015
            job.set_tmp_file_name(os.path.join(testpy_output_dir, "%s.xml" % test))
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1016
            job.set_cwd(os.getcwd())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1017
            job.set_basedir(os.getcwd())
5481
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
  1018
            job.set_tempdir(testpy_output_dir)
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
  1019
            if (options.multiple):
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
  1020
                multiple = " --multiple"
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
  1021
            else:
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
  1022
                multiple = ""
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
  1023
5461
ac47584eca3e Make test.py work on MinGW
Craig Dowell <craigdo@ee.washington.edu>
parents: 5460
diff changeset
  1024
            path_cmd = os.path.join("utils", "test-runner --suite=%s%s" % (test, multiple))
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
  1025
            job.set_shell_command(path_cmd)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1026
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1027
            if options.valgrind and test in core_valgrind_skip_tests:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1028
                job.set_is_skip(True)
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1029
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1030
            if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1031
                print "Queue %s" % test
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1032
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1033
            input_queue.put(job)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1034
            jobs = jobs + 1
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1035
            total_tests = total_tests + 1
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1036
    
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1037
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1038
    # We've taken care of the discovered or specified test suites.  Now we
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1039
    # have to deal with examples run as smoke tests.  We have a list of all of
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1040
    # the example programs it makes sense to try and run.  Each example will
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1041
    # have a condition associated with it that must evaluate to true for us
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1042
    # to try and execute it.  This is used to determine if the example has
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1043
    # a dependency that is not satisfied.  For example, if an example depends
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1044
    # on NSC being configured by waf, that example should have a condition
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1045
    # that evaluates to true if NSC is enabled.  For example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1046
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1047
    #      ("tcp-nsc-zoo", "ENABLE_NSC == True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1048
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1049
    # In this case, the example "tcp-nsc-zoo" will only be run if we find the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1050
    # waf configuration variable "ENABLE_NSC" to be True.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1051
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1052
    # We don't care at all how the trace files come out, so we just write them 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1053
    # to a single temporary directory.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1054
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1055
    # XXX As it stands, all of the trace files have unique names, and so file
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1056
    # collisions can only happen if two instances of an example are running in
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1057
    # two versions of the test.py process concurrently.  We may want to create
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1058
    # uniquely named temporary traces directories to avoid this problem.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1059
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1060
    # We need to figure out what examples to execute.  We are either given one 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1061
    # suite or example explicitly via the --suite or --example option, or we
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1062
    # need to walk the list of examples looking for available example 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1063
    # conditions.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1064
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1065
    # This translates into allowing the following options with respect to the 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1066
    # suites
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1067
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1068
    #  ./test,py:                                           run all of the examples
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1069
    #  ./test.py --constrain=unit                           run no examples
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1070
    #  ./test.py --constrain=example                        run all of the examples
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1071
    #  ./test,py --suite=some-test-suite:                   run no examples
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1072
    #  ./test,py --example=some-example:                    run the single example
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1073
    #  ./test,py --suite=some-suite --example=some-example: run the single example
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1074
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1075
    # XXX could use constrain to separate out examples used for performance 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1076
    # testing
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1077
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1078
    if len(options.suite) == 0 and len(options.example) == 0:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1079
        if len(options.constrain) == 0 or options.constrain == "example":
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1080
            if ENABLE_EXAMPLES:
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1081
                for test, do_run, do_valgrind_run in example_tests:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1082
                    if eval(do_run):
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1083
                        job = Job()
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1084
                        job.set_is_example(True)
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1085
                        job.set_display_name(test)
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1086
                        job.set_tmp_file_name("")
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1087
                        job.set_cwd(testpy_output_dir)
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1088
                        job.set_basedir(os.getcwd())
5481
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
  1089
                        job.set_tempdir(testpy_output_dir)
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1090
                        job.set_shell_command("examples/%s" % test)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1091
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1092
                        if options.valgrind and not eval(do_valgrind_run):
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1093
                            job.set_is_skip (True)
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1094
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1095
                        if options.verbose:
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1096
                            print "Queue %s" % test
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1097
5369
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1098
                        input_queue.put(job)
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1099
                        jobs = jobs + 1
86beb5869f67 split examples, add examples, tweak test.py to minimize builds
Craig Dowell <craigdo@ee.washington.edu>
parents: 5351
diff changeset
  1100
                        total_tests = total_tests + 1
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1101
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1102
    elif len(options.example):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1103
        #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1104
        # If you tell me to run an example, I will try and run the example
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1105
        # irrespective of any condition.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1106
        #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1107
        job = Job()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1108
        job.set_is_example(True)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1109
        job.set_display_name(options.example)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1110
        job.set_tmp_file_name("")
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1111
        job.set_cwd(testpy_output_dir)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1112
        job.set_basedir(os.getcwd())
5481
de01c67af975 pass explicit temp directory to test suites
Craig Dowell <craigdo@ee.washington.edu>
parents: 5470
diff changeset
  1113
        job.set_tempdir(testpy_output_dir)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1114
        job.set_shell_command("examples/%s" % options.example)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1115
        
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1116
        if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1117
            print "Queue %s" % test
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1118
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1119
        input_queue.put(job)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1120
        jobs = jobs + 1
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1121
        total_tests = total_tests + 1
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1122
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1123
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1124
    # Tell the worker threads to pack up and go home for the day.  Each one
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1125
    # will exit when they see their is_break task.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1126
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1127
    for i in range(processors):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1128
        job = Job()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1129
        job.set_is_break(True)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1130
        input_queue.put(job)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1131
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1132
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1133
    # Now all of the tests have been dispatched, so all we have to do here
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1134
    # in the main thread is to wait for them to complete.  Keyboard interrupt
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1135
    # handling is broken as mentioned above.  We use a signal handler to catch
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1136
    # sigint and set a global variable.  When the worker threads sense this
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1137
    # they stop doing real work and will just start throwing jobs back at us
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1138
    # with is_break set to True.  In this case, there are no real results so we 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1139
    # ignore them.  If there are real results, we always print PASS or FAIL to
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1140
    # standard out as a quick indication of what happened.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1141
    #
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1142
    passed_tests = 0
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1143
    failed_tests = 0
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1144
    crashed_tests = 0
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1145
    valgrind_errors = 0
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1146
    for i in range(jobs):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1147
        job = output_queue.get()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1148
        if job.is_break:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1149
            continue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1150
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1151
        if job.is_example:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1152
            kind = "Example"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1153
        else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1154
            kind = "TestSuite"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1155
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1156
        if job.is_skip:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1157
            status = "SKIP"
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1158
            skipped_tests = skipped_tests + 1
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1159
        else:
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1160
            if job.returncode == 0:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1161
                status = "PASS"
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1162
                passed_tests = passed_tests + 1
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1163
            elif job.returncode == 1:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1164
                failed_tests = failed_tests + 1
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1165
                status = "FAIL"
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1166
            elif job.returncode == 2:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1167
                valgrind_errors = valgrind_errors + 1
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1168
                status = "VALGR"
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1169
            else:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1170
                crashed_tests = crashed_tests + 1
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1171
                status = "CRASH"
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1172
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1173
        print "%s: %s %s" % (status, kind, job.display_name)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1174
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1175
        if job.is_example == True:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1176
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1177
            # Examples are the odd man out here.  They are written without any
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1178
            # knowledge that they are going to be run as a test, so we need to 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1179
            # cook up some kind of output for them.  We're writing an xml file,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1180
            # so we do some simple XML that says we ran the example.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1181
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1182
            # XXX We could add some timing information to the examples, i.e. run
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1183
            # them through time and print the results here.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1184
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1185
            f = open(xml_results_file, 'a')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1186
            f.write('<Example>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1187
            example_name = "  <Name>%s</Name>\n" % job.display_name
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1188
            f.write(example_name)
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1189
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1190
            if status == "PASS":
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1191
                f.write('  <Result>PASS</Result>\n')
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1192
            elif status == "FAIL":
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1193
                f.write('  <Result>FAIL</Result>\n')
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1194
            elif status == "VALGR":
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1195
                f.write('  <Result>VALGR</Result>\n')
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1196
            elif status == "SKIP":
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1197
                f.write('  <Result>SKIP</Result>\n')
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1198
            else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1199
                f.write('  <Result>CRASH</Result>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1200
5460
7c6f8ebf4a34 test.py prints example elapsed times
craigdo@ee.washington.edu
parents: 5459
diff changeset
  1201
            f.write('  <ElapsedTime>%.3f</ElapsedTime>\n' % job.elapsed_time)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1202
            f.write('</Example>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1203
            f.close()
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1204
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1205
        else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1206
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1207
            # If we're not running an example, we're running a test suite.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1208
            # These puppies are running concurrently and generating output
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1209
            # that was written to a temporary file to avoid collisions.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1210
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1211
            # Now that we are executing sequentially in the main thread, we can
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1212
            # concatenate the contents of the associated temp file to the main 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1213
            # results file and remove that temp file.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1214
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1215
            # One thing to consider is that a test suite can crash just as
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1216
            # well as any other program, so we need to deal with that 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1217
            # possibility as well.  If it ran correctly it will return 0
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1218
            # if it passed, or 1 if it failed.  In this case, we can count
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1219
            # on the results file it saved being complete.  If it crashed, it 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1220
            # will return some other code, and the file should be considered 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1221
            # corrupt and useless.  If the suite didn't create any XML, then
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1222
            # we're going to have to do it ourselves.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1223
            #
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1224
            # Another issue is how to deal with a valgrind error.  If we run
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1225
            # a test suite under valgrind and it passes, we will get a return
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1226
            # code of 0 and there will be a valid xml results file since the code
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1227
            # ran to completion.  If we get a return code of 1 under valgrind,
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1228
            # the test case failed, but valgrind did not find any problems so the
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1229
            # test case return code was passed through.  We will have a valid xml
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1230
            # results file here as well since the test suite ran.  If we see a 
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1231
            # return code of 2, this means that valgrind found an error (we asked
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1232
            # it to return 2 if it found a problem in run_job_synchronously) but
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1233
            # the suite ran to completion so there is a valid xml results file.
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1234
            # If the suite crashes under valgrind we will see some other error 
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1235
            # return code (like 139).  If valgrind finds an illegal instruction or
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1236
            # some other strange problem, it will die with its own strange return
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1237
            # code (like 132).  However, if the test crashes by itself, not under
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1238
            # valgrind we will also see some other return code.
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1239
            #
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1240
            # If the return code is 0, 1, or 2, we have a valid xml file.  If we 
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1241
            # get another return code, we have no xml and we can't really say what
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1242
            # happened -- maybe the TestSuite crashed, maybe valgrind crashed due
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1243
            # to an illegal instruction.  If we get something beside 0-2, we assume
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1244
            # a crash and fake up an xml entry.  After this is all done, we still
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1245
            # need to indicate a valgrind error somehow, so we fake up an xml entry
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1246
            # with a VALGR result.  Thus, in the case of a working TestSuite that
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1247
            # fails valgrind, we'll see the PASS entry for the working TestSuite
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1248
            # followed by a VALGR failing test suite of the same name.
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1249
            #
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1250
            if job.is_skip:
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1251
                f = open(xml_results_file, 'a')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1252
                f.write("<TestSuite>\n")
5239
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1253
                f.write("  <SuiteName>%s</SuiteName>\n" % job.display_name)
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1254
                f.write('  <SuiteResult>SKIP</SuiteResult>\n')
5239
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1255
                f.write('  <SuiteTime>Execution times not available</SuiteTime>\n')
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1256
                f.write("</TestSuite>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1257
                f.close()
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1258
            else:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1259
                if job.returncode == 0 or job.returncode == 1 or job.returncode == 2:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1260
                    f_to = open(xml_results_file, 'a')
5459
3c240ed38d99 try and make test.py a bit more portable
Craig Dowell <craigdo@ee.washington.edu>
parents: 5415
diff changeset
  1261
                    f_from = open(job.tmp_file_name)
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1262
                    f_to.write(f_from.read())
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1263
                    f_to.close()
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1264
                    f_from.close()
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1265
                else:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1266
                    f = open(xml_results_file, 'a')
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1267
                    f.write("<TestSuite>\n")
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1268
                    f.write("  <SuiteName>%s</SuiteName>\n" % job.display_name)
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1269
                    f.write('  <SuiteResult>CRASH</SuiteResult>\n')
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1270
                    f.write('  <SuiteTime>Execution times not available</SuiteTime>\n')
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1271
                    f.write("</TestSuite>\n")
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1272
                    f.close()
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1273
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1274
                    if job.returncode == 2:
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1275
                        f = open(xml_results_file, 'a')
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1276
                        f.write("<TestSuite>\n")
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1277
                        f.write("  <SuiteName>%s</SuiteName>\n" % job.display_name)
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1278
                        f.write('  <SuiteResult>VALGR</SuiteResult>\n')
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1279
                        f.write('  <SuiteTime>Execution times not available</SuiteTime>\n')
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1280
                        f.write("</TestSuite>\n")
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1281
                        f.close()
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1282
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1283
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1284
    # We have all of the tests run and the results written out.  One final 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1285
    # bit of housekeeping is to wait for all of the threads to close down
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1286
    # so we can exit gracefully.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1287
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1288
    for thread in threads:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1289
        thread.join()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1290
    
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1291
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1292
    # Back at the beginning of time, we started the body of an XML document
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1293
    # since the test suites and examples were going to just write their 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1294
    # individual pieces.  So, we need to finish off and close out the XML 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1295
    # document
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1296
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1297
    f = open(xml_results_file, 'a')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1298
    f.write('</TestResults>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1299
    f.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1300
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1301
    #
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1302
    # Print a quick summary of events
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1303
    #
5402
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1304
    print "%d of %d tests passed (%d passed, %d skipped, %d failed, %d crashed, %d valgrind errors)" % (passed_tests, 
b73b6da8ead6 teach test.py to skip selected tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5390
diff changeset
  1305
        total_tests, passed_tests, skipped_tests, failed_tests, crashed_tests, valgrind_errors)
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1306
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1307
    # The last things to do are to translate the XML results file to "human
5239
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1308
    # readable form" if the user asked for it (or make an XML file somewhere)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1309
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1310
    if len(options.html):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1311
        translate_to_html(xml_results_file, options.html)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1312
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1313
    if len(options.text):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1314
        translate_to_text(xml_results_file, options.text)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1315
5239
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1316
    if len(options.xml):
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1317
        shutil.copyfile(xml_results_file, options.xml)
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1318
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1319
    #
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1320
    # If we have been asked to retain all of the little temporary files, we
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1321
    # don't delete tm.  If we do delete the temporary files, delete only the
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1322
    # directory we just created.  We don't want to happily delete any retained
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1323
    # directories, which will probably surprise the user.
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1324
    #
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1325
    if not options.retain:
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1326
        shutil.rmtree(testpy_output_dir)
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1327
5403
530155ecdbcb fix ns3-tcp-cwnd unit test
Craig Dowell <craigdo@ee.washington.edu>
parents: 5402
diff changeset
  1328
    if passed_tests + skipped_tests == total_tests:
5351
6abced63cd7f keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5350
diff changeset
  1329
        return 0 # success
5350
9f457bebbcf4 have test.py return status
Craig Dowell <craigdo@ee.washington.edu>
parents: 5324
diff changeset
  1330
    else:
5351
6abced63cd7f keep wifi-ap from spewing all over std_out during tests
Craig Dowell <craigdo@ee.washington.edu>
parents: 5350
diff changeset
  1331
        return 1 # catchall for general errors
5350
9f457bebbcf4 have test.py return status
Craig Dowell <craigdo@ee.washington.edu>
parents: 5324
diff changeset
  1332
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1333
def main(argv):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1334
    parser = optparse.OptionParser()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1335
    parser.add_option("-c", "--constrain", action="store", type="string", dest="constrain", default="",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1336
                      metavar="KIND",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1337
                      help="constrain the test-runner by kind of test")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1338
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1339
    parser.add_option("-e", "--example", action="store", type="string", dest="example", default="",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1340
                      metavar="EXAMPLE",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1341
                      help="specify a single example to run")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1342
5370
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1343
    parser.add_option("-g", "--grind", action="store_true", dest="valgrind", default=False,
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1344
                      help="run the test suites and examples using valgrind")
a27163124701 add valgrind powers back to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5369
diff changeset
  1345
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1346
    parser.add_option("-k", "--kinds", action="store_true", dest="kinds", default=False,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1347
                      help="print the kinds of tests available")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1348
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1349
    parser.add_option("-l", "--list", action="store_true", dest="list", default=False,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1350
                      help="print the list of known tests")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1351
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
  1352
    parser.add_option("-m", "--multiple", action="store_true", dest="multiple", default=False,
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
  1353
                      help="report multiple failures from test suites and test cases")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
  1354
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1355
    parser.add_option("-n", "--nowaf", action="store_true", dest="nowaf", default=False,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1356
                      help="do not run waf before starting testing")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1357
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1358
    parser.add_option("-s", "--suite", action="store", type="string", dest="suite", default="",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1359
                      metavar="TEST-SUITE",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1360
                      help="specify a single test suite to run")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1361
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1362
    parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1363
                      help="print progress and informational messages")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1364
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1365
    parser.add_option("-w", "--web", "--html", action="store", type="string", dest="html", default="",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1366
                      metavar="HTML-FILE",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1367
                      help="write detailed test results into HTML-FILE.html")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1368
5412
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1369
    parser.add_option("-r", "--retain", action="store_true", dest="retain", default=False,
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1370
                      help="retain all temporary files (which are normally deleted)")
00d7fe69d024 refalgamize temporary files in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5403
diff changeset
  1371
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1372
    parser.add_option("-t", "--text", action="store", type="string", dest="text", default="",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1373
                      metavar="TEXT-FILE",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1374
                      help="write detailed test results into TEXT-FILE.txt")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1375
5239
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1376
    parser.add_option("-x", "--xml", action="store", type="string", dest="xml", default="",
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1377
                      metavar="XML-FILE",
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1378
                      help="write detailed test results into XML-FILE.xml")
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1379
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1380
    global options
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1381
    options = parser.parse_args()[0]
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1382
    signal.signal(signal.SIGINT, sigint_hook)
5350
9f457bebbcf4 have test.py return status
Craig Dowell <craigdo@ee.washington.edu>
parents: 5324
diff changeset
  1383
    
9f457bebbcf4 have test.py return status
Craig Dowell <craigdo@ee.washington.edu>
parents: 5324
diff changeset
  1384
    return run_tests()
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1385
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1386
if __name__ == '__main__':
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1387
    sys.exit(main(sys.argv))