test.py
author Craig Dowell <craigdo@ee.washington.edu>
Wed, 30 Sep 2009 20:26:33 -0700
changeset 5324 0ba73cdd2a43
parent 5295 c1bd4ffb5e47
child 5350 9f457bebbcf4
permissions -rwxr-xr-x
Refalgamize test framework to allow multiple test failures
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     1
#! /usr/bin/env python
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     2
## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     3
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     4
# Copyright (c) 2009 University of Washington
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     5
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     6
# This program is free software; you can redistribute it and/or modify
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     7
# it under the terms of the GNU General Public License version 2 as
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     8
# published by the Free Software Foundation;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     9
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    10
# This program is distributed in the hope that it will be useful,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    11
# but WITHOUT ANY WARRANTY; without even the implied warranty of
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    12
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    13
# GNU General Public License for more details.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    14
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    15
# You should have received a copy of the GNU General Public License
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    16
# along with this program; if not, write to the Free Software
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    17
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    18
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    19
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    20
import os
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    21
import sys
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    22
import optparse
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    23
import subprocess
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    24
import threading
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    25
import Queue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    26
import signal
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    27
import random
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    28
import xml.dom.minidom
5239
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
    29
import shutil
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    30
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    31
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    32
# XXX This should really be part of a waf command to list the configuration
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    33
# items relative to optional ns-3 pieces.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    34
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    35
# A list of interesting configuration items in the waf configuration 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    36
# cache which we may be interested in when deciding on which examples
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    37
# to run and how to run them.  These are set by waf during the 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    38
# configuration phase and the corresponding assignments are usually
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    39
# found in the associated subdirectory wscript files.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    40
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    41
interesting_config_items = [
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    42
    "NS3_BUILDDIR",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    43
    "NS3_MODULE_PATH",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    44
    "ENABLE_NSC",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    45
    "ENABLE_REAL_TIME",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    46
]
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    47
5295
c1bd4ffb5e47 fix test.py -v
Craig Dowell <craigdo@ee.washington.edu>
parents: 5279
diff changeset
    48
ENABLE_NSC = False
c1bd4ffb5e47 fix test.py -v
Craig Dowell <craigdo@ee.washington.edu>
parents: 5279
diff changeset
    49
ENABLE_REAL_TIME = False
c1bd4ffb5e47 fix test.py -v
Craig Dowell <craigdo@ee.washington.edu>
parents: 5279
diff changeset
    50
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    51
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    52
# A list of examples to run as smoke tests just to ensure that they remain 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    53
# buildable and runnable over time.  Also a condition under which to run
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    54
# the example (from the waf configuration).
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    55
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    56
# XXX Should this not be read from a configuration file somewhere and not
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    57
# hardcoded.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    58
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    59
example_tests = [
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    60
    ("csma-bridge", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    61
    ("csma-bridge-one-hop", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    62
    ("csma-broadcast", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    63
    ("csma-multicast", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    64
    ("csma-one-subnet", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    65
    ("csma-packet-socket", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    66
    ("csma-ping", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    67
    ("csma-raw-ip-socket", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    68
    ("csma-star", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    69
    ("dynamic-global-routing", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    70
    ("first", "True"),
5255
2f3abe6fa8a3 Remove and replace Callback unit test, add examples to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5243
diff changeset
    71
    ("global-injection-slash32", "True"),
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    72
    ("global-routing-slash32", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    73
    ("hello-simulator", "True"),
5255
2f3abe6fa8a3 Remove and replace Callback unit test, add examples to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5243
diff changeset
    74
    ("icmpv6-redirect", "True"),
2f3abe6fa8a3 Remove and replace Callback unit test, add examples to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5243
diff changeset
    75
    ("mesh", "True"),
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    76
    ("mixed-global-routing", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    77
    ("mixed-wireless", "True"),
5257
fc9fd5cbc007 Remove and replace config unit test (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 5255
diff changeset
    78
    ("multirate", "False"), # takes forever to run
5255
2f3abe6fa8a3 Remove and replace Callback unit test, add examples to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5243
diff changeset
    79
    ("nix-simple", "True"),
5257
fc9fd5cbc007 Remove and replace config unit test (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 5255
diff changeset
    80
    ("nms-p2p-nix", "False"), # takes forever to run
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    81
    ("object-names", "True"),
5255
2f3abe6fa8a3 Remove and replace Callback unit test, add examples to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5243
diff changeset
    82
    ("ping6", "True"),
2f3abe6fa8a3 Remove and replace Callback unit test, add examples to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5243
diff changeset
    83
    ("radvd", "True"),
2f3abe6fa8a3 Remove and replace Callback unit test, add examples to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5243
diff changeset
    84
    ("radvd-two-prefix", "True"),    
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    85
    ("realtime-udp-echo", "ENABLE_REAL_TIME == True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    86
    ("second", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    87
    ("simple-alternate-routing", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    88
    ("simple-error-model", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    89
    ("simple-global-routing", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    90
    ("simple-point-to-point-olsr", "True"),
5255
2f3abe6fa8a3 Remove and replace Callback unit test, add examples to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5243
diff changeset
    91
    ("simple-routing-ping6", "True"),
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    92
    ("simple-wifi-frame-aggregation", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    93
    ("star", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    94
    ("static-routing-slash32", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    95
    ("tcp-large-transfer", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    96
    ("tcp-nsc-zoo", "ENABLE_NSC == True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    97
    ("tcp-star-server", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    98
    ("test-ipv6", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    99
    ("third", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   100
    ("udp-echo", "True"),
5255
2f3abe6fa8a3 Remove and replace Callback unit test, add examples to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5243
diff changeset
   101
    ("virtual-net-device", "True"),
5257
fc9fd5cbc007 Remove and replace config unit test (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 5255
diff changeset
   102
    ("wifi-adhoc", "False"), # takes forever to run
5255
2f3abe6fa8a3 Remove and replace Callback unit test, add examples to test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5243
diff changeset
   103
    ("wifi-ap", "True"),
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   104
    ("wifi-wired-bridging", "True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   105
]
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   106
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   107
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   108
# Most of the examples produce gangs of trace files, so we want to find
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   109
# somewhere to put them that won't pollute the current directory.  One
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   110
# obvious place is somewhere in /tmp.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   111
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   112
TMP_TRACES_DIR = "/tmp/unchecked-traces"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   113
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   114
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   115
# The test suites are going to want to output status.  They are running
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   116
# concurrently.  This means that unless we are careful, the output of
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   117
# the test suites will be interleaved.  Rather than introducing a lock
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   118
# file that could unintentionally start serializing execution, we ask
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   119
# the tests to write their output to a temporary directory and then 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   120
# put together the final output file when we "join" the test tasks back
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   121
# to the main thread.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   122
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   123
TMP_OUTPUT_DIR = "/tmp/testpy"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   124
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   125
def get_node_text(node):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   126
    for child in node.childNodes:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   127
        if child.nodeType == child.TEXT_NODE:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   128
            return child.nodeValue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   129
    return "None"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   130
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   131
#
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   132
# A simple example of writing a text file with a test result summary.  It is 
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   133
# expected that this output will be fine for developers looking for problems.
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   134
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   135
def translate_to_text(results_file, text_file):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   136
    f = open(text_file, 'w')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   137
    dom = xml.dom.minidom.parse(results_file)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   138
    for suite in dom.getElementsByTagName("TestSuite"):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   139
        result = get_node_text(suite.getElementsByTagName("SuiteResult")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   140
        name = get_node_text(suite.getElementsByTagName("SuiteName")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   141
        time = get_node_text(suite.getElementsByTagName("SuiteTime")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   142
        output = "%s: Test Suite \"%s\" (%s)\n" % (result, name, time)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   143
        f.write(output)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   144
        if result != "CRASH":
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   145
            for case in suite.getElementsByTagName("TestCase"):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   146
                result = get_node_text(case.getElementsByTagName("CaseResult")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   147
                name = get_node_text(case.getElementsByTagName("CaseName")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   148
                time = get_node_text(case.getElementsByTagName("CaseTime")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   149
                output =   "  %s: Test Case \"%s\" (%s)\n" % (result, name, time)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   150
                f.write(output)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   151
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   152
                if result == "FAIL":
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   153
                    for details in case.getElementsByTagName("FailureDetails"):
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   154
                        f.write("    Details:\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   155
                        f.write("      Message:   %s\n" % get_node_text(details.getElementsByTagName("Message")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   156
                        f.write("      Condition: %s\n" % get_node_text(details.getElementsByTagName("Condition")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   157
                        f.write("      Actual:    %s\n" % get_node_text(details.getElementsByTagName("Actual")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   158
                        f.write("      Limit:     %s\n" % get_node_text(details.getElementsByTagName("Limit")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   159
                        f.write("      File:      %s\n" % get_node_text(details.getElementsByTagName("File")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   160
                        f.write("      Line:      %s\n" % get_node_text(details.getElementsByTagName("Line")[0]))
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   161
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   162
    for example in dom.getElementsByTagName("Example"):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   163
        result = get_node_text(example.getElementsByTagName("Result")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   164
        name = get_node_text(example.getElementsByTagName("Name")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   165
        output = "%s: Example \"%s\"\n" % (result, name)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   166
        f.write(output)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   167
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   168
    f.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   169
    
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   170
#
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   171
# A simple example of writing an HTML file with a test result summary.  It is 
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   172
# expected that this will eventually be made prettier as time progresses and
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   173
# we have time to tweak it.  This may end up being moved to a separate module
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   174
# since it will probably grow over time.
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   175
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   176
def translate_to_html(results_file, html_file):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   177
    f = open(html_file, 'w')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   178
    f.write("<html>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   179
    f.write("<body>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   180
    f.write("<center><h1>ns-3 Test Results</h1></center>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   181
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   182
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   183
    # Read and parse the whole results file.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   184
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   185
    dom = xml.dom.minidom.parse(results_file)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   186
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   187
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   188
    # Iterate through the test suites
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   189
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   190
    f.write("<h2>Test Suites</h2>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   191
    for suite in dom.getElementsByTagName("TestSuite"):
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   192
     
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   193
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   194
        # For each test suite, get its name, result and execution time info
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   195
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   196
        name = get_node_text(suite.getElementsByTagName("SuiteName")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   197
        result = get_node_text(suite.getElementsByTagName("SuiteResult")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   198
        time = get_node_text(suite.getElementsByTagName("SuiteTime")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   199
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   200
        # 
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   201
        # Print a level three header in green with the result, name and time.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   202
        # If the test suite passed, the header is printed in green, otherwise
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   203
        # it is printed in red.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   204
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   205
        if result == "PASS":
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   206
            f.write("<h3 style=\"color:green\">%s: %s (%s)</h3>\n" % (result, name, time))
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   207
        else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   208
            f.write("<h3 style=\"color:red\">%s: %s (%s)</h3>\n" % (result, name, time))
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   209
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   210
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   211
        # The test case information goes in a table.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   212
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   213
        f.write("<table border=\"1\">\n")
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   214
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   215
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   216
        # The first column of the table has the heading Result
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   217
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   218
        f.write("<th> Result </th>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   219
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   220
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   221
        # If the suite crashed, there is no further information, so just
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   222
        # delare a new table row with the result (CRASH) in it.  Looks like:
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   223
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   224
        #   +--------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   225
        #   | Result |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   226
        #   +--------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   227
        #   | CRASH  |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   228
        #   +--------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   229
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   230
        # Then go on to the next test suite
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   231
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   232
        if result == "CRASH":
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   233
            f.write("<tr>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   234
            f.write("<td style=\"color:red\">%s</td>\n" % result)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   235
            f.write("</tr>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   236
            f.write("</table>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   237
            continue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   238
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   239
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   240
        # If the suite didn't crash, we expect more information, so fill out
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   241
        # the table heading row.  Like,
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   242
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   243
        #   +--------+----------------+------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   244
        #   | Result | Test Case Name | Time |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   245
        #   +--------+----------------+------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   246
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   247
        f.write("<th>Test Case Name</th>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   248
        f.write("<th> Time </th>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   249
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   250
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   251
        # If the test case failed, we need to print out some failure details
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   252
        # so extend the heading row again.  Like,
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   253
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   254
        #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   255
        #   | Result | Test Case Name | Time | Failure Details |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   256
        #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   257
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   258
        if result == "FAIL":
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   259
            f.write("<th>Failure Details</th>\n")
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   260
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   261
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   262
        # Now iterate through all of the test cases.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   263
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   264
        for case in suite.getElementsByTagName("TestCase"):
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   265
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   266
            #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   267
            # Get the name, result and timing information from xml to use in
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   268
            # printing table below.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   269
            #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   270
            name = get_node_text(case.getElementsByTagName("CaseName")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   271
            result = get_node_text(case.getElementsByTagName("CaseResult")[0])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   272
            time = get_node_text(case.getElementsByTagName("CaseTime")[0])
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   273
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   274
            #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   275
            # If the test case failed, we iterate through possibly multiple
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   276
            # failure details
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   277
            #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   278
            if result == "FAIL":
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   279
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   280
                # There can be multiple failures for each test case.  The first
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   281
                # row always gets the result, name and timing information along
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   282
                # with the failure details.  Remaining failures don't duplicate
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   283
                # this information but just get blanks for readability.  Like,
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   284
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   285
                #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   286
                #   | Result | Test Case Name | Time | Failure Details |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   287
                #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   288
                #   |  FAIL  | The name       | time | It's busted     |   
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   289
                #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   290
                #   |        |                |      | Really broken   |   
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   291
                #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   292
                #   |        |                |      | Busted bad      |   
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   293
                #   +--------+----------------+------+-----------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   294
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   295
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   296
                first_row = True
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   297
                for details in case.getElementsByTagName("FailureDetails"):
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   298
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   299
                    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   300
                    # Start a new row in the table for each possible Failure Detail
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   301
                    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   302
                    f.write("<tr>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   303
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   304
                    if first_row:
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   305
                        first_row = False
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   306
                        f.write("<td style=\"color:red\">%s</td>\n" % result)
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   307
                        f.write("<td>%s</td>\n" % name)
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   308
                        f.write("<td>%s</td>\n" % time)
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   309
                    else:
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   310
                        f.write("<td></td>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   311
                        f.write("<td></td>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   312
                        f.write("<td></td>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   313
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   314
                    f.write("<td>")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   315
                    f.write("<b>Message: </b>%s, " % get_node_text(details.getElementsByTagName("Message")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   316
                    f.write("<b>Condition: </b>%s, " % get_node_text(details.getElementsByTagName("Condition")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   317
                    f.write("<b>Actual: </b>%s, " % get_node_text(details.getElementsByTagName("Actual")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   318
                    f.write("<b>Limit: </b>%s, " % get_node_text(details.getElementsByTagName("Limit")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   319
                    f.write("<b>File: </b>%s, " % get_node_text(details.getElementsByTagName("File")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   320
                    f.write("<b>Line: </b>%s" % get_node_text(details.getElementsByTagName("Line")[0]))
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   321
                    f.write("</td>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   322
                    
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   323
                    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   324
                    # End the table row
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   325
                    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   326
                    f.write("</td>\n")
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   327
            else:
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   328
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   329
                # If this particular test case passed, then we just print the PASS
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   330
                # result in green, followed by the test case name and its execution
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   331
                # time information.  These go off in <td> ... </td> table data.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   332
                # The details table entry is left blank.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   333
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   334
                #   +--------+----------------+------+---------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   335
                #   | Result | Test Case Name | Time | Details |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   336
                #   +--------+----------------+------+---------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   337
                #   |  PASS  | The name       | time |         |   
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   338
                #   +--------+----------------+------+---------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   339
                #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   340
                f.write("<tr>\n")
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   341
                f.write("<td style=\"color:green\">%s</td>\n" % result)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   342
                f.write("<td>%s</td>\n" % name)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   343
                f.write("<td>%s</td>\n" % time)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   344
                f.write("<td></td>\n")
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   345
                f.write("</tr>\n")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   346
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   347
        # All of the rows are written, so we need to end the table.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   348
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   349
        f.write("</table>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   350
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   351
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   352
    # That's it for all of the test suites.  Now we have to do something about 
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   353
    # our examples.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   354
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   355
    f.write("<h2>Examples</h2>\n")
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   356
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   357
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   358
    # Example status is rendered in a table just like the suites.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   359
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   360
    f.write("<table border=\"1\">\n")
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   361
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   362
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   363
    # The table headings look like,
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   364
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   365
    #   +--------+--------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   366
    #   | Result | Example Name |
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   367
    #   +--------+--------------+
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   368
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   369
    f.write("<th> Result </th>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   370
    f.write("<th>Example Name</th>\n")
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   371
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   372
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   373
    # Now iterate through all of the examples
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   374
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   375
    for example in dom.getElementsByTagName("Example"):
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   376
        
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   377
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   378
        # Start a new row for each example
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   379
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   380
        f.write("<tr>\n")
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   381
        
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   382
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   383
        # Get the result and name of the example in question
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   384
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   385
        result = get_node_text(example.getElementsByTagName("Result")[0])
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   386
        name =   get_node_text(example.getElementsByTagName("Name")[0])
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   387
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   388
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   389
        # If the example eitehr failed or crashed, print its result status
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   390
        # in red; otherwise green.  This goes in a <td> ... </td> table data
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   391
        #
5243
46a3f1348d4d crashing examples should print red in html
Craig Dowell <craigdo@ee.washington.edu>
parents: 5239
diff changeset
   392
        if result in ["FAIL", "CRASH"]:
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   393
            f.write("<td style=\"color:red\">%s</td>\n" % result)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   394
        else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   395
            f.write("<td style=\"color:green\">%s</td>\n" % result)
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   396
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   397
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   398
        # Write the example name as a new tagle data.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   399
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   400
        f.write("<td>%s</td>\n" % name)
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   401
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   402
        #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   403
        # That's it for the current example, so terminate the row.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   404
        #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   405
        f.write("</tr>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   406
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   407
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   408
    # That's it for the table of examples, so terminate the table.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   409
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   410
    f.write("</table>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   411
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   412
    #
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   413
    # And that's it for the report, so finish up.
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   414
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   415
    f.write("</body>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   416
    f.write("</html>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   417
    f.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   418
    
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   419
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   420
# Python Control-C handling is broken in the presence of multiple threads.  
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   421
# Signals get delivered to the runnable/running thread by default and if 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   422
# it is blocked, the signal is simply ignored.  So we hook sigint and set 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   423
# a global variable telling the system to shut down gracefully.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   424
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   425
thread_exit = False
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   426
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   427
def sigint_hook(signal, frame):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   428
    global thread_exit
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   429
    thread_exit = True
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   430
    return 0
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   431
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   432
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   433
# Waf can be configured to compile in debug or optimized modes.  In each
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   434
# case, the resulting built goes into a different directory.  If we want
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   435
# test tests to run from the correct code-base, we have to figure out which
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   436
# mode waf is running in.  This is called its active variant.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   437
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   438
# XXX This function pokes around in the waf internal state file.  To be a
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   439
# little less hacky, we should add a commmand to waf to return this info
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   440
# and use that result.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   441
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   442
def read_waf_active_variant():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   443
    for line in open("build/c4che/default.cache.py").readlines():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   444
        if line.startswith("NS3_ACTIVE_VARIANT"):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   445
            exec(line, globals())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   446
            break
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   447
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   448
    if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   449
        print "NS3_ACTIVE_VARIANT == %s" % NS3_ACTIVE_VARIANT
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   450
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   451
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   452
# In general, the build process itself naturally takes care of figuring out
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   453
# which tests are built into the test runner.  For example, if waf configure
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   454
# determines that ENABLE_EMU is false due to some missing dependency,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   455
# the tests for the emu net device simply will not be built and will 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   456
# therefore not be included in the built test runner.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   457
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   458
# Examples, however, are a different story.  In that case, we are just given
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   459
# a list of examples that could be run.  Instead of just failing, for example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   460
# nsc-tcp-zoo if NSC is not present, we look into the waf saved configuration
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   461
# for relevant configuration items.  
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   462
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   463
# XXX This function pokes around in the waf internal state file.  To be a
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   464
# little less hacky, we should add a commmand to waf to return this info
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   465
# and use that result.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   466
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   467
def read_waf_config():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   468
    for line in open("build/c4che/%s.cache.py" % NS3_ACTIVE_VARIANT).readlines():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   469
        for item in interesting_config_items:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   470
            if line.startswith(item):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   471
                exec(line, globals())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   472
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   473
    if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   474
        for item in interesting_config_items:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   475
            print "%s ==" % item, eval(item)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   476
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   477
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   478
# It seems pointless to fork a process to run waf to fork a process to run
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   479
# the test runner, so we just run the test runner directly.  The main thing 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   480
# that waf would do for us would be to sort out the shared library path but
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   481
# we can deal with that easily and do here.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   482
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   483
# There can be many different ns-3 repositories on a system, and each has 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   484
# its own shared libraries, so ns-3 doesn't hardcode a shared library search
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   485
# path -- it is cooked up dynamically, so we do that too.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   486
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   487
def make_library_path():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   488
    global LIBRARY_PATH
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   489
5275
d4008f2981ba Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents: 5274
diff changeset
   490
    LIBRARY_PATH = "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:'"        
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   491
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   492
    if sys.platform == "darwin":
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   493
        LIBRARY_PATH = "DYLD_LIBRARY_PATH='"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   494
    elif sys.platform == "win32":
5275
d4008f2981ba Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents: 5274
diff changeset
   495
        LIBRARY_PATH = "PATH=$PATH:'"
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   496
    elif sys.platform == "cygwin":
5275
d4008f2981ba Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents: 5274
diff changeset
   497
        LIBRARY_PATH = "PATH=$PATH:'"
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   498
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   499
    for path in NS3_MODULE_PATH:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   500
        LIBRARY_PATH = LIBRARY_PATH + path + ":"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   501
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   502
    LIBRARY_PATH = LIBRARY_PATH + "'"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   503
5275
d4008f2981ba Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents: 5274
diff changeset
   504
    if options.verbose:
d4008f2981ba Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents: 5274
diff changeset
   505
        print "LIBRARY_PATH == %s" % LIBRARY_PATH
d4008f2981ba Windows needs entire path
Craig Dowell <craigdo@ee.washington.edu>
parents: 5274
diff changeset
   506
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   507
def run_job_synchronously(shell_command, directory):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   508
    cmd = "%s %s/%s/%s" % (LIBRARY_PATH, NS3_BUILDDIR, NS3_ACTIVE_VARIANT, shell_command)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   509
    if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   510
        print "Synchronously execute %s" % cmd
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   511
    proc = subprocess.Popen(cmd, shell=True, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   512
    stdout_results = proc.communicate()[0]
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   513
    return (proc.returncode, stdout_results)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   514
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   515
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   516
# This class defines a unit of testing work.  It will typically refer to
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   517
# a test suite to run using the test-runner, or an example to run directly.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   518
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   519
class Job():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   520
    def __init__(self):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   521
        self.is_break = False
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   522
        self.is_example = False
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   523
        self.shell_command = ""
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   524
        self.display_name = ""
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   525
        self.cwd = ""
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   526
        self.tmp_file_name = ""
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   527
        self.returncode = False
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   528
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   529
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   530
    # A job is either a standard job or a special job indicating that a worker
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   531
    # thread should exist.  This special job is indicated by setting is_break 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   532
    # to true.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   533
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   534
    def set_is_break(self, is_break):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   535
        self.is_break = is_break
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   536
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   537
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   538
    # Examples are treated differently than standard test suites.  This is
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   539
    # mostly because they are completely unaware that they are being run as 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   540
    # tests.  So we have to do some special case processing to make them look
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   541
    # like tests.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   542
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   543
    def set_is_example(self, is_example):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   544
        self.is_example = is_example
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   545
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   546
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   547
    # This is the shell command that will be executed in the job.  For example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   548
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   549
    #  "utils/test-runner --suite=some-test-suite"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   550
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   551
    def set_shell_command(self, shell_command):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   552
        self.shell_command = shell_command
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   553
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   554
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   555
    # This is the dispaly name of the job, typically the test suite or example 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   556
    # name.  For example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   557
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   558
    #  "some-test-suite" or "udp-echo"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   559
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   560
    def set_display_name(self, display_name):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   561
        self.display_name = display_name
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   562
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   563
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   564
    # This is the base directory of the repository out of which the tests are
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   565
    # being run.  It will be used deep down in the testing framework to determine
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   566
    # where the source directory of the test was, and therefore where to find 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   567
    # provided test vectors.  For example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   568
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   569
    #  "/home/user/repos/ns-3-dev"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   570
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   571
    def set_basedir(self, basedir):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   572
        self.basedir = basedir
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   573
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   574
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   575
    # This is the current working directory that will be given to an executing
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   576
    # test as it is being run.  It will be used for examples to tell them where
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   577
    # to write all of the pcap files that we will be carefully ignoring.  For
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   578
    # example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   579
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   580
    #  "/tmp/unchecked-traces"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   581
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   582
    def set_cwd(self, cwd):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   583
        self.cwd = cwd
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   584
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   585
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   586
    # This is the temporary results file name that will be given to an executing 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   587
    # test as it is being run.  We will be running all of our tests in parallel
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   588
    # so there must be multiple temporary output files.  These will be collected
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   589
    # into a single XML file at the end and then be deleted.  The file names are
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   590
    # just giant random numbers, for example
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   591
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   592
    #  "/tmp/testpy/5437925246732857"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   593
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   594
    def set_tmp_file_name(self, tmp_file_name):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   595
        self.tmp_file_name = tmp_file_name
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   596
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   597
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   598
    # The return code received when the job process is executed.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   599
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   600
    def set_returncode(self, returncode):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   601
        self.returncode = returncode
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   602
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   603
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   604
# The worker thread class that handles the actual running of a given test.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   605
# Once spawned, it receives requests for work through its input_queue and
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   606
# ships the results back through the output_queue.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   607
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   608
class worker_thread(threading.Thread):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   609
    def __init__(self, input_queue, output_queue):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   610
        threading.Thread.__init__(self)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   611
        self.input_queue = input_queue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   612
        self.output_queue = output_queue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   613
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   614
    def run(self):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   615
        while True:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   616
            job = self.input_queue.get()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   617
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   618
            # Worker threads continue running until explicitly told to stop with
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   619
            # a special job.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   620
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   621
            if job.is_break:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   622
                return
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   623
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   624
            # If the global interrupt handler sets the thread_exit variable,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   625
            # we stop doing real work and just report back a "break" in the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   626
            # normal command processing has happened.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   627
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   628
            if thread_exit == True:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   629
                job.set_is_break(True)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   630
                self.output_queue.put(job)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   631
                continue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   632
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   633
            # Otherwise go about the business of running tests as normal.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   634
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   635
            else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   636
                if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   637
                    print "Launch %s" % job.shell_command
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   638
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   639
                if job.is_example:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   640
                    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   641
                    # If we have an example, the shell command is all we need to
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   642
                    # know.  It will be something like "examples/udp-echo"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   643
                    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   644
                    (job.returncode, standard_out) = run_job_synchronously(job.shell_command, job.cwd)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   645
                else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   646
                    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   647
                    # If we're a test suite, we need to provide a little more info
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   648
                    # to the test runner, specifically the base directory and temp
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   649
                    # file name
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   650
                    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   651
                    (job.returncode, standard_out) = run_job_synchronously(job.shell_command + " --basedir=%s --out=%s" %
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   652
                        (job.basedir, job.tmp_file_name), job.cwd)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   653
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   654
                if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   655
                    print standard_out
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   656
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   657
                self.output_queue.put(job)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   658
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   659
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   660
# This is the main function that does the work of interacting with the test-runner
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   661
# itself.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   662
#
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   663
def run_tests():
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   664
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   665
    # Run waf to make sure that everything is built, configured and ready to go
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   666
    # unless we are explicitly told not to.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   667
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   668
    if options.nowaf == False:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   669
        proc = subprocess.Popen("./waf", shell=True)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   670
        proc.communicate()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   671
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   672
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   673
    # Pull some interesting configuration information out of waf, primarily
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   674
    # so we can know where executables can be found, but also to tell us what
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   675
    # pieces of the system have been built.  This will tell us what examples 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   676
    # are runnable.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   677
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   678
    read_waf_active_variant()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   679
    read_waf_config()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   680
    make_library_path()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   681
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   682
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   683
    # There are a couple of options that imply we can to exit before starting
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   684
    # up a bunch of threads and running tests.  Let's detect these cases and 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   685
    # handle them without doing all of the hard work.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   686
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   687
    if options.kinds:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   688
        (rc, standard_out) = run_job_synchronously("utils/test-runner --kinds", os.getcwd())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   689
        print standard_out
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   690
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   691
    if options.list:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   692
        (rc, standard_out) = run_job_synchronously("utils/test-runner --list", os.getcwd())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   693
        print standard_out
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   694
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   695
    if options.kinds or options.list:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   696
        return
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   697
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   698
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   699
    # We communicate results in two ways.  First, a simple message relating 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   700
    # PASS, FAIL, or SKIP is always written to the standard output.  It is 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   701
    # expected that this will be one of the main use cases.  A developer can
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   702
    # just run test.py with no options and see that all of the tests still 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   703
    # pass.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   704
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   705
    # The second main use case is when detailed status is requested (with the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   706
    # --text or --html options).  Typicall this will be text if a developer
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   707
    # finds a problem, or HTML for nightly builds.  In these cases, an
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   708
    # XML file is written containing the status messages from the test suites.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   709
    # This file is then read and translated into text or HTML.  It is expected
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   710
    # that nobody will really be interested in the XML, so we write it to 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   711
    # somewhere in /tmp with a random name to avoid collisions.  Just in case 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   712
    # some strange once-in-a-lifetime error occurs, we always write the info
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   713
    # so it can be found, we just may not use it.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   714
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   715
    # When we run examples as smoke tests, they are going to want to create
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   716
    # lots and lots of trace files.  We aren't really interested in the contents
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   717
    # of the trace files, so we also just stash them off in /tmp somewhere.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   718
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   719
    if not os.path.exists(TMP_OUTPUT_DIR):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   720
        os.makedirs(TMP_OUTPUT_DIR)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   721
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   722
    if not os.path.exists(TMP_TRACES_DIR):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   723
        os.makedirs(TMP_TRACES_DIR)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   724
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   725
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   726
    # Create the main output file and start filling it with XML.  We need to 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   727
    # do this since the tests will just append individual results to this file.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   728
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   729
    xml_results_file = TMP_OUTPUT_DIR + "%d.xml" % random.randint(0, sys.maxint)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   730
    f = open(xml_results_file, 'w')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   731
    f.write('<?xml version="1.0"?>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   732
    f.write('<TestResults>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   733
    f.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   734
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   735
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   736
    # We need to figure out what test suites to execute.  We are either given one 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   737
    # suite or example explicitly via the --suite or --example option, or we
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   738
    # need to call into the test runner and ask it to list all of the available
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   739
    # test suites.  Further, we need to provide the constraint information if it
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   740
    # has been given to us.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   741
    # 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   742
    # This translates into allowing the following options with respect to the 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   743
    # suites
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   744
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   745
    #  ./test,py:                                           run all of the suites
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   746
    #  ./test.py --constrain=unit:                          run all unit suites
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   747
    #  ./test,py --suite=some-test-suite:                   run the single suite
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   748
    #  ./test,py --example=udp-echo:                        run no test suites
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   749
    #  ./test,py --suite=some-suite --example=some-example: run the single suite
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   750
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   751
    # We can also use the --constrain option to provide an ordering of test 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   752
    # execution quite easily.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   753
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   754
    if len(options.suite):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   755
        suites = options.suite + "\n"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   756
    elif len(options.example) == 0:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   757
        if len(options.constrain):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   758
            (rc, suites) = run_job_synchronously("utils/test-runner --list --constrain=%s" % options.constrain, os.getcwd())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   759
        else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   760
            (rc, suites) = run_job_synchronously("utils/test-runner --list", os.getcwd())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   761
    else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   762
        suites = ""
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   763
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   764
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   765
    # suite_list will either a single test suite name that the user has 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   766
    # indicated she wants to run or a list of test suites provided by
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   767
    # the test-runner possibly according to user provided constraints.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   768
    # We go through the trouble of setting up the parallel execution 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   769
    # even in the case of a single suite to avoid having two process the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   770
    # results in two different places.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   771
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   772
    suite_list = suites.split('\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   773
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   774
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   775
    # We now have a possibly large number of test suites to run, so we want to
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   776
    # run them in parallel.  We're going to spin up a number of worker threads
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   777
    # that will run our test jobs for us.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   778
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   779
    input_queue = Queue.Queue(0)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   780
    output_queue = Queue.Queue(0)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   781
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   782
    jobs = 0
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   783
    threads=[]
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   784
5273
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   785
    #
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   786
    # In Python 2.6 you can just use multiprocessing module, but we don't want
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   787
    # to introduce that dependency yet; so we jump through a few hoops.
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   788
    #
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   789
    processors = 1
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   790
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   791
    if 'SC_NPROCESSORS_ONLN'in os.sysconf_names:
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   792
        processors = os.sysconf('SC_NPROCESSORS_ONLN')
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   793
    else:
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   794
        proc = subprocess.Popen("sysctl -n hw.ncpu", shell = True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   795
        stdout_results, stderr_results = proc.communicate()
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   796
        if len(stderr_results) == 0:
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   797
            processors = int(stdout_results)
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   798
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   799
    #
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   800
    # Now, spin up one thread per processor which will eventually mean one test
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   801
    # per processor running concurrently.
c95bd39cc980 Remove multiprocessing dependency in test.py
Craig Dowell <craigdo@ee.washington.edu>
parents: 5257
diff changeset
   802
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   803
    for i in range(processors):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   804
        thread = worker_thread(input_queue, output_queue)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   805
        threads.append(thread)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   806
        thread.start()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   807
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   808
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   809
    # We now have worker threads spun up, and a list of work to do.  So, run 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   810
    # through the list of test suites and dispatch a job to run each one.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   811
    # 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   812
    # Dispatching will run with unlimited speed and the worker threads will 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   813
    # execute as fast as possible from the queue.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   814
    #
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   815
    total_tests = 0
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   816
    for test in suite_list:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   817
        if len(test):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   818
            job = Job()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   819
            job.set_is_example(False)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   820
            job.set_display_name(test)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   821
            job.set_tmp_file_name(TMP_OUTPUT_DIR + "%d" % random.randint(0, sys.maxint))
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   822
            job.set_cwd(os.getcwd())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   823
            job.set_basedir(os.getcwd())
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   824
            if (options.multiple):
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   825
                multiple = " --multiple"
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   826
            else:
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   827
                multiple = ""
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   828
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
   829
            job.set_shell_command("utils/test-runner --suite='%s'%s" % (test, multiple))
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   830
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   831
            if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   832
                print "Queue %s" % test
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   833
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   834
            input_queue.put(job)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   835
            jobs = jobs + 1
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   836
            total_tests = total_tests + 1
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   837
    
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   838
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   839
    # We've taken care of the discovered or specified test suites.  Now we
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   840
    # have to deal with examples run as smoke tests.  We have a list of all of
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   841
    # the example programs it makes sense to try and run.  Each example will
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   842
    # have a condition associated with it that must evaluate to true for us
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   843
    # to try and execute it.  This is used to determine if the example has
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   844
    # a dependency that is not satisfied.  For example, if an example depends
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   845
    # on NSC being configured by waf, that example should have a condition
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   846
    # that evaluates to true if NSC is enabled.  For example,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   847
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   848
    #      ("tcp-nsc-zoo", "ENABLE_NSC == True"),
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   849
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   850
    # In this case, the example "tcp-nsc-zoo" will only be run if we find the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   851
    # waf configuration variable "ENABLE_NSC" to be True.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   852
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   853
    # We don't care at all how the trace files come out, so we just write them 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   854
    # to a single temporary directory.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   855
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   856
    # XXX As it stands, all of the trace files have unique names, and so file
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   857
    # collisions can only happen if two instances of an example are running in
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   858
    # two versions of the test.py process concurrently.  We may want to create
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   859
    # uniquely named temporary traces directories to avoid this problem.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   860
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   861
    # We need to figure out what examples to execute.  We are either given one 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   862
    # suite or example explicitly via the --suite or --example option, or we
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   863
    # need to walk the list of examples looking for available example 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   864
    # conditions.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   865
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   866
    # This translates into allowing the following options with respect to the 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   867
    # suites
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   868
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   869
    #  ./test,py:                                           run all of the examples
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   870
    #  ./test.py --constrain=unit                           run no examples
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   871
    #  ./test.py --constrain=example                       run all of the examples
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   872
    #  ./test,py --suite=some-test-suite:                   run no examples
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   873
    #  ./test,py --example=some-example:                    run the single example
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   874
    #  ./test,py --suite=some-suite --example=some-example: run the single example
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   875
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   876
    # XXX could use constrain to separate out examples used for performance 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   877
    # testing
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   878
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   879
    if len(options.suite) == 0 and len(options.example) == 0:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   880
        if len(options.constrain) == 0 or options.constrain == "example":
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   881
            for test, condition in example_tests:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   882
                if eval(condition) == True:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   883
                    job = Job()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   884
                    job.set_is_example(True)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   885
                    job.set_display_name(test)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   886
                    job.set_tmp_file_name("")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   887
                    job.set_cwd(TMP_TRACES_DIR)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   888
                    job.set_basedir(os.getcwd())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   889
                    job.set_shell_command("examples/%s" % test)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   890
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   891
                    if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   892
                        print "Queue %s" % test
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   893
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   894
                    input_queue.put(job)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   895
                    jobs = jobs + 1
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   896
                    total_tests = total_tests + 1
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   897
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   898
    elif len(options.example):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   899
        #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   900
        # If you tell me to run an example, I will try and run the example
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   901
        # irrespective of any condition.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   902
        #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   903
        job = Job()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   904
        job.set_is_example(True)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   905
        job.set_display_name(options.example)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   906
        job.set_tmp_file_name("")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   907
        job.set_cwd(TMP_TRACES_DIR)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   908
        job.set_basedir(os.getcwd())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   909
        job.set_shell_command("examples/%s" % options.example)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   910
        
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   911
        if options.verbose:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   912
            print "Queue %s" % test
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   913
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   914
        input_queue.put(job)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   915
        jobs = jobs + 1
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   916
        total_tests = total_tests + 1
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   917
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   918
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   919
    # Tell the worker threads to pack up and go home for the day.  Each one
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   920
    # will exit when they see their is_break task.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   921
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   922
    for i in range(processors):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   923
        job = Job()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   924
        job.set_is_break(True)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   925
        input_queue.put(job)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   926
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   927
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   928
    # Now all of the tests have been dispatched, so all we have to do here
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   929
    # in the main thread is to wait for them to complete.  Keyboard interrupt
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   930
    # handling is broken as mentioned above.  We use a signal handler to catch
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   931
    # sigint and set a global variable.  When the worker threads sense this
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   932
    # they stop doing real work and will just start throwing jobs back at us
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   933
    # with is_break set to True.  In this case, there are no real results so we 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   934
    # ignore them.  If there are real results, we always print PASS or FAIL to
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   935
    # standard out as a quick indication of what happened.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   936
    #
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   937
    passed_tests = 0
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   938
    failed_tests = 0
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   939
    crashed_tests = 0
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   940
    for i in range(jobs):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   941
        job = output_queue.get()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   942
        if job.is_break:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   943
            continue
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   944
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   945
        if job.is_example:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   946
            kind = "Example"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   947
        else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   948
            kind = "TestSuite"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   949
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   950
        if job.returncode == 0:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   951
            status = "PASS"
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   952
            passed_tests = passed_tests + 1
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   953
        elif job.returncode == 1:
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   954
            failed_tests = failed_tests + 1
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   955
            status = "FAIL"
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   956
        else:
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   957
            crashed_tests = crashed_tests + 1
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
   958
            status = "CRASH"
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   959
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   960
        print "%s: %s %s" % (status, kind, job.display_name)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   961
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   962
        if job.is_example == True:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   963
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   964
            # Examples are the odd man out here.  They are written without any
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   965
            # knowledge that they are going to be run as a test, so we need to 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   966
            # cook up some kind of output for them.  We're writing an xml file,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   967
            # so we do some simple XML that says we ran the example.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   968
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   969
            # XXX We could add some timing information to the examples, i.e. run
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   970
            # them through time and print the results here.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   971
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   972
            f = open(xml_results_file, 'a')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   973
            f.write('<Example>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   974
            example_name = "  <Name>%s</Name>\n" % job.display_name
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   975
            f.write(example_name)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   976
            if job.returncode == 0:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   977
                f.write('  <Result>PASS</Result>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   978
            elif job.returncode == 1:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   979
                f.write('  <Result>FAIL</Result>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   980
            else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   981
                f.write('  <Result>CRASH</Result>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   982
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   983
            f.write('</Example>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   984
            f.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   985
        else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   986
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   987
            # If we're not running an example, we're running a test suite.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   988
            # These puppies are running concurrently and generating output
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   989
            # that was written to a temporary file to avoid collisions.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   990
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   991
            # Now that we are executing sequentially in the main thread, we can
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   992
            # concatenate the contents of the associated temp file to the main 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   993
            # results file and remove that temp file.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   994
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   995
            # One thing to consider is that a test suite can crash just as
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   996
            # well as any other program, so we need to deal with that 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   997
            # possibility as well.  If it ran correctly it will return 0
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   998
            # if it passed, or 1 if it failed.  In this case, we can count
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   999
            # on the results file it saved being complete.  If it crashed, it 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1000
            # will return some other code, and the file should be considered 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1001
            # corrupt and useless.  If the suite didn't create any XML, then
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1002
            # we're going to have to do it ourselves.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1003
            #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1004
            if job.returncode == 0 or job.returncode == 1:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1005
                f_to = open(xml_results_file, 'a')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1006
                f_from = open(job.tmp_file_name, 'r')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1007
                f_to.write(f_from.read())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1008
                f_to.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1009
                f_from.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1010
            else:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1011
                f = open(xml_results_file, 'a')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1012
                f.write("<TestSuite>\n")
5239
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1013
                f.write("  <SuiteName>%s</SuiteName>\n" % job.display_name)
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1014
                f.write('  <SuiteResult>CRASH</SuiteResult>\n')
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1015
                f.write('  <SuiteTime>Execution times not available</SuiteTime>\n')
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1016
                f.write("</TestSuite>\n")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1017
                f.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1018
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1019
            os.remove(job.tmp_file_name)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1020
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1021
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1022
    # We have all of the tests run and the results written out.  One final 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1023
    # bit of housekeeping is to wait for all of the threads to close down
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1024
    # so we can exit gracefully.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1025
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1026
    for thread in threads:
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1027
        thread.join()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1028
    
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1029
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1030
    # Back at the beginning of time, we started the body of an XML document
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1031
    # since the test suites and examples were going to just write their 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1032
    # individual pieces.  So, we need to finish off and close out the XML 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1033
    # document
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1034
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1035
    f = open(xml_results_file, 'a')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1036
    f.write('</TestResults>\n')
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1037
    f.close()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1038
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1039
    #
5279
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1040
    # Print a quick summary of events
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1041
    #
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1042
    print "%d of %d tests passed (%d passed, %d failed, %d crashed)" % (passed_tests, total_tests, passed_tests, 
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1043
                                                                        failed_tests, crashed_tests)
1aec8ea805d2 update tcp interop response vectors
Craig Dowell <craigdo@ee.washington.edu>
parents: 5275
diff changeset
  1044
    #
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1045
    # The last things to do are to translate the XML results file to "human
5239
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1046
    # readable form" if the user asked for it (or make an XML file somewhere)
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1047
    #
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1048
    if len(options.html):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1049
        translate_to_html(xml_results_file, options.html)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1050
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1051
    if len(options.text):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1052
        translate_to_text(xml_results_file, options.text)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1053
5239
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1054
    if len(options.xml):
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1055
        shutil.copyfile(xml_results_file, options.xml)
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1056
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1057
def main(argv):
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1058
    random.seed()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1059
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1060
    parser = optparse.OptionParser()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1061
    parser.add_option("-c", "--constrain", action="store", type="string", dest="constrain", default="",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1062
                      metavar="KIND",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1063
                      help="constrain the test-runner by kind of test")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1064
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1065
    parser.add_option("-e", "--example", action="store", type="string", dest="example", default="",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1066
                      metavar="EXAMPLE",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1067
                      help="specify a single example to run")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1068
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1069
    parser.add_option("-k", "--kinds", action="store_true", dest="kinds", default=False,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1070
                      help="print the kinds of tests available")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1071
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1072
    parser.add_option("-l", "--list", action="store_true", dest="list", default=False,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1073
                      help="print the list of known tests")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1074
5324
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
  1075
    parser.add_option("-m", "--multiple", action="store_true", dest="multiple", default=False,
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
  1076
                      help="report multiple failures from test suites and test cases")
0ba73cdd2a43 Refalgamize test framework to allow multiple test failures
Craig Dowell <craigdo@ee.washington.edu>
parents: 5295
diff changeset
  1077
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1078
    parser.add_option("-n", "--nowaf", action="store_true", dest="nowaf", default=False,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1079
                      help="do not run waf before starting testing")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1080
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1081
    parser.add_option("-s", "--suite", action="store", type="string", dest="suite", default="",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1082
                      metavar="TEST-SUITE",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1083
                      help="specify a single test suite to run")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1084
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1085
    parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1086
                      help="print progress and informational messages")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1087
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1088
    parser.add_option("-w", "--web", "--html", action="store", type="string", dest="html", default="",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1089
                      metavar="HTML-FILE",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1090
                      help="write detailed test results into HTML-FILE.html")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1091
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1092
    parser.add_option("-t", "--text", action="store", type="string", dest="text", default="",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1093
                      metavar="TEXT-FILE",
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1094
                      help="write detailed test results into TEXT-FILE.txt")
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1095
5239
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1096
    parser.add_option("-x", "--xml", action="store", type="string", dest="xml", default="",
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1097
                      metavar="XML-FILE",
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1098
                      help="write detailed test results into XML-FILE.xml")
2e753de86174 Remove and replace attribute unit tests (bug 675)
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
  1099
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1100
    global options
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1101
    options = parser.parse_args()[0]
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1102
    signal.signal(signal.SIGINT, sigint_hook)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1103
    run_tests()
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1104
    return 0
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1105
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1106
if __name__ == '__main__':
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
  1107
    sys.exit(main(sys.argv))