#! /usr/bin/env python

## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-

#

# Copyright (c) 2009 University of Washington

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License version 2 as

# published by the Free Software Foundation;

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

#

import os

import sys

import time

import optparse

import subprocess

import threading

import Queue

import signal

import xml.dom.minidom

import shutil

import re

#

# XXX This should really be part of a waf command to list the configuration

# items relative to optional ns-3 pieces.

#

# A list of interesting configuration items in the waf configuration 

# cache which we may be interested in when deciding on which examples

# to run and how to run them.  These are set by waf during the 

# configuration phase and the corresponding assignments are usually

# found in the associated subdirectory wscript files.

#

interesting_config_items = [

    "NS3_BUILDDIR",

    "NS3_MODULE_PATH",

    "ENABLE_NSC",

    "ENABLE_REAL_TIME",

    "ENABLE_EXAMPLES",

]

ENABLE_NSC = False

ENABLE_REAL_TIME = False

ENABLE_EXAMPLES = True

#

# If the user has constrained us to run certain kinds of tests, we can tell waf

# to only build

#

core_kinds = ["bvt", "core", "system", "unit"]

#

# There are some special cases for test suites that kill valgrind.  This is

# because NSC causes illegal instruction crashes when run under valgrind.

#

core_valgrind_skip_tests = [

    "ns3-tcp-cwnd",

    "ns3-tcp-interoperability",

]

#

# A list of examples to run as smoke tests just to ensure that they remain 

# buildable and runnable over time.  Also a condition under which to run

# the example (from the waf configuration), and a condition under which to

# run the example under valgrind.  This is because NSC causes illegal 

# instruction crashes when run under valgrind.

#

# XXX Should this not be read from a configuration file somewhere and not

# hardcoded.

#

example_tests = [

    ("csma/csma-bridge", "True", "True"),

    ("csma/csma-bridge-one-hop", "True", "True"),

    ("csma/csma-broadcast", "True", "True"),

    ("csma/csma-multicast", "True", "True"),

    ("csma/csma-one-subnet", "True", "True"),

    ("csma/csma-packet-socket", "True", "True"),

    ("csma/csma-ping", "True", "True"),

    ("csma/csma-raw-ip-socket", "True", "True"),

    ("csma/csma-star", "True", "True"),

    ("emulation/emu-ping", "False", "True"),

    ("emulation/emu-udp-echo", "False", "True"),

    ("error-model/simple-error-model", "True", "True"),

    ("ipv6/icmpv6-redirect", "True", "True"),

    ("ipv6/ping6", "True", "True"),

    ("ipv6/radvd", "True", "True"),

    ("ipv6/radvd-two-prefix", "True", "True"),    

    ("ipv6/test-ipv6", "True", "True"),

    ("mesh/mesh", "True", "True"),

    ("naming/object-names", "True", "True"),

    ("realtime/realtime-udp-echo", "ENABLE_REAL_TIME == True", "True"),

    ("routing/dynamic-global-routing", "True", "True"),

    ("routing/global-injection-slash32", "True", "True"),

    ("routing/global-routing-slash32", "True", "True"),

    ("routing/mixed-global-routing", "True", "True"),

    ("routing/nix-simple", "True", "True"),

    ("routing/nms-p2p-nix", "False", "True"), # Takes too long to run

    ("routing/simple-alternate-routing", "True", "True"),

    ("routing/simple-global-routing", "True", "True"),

    ("routing/simple-point-to-point-olsr", "True", "True"),

    ("routing/simple-routing-ping6", "True", "True"),

    ("routing/static-routing-slash32", "True", "True"),

    ("routing/aodv", "True", "True"),

    ("stats/wifi-example-sim", "True", "True"),

    ("tap/tap-wifi-dumbbell", "False", "True"), # Requires manual configuration

    ("tcp/star", "True", "True"),

    ("tcp/tcp-large-transfer", "True", "True"),

    ("tcp/tcp-nsc-lfn", "ENABLE_NSC == True", "True"),

    ("tcp/tcp-nsc-zoo", "ENABLE_NSC == True", "True"),

    ("tcp/tcp-star-server", "True", "True"),

    ("tunneling/virtual-net-device", "True", "True"),

    ("tutorial/first", "True", "True"),

    ("tutorial/hello-simulator", "True", "True"),

    ("tutorial/second", "True", "True"),

    ("tutorial/third", "True", "True"),

    ("tutorial/fourth", "True", "True"),

    ("tutorial/fifth", "True", "True"),

    ("tutorial/sixth", "True", "True"),

    ("udp/udp-echo", "True", "True"),

    ("wireless/mixed-wireless", "True", "True"),

    ("wireless/multirate", "False", "True"), # Takes too long to run

    ("wireless/simple-wifi-frame-aggregation", "True", "True"),

    ("wireless/wifi-adhoc", "False", "True"), # Takes too long to run

    ("wireless/wifi-ap --verbose=0", "True", "True"), # Don't let it spew to stdout

    ("wireless/wifi-clear-channel-cmu", "False", "True"), # Requires specific hardware

    ("wireless/wifi-simple-adhoc", "True", "True"),

    ("wireless/wifi-simple-adhoc-grid", "True", "True"),

    ("wireless/wifi-simple-infra", "True", "True"),

    ("wireless/wifi-simple-interference", "True", "True"),

    ("wireless/wifi-wired-bridging", "True", "True"),

    ("wimax/wimax-simple", "True", "True"),

    ("wimax/wimax-ipv4", "True", "True"),

    ("wimax/wimax-multicast", "True", "True"),

]

#

# The test suites are going to want to output status.  They are running

# concurrently.  This means that unless we are careful, the output of

# the test suites will be interleaved.  Rather than introducing a lock

# file that could unintentionally start serializing execution, we ask

# the tests to write their output to a temporary directory and then 

# put together the final output file when we "join" the test tasks back

# to the main thread.  In addition to this issue, the example programs

# often write lots and lots of trace files which we will just ignore.

# We put all of them into the temp directory as well, so they can be

# easily deleted.

#

TMP_OUTPUT_DIR = "testpy-output"

def get_node_text(node):

    for child in node.childNodes:

        if child.nodeType == child.TEXT_NODE:

            return child.nodeValue

    return "None"

#

# A simple example of writing a text file with a test result summary.  It is 

# expected that this output will be fine for developers looking for problems.

#

def translate_to_text(results_file, text_file):

    f = open(text_file, 'w')

    dom = xml.dom.minidom.parse(results_file)

    for suite in dom.getElementsByTagName("TestSuite"):

        result = get_node_text(suite.getElementsByTagName("SuiteResult")[0])

        name = get_node_text(suite.getElementsByTagName("SuiteName")[0])

        time = get_node_text(suite.getElementsByTagName("SuiteTime")[0])

        output = "%s: Test Suite \"%s\" (%s)\n" % (result, name, time)

        f.write(output)

        if result != "CRASH":

            for case in suite.getElementsByTagName("TestCase"):

                result = get_node_text(case.getElementsByTagName("CaseResult")[0])

                name = get_node_text(case.getElementsByTagName("CaseName")[0])

                time = get_node_text(case.getElementsByTagName("CaseTime")[0])

                output =   "  %s: Test Case \"%s\" (%s)\n" % (result, name, time)

                f.write(output)

                if result == "FAIL":

                    for details in case.getElementsByTagName("FailureDetails"):

                        f.write("    Details:\n")

                        f.write("      Message:   %s\n" % get_node_text(details.getElementsByTagName("Message")[0]))

                        f.write("      Condition: %s\n" % get_node_text(details.getElementsByTagName("Condition")[0]))

                        f.write("      Actual:    %s\n" % get_node_text(details.getElementsByTagName("Actual")[0]))

                        f.write("      Limit:     %s\n" % get_node_text(details.getElementsByTagName("Limit")[0]))

                        f.write("      File:      %s\n" % get_node_text(details.getElementsByTagName("File")[0]))

                        f.write("      Line:      %s\n" % get_node_text(details.getElementsByTagName("Line")[0]))

    for example in dom.getElementsByTagName("Example"):

        result = get_node_text(example.getElementsByTagName("Result")[0])

        name = get_node_text(example.getElementsByTagName("Name")[0])

        time = get_node_text(example.getElementsByTagName("ElapsedTime")[0])

        output = "%s: Example \"%s\" (%s)\n" % (result, name, time)

        f.write(output)

    f.close()

#

# A simple example of writing an HTML file with a test result summary.  It is 

# expected that this will eventually be made prettier as time progresses and

# we have time to tweak it.  This may end up being moved to a separate module

# since it will probably grow over time.

#

def translate_to_html(results_file, html_file):

    f = open(html_file, 'w')

    f.write("<html>\n")

    f.write("<body>\n")

    f.write("<center><h1>ns-3 Test Results</h1></center>\n")

    #

    # Read and parse the whole results file.

    #

    dom = xml.dom.minidom.parse(results_file)

    #

    # Iterate through the test suites

    #

    f.write("<h2>Test Suites</h2>\n")

    for suite in dom.getElementsByTagName("TestSuite"):

        #

        # For each test suite, get its name, result and execution time info

        #

        name = get_node_text(suite.getElementsByTagName("SuiteName")[0])

        result = get_node_text(suite.getElementsByTagName("SuiteResult")[0])

        time = get_node_text(suite.getElementsByTagName("SuiteTime")[0])

        # 

        # Print a level three header with the result, name and time.  If the 

        # test suite passed, the header is printed in green. If the suite was

        # skipped, print it in orange, otherwise assume something bad happened

        # and print in red.

        #

        if result == "PASS":

            f.write("<h3 style=\"color:green\">%s: %s (%s)</h3>\n" % (result, name, time))

        elif result == "SKIP":

            f.write("<h3 style=\"color:#ff6600\">%s: %s (%s)</h3>\n" % (result, name, time))

        else:

            f.write("<h3 style=\"color:red\">%s: %s (%s)</h3>\n" % (result, name, time))

        #

        # The test case information goes in a table.

        #

        f.write("<table border=\"1\">\n")

        #

        # The first column of the table has the heading Result

        #

        f.write("<th> Result </th>\n")

        #

        # If the suite crashed or is skipped, there is no further information, so just

        # delare a new table row with the result (CRASH or SKIP) in it.  Looks like:

        #

        #   +--------+

        #   | Result |

        #   +--------+

        #   | CRASH  |

        #   +--------+

        #

        # Then go on to the next test suite.  Valgrind and skipped errors look the same.

        #

        if result in ["CRASH", "SKIP", "VALGR"]:

            f.write("<tr>\n")

            if result == "SKIP":

                f.write("<td style=\"color:#ff6600\">%s</td>\n" % result)

            else:

                f.write("<td style=\"color:red\">%s</td>\n" % result)

            f.write("</tr>\n")

            f.write("</table>\n")

            continue

        #

        # If the suite didn't crash, we expect more information, so fill out

        # the table heading row.  Like,

        #

        #   +--------+----------------+------+

        #   | Result | Test Case Name | Time |

        #   +--------+----------------+------+

        #

        f.write("<th>Test Case Name</th>\n")

        f.write("<th> Time </th>\n")

        #

        # If the test case failed, we need to print out some failure details

        # so extend the heading row again.  Like,

        #

        #   +--------+----------------+------+-----------------+

        #   | Result | Test Case Name | Time | Failure Details |

        #   +--------+----------------+------+-----------------+

        #

        if result == "FAIL":

            f.write("<th>Failure Details</th>\n")

        #

        # Now iterate through all of the test cases.

        #

        for case in suite.getElementsByTagName("TestCase"):

            #

            # Get the name, result and timing information from xml to use in

            # printing table below.

            #

            name = get_node_text(case.getElementsByTagName("CaseName")[0])

            result = get_node_text(case.getElementsByTagName("CaseResult")[0])

            time = get_node_text(case.getElementsByTagName("CaseTime")[0])

            #

            # If the test case failed, we iterate through possibly multiple

            # failure details

            #

            if result == "FAIL":

                #

                # There can be multiple failures for each test case.  The first

                # row always gets the result, name and timing information along

                # with the failure details.  Remaining failures don't duplicate

                # this information but just get blanks for readability.  Like,

                #

                #   +--------+----------------+------+-----------------+

                #   | Result | Test Case Name | Time | Failure Details |

                #   +--------+----------------+------+-----------------+

                #   |  FAIL  | The name       | time | It's busted     |   

                #   +--------+----------------+------+-----------------+

                #   |        |                |      | Really broken   |   

                #   +--------+----------------+------+-----------------+

                #   |        |                |      | Busted bad      |   

                #   +--------+----------------+------+-----------------+

                #

                first_row = True

                for details in case.getElementsByTagName("FailureDetails"):

                    #

                    # Start a new row in the table for each possible Failure Detail

                    #

                    f.write("<tr>\n")

                    if first_row:

                        first_row = False

                        f.write("<td style=\"color:red\">%s</td>\n" % result)

                        f.write("<td>%s</td>\n" % name)

                        f.write("<td>%s</td>\n" % time)

                    else:

                        f.write("<td></td>\n")

                        f.write("<td></td>\n")

                        f.write("<td></td>\n")

                    f.write("<td>")

                    f.write("<b>Message: </b>%s, " % get_node_text(details.getElementsByTagName("Message")[0]))

                    f.write("<b>Condition: </b>%s, " % get_node_text(details.getElementsByTagName("Condition")[0]))

                    f.write("<b>Actual: </b>%s, " % get_node_text(details.getElementsByTagName("Actual")[0]))

                    f.write("<b>Limit: </b>%s, " % get_node_text(details.getElementsByTagName("Limit")[0]))

                    f.write("<b>File: </b>%s, " % get_node_text(details.getElementsByTagName("File")[0]))

                    f.write("<b>Line: </b>%s" % get_node_text(details.getElementsByTagName("Line")[0]))

                    f.write("</td>\n")

                    #

                    # End the table row

                    #

                    f.write("</td>\n")

            else:

                #

                # If this particular test case passed, then we just print the PASS

                # result in green, followed by the test case name and its execution

                # time information.  These go off in <td> ... </td> table data.

                # The details table entry is left blank.

                #

                #   +--------+----------------+------+---------+

                #   | Result | Test Case Name | Time | Details |

                #   +--------+----------------+------+---------+

                #   |  PASS  | The name       | time |         |   

                #   +--------+----------------+------+---------+

                #

                f.write("<tr>\n")

                f.write("<td style=\"color:green\">%s</td>\n" % result)

                f.write("<td>%s</td>\n" % name)

                f.write("<td>%s</td>\n" % time)

                f.write("<td></td>\n")

                f.write("</tr>\n")

        #

        # All of the rows are written, so we need to end the table.

        #

        f.write("</table>\n")

    #

    # That's it for all of the test suites.  Now we have to do something about 

    # our examples.

    #

    f.write("<h2>Examples</h2>\n")

    #

    # Example status is rendered in a table just like the suites.

    #

    f.write("<table border=\"1\">\n")

    #

    # The table headings look like,

    #

    #   +--------+--------------+--------------+

    #   | Result | Example Name | Elapsed Time |

    #   +--------+--------------+--------------+

    #

    f.write("<th> Result </th>\n")

    f.write("<th>Example Name</th>\n")

    f.write("<th>Elapsed Time</th>\n")

    #

    # Now iterate through all of the examples

    #

    for example in dom.getElementsByTagName("Example"):

        #

        # Start a new row for each example

        #

        f.write("<tr>\n")

        #

        # Get the result and name of the example in question

        #

        result = get_node_text(example.getElementsByTagName("Result")[0])

        name =   get_node_text(example.getElementsByTagName("Name")[0])

        time =   get_node_text(example.getElementsByTagName("ElapsedTime")[0])

        #

        # If the example either failed or crashed, print its result status

        # in red; otherwise green.  This goes in a <td> ... </td> table data

        #

        if result == "PASS":

            f.write("<td style=\"color:green\">%s</td>\n" % result)

        elif result == "SKIP":

            f.write("<td style=\"color:#ff6600\">%s</fd>\n" % result)

        else:

            f.write("<td style=\"color:red\">%s</td>\n" % result)

        #

        # Write the example name as a new tag data.

        #

        f.write("<td>%s</td>\n" % name)

        #

        # Write the elapsed time as a new tag data.

        #

        f.write("<td>%s</td>\n" % time)

        #

        # That's it for the current example, so terminate the row.

        #

        f.write("</tr>\n")

    #

    # That's it for the table of examples, so terminate the table.

    #

    f.write("</table>\n")

    #

    # And that's it for the report, so finish up.

    #

    f.write("</body>\n")

    f.write("</html>\n")

    f.close()

#

# Python Control-C handling is broken in the presence of multiple threads.  

# Signals get delivered to the runnable/running thread by default and if 

# it is blocked, the signal is simply ignored.  So we hook sigint and set 

# a global variable telling the system to shut down gracefully.

#

thread_exit = False

def sigint_hook(signal, frame):

    global thread_exit

    thread_exit = True

    return 0