#! /usr/bin/env python

## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-

#

# Copyright (c) 2009 University of Washington

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License version 2 as

# published by the Free Software Foundation;

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

#

import os

import sys

import optparse

import subprocess

import threading

import Queue

import signal

import random

import xml.dom.minidom

import shutil

#

# XXX This should really be part of a waf command to list the configuration

# items relative to optional ns-3 pieces.

#

# A list of interesting configuration items in the waf configuration 

# cache which we may be interested in when deciding on which examples

# to run and how to run them.  These are set by waf during the 

# configuration phase and the corresponding assignments are usually

# found in the associated subdirectory wscript files.

#

interesting_config_items = [

    "NS3_BUILDDIR",

    "NS3_MODULE_PATH",

    "ENABLE_NSC",

    "ENABLE_REAL_TIME",

]

ENABLE_NSC = False

ENABLE_REAL_TIME = False

#

# A list of examples to run as smoke tests just to ensure that they remain 

# buildable and runnable over time.  Also a condition under which to run

# the example (from the waf configuration).

#

# XXX Should this not be read from a configuration file somewhere and not

# hardcoded.

#

example_tests = [

]

#

# Most of the examples produce gangs of trace files, so we want to find

# somewhere to put them that won't pollute the current directory.  One

# obvious place is somewhere in /tmp.

#

TMP_TRACES_DIR = "/tmp/unchecked-traces"

#

# The test suites are going to want to output status.  They are running

# concurrently.  This means that unless we are careful, the output of

# the test suites will be interleaved.  Rather than introducing a lock

# file that could unintentionally start serializing execution, we ask

# the tests to write their output to a temporary directory and then 

# put together the final output file when we "join" the test tasks back

# to the main thread.

#

TMP_OUTPUT_DIR = "/tmp/testpy"

def get_node_text(node):

    for child in node.childNodes:

        if child.nodeType == child.TEXT_NODE:

            return child.nodeValue

    return "None"

#

# A simple example of writing a text file with a test result summary.  It is 

# expected that this output will be fine for developers looking for problems.

#

def translate_to_text(results_file, text_file):

    f = open(text_file, 'w')

    dom = xml.dom.minidom.parse(results_file)

    for suite in dom.getElementsByTagName("TestSuite"):

        result = get_node_text(suite.getElementsByTagName("SuiteResult")[0])

        name = get_node_text(suite.getElementsByTagName("SuiteName")[0])

        time = get_node_text(suite.getElementsByTagName("SuiteTime")[0])

        output = "%s: Test Suite \"%s\" (%s)\n" % (result, name, time)

        f.write(output)

        if result != "CRASH":

            for case in suite.getElementsByTagName("TestCase"):

                result = get_node_text(case.getElementsByTagName("CaseResult")[0])

                name = get_node_text(case.getElementsByTagName("CaseName")[0])

                time = get_node_text(case.getElementsByTagName("CaseTime")[0])

                output =   "  %s: Test Case \"%s\" (%s)\n" % (result, name, time)

                f.write(output)

                if result == "FAIL":

                    for details in case.getElementsByTagName("FailureDetails"):

                        f.write("    Details:\n")

                        f.write("      Message:   %s\n" % get_node_text(details.getElementsByTagName("Message")[0]))

                        f.write("      Condition: %s\n" % get_node_text(details.getElementsByTagName("Condition")[0]))

                        f.write("      Actual:    %s\n" % get_node_text(details.getElementsByTagName("Actual")[0]))

                        f.write("      Limit:     %s\n" % get_node_text(details.getElementsByTagName("Limit")[0]))

                        f.write("      File:      %s\n" % get_node_text(details.getElementsByTagName("File")[0]))

                        f.write("      Line:      %s\n" % get_node_text(details.getElementsByTagName("Line")[0]))

    for example in dom.getElementsByTagName("Example"):

        result = get_node_text(example.getElementsByTagName("Result")[0])

        name = get_node_text(example.getElementsByTagName("Name")[0])

        output = "%s: Example \"%s\"\n" % (result, name)

        f.write(output)

    f.close()

#

# A simple example of writing an HTML file with a test result summary.  It is 

# expected that this will eventually be made prettier as time progresses and

# we have time to tweak it.  This may end up being moved to a separate module

# since it will probably grow over time.

#

def translate_to_html(results_file, html_file):

    f = open(html_file, 'w')

    f.write("<html>\n")

    f.write("<body>\n")

    f.write("<center><h1>ns-3 Test Results</h1></center>\n")

    #

    # Read and parse the whole results file.

    #

    dom = xml.dom.minidom.parse(results_file)

    #

    # Iterate through the test suites

    #

    f.write("<h2>Test Suites</h2>\n")

    for suite in dom.getElementsByTagName("TestSuite"):

        #

        # For each test suite, get its name, result and execution time info

        #

        name = get_node_text(suite.getElementsByTagName("SuiteName")[0])

        result = get_node_text(suite.getElementsByTagName("SuiteResult")[0])

        time = get_node_text(suite.getElementsByTagName("SuiteTime")[0])

        # 

        # Print a level three header in green with the result, name and time.

        # If the test suite passed, the header is printed in green, otherwise

        # it is printed in red.

        #

        if result == "PASS":

            f.write("<h3 style=\"color:green\">%s: %s (%s)</h3>\n" % (result, name, time))

        else:

            f.write("<h3 style=\"color:red\">%s: %s (%s)</h3>\n" % (result, name, time))

        #

        # The test case information goes in a table.

        #

        f.write("<table border=\"1\">\n")

        #

        # The first column of the table has the heading Result

        #

        f.write("<th> Result </th>\n")

        #

        # If the suite crashed, there is no further information, so just

        # delare a new table row with the result (CRASH) in it.  Looks like:

        #

        #   +--------+

        #   | Result |

        #   +--------+

        #   | CRASH  |

        #   +--------+

        #

        # Then go on to the next test suite

        #

        if result == "CRASH":

            f.write("<tr>\n")

            f.write("<td style=\"color:red\">%s</td>\n" % result)

            f.write("</tr>\n")

            f.write("</table>\n")

            continue

        #

        # If the suite didn't crash, we expect more information, so fill out

        # the table heading row.  Like,

        #

        #   +--------+----------------+------+

        #   | Result | Test Case Name | Time |

        #   +--------+----------------+------+

        #

        f.write("<th>Test Case Name</th>\n")

        f.write("<th> Time </th>\n")

        #

        # If the test case failed, we need to print out some failure details

        # so extend the heading row again.  Like,

        #

        #   +--------+----------------+------+-----------------+

        #   | Result | Test Case Name | Time | Failure Details |

        #   +--------+----------------+------+-----------------+

        #

        if result == "FAIL":

            f.write("<th>Failure Details</th>\n")

        #

        # Now iterate through all of the test cases.

        #

        for case in suite.getElementsByTagName("TestCase"):

            #

            # Get the name, result and timing information from xml to use in

            # printing table below.

            #

            name = get_node_text(case.getElementsByTagName("CaseName")[0])

            result = get_node_text(case.getElementsByTagName("CaseResult")[0])

            time = get_node_text(case.getElementsByTagName("CaseTime")[0])

            #

            # If the test case failed, we iterate through possibly multiple

            # failure details

            #

            if result == "FAIL":

                #

                # There can be multiple failures for each test case.  The first

                # row always gets the result, name and timing information along

                # with the failure details.  Remaining failures don't duplicate

                # this information but just get blanks for readability.  Like,

                #

                #   +--------+----------------+------+-----------------+

                #   | Result | Test Case Name | Time | Failure Details |

                #   +--------+----------------+------+-----------------+

                #   |  FAIL  | The name       | time | It's busted     |   

                #   +--------+----------------+------+-----------------+

                #   |        |                |      | Really broken   |   

                #   +--------+----------------+------+-----------------+

                #   |        |                |      | Busted bad      |   

                #   +--------+----------------+------+-----------------+

                #

                first_row = True

                for details in case.getElementsByTagName("FailureDetails"):

                    #

                    # Start a new row in the table for each possible Failure Detail

                    #

                    f.write("<tr>\n")

                    if first_row:

                        first_row = False

                        f.write("<td style=\"color:red\">%s</td>\n" % result)

                        f.write("<td>%s</td>\n" % name)

                        f.write("<td>%s</td>\n" % time)

                    else:

                        f.write("<td></td>\n")

                        f.write("<td></td>\n")

                        f.write("<td></td>\n")

                    f.write("<td>")

                    f.write("<b>Message: </b>%s, " % get_node_text(details.getElementsByTagName("Message")[0]))

                    f.write("<b>Condition: </b>%s, " % get_node_text(details.getElementsByTagName("Condition")[0]))

                    f.write("<b>Actual: </b>%s, " % get_node_text(details.getElementsByTagName("Actual")[0]))

                    f.write("<b>Limit: </b>%s, " % get_node_text(details.getElementsByTagName("Limit")[0]))

                    f.write("<b>File: </b>%s, " % get_node_text(details.getElementsByTagName("File")[0]))

                    f.write("<b>Line: </b>%s" % get_node_text(details.getElementsByTagName("Line")[0]))

                    f.write("</td>\n")

                    #

                    # End the table row

                    #

                    f.write("</td>\n")

            else:

                #

                # If this particular test case passed, then we just print the PASS

                # result in green, followed by the test case name and its execution

                # time information.  These go off in <td> ... </td> table data.

                # The details table entry is left blank.

                #

                #   +--------+----------------+------+---------+

                #   | Result | Test Case Name | Time | Details |

                #   +--------+----------------+------+---------+

                #   |  PASS  | The name       | time |         |   

                #   +--------+----------------+------+---------+

                #

                f.write("<tr>\n")

                f.write("<td style=\"color:green\">%s</td>\n" % result)

                f.write("<td>%s</td>\n" % name)

                f.write("<td>%s</td>\n" % time)

                f.write("<td></td>\n")

                f.write("</tr>\n")

        #

        # All of the rows are written, so we need to end the table.

        #

        f.write("</table>\n")

    #

    # That's it for all of the test suites.  Now we have to do something about 

    # our examples.

    #

    f.write("<h2>Examples</h2>\n")

    #

    # Example status is rendered in a table just like the suites.

    #

    f.write("<table border=\"1\">\n")

    #

    # The table headings look like,

    #

    #   +--------+--------------+

    #   | Result | Example Name |

    #   +--------+--------------+

    #

    f.write("<th> Result </th>\n")

    f.write("<th>Example Name</th>\n")

    #

    # Now iterate through all of the examples

    #

    for example in dom.getElementsByTagName("Example"):

        #

        # Start a new row for each example

        #

        f.write("<tr>\n")

        #

        # Get the result and name of the example in question

        #

        result = get_node_text(example.getElementsByTagName("Result")[0])

        name =   get_node_text(example.getElementsByTagName("Name")[0])

        #

        # If the example eitehr failed or crashed, print its result status

        # in red; otherwise green.  This goes in a <td> ... </td> table data

        #

        if result in ["FAIL", "CRASH"]:

            f.write("<td style=\"color:red\">%s</td>\n" % result)

        else:

            f.write("<td style=\"color:green\">%s</td>\n" % result)

        #

        # Write the example name as a new tagle data.

        #

        f.write("<td>%s</td>\n" % name)

        #

        # That's it for the current example, so terminate the row.

        #

        f.write("</tr>\n")

    #

    # That's it for the table of examples, so terminate the table.

    #

    f.write("</table>\n")

    #

    # And that's it for the report, so finish up.

    #

    f.write("</body>\n")

    f.write("</html>\n")

    f.close()

#

# Python Control-C handling is broken in the presence of multiple threads.  

# Signals get delivered to the runnable/running thread by default and if 

# it is blocked, the signal is simply ignored.  So we hook sigint and set 

# a global variable telling the system to shut down gracefully.

#

thread_exit = False

def sigint_hook(signal, frame):

    global thread_exit

    thread_exit = True

    return 0

#

# Waf can be configured to compile in debug or optimized modes.  In each

# case, the resulting built goes into a different directory.  If we want

# test tests to run from the correct code-base, we have to figure out which

# mode waf is running in.  This is called its active variant.

#

# XXX This function pokes around in the waf internal state file.  To be a

# little less hacky, we should add a commmand to waf to return this info

# and use that result.

#

def read_waf_active_variant():

    for line in open("build/c4che/default.cache.py").readlines():

        if line.startswith("NS3_ACTIVE_VARIANT"):

            exec(line, globals())

            break

    if options.verbose:

        print "NS3_ACTIVE_VARIANT == %s" % NS3_ACTIVE_VARIANT

#

# In general, the build process itself naturally takes care of figuring out

# which tests are built into the test runner.  For example, if waf configure

# determines that ENABLE_EMU is false due to some missing dependency,

# the tests for the emu net device simply will not be built and will 

# therefore not be included in the built test runner.

#

# Examples, however, are a different story.  In that case, we are just given

# a list of examples that could be run.  Instead of just failing, for example,

# nsc-tcp-zoo if NSC is not present, we look into the waf saved configuration

# for relevant configuration items.  

#

# XXX This function pokes around in the waf internal state file.  To be a

# little less hacky, we should add a commmand to waf to return this info

# and use that result.

#

def read_waf_config():

    for line in open("build/c4che/%s.cache.py" % NS3_ACTIVE_VARIANT).readlines():

        for item in interesting_config_items:

            if line.startswith(item):

                exec(line, globals())

    if options.verbose:

        for item in interesting_config_items: