#! /usr/bin/env python

## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-

#

# Copyright (c) 2009 University of Washington

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License version 2 as

# published by the Free Software Foundation;

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

#

import os

import sys

import time

import optparse

import subprocess

import threading

import Queue

import signal

import xml.dom.minidom

import shutil

import re

from utils import get_list_from_file

#

# XXX This should really be part of a waf command to list the configuration

# items relative to optional ns-3 pieces.

#

# A list of interesting configuration items in the waf configuration 

# cache which we may be interested in when deciding on which examples

# to run and how to run them.  These are set by waf during the 

# configuration phase and the corresponding assignments are usually

# found in the associated subdirectory wscript files.

#

interesting_config_items = [

    "NS3_ENABLED_MODULES",

    "NS3_MODULE_PATH",

    "NSC_ENABLED",

    "ENABLE_REAL_TIME",

    "ENABLE_THREADING",

    "ENABLE_EXAMPLES",

    "ENABLE_TESTS",

    "EXAMPLE_DIRECTORIES",

    "ENABLE_PYTHON_BINDINGS",

    "ENABLE_CLICK",

    "ENABLE_BRITE",

    "ENABLE_OPENFLOW",

    "APPNAME",

    "BUILD_PROFILE",

    "VERSION",

    "PYTHON",

    "VALGRIND_FOUND",

]

NSC_ENABLED = False

ENABLE_REAL_TIME = False

ENABLE_THREADING = False

ENABLE_EXAMPLES = True

ENABLE_TESTS = True

ENABLE_CLICK = False

ENABLE_BRITE = False

ENABLE_OPENFLOW = False

EXAMPLE_DIRECTORIES = []

APPNAME = ""

BUILD_PROFILE = ""

BUILD_PROFILE_SUFFIX = ""

VERSION = ""

PYTHON = ""

VALGRIND_FOUND = True

#

# This will be given a prefix and a suffix when the waf config file is

# read.

#

test_runner_name = "test-runner"

#

# If the user has constrained us to run certain kinds of tests, we can tell waf

# to only build

#

core_kinds = ["bvt", "core", "performance", "system", "unit"]

#

# There are some special cases for test suites that kill valgrind.  This is

# because NSC causes illegal instruction crashes when run under valgrind.

#

core_valgrind_skip_tests = [

    "ns3-tcp-cwnd",

    "nsc-tcp-loss",

    "ns3-tcp-interoperability",

    "routing-click",

    "lte-rr-ff-mac-scheduler",

    "lte-tdmt-ff-mac-scheduler",

    "lte-fdmt-ff-mac-scheduler",

    "lte-pf-ff-mac-scheduler",

    "lte-tta-ff-mac-scheduler",

    "lte-fdbet-ff-mac-scheduler",

    "lte-ttbet-ff-mac-scheduler",

    "lte-fdtbfq-ff-mac-scheduler",

    "lte-tdtbfq-ff-mac-scheduler",

    "lte-pss-ff-mac-scheduler",

]

# 

# There are some special cases for test suites that fail when NSC is

# missing.

#

core_nsc_missing_skip_tests = [

    "ns3-tcp-cwnd",

    "nsc-tcp-loss",

    "ns3-tcp-interoperability",

]

#

# Parse the examples-to-run file if it exists.

#

# This function adds any C++ examples or Python examples that are to be run

# to the lists in example_tests and python_tests, respectively.

#

def parse_examples_to_run_file(

    examples_to_run_path,

    cpp_executable_dir,

    python_script_dir,

    example_tests,

    example_names_original,

    python_tests):

    # Look for the examples-to-run file exists.

    if os.path.exists(examples_to_run_path):

        # Each tuple in the C++ list of examples to run contains

        #

        #     (example_name, do_run, do_valgrind_run)

        #

        # where example_name is the executable to be run, do_run is a

        # condition under which to run the example, and do_valgrind_run is

        # a condition under which to run the example under valgrind.  This

        # is needed because NSC causes illegal instruction crashes with

        # some tests when they are run under valgrind.

        #

        # Note that the two conditions are Python statements that

        # can depend on waf configuration variables.  For example,

        #

        #     ("tcp-nsc-lfn", "NSC_ENABLED == True", "NSC_ENABLED == False"),

        #

        cpp_examples = get_list_from_file(examples_to_run_path, "cpp_examples")

        for example_name, do_run, do_valgrind_run in cpp_examples:

            # Seperate the example name from its arguments.

            example_name_original = example_name

            example_name_parts = example_name.split(' ', 1)

            if len(example_name_parts) == 1:

                example_name      = example_name_parts[0]

                example_arguments = ""

            else:

                example_name      = example_name_parts[0]

                example_arguments = example_name_parts[1]

            # Add the proper prefix and suffix to the example name to

            # match what is done in the wscript file.

            example_path = "%s%s-%s%s" % (APPNAME, VERSION, example_name, BUILD_PROFILE_SUFFIX)

            # Set the full path for the example.

            example_path = os.path.join(cpp_executable_dir, example_path)

            example_name = os.path.join(

                os.path.relpath(cpp_executable_dir, NS3_BUILDDIR),

                example_name)

            # Add all of the C++ examples that were built, i.e. found

            # in the directory, to the list of C++ examples to run.

            if os.path.exists(example_path):

                # Add any arguments to the path.

                if len(example_name_parts) != 1:

                    example_path = "%s %s" % (example_path, example_arguments)

                    example_name = "%s %s" % (example_name, example_arguments)

                # Add this example.

                example_tests.append((example_name, example_path, do_run, do_valgrind_run))

                example_names_original.append(example_name_original)

        # Each tuple in the Python list of examples to run contains

        #

        #     (example_name, do_run)

        #

        # where example_name is the Python script to be run and

        # do_run is a condition under which to run the example.

        #

        # Note that the condition is a Python statement that can

        # depend on waf configuration variables.  For example,

        #

        #     ("realtime-udp-echo.py", "ENABLE_REAL_TIME == True"),

        #

        python_examples = get_list_from_file(examples_to_run_path, "python_examples")

        for example_name, do_run in python_examples:

            # Seperate the example name from its arguments.

            example_name_parts = example_name.split(' ', 1)

            if len(example_name_parts) == 1:

                example_name      = example_name_parts[0]

                example_arguments = ""

            else:

                example_name      = example_name_parts[0]

                example_arguments = example_name_parts[1]

            # Set the full path for the example.

            example_path = os.path.join(python_script_dir, example_name)

            # Add all of the Python examples that were found to the

            # list of Python examples to run.

            if os.path.exists(example_path):

                # Add any arguments to the path.

                if len(example_name_parts) != 1:

                    example_path = "%s %s" % (example_path, example_arguments)

                # Add this example.

                python_tests.append((example_path, do_run))

#

# The test suites are going to want to output status.  They are running

# concurrently.  This means that unless we are careful, the output of

# the test suites will be interleaved.  Rather than introducing a lock

# file that could unintentionally start serializing execution, we ask

# the tests to write their output to a temporary directory and then 

# put together the final output file when we "join" the test tasks back

# to the main thread.  In addition to this issue, the example programs

# often write lots and lots of trace files which we will just ignore.

# We put all of them into the temp directory as well, so they can be

# easily deleted.

#

TMP_OUTPUT_DIR = "testpy-output"

def read_test(test):

    result = test.find('Result').text

    name = test.find('Name').text

    if not test.find('Time') is None:

        time_real = test.find('Time').get('real')

    else:

        time_real = ''

    return (result, name, time_real)

#

# A simple example of writing a text file with a test result summary.  It is 

# expected that this output will be fine for developers looking for problems.

#

def node_to_text (test, f):

    (result, name, time_real) = read_test(test)

    output = "%s: Test Suite \"%s\" (%s)\n" % (result, name, time_real)

    f.write(output)

    for details in test.findall('FailureDetails'):

        f.write("    Details:\n")

        f.write("      Message:   %s\n" % details.find('Message').text)

        f.write("      Condition: %s\n" % details.find('Condition').text)

        f.write("      Actual:    %s\n" % details.find('Actual').text)

        f.write("      Limit:     %s\n" % details.find('Limit').text)

        f.write("      File:      %s\n" % details.find('File').text)

        f.write("      Line:      %s\n" % details.find('Line').text)

    for child in test.findall('Test'):

        node_to_text(child, f)

def translate_to_text(results_file, text_file):

    f = open(text_file, 'w')

    import xml.etree.ElementTree as ET

    et = ET.parse (results_file)

    for test in et.findall('Test'):

        node_to_text (test, f)

    for example in et.findall('Example'):

        result = example.find('Result').text

        name = example.find('Name').text

        if not example.find('Time') is None:

            time_real = example.find('Time').get('real')

        else:

            time_real = ''

        output = "%s: Example \"%s\" (%s)\n" % (result, name, time_real)

        f.write(output)

    f.close()

#

# A simple example of writing an HTML file with a test result summary.  It is 

# expected that this will eventually be made prettier as time progresses and

# we have time to tweak it.  This may end up being moved to a separate module

# since it will probably grow over time.

#

def translate_to_html(results_file, html_file):

    f = open(html_file, 'w')

    f.write("<html>\n")

    f.write("<body>\n")

    f.write("<center><h1>ns-3 Test Results</h1></center>\n")

    #

    # Read and parse the whole results file.

    #

    import xml.etree.ElementTree as ET

    et = ET.parse(results_file)

    #

    # Iterate through the test suites

    #

    f.write("<h2>Test Suites</h2>\n")

    for suite in et.findall('Test'):     

        #

        # For each test suite, get its name, result and execution time info

        #

        (result, name, time) = read_test (suite)

        # 

        # Print a level three header with the result, name and time.  If the 

        # test suite passed, the header is printed in green. If the suite was

        # skipped, print it in orange, otherwise assume something bad happened

        # and print in red.

        #

        if result == "PASS":

            f.write("<h3 style=\"color:green\">%s: %s (%s)</h3>\n" % (result, name, time))

        elif result == "SKIP":

            f.write("<h3 style=\"color:#ff6600\">%s: %s (%s)</h3>\n" % (result, name, time))

        else:

            f.write("<h3 style=\"color:red\">%s: %s (%s)</h3>\n" % (result, name, time))

        #

        # The test case information goes in a table.

        #

        f.write("<table border=\"1\">\n")

        #

        # The first column of the table has the heading Result

        #

        f.write("<th> Result </th>\n")

        #

        # If the suite crashed or is skipped, there is no further information, so just

        # delare a new table row with the result (CRASH or SKIP) in it.  Looks like:

        #

        #   +--------+

        #   | Result |

        #   +--------+

        #   | CRASH  |

        #   +--------+

        #

        # Then go on to the next test suite.  Valgrind and skipped errors look the same.

        #

        if result in ["CRASH", "SKIP", "VALGR"]:

            f.write("<tr>\n")

            if result == "SKIP":

                f.write("<td style=\"color:#ff6600\">%s</td>\n" % result)

            else:

                f.write("<td style=\"color:red\">%s</td>\n" % result)

            f.write("</tr>\n")

            f.write("</table>\n")

            continue

        #

        # If the suite didn't crash, we expect more information, so fill out

        # the table heading row.  Like,

        #

        #   +--------+----------------+------+

        #   | Result | Test Case Name | Time |

        #   +--------+----------------+------+

        #

        f.write("<th>Test Case Name</th>\n")

        f.write("<th> Time </th>\n")

        #

        # If the test case failed, we need to print out some failure details

        # so extend the heading row again.  Like,

        #

        #   +--------+----------------+------+-----------------+

        #   | Result | Test Case Name | Time | Failure Details |

        #   +--------+----------------+------+-----------------+

        #

        if result == "FAIL":

            f.write("<th>Failure Details</th>\n")

        #

        # Now iterate through all of the test cases.

        #

        for case in suite.findall('Test'):

            #

            # Get the name, result and timing information from xml to use in

            # printing table below.

            #

            (result, name, time) = read_test(case)

            #

            # If the test case failed, we iterate through possibly multiple

            # failure details

            #

            if result == "FAIL":

                #

                # There can be multiple failures for each test case.  The first

                # row always gets the result, name and timing information along

                # with the failure details.  Remaining failures don't duplicate

                # this information but just get blanks for readability.  Like,

                #

                #   +--------+----------------+------+-----------------+

                #   | Result | Test Case Name | Time | Failure Details |

                #   +--------+----------------+------+-----------------+

                #   |  FAIL  | The name       | time | It's busted     |   

                #   +--------+----------------+------+-----------------+

                #   |        |                |      | Really broken   |   

                #   +--------+----------------+------+-----------------+

                #   |        |                |      | Busted bad      |   

                #   +--------+----------------+------+-----------------+

                #

                first_row = True

                for details in case.findall('FailureDetails'):

                    #

                    # Start a new row in the table for each possible Failure Detail

                    #

                    f.write("<tr>\n")

                    if first_row:

                        first_row = False

                        f.write("<td style=\"color:red\">%s</td>\n" % result)

                        f.write("<td>%s</td>\n" % name)

                        f.write("<td>%s</td>\n" % time)

                    else:

                        f.write("<td></td>\n")

                        f.write("<td></td>\n")

                        f.write("<td></td>\n")

                    f.write("<td>")

                    f.write("<b>Message: </b>%s, " % details.find('Message').text)

                    f.write("<b>Condition: </b>%s, " % details.find('Condition').text)

                    f.write("<b>Actual: </b>%s, " % details.find('Actual').text)

                    f.write("<b>Limit: </b>%s, " % details.find('Limit').text)

                    f.write("<b>File: </b>%s, " % details.find('File').text)

                    f.write("<b>Line: </b>%s" % details.find('Line').text)

                    f.write("</td>\n")

                    #

                    # End the table row

                    #

                    f.write("</td>\n")

            else:

                #

                # If this particular test case passed, then we just print the PASS

                # result in green, followed by the test case name and its execution

                # time information.  These go off in <td> ... </td> table data.

                # The details table entry is left blank.

                #

                #   +--------+----------------+------+---------+

                #   | Result | Test Case Name | Time | Details |

                #   +--------+----------------+------+---------+

                #   |  PASS  | The name       | time |         |   

                #   +--------+----------------+------+---------+

                #

                f.write("<tr>\n")

                f.write("<td style=\"color:green\">%s</td>\n" % result)

                f.write("<td>%s</td>\n" % name)

                f.write("<td>%s</td>\n" % time)

                f.write("<td></td>\n")

                f.write("</tr>\n")

        #

        # All of the rows are written, so we need to end the table.

        #

        f.write("</table>\n")

    #

    # That's it for all of the test suites.  Now we have to do something about 

    # our examples.

    #

    f.write("<h2>Examples</h2>\n")

    #

    # Example status is rendered in a table just like the suites.

    #

    f.write("<table border=\"1\">\n")

    #

    # The table headings look like,

    #

    #   +--------+--------------+--------------+

    #   | Result | Example Name | Elapsed Time |

    #   +--------+--------------+--------------+

    #

    f.write("<th> Result </th>\n")

    f.write("<th>Example Name</th>\n")

    f.write("<th>Elapsed Time</th>\n")

    #

    # Now iterate through all of the examples