#! /usr/bin/env python

## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-

#

# Copyright (c) 2009 University of Washington

#

# This program is free software; you can redistribute it and/or modify

# it under the terms of the GNU General Public License version 2 as

# published by the Free Software Foundation;

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU General Public License for more details.

#

# You should have received a copy of the GNU General Public License

# along with this program; if not, write to the Free Software

# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

#

import os

import sys

import optparse

import subprocess

import multiprocessing

import threading

import Queue

import signal

import random

import xml.dom.minidom

import shutil

#

# XXX This should really be part of a waf command to list the configuration

# items relative to optional ns-3 pieces.

#

# A list of interesting configuration items in the waf configuration 

# cache which we may be interested in when deciding on which examples

# to run and how to run them.  These are set by waf during the 

# configuration phase and the corresponding assignments are usually

# found in the associated subdirectory wscript files.

#

interesting_config_items = [

    "NS3_BUILDDIR",

    "NS3_MODULE_PATH",

    "ENABLE_EMU",

    "ENABLE_GSL",

    "ENABLE_GTK_CONFIG_STORE",

    "ENABLE_LIBXML2",

    "ENABLE_NSC",

    "ENABLE_PYTHON_BINDINGS",

    "ENABLE_PYTHON_SCANNING",

    "ENABLE_REAL_TIME",

    "ENABLE_STATIC_NS3",

    "ENABLE_SUDO",

    "ENABLE_TAP",

    "ENABLE_THREADING",

]

#

# A list of examples to run as smoke tests just to ensure that they remain 

# buildable and runnable over time.  Also a condition under which to run

# the example (from the waf configuration).

#

# XXX Should this not be read from a configuration file somewhere and not

# hardcoded.

#

example_tests = [

    ("run-tests", "False"),

    ("csma-bridge", "True"),

    ("csma-bridge-one-hop", "True"),

    ("csma-broadcast", "True"),

    ("csma-multicast", "True"),

    ("csma-one-subnet", "True"),

    ("csma-packet-socket", "True"),

    ("csma-ping", "True"),

    ("csma-raw-ip-socket", "True"),

    ("csma-star", "True"),

    ("dynamic-global-routing", "True"),

    ("first", "True"),

    ("global-routing-slash32", "True"),

    ("hello-simulator", "True"),

    ("mixed-global-routing", "True"),

    ("mixed-wireless", "True"),

    ("object-names", "True"),

    ("realtime-udp-echo", "ENABLE_REAL_TIME == True"),

    ("second", "True"),

    ("simple-alternate-routing", "True"),

    ("simple-error-model", "True"),

    ("simple-global-routing", "True"),

    ("simple-point-to-point-olsr", "True"),

    ("simple-wifi-frame-aggregation", "True"),

    ("star", "True"),

    ("static-routing-slash32", "True"),

    ("tcp-large-transfer", "True"),

    ("tcp-nsc-zoo", "ENABLE_NSC == True"),

    ("tcp-star-server", "True"),

    ("test-ipv6", "True"),

    ("third", "True"),

    ("udp-echo", "True"),

    ("wifi-wired-bridging", "True"),

]

#

# Most of the examples produce gangs of trace files, so we want to find

# somewhere to put them that won't pollute the current directory.  One

# obvious place is somewhere in /tmp.

#

TMP_TRACES_DIR = "/tmp/unchecked-traces"

#

# The test suites are going to want to output status.  They are running

# concurrently.  This means that unless we are careful, the output of

# the test suites will be interleaved.  Rather than introducing a lock

# file that could unintentionally start serializing execution, we ask

# the tests to write their output to a temporary directory and then 

# put together the final output file when we "join" the test tasks back

# to the main thread.

#

TMP_OUTPUT_DIR = "/tmp/testpy"

def get_node_text(node):

    for child in node.childNodes:

        if child.nodeType == child.TEXT_NODE:

            return child.nodeValue

    return "None"

#

# A simple example of writing a text file with a test result summary.

#

def translate_to_text(results_file, text_file):

    f = open(text_file, 'w')

    dom = xml.dom.minidom.parse(results_file)

    for suite in dom.getElementsByTagName("TestSuite"):

        result = get_node_text(suite.getElementsByTagName("SuiteResult")[0])

        name = get_node_text(suite.getElementsByTagName("SuiteName")[0])

        time = get_node_text(suite.getElementsByTagName("SuiteTime")[0])

        output = "%s: Test Suite \"%s\" (%s)\n" % (result, name, time)

        f.write(output)

        if result != "CRASH":

            for case in suite.getElementsByTagName("TestCase"):

                result = get_node_text(case.getElementsByTagName("CaseResult")[0])

                name = get_node_text(case.getElementsByTagName("CaseName")[0])

                time = get_node_text(case.getElementsByTagName("CaseTime")[0])

                output =   "  %s: Test Case \"%s\" (%s)\n" % (result, name, time)

                f.write(output)

                if result == "FAIL":

                    f.write("    Details:\n")

                    f.write("      Message:   %s\n" % get_node_text(case.getElementsByTagName("CaseMessage")[0]))

                    f.write("      Condition: %s\n" % get_node_text(case.getElementsByTagName("CaseCondition")[0]))

                    f.write("      Actual:    %s\n" % get_node_text(case.getElementsByTagName("CaseActual")[0]))

                    f.write("      Limit:     %s\n" % get_node_text(case.getElementsByTagName("CaseLimit")[0]))

                    f.write("      File:      %s\n" % get_node_text(case.getElementsByTagName("CaseFile")[0]))

                    f.write("      Line:      %s\n" % get_node_text(case.getElementsByTagName("CaseLine")[0]))

    for example in dom.getElementsByTagName("Example"):

        result = get_node_text(example.getElementsByTagName("Result")[0])

        name = get_node_text(example.getElementsByTagName("Name")[0])

        output = "%s: Example \"%s\"\n" % (result, name)

        f.write(output)

    f.close()

#

# A simple example of writing an HTML file with a test result summary.

#

def translate_to_html(results_file, html_file):

    f = open(html_file, 'w')

    f.write("<html>\n")

    f.write("<body>\n")

    f.write("<center><h1>ns-3 Test Results</h1></center>\n")

    dom = xml.dom.minidom.parse(results_file)

    f.write("<h2>Test Suites</h2>\n")

    for suite in dom.getElementsByTagName("TestSuite"):

        name = get_node_text(suite.getElementsByTagName("SuiteName")[0])

        result = get_node_text(suite.getElementsByTagName("SuiteResult")[0])

        time = get_node_text(suite.getElementsByTagName("SuiteTime")[0])

        if result == "PASS":

            f.write("<h3 style=\"color:green\">%s: %s (%s)</h3>\n" % (result, name, time))

        else:

            f.write("<h3 style=\"color:red\">%s: %s (%s)</h3>\n" % (result, name, time))

        f.write("<table border=\"1\">\n")

        f.write("<th> Result </th>\n")

        if result == "CRASH":

            f.write("<tr>\n")

            f.write("<td style=\"color:red\">%s</td>\n" % result)

            f.write("</tr>\n")

            f.write("</table>\n")

            continue

        f.write("<th>Test Case Name</th>\n")

        f.write("<th> Time </th>\n")

        if result == "FAIL":

            f.write("<th>Details</th>\n")

        for case in suite.getElementsByTagName("TestCase"):

            f.write("<tr>\n")

            name = get_node_text(case.getElementsByTagName("CaseName")[0])

            result = get_node_text(case.getElementsByTagName("CaseResult")[0])

            time = get_node_text(case.getElementsByTagName("CaseTime")[0])

            if result == "FAIL":

                f.write("<td style=\"color:red\">%s</td>\n" % result)

                f.write("<td>%s</td>\n" % name)

                f.write("<td>%s</td>\n" % time)

                f.write("<td>")

                f.write("<b>Message: </b>%s, " % get_node_text(case.getElementsByTagName("CaseMessage")[0]))

                f.write("<b>Condition: </b>%s, " % get_node_text(case.getElementsByTagName("CaseCondition")[0]))

                f.write("<b>Actual: </b>%s, " % get_node_text(case.getElementsByTagName("CaseActual")[0]))

                f.write("<b>Limit: </b>%s, " % get_node_text(case.getElementsByTagName("CaseLimit")[0]))

                f.write("<b>File: </b>%s, " % get_node_text(case.getElementsByTagName("CaseFile")[0]))

                f.write("<b>Line: </b>%s" % get_node_text(case.getElementsByTagName("CaseLine")[0]))

                f.write("</td>\n")

            else:

                f.write("<td style=\"color:green\">%s</td>\n" % result)

                f.write("<td>%s</td>\n" % name)

                f.write("<td>%s</td>\n" % time)

                f.write("<td></td>\n")

            f.write("</tr>\n")

        f.write("</table>\n")

    f.write("<h2>Examples</h2>\n")

    f.write("<table border=\"1\">\n")

    f.write("<th> Result </th>\n")

    f.write("<th>Example Name</th>\n")

    for example in dom.getElementsByTagName("Example"):

        f.write("<tr>\n")

        result = get_node_text(example.getElementsByTagName("Result")[0])

            f.write("<td style=\"color:red\">%s</td>\n" % result)

        else:

            f.write("<td style=\"color:green\">%s</td>\n" % result)

        name =   get_node_text(example.getElementsByTagName("Name")[0])

        f.write("<td>%s</td>\n" % name)

        f.write("</tr>\n")

    f.write("</table>\n")

    f.write("</body>\n")

    f.write("</html>\n")

    f.close()

#

# Python Control-C handling is broken in the presence of multiple threads.  

# Signals get delivered to the runnable/running thread by default and if 

# it is blocked, the signal is simply ignored.  So we hook sigint and set 

# a global variable telling the system to shut down gracefully.

#

thread_exit = False

def sigint_hook(signal, frame):

    global thread_exit

    thread_exit = True

    return 0

#

# Waf can be configured to compile in debug or optimized modes.  In each

# case, the resulting built goes into a different directory.  If we want

# test tests to run from the correct code-base, we have to figure out which

# mode waf is running in.  This is called its active variant.

#

# XXX This function pokes around in the waf internal state file.  To be a

# little less hacky, we should add a commmand to waf to return this info

# and use that result.

#

def read_waf_active_variant():

    for line in open("build/c4che/default.cache.py").readlines():

        if line.startswith("NS3_ACTIVE_VARIANT"):

            exec(line, globals())

            break

    if options.verbose:

        print "NS3_ACTIVE_VARIANT == %s" % NS3_ACTIVE_VARIANT

#

# In general, the build process itself naturally takes care of figuring out

# which tests are built into the test runner.  For example, if waf configure

# determines that ENABLE_EMU is false due to some missing dependency,

# the tests for the emu net device simply will not be built and will 

# therefore not be included in the built test runner.

#

# Examples, however, are a different story.  In that case, we are just given

# a list of examples that could be run.  Instead of just failing, for example,

# nsc-tcp-zoo if NSC is not present, we look into the waf saved configuration

# for relevant configuration items.  

#

# XXX This function pokes around in the waf internal state file.  To be a

# little less hacky, we should add a commmand to waf to return this info

# and use that result.

#

def read_waf_config():

    for line in open("build/c4che/%s.cache.py" % NS3_ACTIVE_VARIANT).readlines():

        for item in interesting_config_items:

            if line.startswith(item):

                exec(line, globals())

    if options.verbose:

        for item in interesting_config_items:

            print "%s ==" % item, eval(item)

#

# It seems pointless to fork a process to run waf to fork a process to run

# the test runner, so we just run the test runner directly.  The main thing 

# that waf would do for us would be to sort out the shared library path but

# we can deal with that easily and do here.

#

# There can be many different ns-3 repositories on a system, and each has 

# its own shared libraries, so ns-3 doesn't hardcode a shared library search

# path -- it is cooked up dynamically, so we do that too.

#

def make_library_path():

    global LIBRARY_PATH

    LIBRARY_PATH = "LD_LIBRARY_PATH='"        

    if sys.platform == "darwin":

        LIBRARY_PATH = "DYLD_LIBRARY_PATH='"

    elif sys.platform == "win32":

        LIBRARY_PATH = "PATH='"

    elif sys.platform == "cygwin":

        LIBRARY_PATH = "PATH='"

    for path in NS3_MODULE_PATH:

        LIBRARY_PATH = LIBRARY_PATH + path + ":"

    LIBRARY_PATH = LIBRARY_PATH + "'"

def run_job_synchronously(shell_command, directory):

    cmd = "%s %s/%s/%s" % (LIBRARY_PATH, NS3_BUILDDIR, NS3_ACTIVE_VARIANT, shell_command)

    if options.verbose:

        print "Synchronously execute %s" % cmd

    proc = subprocess.Popen(cmd, shell=True, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    stdout_results = proc.communicate()[0]

    return (proc.returncode, stdout_results)

#

# This class defines a unit of testing work.  It will typically refer to

# a test suite to run using the test-runner, or an example to run directly.

#

class Job():

    def __init__(self):

        self.is_break = False

        self.is_example = False

        self.shell_command = ""

        self.display_name = ""

        self.cwd = ""

        self.tmp_file_name = ""

        self.returncode = False

    #

    # A job is either a standard job or a special job indicating that a worker

    # thread should exist.  This special job is indicated by setting is_break 

    # to true.

    #

    def set_is_break(self, is_break):

        self.is_break = is_break

    #

    # Examples are treated differently than standard test suites.  This is

    # mostly because they are completely unaware that they are being run as 

    # tests.  So we have to do some special case processing to make them look

    # like tests.

    #

    def set_is_example(self, is_example):

        self.is_example = is_example

    #

    # This is the shell command that will be executed in the job.  For example,

    #

    #  "utils/test-runner --suite=some-test-suite"

    #

    def set_shell_command(self, shell_command):

        self.shell_command = shell_command

    #

    # This is the dispaly name of the job, typically the test suite or example 

    # name.  For example,

    #

    #  "some-test-suite" or "udp-echo"

    #

    def set_display_name(self, display_name):

        self.display_name = display_name

    #

    # This is the base directory of the repository out of which the tests are

    # being run.  It will be used deep down in the testing framework to determine

    # where the source directory of the test was, and therefore where to find 

    # provided test vectors.  For example,

    #

    #  "/home/user/repos/ns-3-dev"

    #

    def set_basedir(self, basedir):

        self.basedir = basedir

    #

    # This is the current working directory that will be given to an executing

    # test as it is being run.  It will be used for examples to tell them where

    # to write all of the pcap files that we will be carefully ignoring.  For

    # example,

    #

    #  "/tmp/unchecked-traces"

    #

    def set_cwd(self, cwd):

        self.cwd = cwd

    #

    # This is the temporary results file name that will be given to an executing 

    # test as it is being run.  We will be running all of our tests in parallel

    # so there must be multiple temporary output files.  These will be collected

    # into a single XML file at the end and then be deleted.  The file names are

    # just giant random numbers, for example

    #

    #  "/tmp/testpy/5437925246732857"

    #

    def set_tmp_file_name(self, tmp_file_name):

        self.tmp_file_name = tmp_file_name

    #

    # The return code received when the job process is executed.

    #

    def set_returncode(self, returncode):

        self.returncode = returncode

#

# The worker thread class that handles the actual running of a given test.

# Once spawned, it receives requests for work through its input_queue and

# ships the results back through the output_queue.

#

class worker_thread(threading.Thread):

    def __init__(self, input_queue, output_queue):

        threading.Thread.__init__(self)

        self.input_queue = input_queue

        self.output_queue = output_queue

    def run(self):

        while True:

            job = self.input_queue.get()

            #

            # Worker threads continue running until explicitly told to stop with

            # a special job.

            #

            if job.is_break:

                return

            #

            # If the global interrupt handler sets the thread_exit variable,

            # we stop doing real work and just report back a "break" in the

            # normal command processing has happened.

            #

            if thread_exit == True:

                job.set_is_break(True)

                self.output_queue.put(job)

                continue

            #

            # Otherwise go about the business of running tests as normal.

            #

            else:

                if options.verbose:

                    print "Launch %s" % job.shell_command

                if job.is_example:

                    #

                    # If we have an example, the shell command is all we need to

                    # know.  It will be something like "examples/udp-echo"

                    #

                    (job.returncode, standard_out) = run_job_synchronously(job.shell_command, job.cwd)

                else:

                    #

                    # If we're a test suite, we need to provide a little more info

                    # to the test runner, specifically the base directory and temp

                    # file name

                    #

                    (job.returncode, standard_out) = run_job_synchronously(job.shell_command + " --basedir=%s --out=%s" %

                        (job.basedir, job.tmp_file_name), job.cwd)

                if options.verbose:

                    print standard_out

                self.output_queue.put(job)

#

# This is the main function that does the work of interacting with the test-runner

# itself.

#

def run_tests():

    #

    # Run waf to make sure that everything is built, configured and ready to go

    # unless we are explicitly told not to.

    #

    if options.nowaf == False:

        proc = subprocess.Popen("./waf", shell=True)

        proc.communicate()

    #

    # Pull some interesting configuration information out of waf, primarily

    # so we can know where executables can be found, but also to tell us what

    # pieces of the system have been built.  This will tell us what examples 

    # are runnable.

    #

    read_waf_active_variant()

    read_waf_config()

    make_library_path()

    #

    # There are a couple of options that imply we can to exit before starting

    # up a bunch of threads and running tests.  Let's detect these cases and 

    # handle them without doing all of the hard work.

    #

    if options.kinds:

        (rc, standard_out) = run_job_synchronously("utils/test-runner --kinds", os.getcwd())

        print standard_out

    if options.list:

        (rc, standard_out) = run_job_synchronously("utils/test-runner --list", os.getcwd())

        print standard_out

    if options.kinds or options.list:

        return

    #

    # We communicate results in two ways.  First, a simple message relating 

    # PASS, FAIL, or SKIP is always written to the standard output.  It is