utils/test-runner.cc
author Craig Dowell <craigdo@ee.washington.edu>
Tue, 29 Sep 2009 19:58:29 -0700
changeset 5297 a7f197a2410b
parent 4772 7b6ae6bf0055
child 5324 0ba73cdd2a43
permissions -rw-r--r--
If test suite not found, return failure
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     1
/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     2
/*
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     3
 * Copyright (c) 2009 University of Washington
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     4
 *
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     5
 * This program is free software; you can redistribute it and/or modify
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     6
 * it under the terms of the GNU General Public License version 2 as
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     7
 * published by the Free Software Foundation;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     8
 *
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
     9
 * This program is distributed in the hope that it will be useful,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    10
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    11
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    12
 * GNU General Public License for more details.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    13
 *
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    14
 * You should have received a copy of the GNU General Public License
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    15
 * along with this program; if not, write to the Free Software
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    16
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    17
 */
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    18
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    19
#include "ns3/test.h"
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    20
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    21
#include <iostream>
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    22
#include <fstream>
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    23
#include <string>
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    24
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    25
using namespace ns3;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    26
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    27
//
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    28
// Run one of the test suites.  Returns an integer with the boolean sense of
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    29
// "an error has occurred."  That is, 0 == false -> no error; 1 == true -> an
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    30
// error occurred.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    31
//
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    32
int 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    33
main (int argc, char *argv[])
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    34
{
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    35
  bool doVerbose = false;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    36
  bool doList = false;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    37
  bool doHelp = false;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    38
  bool doSuite = false;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    39
  bool doKinds = false;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    40
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    41
  bool haveBasedir = false;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    42
  bool haveOutfile = false;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    43
  bool haveType = false;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    44
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    45
  std::string suiteName;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    46
  std::string basedir;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    47
  std::string outfileName;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    48
  std::string typeName;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    49
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    50
  for (int i = 1; i < argc; ++i)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    51
    {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    52
      std::string arg(argv[i]);
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    53
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    54
      if (arg.find ("--basedir=") != std::string::npos)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    55
        {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    56
          basedir = arg.substr (arg.find_first_of ("=") + 1, 9999);
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    57
          haveBasedir = true;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    58
        }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    59
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    60
      if (arg.find ("--constrain=") != std::string::npos)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    61
        {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    62
          typeName = arg.substr (arg.find_first_of ("=") + 1, 9999);
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    63
          haveType = true;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    64
        }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    65
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    66
      if (arg.compare ("--help") == 0)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    67
        {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    68
          doHelp = true;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    69
        }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    70
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    71
      if (arg.compare ("--kinds") == 0)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    72
        {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    73
          doKinds = true;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    74
        }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    75
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    76
      if (arg.compare ("--list") == 0)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    77
        {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    78
          doList = true;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    79
        }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    80
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    81
      if (arg.find ("--out=") != std::string::npos)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    82
        {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    83
          outfileName = arg.substr (arg.find_first_of ("=") + 1, 9999);
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    84
          haveOutfile = true;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    85
        }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    86
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    87
      if (arg.find ("--suite=") != std::string::npos)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    88
        {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    89
          suiteName = arg.substr (arg.find_first_of ("=") + 1, 9999);
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    90
          doSuite = true;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    91
        }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    92
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    93
      if (arg.compare ("--verbose") == 0)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    94
        {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    95
          doVerbose = true;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    96
        }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    97
    }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    98
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
    99
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   100
  // A help request trumps everything else.  If we have one, just print the help
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   101
  // and leave.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   102
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   103
  if (doHelp)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   104
    {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   105
      std::cout << "  --basedir=dir:          Set the base directory (where to find src) to \"dir\"" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   106
      std::cout << "  --constrain=test-type:  Constrain checks to test suites of type \"test-type\"" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   107
      std::cout << "  --help:                 Print this message" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   108
      std::cout << "  --kinds:                List all of the available kinds of tests" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   109
      std::cout << "  --list:                 List all of the test suites (optionally constrained by test-type)" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   110
      std::cout << "  --out=file-name:        Set the test status output file to \"file-name\"" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   111
      std::cout << "  --suite=suite-name:     Run the test suite named \"suite-name\"" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   112
      std::cout << "  --verbose:              Turn on messages in the run test suites" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   113
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   114
      return false;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   115
    }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   116
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   117
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   118
  // A kinds request trumps everything remaining.  If we are asked, just 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   119
  // print the list of types and leave.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   120
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   121
  if (doKinds)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   122
    {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   123
      //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   124
      // Coming up with a string to represent a test type is completely up to
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   125
      // us here.  We just define the types as being a string composed of the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   126
      // enum defined in test.h converted to lower case.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   127
      //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   128
      std::cout << "  bvt:         Build Verification Tests (to see if build completed successfully)" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   129
      std::cout << "  unit:        Unit Tests (within modules to check basic functionality)" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   130
      std::cout << "  system:      System Tests (spans modules to check integration of modules)" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   131
      std::cout << "  example:     Examples (to see if example programs run successfully)" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   132
      std::cout << "  performance: Performance Tests (check to see if the system is as fast as expected)" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   133
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   134
      return false;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   135
    }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   136
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   137
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   138
  // A list request is the first functional request.  It trumps running the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   139
  // actual tests.  If we get a list request, we don't run anything, we just
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   140
  // do the requested list which may or may not be qualified by a typename.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   141
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   142
  if (doList)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   143
    {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   144
      for (uint32_t i = 0; i < TestRunner::GetNTestSuites (); ++i)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   145
        {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   146
          TestSuite *suite = TestRunner::GetTestSuite (i);
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   147
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   148
          //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   149
          // Filter the tests listed by type if requested.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   150
          //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   151
          if (haveType)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   152
            {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   153
              TestSuite::TestType type = suite->GetTestType ();
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   154
              if (typeName == "bvt" && type != TestSuite::BVT)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   155
                {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   156
                  continue;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   157
                }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   158
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   159
              if (typeName == "unit" && type != TestSuite::UNIT)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   160
                {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   161
                  continue;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   162
                }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   163
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   164
              if (typeName == "system" && type != TestSuite::SYSTEM)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   165
                {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   166
                  continue;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   167
                }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   168
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   169
              if (typeName == "example" && type != TestSuite::EXAMPLE)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   170
                {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   171
                  continue;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   172
                }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   173
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   174
              if (typeName == "performance" && type != TestSuite::PERFORMANCE)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   175
                {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   176
                  continue;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   177
                }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   178
            }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   179
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   180
          //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   181
          // This creates a list of test suite names that can be used by the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   182
          // high level test manager to get a list of all tests.  It will then
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   183
          // typically launch individual tests in parallel, calling back here
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   184
          // with a specific "suite=" to run.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   185
          //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   186
          std::cout << suite->GetName () << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   187
        }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   188
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   189
      return false;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   190
    }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   191
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   192
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   193
  // If we haven't been asked to run a test suite, we are just going to happily
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   194
  // try and run everything.  Test suites are possibly going to need to figure
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   195
  // out where there source directory is, and to do that they will need to know
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   196
  // where the base directory of the distribution is (the directory in which 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   197
  // "src" is found).  We could try and run without it, but when it is needed,
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   198
  // the test will fail with an assertion.  So to be safe, we require a basedir
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   199
  // to proceed.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   200
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   201
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   202
  if (haveBasedir == false)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   203
    {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   204
      std::cout << "Must specify a base directory to run tests (use --basedir option)" << std::endl;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   205
      return true;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   206
    }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   207
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   208
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   209
  // If given an output file, we just append the output of each test suite 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   210
  // we're asked to run to the end of that file.  We need to append since the
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   211
  // higher level test runner may be just running a number of tests back to 
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   212
  // back.  We leave it up to that code to decide how to deal with possible
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   213
  // parallel operation -- we just append to a file here.  If no output file
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   214
  // is specified, we don't do any output and just return the sense of error
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   215
  // given by the test.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   216
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   217
  std::ofstream *pofs = 0;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   218
  std::ofstream ofs;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   219
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   220
  if (!outfileName.empty ())
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   221
    {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   222
      ofs.open (outfileName.c_str (), std::fstream::out | std::fstream::app);
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   223
      pofs = &ofs;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   224
    }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   225
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   226
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   227
  // If we have a specified test suite to run, then we only run that suite.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   228
  // The default case is to "run everything.  We don't expect this to be done
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   229
  // much since typically higher level code will be running suites in parallel
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   230
  // but we'll do it if asked.
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   231
  //
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   232
  bool result = false;
5297
a7f197a2410b If test suite not found, return failure
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
   233
  bool suiteRan = false;
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   234
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   235
  for (uint32_t i = 0; i < TestRunner::GetNTestSuites (); ++i)
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   236
    {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   237
      TestSuite *testSuite = TestRunner::GetTestSuite (i);
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   238
      if (doSuite == false || (doSuite == true && suiteName == testSuite->GetName ()))
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   239
        {
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   240
          testSuite->SetBaseDir (basedir);
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   241
          testSuite->SetStream (pofs);
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   242
          testSuite->SetVerbose (doVerbose);
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   243
          result |= testSuite->Run ();
5297
a7f197a2410b If test suite not found, return failure
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
   244
          suiteRan = true;
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   245
        }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   246
    }
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   247
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   248
  ofs.close();
5297
a7f197a2410b If test suite not found, return failure
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
   249
a7f197a2410b If test suite not found, return failure
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
   250
  //
a7f197a2410b If test suite not found, return failure
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
   251
  // If we couldn't figure out how to run at least one test, then return an error
a7f197a2410b If test suite not found, return failure
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
   252
  //
a7f197a2410b If test suite not found, return failure
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
   253
  if (suiteRan == false)
a7f197a2410b If test suite not found, return failure
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
   254
    {
a7f197a2410b If test suite not found, return failure
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
   255
      return true;
a7f197a2410b If test suite not found, return failure
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
   256
    }
a7f197a2410b If test suite not found, return failure
Craig Dowell <craigdo@ee.washington.edu>
parents: 4772
diff changeset
   257
4772
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   258
  return result;
7b6ae6bf0055 add test and validation framework
Craig Dowell <craigdo@ee.washington.edu>
parents:
diff changeset
   259
}