utils/test-runner.cc
author Josh Pelkey <jpelkey@gatech.edu>
Wed, 11 Aug 2010 11:37:37 -0400
changeset 6553 fb5ad9c7755a
parent 6251 ff3501de65dc
permissions -rw-r--r--
update release notes and fix doxygen warnings
craigdo@4772
     1
/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
craigdo@4772
     2
/*
craigdo@4772
     3
 * Copyright (c) 2009 University of Washington
craigdo@4772
     4
 *
craigdo@4772
     5
 * This program is free software; you can redistribute it and/or modify
craigdo@4772
     6
 * it under the terms of the GNU General Public License version 2 as
craigdo@4772
     7
 * published by the Free Software Foundation;
craigdo@4772
     8
 *
craigdo@4772
     9
 * This program is distributed in the hope that it will be useful,
craigdo@4772
    10
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
craigdo@4772
    11
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
craigdo@4772
    12
 * GNU General Public License for more details.
craigdo@4772
    13
 *
craigdo@4772
    14
 * You should have received a copy of the GNU General Public License
craigdo@4772
    15
 * along with this program; if not, write to the Free Software
craigdo@4772
    16
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
craigdo@4772
    17
 */
craigdo@4772
    18
craigdo@4772
    19
#include "ns3/test.h"
craigdo@6194
    20
#include "ns3/assert.h"
craigdo@6248
    21
#include "ns3/abort.h"
craigdo@4772
    22
craigdo@4772
    23
#include <iostream>
craigdo@4772
    24
#include <fstream>
craigdo@4772
    25
#include <string>
craigdo@6194
    26
#include <stdlib.h>
craigdo@6194
    27
#include <stdio.h>
craigdo@6194
    28
#include <time.h>
craigdo@6194
    29
#include <sys/types.h>
craigdo@6194
    30
#include <sys/stat.h>
craigdo@6194
    31
#include <dirent.h>
craigdo@6194
    32
#include <string.h>
craigdo@4772
    33
craigdo@6204
    34
extern bool gBreakOnFailure;
craigdo@6204
    35
craigdo@4772
    36
using namespace ns3;
craigdo@4772
    37
craigdo@4772
    38
//
craigdo@6194
    39
// Create a temporary directory for use by test programs.  This is not a 
craigdo@6194
    40
// foolproof thing, but a reasonably good way to get a throwaway directory
craigdo@6194
    41
// while running tests in a debugger.
craigdo@6194
    42
//  
craigdo@6194
    43
std::string
craigdo@6194
    44
TempDir (void)
craigdo@6194
    45
{
craigdo@6194
    46
  char *path = NULL;
craigdo@6194
    47
craigdo@6194
    48
  path = getenv ("TMP");
craigdo@6194
    49
  if (path == NULL)
craigdo@6194
    50
    {
craigdo@6194
    51
      path = getenv ("TEMP");
craigdo@6194
    52
      if (path == NULL)
craigdo@6194
    53
        {
craigdo@6194
    54
          path = const_cast<char *> ("/tmp");
craigdo@6194
    55
        }
craigdo@6194
    56
    }
craigdo@6194
    57
craigdo@6194
    58
  //
craigdo@6194
    59
  // Just in case the user wants to go back and find the output, we give
craigdo@6194
    60
  // a hint as to which dir we created by including a time hint.
craigdo@6194
    61
  //
craigdo@6194
    62
  time_t now = time (NULL);
craigdo@6194
    63
  struct tm *tm_now = localtime (&now);
craigdo@6194
    64
  
craigdo@6194
    65
  //
craigdo@6194
    66
  // But we also randomize the name in case there are multiple users doing
craigdo@6194
    67
  // this at the same time
craigdo@6194
    68
  //
vincent@6316
    69
  srand (time (0));
vincent@6316
    70
  long int n = rand ();
craigdo@6194
    71
craigdo@6194
    72
  //
craigdo@6194
    73
  // The final path to the directory is going to look something like
craigdo@6194
    74
  // 
craigdo@6228
    75
  //   /tmp/ns3-14.30.29.32767
craigdo@6194
    76
  //
craigdo@6194
    77
  // The first segment comes from one of the temporary directory env 
craigdo@6228
    78
  // variables or /tmp if not found.  The directory name starts with an
craigdo@6228
    79
  // identifier telling folks who is making all of the temp directories
craigdo@6228
    80
  // and then the local time (in this case 14.30.29 -- which is 2:30 and
craigdo@6228
    81
  // 29 seconds PM).
craigdo@6194
    82
  //
craigdo@6194
    83
  char dirname[1024];
craigdo@6228
    84
  snprintf (dirname, sizeof(dirname),  "%s/ns-3.%d.%d.%d.%ld", path, tm_now->tm_hour, tm_now->tm_min, tm_now->tm_sec, n);
craigdo@6194
    85
vincent@6316
    86
#if (defined(_WIN32) || defined(_WIN64)) && !defined(__CYGWIN__)
vincent@6316
    87
  if(mkdir(dirname) == 0)
vincent@6316
    88
#else
craigdo@6194
    89
  if (mkdir (dirname, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == 0)
vincent@6316
    90
#endif
craigdo@6194
    91
    {
craigdo@6194
    92
      return dirname;
craigdo@6194
    93
    } 
craigdo@6194
    94
  else
craigdo@6194
    95
    {
craigdo@6194
    96
      return "";
craigdo@6194
    97
    }
craigdo@6194
    98
}
craigdo@6194
    99
craigdo@6228
   100
//
craigdo@6228
   101
// Test suites may need to figure out where their source directory is in order
craigdo@6228
   102
// to find test vectors.  To do that they will need to know  where the base 
craigdo@6228
   103
// directory of the distribution is (the directory in which "src" is found).
craigdo@6228
   104
// It is painful for a user debugging test suites to always provide that dir
craigdo@6228
   105
// so we try and find it in the current directory tree.
craigdo@6228
   106
//
craigdo@6194
   107
std::string
craigdo@6194
   108
BaseDir (void)
craigdo@6194
   109
{
craigdo@6194
   110
  //
craigdo@6194
   111
  // Get an absolute path to the current working directory.  Following code
craigdo@6194
   112
  // depends on the leading '/'
craigdo@6194
   113
  //
craigdo@6194
   114
  char pathbuf[PATH_MAX];
craigdo@6248
   115
  if (getcwd (pathbuf, sizeof(pathbuf)) == NULL)
craigdo@6248
   116
    {
craigdo@6248
   117
      NS_ABORT_MSG ("Basedir():  unable to getcwd()");
craigdo@6248
   118
    }
craigdo@6194
   119
craigdo@6194
   120
  //
craigdo@6194
   121
  // Walk up the directory tree looking for a directory that has files that
craigdo@6194
   122
  // indicate it is the base of an ns-3 distribution.  We use VERSION and
craigdo@6194
   123
  // LICENSE which have been there from the beginning of time.
craigdo@6194
   124
  //
craigdo@6194
   125
  for (;;)
craigdo@6194
   126
    {
craigdo@6194
   127
      bool haveVersion = false;
craigdo@6194
   128
      bool haveLicense = false;
craigdo@6194
   129
craigdo@6194
   130
      //
craigdo@6194
   131
      // Open the directory file for the current directory and loop through
craigdo@6194
   132
      // the directory entries.
craigdo@6194
   133
      //
craigdo@6194
   134
      DIR *dp = opendir (pathbuf);
craigdo@6194
   135
      if (dp != NULL)
craigdo@6194
   136
        {
craigdo@6194
   137
          while (struct dirent *de = readdir (dp))
craigdo@6194
   138
            {
craigdo@6194
   139
              if (strcmp (de->d_name, "VERSION") == 0)
craigdo@6194
   140
                {
craigdo@6194
   141
                  haveVersion = true;
craigdo@6194
   142
                }
craigdo@6194
   143
craigdo@6194
   144
              if (strcmp (de->d_name, "LICENSE") == 0)
craigdo@6194
   145
                {
craigdo@6194
   146
                  haveLicense = true;
craigdo@6194
   147
                }
craigdo@6194
   148
            }
craigdo@6194
   149
        }
mathieu@6251
   150
      closedir (dp);
craigdo@6194
   151
craigdo@6194
   152
      //
craigdo@6194
   153
      // If there's a file named VERSION and a file named LICENSE in this
craigdo@6194
   154
      // directory, we assume it's our base directory.
craigdo@6194
   155
      //
craigdo@6194
   156
      if (haveVersion && haveLicense)
craigdo@6194
   157
        {
craigdo@6194
   158
          return pathbuf;
craigdo@6194
   159
        }
craigdo@6194
   160
craigdo@6194
   161
      //
craigdo@6194
   162
      // Strip off the last segment of the current directory.
craigdo@6194
   163
      //
craigdo@6194
   164
      char *last = strrchr (pathbuf, '/');
craigdo@6194
   165
      NS_ASSERT_MSG (last, "No \"/\" found in absolute path ???");
craigdo@6194
   166
      *last = '\0';
craigdo@6194
   167
craigdo@6194
   168
      if (strlen(pathbuf) == 0)
craigdo@6194
   169
        {
craigdo@6194
   170
          return "";
craigdo@6194
   171
        }
craigdo@6194
   172
    }
craigdo@6194
   173
craigdo@6194
   174
  //
craigdo@6194
   175
  // Quiet the compiler.
craigdo@6194
   176
  //
craigdo@6194
   177
  return "";
craigdo@6194
   178
}
craigdo@6194
   179
craigdo@6194
   180
//
craigdo@4772
   181
// Run one of the test suites.  Returns an integer with the boolean sense of
craigdo@4772
   182
// "an error has occurred."  That is, 0 == false -> no error; 1 == true -> an
craigdo@4772
   183
// error occurred.
craigdo@4772
   184
//
craigdo@4772
   185
int 
craigdo@4772
   186
main (int argc, char *argv[])
craigdo@4772
   187
{
craigdo@4772
   188
  bool doVerbose = false;
craigdo@4772
   189
  bool doList = false;
craigdo@5324
   190
  bool doMultiple = false;
craigdo@4772
   191
  bool doHelp = false;
craigdo@4772
   192
  bool doSuite = false;
craigdo@4772
   193
  bool doKinds = false;
craigdo@4772
   194
craigdo@6204
   195
  gBreakOnFailure = false;
craigdo@6204
   196
craigdo@4772
   197
  bool haveBasedir = false;
craigdo@5481
   198
  bool haveTempdir = false;
craigdo@4772
   199
  bool haveOutfile = false;
craigdo@4772
   200
  bool haveType = false;
craigdo@4772
   201
craigdo@4772
   202
  std::string suiteName;
craigdo@4772
   203
  std::string basedir;
craigdo@5481
   204
  std::string tempdir;
craigdo@4772
   205
  std::string outfileName;
craigdo@4772
   206
  std::string typeName;
craigdo@4772
   207
craigdo@6204
   208
craigdo@4772
   209
  for (int i = 1; i < argc; ++i)
craigdo@4772
   210
    {
craigdo@4772
   211
      std::string arg(argv[i]);
craigdo@4772
   212
craigdo@6204
   213
      if (arg.find ("--assert") != std::string::npos)
craigdo@6204
   214
        {
craigdo@6204
   215
          gBreakOnFailure = true;
craigdo@6204
   216
        }
craigdo@6204
   217
craigdo@4772
   218
      if (arg.find ("--basedir=") != std::string::npos)
craigdo@4772
   219
        {
craigdo@4772
   220
          basedir = arg.substr (arg.find_first_of ("=") + 1, 9999);
craigdo@4772
   221
          haveBasedir = true;
craigdo@4772
   222
        }
craigdo@4772
   223
craigdo@4772
   224
      if (arg.find ("--constrain=") != std::string::npos)
craigdo@4772
   225
        {
craigdo@4772
   226
          typeName = arg.substr (arg.find_first_of ("=") + 1, 9999);
craigdo@4772
   227
          haveType = true;
craigdo@4772
   228
        }
craigdo@4772
   229
craigdo@4772
   230
      if (arg.compare ("--help") == 0)
craigdo@4772
   231
        {
craigdo@4772
   232
          doHelp = true;
craigdo@4772
   233
        }
craigdo@4772
   234
craigdo@4772
   235
      if (arg.compare ("--kinds") == 0)
craigdo@4772
   236
        {
craigdo@4772
   237
          doKinds = true;
craigdo@4772
   238
        }
craigdo@4772
   239
craigdo@4772
   240
      if (arg.compare ("--list") == 0)
craigdo@4772
   241
        {
craigdo@4772
   242
          doList = true;
craigdo@4772
   243
        }
craigdo@4772
   244
craigdo@5324
   245
      if (arg.compare ("--multiple") == 0)
craigdo@5324
   246
        {
craigdo@5324
   247
          doMultiple = true;
craigdo@5324
   248
        }
craigdo@5324
   249
craigdo@4772
   250
      if (arg.find ("--out=") != std::string::npos)
craigdo@4772
   251
        {
craigdo@4772
   252
          outfileName = arg.substr (arg.find_first_of ("=") + 1, 9999);
craigdo@4772
   253
          haveOutfile = true;
craigdo@4772
   254
        }
craigdo@4772
   255
craigdo@4772
   256
      if (arg.find ("--suite=") != std::string::npos)
craigdo@4772
   257
        {
craigdo@4772
   258
          suiteName = arg.substr (arg.find_first_of ("=") + 1, 9999);
craigdo@4772
   259
          doSuite = true;
craigdo@4772
   260
        }
craigdo@4772
   261
craigdo@5481
   262
      if (arg.find ("--tempdir=") != std::string::npos)
craigdo@5481
   263
        {
craigdo@5481
   264
          tempdir = arg.substr (arg.find_first_of ("=") + 1, 9999);
craigdo@5481
   265
          haveTempdir = true;
craigdo@5481
   266
        }
craigdo@5481
   267
craigdo@4772
   268
      if (arg.compare ("--verbose") == 0)
craigdo@4772
   269
        {
craigdo@4772
   270
          doVerbose = true;
craigdo@4772
   271
        }
craigdo@4772
   272
    }
craigdo@4772
   273
craigdo@4772
   274
  //
craigdo@4772
   275
  // A help request trumps everything else.  If we have one, just print the help
craigdo@4772
   276
  // and leave.
craigdo@4772
   277
  //
craigdo@4772
   278
  if (doHelp)
craigdo@4772
   279
    {
craigdo@6204
   280
      std::cout << "  --assert:               Tell tests to segfault (like assert) if an error is detected" << std::endl;
craigdo@4772
   281
      std::cout << "  --basedir=dir:          Set the base directory (where to find src) to \"dir\"" << std::endl;
mathieu@5780
   282
      std::cout << "  --tempdir=dir:          Set the temporary directory (where to find data files) to \"dir\"" << std::endl;
craigdo@4772
   283
      std::cout << "  --constrain=test-type:  Constrain checks to test suites of type \"test-type\"" << std::endl;
craigdo@4772
   284
      std::cout << "  --help:                 Print this message" << std::endl;
craigdo@4772
   285
      std::cout << "  --kinds:                List all of the available kinds of tests" << std::endl;
craigdo@4772
   286
      std::cout << "  --list:                 List all of the test suites (optionally constrained by test-type)" << std::endl;
craigdo@5324
   287
      std::cout << "  --multiple:             Allow test suites and cases to produce multiple failures" << std::endl;
craigdo@4772
   288
      std::cout << "  --out=file-name:        Set the test status output file to \"file-name\"" << std::endl;
craigdo@4772
   289
      std::cout << "  --suite=suite-name:     Run the test suite named \"suite-name\"" << std::endl;
craigdo@4772
   290
      std::cout << "  --verbose:              Turn on messages in the run test suites" << std::endl;
craigdo@4772
   291
craigdo@4772
   292
      return false;
craigdo@4772
   293
    }
craigdo@4772
   294
craigdo@4772
   295
  //
craigdo@4772
   296
  // A kinds request trumps everything remaining.  If we are asked, just 
craigdo@4772
   297
  // print the list of types and leave.
craigdo@4772
   298
  //
craigdo@4772
   299
  if (doKinds)
craigdo@4772
   300
    {
craigdo@4772
   301
      //
craigdo@4772
   302
      // Coming up with a string to represent a test type is completely up to
craigdo@4772
   303
      // us here.  We just define the types as being a string composed of the
craigdo@4772
   304
      // enum defined in test.h converted to lower case.
craigdo@4772
   305
      //
craigdo@4772
   306
      std::cout << "  bvt:         Build Verification Tests (to see if build completed successfully)" << std::endl;
craigdo@5369
   307
      std::cout << "  core:        Run all TestSuite-based tests (exclude examples)" << std::endl;
craigdo@4772
   308
      std::cout << "  example:     Examples (to see if example programs run successfully)" << std::endl;
craigdo@4772
   309
      std::cout << "  performance: Performance Tests (check to see if the system is as fast as expected)" << std::endl;
craigdo@5369
   310
      std::cout << "  system:      System Tests (spans modules to check integration of modules)" << std::endl;
craigdo@5369
   311
      std::cout << "  unit:        Unit Tests (within modules to check basic functionality)" << std::endl;
craigdo@4772
   312
craigdo@4772
   313
      return false;
craigdo@4772
   314
    }
craigdo@4772
   315
craigdo@4772
   316
  //
craigdo@4772
   317
  // A list request is the first functional request.  It trumps running the
craigdo@4772
   318
  // actual tests.  If we get a list request, we don't run anything, we just
craigdo@4772
   319
  // do the requested list which may or may not be qualified by a typename.
craigdo@4772
   320
  //
craigdo@4772
   321
  if (doList)
craigdo@4772
   322
    {
craigdo@4772
   323
      for (uint32_t i = 0; i < TestRunner::GetNTestSuites (); ++i)
craigdo@4772
   324
        {
craigdo@4772
   325
          TestSuite *suite = TestRunner::GetTestSuite (i);
craigdo@4772
   326
craigdo@4772
   327
          //
craigdo@5369
   328
          // Filter the tests listed by type if requested.  The special typeName 
craigdo@5369
   329
          // "core" means any TestSuite.
craigdo@4772
   330
          //
craigdo@5369
   331
          if (haveType && typeName != "core")
craigdo@4772
   332
            {
craigdo@4772
   333
              TestSuite::TestType type = suite->GetTestType ();
craigdo@4772
   334
              if (typeName == "bvt" && type != TestSuite::BVT)
craigdo@4772
   335
                {
craigdo@4772
   336
                  continue;
craigdo@4772
   337
                }
craigdo@4772
   338
craigdo@4772
   339
              if (typeName == "unit" && type != TestSuite::UNIT)
craigdo@4772
   340
                {
craigdo@4772
   341
                  continue;
craigdo@4772
   342
                }
craigdo@4772
   343
craigdo@4772
   344
              if (typeName == "system" && type != TestSuite::SYSTEM)
craigdo@4772
   345
                {
craigdo@4772
   346
                  continue;
craigdo@4772
   347
                }
craigdo@4772
   348
craigdo@4772
   349
              if (typeName == "example" && type != TestSuite::EXAMPLE)
craigdo@4772
   350
                {
craigdo@4772
   351
                  continue;
craigdo@4772
   352
                }
craigdo@4772
   353
craigdo@4772
   354
              if (typeName == "performance" && type != TestSuite::PERFORMANCE)
craigdo@4772
   355
                {
craigdo@4772
   356
                  continue;
craigdo@4772
   357
                }
craigdo@4772
   358
            }
craigdo@4772
   359
craigdo@4772
   360
          //
craigdo@4772
   361
          // This creates a list of test suite names that can be used by the
craigdo@4772
   362
          // high level test manager to get a list of all tests.  It will then
craigdo@4772
   363
          // typically launch individual tests in parallel, calling back here
craigdo@4772
   364
          // with a specific "suite=" to run.
craigdo@4772
   365
          //
craigdo@4772
   366
          std::cout << suite->GetName () << std::endl;
craigdo@4772
   367
        }
craigdo@4772
   368
craigdo@4772
   369
      return false;
craigdo@4772
   370
    }
craigdo@4772
   371
craigdo@4772
   372
  //
craigdo@6194
   373
  // We have a lot of options possible to provide flexibility.  It can become
craigdo@6194
   374
  // painful, however, to provide all of the options when debugging, and it 
craigdo@6194
   375
  // turns out that not all tests require all options.  It is really helpful
craigdo@6194
   376
  // to try and put together some reasonable defaults if we're not provided
craigdo@6194
   377
  // them.
craigdo@4772
   378
  //
craigdo@6194
   379
  if (!haveTempdir)
craigdo@6194
   380
    {
craigdo@6194
   381
      //
craigdo@6194
   382
      // No temporary directory was provided.  We don't know if the selected
craigdo@6194
   383
      // test or tests will need one, but we can cook something up.  The 
craigdo@6194
   384
      // tmpnam function has its own set of problems, so we'll just do our 
craigdo@6194
   385
      // own thing.
craigdo@6194
   386
      //
craigdo@6194
   387
      tempdir = TempDir ();
craigdo@6194
   388
      if (tempdir.size ()) 
craigdo@6194
   389
        {
craigdo@6194
   390
          std::cout << "Temporary directory not provided.  Using \"" << tempdir << "\"" << std::endl;
craigdo@6194
   391
          haveTempdir = true;
craigdo@6194
   392
        }
craigdo@6194
   393
      else
craigdo@6194
   394
        {
craigdo@6194
   395
          std::cout << "Temporary directory not provided and unable to create one." << std::endl;
craigdo@6194
   396
          return true;
craigdo@6194
   397
        }
craigdo@6194
   398
    }
craigdo@4772
   399
craigdo@4772
   400
  if (haveBasedir == false)
craigdo@4772
   401
    {
craigdo@6194
   402
      //
craigdo@6228
   403
      // No basedir was provided.  If we don't have it, we can try and find it 
craigdo@6228
   404
      // in the current directory tree.
craigdo@6194
   405
      //
craigdo@6194
   406
      basedir = BaseDir ();
craigdo@6194
   407
      if (basedir.size ()) 
craigdo@6194
   408
        {
craigdo@6194
   409
          std::cout << "Base directory not provided.  Using \"" << basedir << "\"" << std::endl;
craigdo@6194
   410
          haveBasedir = true;
craigdo@6194
   411
        }
craigdo@6194
   412
      else
craigdo@6194
   413
        {
craigdo@6194
   414
          std::cout << "Base directory not provided and unable to find one." << std::endl;
craigdo@6194
   415
          return true;
craigdo@6194
   416
        }
craigdo@4772
   417
    }
craigdo@4772
   418
craigdo@4772
   419
  //
craigdo@4772
   420
  // If given an output file, we just append the output of each test suite 
craigdo@4772
   421
  // we're asked to run to the end of that file.  We need to append since the
craigdo@4772
   422
  // higher level test runner may be just running a number of tests back to 
craigdo@4772
   423
  // back.  We leave it up to that code to decide how to deal with possible
craigdo@4772
   424
  // parallel operation -- we just append to a file here.  If no output file
craigdo@4772
   425
  // is specified, we don't do any output and just return the sense of error
craigdo@4772
   426
  // given by the test.
craigdo@4772
   427
  //
craigdo@4772
   428
  std::ofstream *pofs = 0;
craigdo@4772
   429
  std::ofstream ofs;
craigdo@4772
   430
craigdo@4772
   431
  if (!outfileName.empty ())
craigdo@4772
   432
    {
craigdo@4772
   433
      ofs.open (outfileName.c_str (), std::fstream::out | std::fstream::app);
craigdo@4772
   434
      pofs = &ofs;
craigdo@4772
   435
    }
craigdo@4772
   436
craigdo@4772
   437
  //
craigdo@4772
   438
  // If we have a specified test suite to run, then we only run that suite.
craigdo@4772
   439
  // The default case is to "run everything.  We don't expect this to be done
craigdo@4772
   440
  // much since typically higher level code will be running suites in parallel
craigdo@4772
   441
  // but we'll do it if asked.
craigdo@4772
   442
  //
craigdo@4772
   443
  bool result = false;
craigdo@5297
   444
  bool suiteRan = false;
craigdo@4772
   445
craigdo@4772
   446
  for (uint32_t i = 0; i < TestRunner::GetNTestSuites (); ++i)
craigdo@4772
   447
    {
craigdo@4772
   448
      TestSuite *testSuite = TestRunner::GetTestSuite (i);
craigdo@4772
   449
      if (doSuite == false || (doSuite == true && suiteName == testSuite->GetName ()))
craigdo@4772
   450
        {
craigdo@4772
   451
          testSuite->SetBaseDir (basedir);
craigdo@5481
   452
          testSuite->SetTempDir (tempdir);
craigdo@4772
   453
          testSuite->SetStream (pofs);
craigdo@4772
   454
          testSuite->SetVerbose (doVerbose);
craigdo@5324
   455
          testSuite->SetContinueOnFailure (doMultiple);
craigdo@4772
   456
          result |= testSuite->Run ();
craigdo@5297
   457
          suiteRan = true;
craigdo@4772
   458
        }
craigdo@4772
   459
    }
craigdo@4772
   460
craigdo@4772
   461
  ofs.close();
craigdo@5297
   462
craigdo@5297
   463
  //
craigdo@5297
   464
  // If we couldn't figure out how to run at least one test, then return an error
craigdo@5297
   465
  //
craigdo@5297
   466
  if (suiteRan == false)
craigdo@5297
   467
    {
craigdo@6194
   468
      std::cout << "Unable to find a test to run (constraints too severe or test not found)" << std::endl;
craigdo@5297
   469
      return true;
craigdo@5297
   470
    }
craigdo@5297
   471
craigdo@4772
   472
  return result;
craigdo@4772
   473
}