utils/test-runner.cc
author Josh Pelkey <jpelkey@gatech.edu>
Wed, 11 Aug 2010 11:37:37 -0400
changeset 6553 fb5ad9c7755a
parent 6251 ff3501de65dc
permissions -rw-r--r--
update release notes and fix doxygen warnings
     1 /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */
     2 /*
     3  * Copyright (c) 2009 University of Washington
     4  *
     5  * This program is free software; you can redistribute it and/or modify
     6  * it under the terms of the GNU General Public License version 2 as
     7  * published by the Free Software Foundation;
     8  *
     9  * This program is distributed in the hope that it will be useful,
    10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
    11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    12  * GNU General Public License for more details.
    13  *
    14  * You should have received a copy of the GNU General Public License
    15  * along with this program; if not, write to the Free Software
    16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
    17  */
    18 
    19 #include "ns3/test.h"
    20 #include "ns3/assert.h"
    21 #include "ns3/abort.h"
    22 
    23 #include <iostream>
    24 #include <fstream>
    25 #include <string>
    26 #include <stdlib.h>
    27 #include <stdio.h>
    28 #include <time.h>
    29 #include <sys/types.h>
    30 #include <sys/stat.h>
    31 #include <dirent.h>
    32 #include <string.h>
    33 
    34 extern bool gBreakOnFailure;
    35 
    36 using namespace ns3;
    37 
    38 //
    39 // Create a temporary directory for use by test programs.  This is not a 
    40 // foolproof thing, but a reasonably good way to get a throwaway directory
    41 // while running tests in a debugger.
    42 //  
    43 std::string
    44 TempDir (void)
    45 {
    46   char *path = NULL;
    47 
    48   path = getenv ("TMP");
    49   if (path == NULL)
    50     {
    51       path = getenv ("TEMP");
    52       if (path == NULL)
    53         {
    54           path = const_cast<char *> ("/tmp");
    55         }
    56     }
    57 
    58   //
    59   // Just in case the user wants to go back and find the output, we give
    60   // a hint as to which dir we created by including a time hint.
    61   //
    62   time_t now = time (NULL);
    63   struct tm *tm_now = localtime (&now);
    64   
    65   //
    66   // But we also randomize the name in case there are multiple users doing
    67   // this at the same time
    68   //
    69   srand (time (0));
    70   long int n = rand ();
    71 
    72   //
    73   // The final path to the directory is going to look something like
    74   // 
    75   //   /tmp/ns3-14.30.29.32767
    76   //
    77   // The first segment comes from one of the temporary directory env 
    78   // variables or /tmp if not found.  The directory name starts with an
    79   // identifier telling folks who is making all of the temp directories
    80   // and then the local time (in this case 14.30.29 -- which is 2:30 and
    81   // 29 seconds PM).
    82   //
    83   char dirname[1024];
    84   snprintf (dirname, sizeof(dirname),  "%s/ns-3.%d.%d.%d.%ld", path, tm_now->tm_hour, tm_now->tm_min, tm_now->tm_sec, n);
    85 
    86 #if (defined(_WIN32) || defined(_WIN64)) && !defined(__CYGWIN__)
    87   if(mkdir(dirname) == 0)
    88 #else
    89   if (mkdir (dirname, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH) == 0)
    90 #endif
    91     {
    92       return dirname;
    93     } 
    94   else
    95     {
    96       return "";
    97     }
    98 }
    99 
   100 //
   101 // Test suites may need to figure out where their source directory is in order
   102 // to find test vectors.  To do that they will need to know  where the base 
   103 // directory of the distribution is (the directory in which "src" is found).
   104 // It is painful for a user debugging test suites to always provide that dir
   105 // so we try and find it in the current directory tree.
   106 //
   107 std::string
   108 BaseDir (void)
   109 {
   110   //
   111   // Get an absolute path to the current working directory.  Following code
   112   // depends on the leading '/'
   113   //
   114   char pathbuf[PATH_MAX];
   115   if (getcwd (pathbuf, sizeof(pathbuf)) == NULL)
   116     {
   117       NS_ABORT_MSG ("Basedir():  unable to getcwd()");
   118     }
   119 
   120   //
   121   // Walk up the directory tree looking for a directory that has files that
   122   // indicate it is the base of an ns-3 distribution.  We use VERSION and
   123   // LICENSE which have been there from the beginning of time.
   124   //
   125   for (;;)
   126     {
   127       bool haveVersion = false;
   128       bool haveLicense = false;
   129 
   130       //
   131       // Open the directory file for the current directory and loop through
   132       // the directory entries.
   133       //
   134       DIR *dp = opendir (pathbuf);
   135       if (dp != NULL)
   136         {
   137           while (struct dirent *de = readdir (dp))
   138             {
   139               if (strcmp (de->d_name, "VERSION") == 0)
   140                 {
   141                   haveVersion = true;
   142                 }
   143 
   144               if (strcmp (de->d_name, "LICENSE") == 0)
   145                 {
   146                   haveLicense = true;
   147                 }
   148             }
   149         }
   150       closedir (dp);
   151 
   152       //
   153       // If there's a file named VERSION and a file named LICENSE in this
   154       // directory, we assume it's our base directory.
   155       //
   156       if (haveVersion && haveLicense)
   157         {
   158           return pathbuf;
   159         }
   160 
   161       //
   162       // Strip off the last segment of the current directory.
   163       //
   164       char *last = strrchr (pathbuf, '/');
   165       NS_ASSERT_MSG (last, "No \"/\" found in absolute path ???");
   166       *last = '\0';
   167 
   168       if (strlen(pathbuf) == 0)
   169         {
   170           return "";
   171         }
   172     }
   173 
   174   //
   175   // Quiet the compiler.
   176   //
   177   return "";
   178 }
   179 
   180 //
   181 // Run one of the test suites.  Returns an integer with the boolean sense of
   182 // "an error has occurred."  That is, 0 == false -> no error; 1 == true -> an
   183 // error occurred.
   184 //
   185 int 
   186 main (int argc, char *argv[])
   187 {
   188   bool doVerbose = false;
   189   bool doList = false;
   190   bool doMultiple = false;
   191   bool doHelp = false;
   192   bool doSuite = false;
   193   bool doKinds = false;
   194 
   195   gBreakOnFailure = false;
   196 
   197   bool haveBasedir = false;
   198   bool haveTempdir = false;
   199   bool haveOutfile = false;
   200   bool haveType = false;
   201 
   202   std::string suiteName;
   203   std::string basedir;
   204   std::string tempdir;
   205   std::string outfileName;
   206   std::string typeName;
   207 
   208 
   209   for (int i = 1; i < argc; ++i)
   210     {
   211       std::string arg(argv[i]);
   212 
   213       if (arg.find ("--assert") != std::string::npos)
   214         {
   215           gBreakOnFailure = true;
   216         }
   217 
   218       if (arg.find ("--basedir=") != std::string::npos)
   219         {
   220           basedir = arg.substr (arg.find_first_of ("=") + 1, 9999);
   221           haveBasedir = true;
   222         }
   223 
   224       if (arg.find ("--constrain=") != std::string::npos)
   225         {
   226           typeName = arg.substr (arg.find_first_of ("=") + 1, 9999);
   227           haveType = true;
   228         }
   229 
   230       if (arg.compare ("--help") == 0)
   231         {
   232           doHelp = true;
   233         }
   234 
   235       if (arg.compare ("--kinds") == 0)
   236         {
   237           doKinds = true;
   238         }
   239 
   240       if (arg.compare ("--list") == 0)
   241         {
   242           doList = true;
   243         }
   244 
   245       if (arg.compare ("--multiple") == 0)
   246         {
   247           doMultiple = true;
   248         }
   249 
   250       if (arg.find ("--out=") != std::string::npos)
   251         {
   252           outfileName = arg.substr (arg.find_first_of ("=") + 1, 9999);
   253           haveOutfile = true;
   254         }
   255 
   256       if (arg.find ("--suite=") != std::string::npos)
   257         {
   258           suiteName = arg.substr (arg.find_first_of ("=") + 1, 9999);
   259           doSuite = true;
   260         }
   261 
   262       if (arg.find ("--tempdir=") != std::string::npos)
   263         {
   264           tempdir = arg.substr (arg.find_first_of ("=") + 1, 9999);
   265           haveTempdir = true;
   266         }
   267 
   268       if (arg.compare ("--verbose") == 0)
   269         {
   270           doVerbose = true;
   271         }
   272     }
   273 
   274   //
   275   // A help request trumps everything else.  If we have one, just print the help
   276   // and leave.
   277   //
   278   if (doHelp)
   279     {
   280       std::cout << "  --assert:               Tell tests to segfault (like assert) if an error is detected" << std::endl;
   281       std::cout << "  --basedir=dir:          Set the base directory (where to find src) to \"dir\"" << std::endl;
   282       std::cout << "  --tempdir=dir:          Set the temporary directory (where to find data files) to \"dir\"" << std::endl;
   283       std::cout << "  --constrain=test-type:  Constrain checks to test suites of type \"test-type\"" << std::endl;
   284       std::cout << "  --help:                 Print this message" << std::endl;
   285       std::cout << "  --kinds:                List all of the available kinds of tests" << std::endl;
   286       std::cout << "  --list:                 List all of the test suites (optionally constrained by test-type)" << std::endl;
   287       std::cout << "  --multiple:             Allow test suites and cases to produce multiple failures" << std::endl;
   288       std::cout << "  --out=file-name:        Set the test status output file to \"file-name\"" << std::endl;
   289       std::cout << "  --suite=suite-name:     Run the test suite named \"suite-name\"" << std::endl;
   290       std::cout << "  --verbose:              Turn on messages in the run test suites" << std::endl;
   291 
   292       return false;
   293     }
   294 
   295   //
   296   // A kinds request trumps everything remaining.  If we are asked, just 
   297   // print the list of types and leave.
   298   //
   299   if (doKinds)
   300     {
   301       //
   302       // Coming up with a string to represent a test type is completely up to
   303       // us here.  We just define the types as being a string composed of the
   304       // enum defined in test.h converted to lower case.
   305       //
   306       std::cout << "  bvt:         Build Verification Tests (to see if build completed successfully)" << std::endl;
   307       std::cout << "  core:        Run all TestSuite-based tests (exclude examples)" << std::endl;
   308       std::cout << "  example:     Examples (to see if example programs run successfully)" << std::endl;
   309       std::cout << "  performance: Performance Tests (check to see if the system is as fast as expected)" << std::endl;
   310       std::cout << "  system:      System Tests (spans modules to check integration of modules)" << std::endl;
   311       std::cout << "  unit:        Unit Tests (within modules to check basic functionality)" << std::endl;
   312 
   313       return false;
   314     }
   315 
   316   //
   317   // A list request is the first functional request.  It trumps running the
   318   // actual tests.  If we get a list request, we don't run anything, we just
   319   // do the requested list which may or may not be qualified by a typename.
   320   //
   321   if (doList)
   322     {
   323       for (uint32_t i = 0; i < TestRunner::GetNTestSuites (); ++i)
   324         {
   325           TestSuite *suite = TestRunner::GetTestSuite (i);
   326 
   327           //
   328           // Filter the tests listed by type if requested.  The special typeName 
   329           // "core" means any TestSuite.
   330           //
   331           if (haveType && typeName != "core")
   332             {
   333               TestSuite::TestType type = suite->GetTestType ();
   334               if (typeName == "bvt" && type != TestSuite::BVT)
   335                 {
   336                   continue;
   337                 }
   338 
   339               if (typeName == "unit" && type != TestSuite::UNIT)
   340                 {
   341                   continue;
   342                 }
   343 
   344               if (typeName == "system" && type != TestSuite::SYSTEM)
   345                 {
   346                   continue;
   347                 }
   348 
   349               if (typeName == "example" && type != TestSuite::EXAMPLE)
   350                 {
   351                   continue;
   352                 }
   353 
   354               if (typeName == "performance" && type != TestSuite::PERFORMANCE)
   355                 {
   356                   continue;
   357                 }
   358             }
   359 
   360           //
   361           // This creates a list of test suite names that can be used by the
   362           // high level test manager to get a list of all tests.  It will then
   363           // typically launch individual tests in parallel, calling back here
   364           // with a specific "suite=" to run.
   365           //
   366           std::cout << suite->GetName () << std::endl;
   367         }
   368 
   369       return false;
   370     }
   371 
   372   //
   373   // We have a lot of options possible to provide flexibility.  It can become
   374   // painful, however, to provide all of the options when debugging, and it 
   375   // turns out that not all tests require all options.  It is really helpful
   376   // to try and put together some reasonable defaults if we're not provided
   377   // them.
   378   //
   379   if (!haveTempdir)
   380     {
   381       //
   382       // No temporary directory was provided.  We don't know if the selected
   383       // test or tests will need one, but we can cook something up.  The 
   384       // tmpnam function has its own set of problems, so we'll just do our 
   385       // own thing.
   386       //
   387       tempdir = TempDir ();
   388       if (tempdir.size ()) 
   389         {
   390           std::cout << "Temporary directory not provided.  Using \"" << tempdir << "\"" << std::endl;
   391           haveTempdir = true;
   392         }
   393       else
   394         {
   395           std::cout << "Temporary directory not provided and unable to create one." << std::endl;
   396           return true;
   397         }
   398     }
   399 
   400   if (haveBasedir == false)
   401     {
   402       //
   403       // No basedir was provided.  If we don't have it, we can try and find it 
   404       // in the current directory tree.
   405       //
   406       basedir = BaseDir ();
   407       if (basedir.size ()) 
   408         {
   409           std::cout << "Base directory not provided.  Using \"" << basedir << "\"" << std::endl;
   410           haveBasedir = true;
   411         }
   412       else
   413         {
   414           std::cout << "Base directory not provided and unable to find one." << std::endl;
   415           return true;
   416         }
   417     }
   418 
   419   //
   420   // If given an output file, we just append the output of each test suite 
   421   // we're asked to run to the end of that file.  We need to append since the
   422   // higher level test runner may be just running a number of tests back to 
   423   // back.  We leave it up to that code to decide how to deal with possible
   424   // parallel operation -- we just append to a file here.  If no output file
   425   // is specified, we don't do any output and just return the sense of error
   426   // given by the test.
   427   //
   428   std::ofstream *pofs = 0;
   429   std::ofstream ofs;
   430 
   431   if (!outfileName.empty ())
   432     {
   433       ofs.open (outfileName.c_str (), std::fstream::out | std::fstream::app);
   434       pofs = &ofs;
   435     }
   436 
   437   //
   438   // If we have a specified test suite to run, then we only run that suite.
   439   // The default case is to "run everything.  We don't expect this to be done
   440   // much since typically higher level code will be running suites in parallel
   441   // but we'll do it if asked.
   442   //
   443   bool result = false;
   444   bool suiteRan = false;
   445 
   446   for (uint32_t i = 0; i < TestRunner::GetNTestSuites (); ++i)
   447     {
   448       TestSuite *testSuite = TestRunner::GetTestSuite (i);
   449       if (doSuite == false || (doSuite == true && suiteName == testSuite->GetName ()))
   450         {
   451           testSuite->SetBaseDir (basedir);
   452           testSuite->SetTempDir (tempdir);
   453           testSuite->SetStream (pofs);
   454           testSuite->SetVerbose (doVerbose);
   455           testSuite->SetContinueOnFailure (doMultiple);
   456           result |= testSuite->Run ();
   457           suiteRan = true;
   458         }
   459     }
   460 
   461   ofs.close();
   462 
   463   //
   464   // If we couldn't figure out how to run at least one test, then return an error
   465   //
   466   if (suiteRan == false)
   467     {
   468       std::cout << "Unable to find a test to run (constraints too severe or test not found)" << std::endl;
   469       return true;
   470     }
   471 
   472   return result;
   473 }