--- a/.hgignore Fri Oct 09 23:51:23 2009 -0700
+++ b/.hgignore Mon Oct 12 14:01:36 2009 -0700
@@ -4,6 +4,7 @@
~$
^build-dir
^build
+^testpy-output
^doc/html
^doc/latex
^\.lock-wscript
--- a/doc/testing/testing-framework.texi Fri Oct 09 23:51:23 2009 -0700
+++ b/doc/testing/testing-framework.texi Mon Oct 12 14:01:36 2009 -0700
@@ -107,6 +107,37 @@
their distribution is working correctly, and by developers who are interested
in determining if changes they have made have caused any regressions.
+There are a number of options available to control the behavir of @code{test.py}.
+if you run @code{test.py --help} you should see a command summary like:
+
+@verbatim
+ Usage: test.py [options]
+
+ Options:
+ -h, --help show this help message and exit
+ -c KIND, --constrain=KIND
+ constrain the test-runner by kind of test
+ -e EXAMPLE, --example=EXAMPLE
+ specify a single example to run
+ -g, --grind run the test suites and examples using valgrind
+ -k, --kinds print the kinds of tests available
+ -l, --list print the list of known tests
+ -m, --multiple report multiple failures from test suites and test
+ cases
+ -n, --nowaf do not run waf before starting testing
+ -s TEST-SUITE, --suite=TEST-SUITE
+ specify a single test suite to run
+ -v, --verbose print progress and informational messages
+ -w HTML-FILE, --web=HTML-FILE, --html=HTML-FILE
+ write detailed test results into HTML-FILE.html
+ -r, --retain retain all temporary files (which are normally
+ deleted)
+ -t TEXT-FILE, --text=TEXT-FILE
+ write detailed test results into TEXT-FILE.txt
+ -x XML-FILE, --xml=XML-FILE
+ write detailed test results into XML-FILE.xml
+@end verbatim
+
If one specifies an optional output style, one can generate detailed descriptions
of the tests and status. Available styles are @command{text} and @command{HTML}.
The buildbots will select the HTML option to generate HTML test reports for the
@@ -324,6 +355,18 @@
./test.py --grind
@end verbatim
+As it runs, @code{test.py} and the programs that it runs indirectly, generate large
+numbers of temporary files. Usually, the content of these files is not interesting,
+however in some cases it can be useful (for debugging purposes) to view these files.
+@code{test.py} provides a @command{--retain} option which will cause these temporary
+files to be kept after the run is completed. The files are saved in a directory
+named @code{testpy} under a subdirectory named according to the current Coordinated
+Universal Time (also known as Greenwich Mean Time).
+
+@verbatim
+ ./test.py --retain
+@end verbatim
+
Finally, @code{test.py} provides a @command{--verbose} option which will print
large amounts of information about its progress. It is not expected that this
will be terribly useful unless there is an error. In this case, you can get
--- a/test.py Fri Oct 09 23:51:23 2009 -0700
+++ b/test.py Mon Oct 12 14:01:36 2009 -0700
@@ -19,12 +19,12 @@
import os
import sys
+import time
import optparse
import subprocess
import threading
import Queue
import signal
-import random
import xml.dom.minidom
import shutil
@@ -149,22 +149,18 @@
]
#
-# Most of the examples produce gangs of trace files, so we want to find
-# somewhere to put them that won't pollute the current directory. One
-# obvious place is somewhere in /tmp.
-#
-TMP_TRACES_DIR = "/tmp/unchecked-traces"
-
-#
# The test suites are going to want to output status. They are running
# concurrently. This means that unless we are careful, the output of
# the test suites will be interleaved. Rather than introducing a lock
# file that could unintentionally start serializing execution, we ask
# the tests to write their output to a temporary directory and then
# put together the final output file when we "join" the test tasks back
-# to the main thread.
+# to the main thread. In addition to this issue, the example programs
+# often write lots and lots of trace files which we will just ignore.
+# We put all of them into the temp directory as well, so they can be
+# easily deleted.
#
-TMP_OUTPUT_DIR = "/tmp/testpy"
+TMP_OUTPUT_DIR = "testpy-output"
def get_node_text(node):
for child in node.childNodes:
@@ -652,10 +648,7 @@
# This is the temporary results file name that will be given to an executing
# test as it is being run. We will be running all of our tests in parallel
# so there must be multiple temporary output files. These will be collected
- # into a single XML file at the end and then be deleted. The file names are
- # just giant random numbers, for example
- #
- # "/tmp/testpy/5437925246732857"
+ # into a single XML file at the end and then be deleted.
#
def set_tmp_file_name(self, tmp_file_name):
self.tmp_file_name = tmp_file_name
@@ -808,26 +801,30 @@
# finds a problem, or HTML for nightly builds. In these cases, an
# XML file is written containing the status messages from the test suites.
# This file is then read and translated into text or HTML. It is expected
- # that nobody will really be interested in the XML, so we write it to
- # somewhere in /tmp with a random name to avoid collisions. Just in case
- # some strange once-in-a-lifetime error occurs, we always write the info
- # so it can be found, we just may not use it.
+ # that nobody will really be interested in the XML, so we write it somewhere
+ # with a unique name (time) to avoid collisions. In case an error happens, we
+ # provide a runtime option to retain the temporary files.
#
# When we run examples as smoke tests, they are going to want to create
# lots and lots of trace files. We aren't really interested in the contents
- # of the trace files, so we also just stash them off in /tmp somewhere.
+ # of the trace files, so we also just stash them off in the temporary dir.
+ # The retain option also causes these unchecked trace files to be kept.
#
+ date_and_time = time.strftime("%Y-%m-%d-%H-%M-%S-CUT", time.gmtime())
+
if not os.path.exists(TMP_OUTPUT_DIR):
os.makedirs(TMP_OUTPUT_DIR)
- if not os.path.exists(TMP_TRACES_DIR):
- os.makedirs(TMP_TRACES_DIR)
+ testpy_output_dir = os.path.join(TMP_OUTPUT_DIR, date_and_time);
+
+ if not os.path.exists(testpy_output_dir):
+ os.makedirs(testpy_output_dir)
#
# Create the main output file and start filling it with XML. We need to
# do this since the tests will just append individual results to this file.
#
- xml_results_file = TMP_OUTPUT_DIR + "%d.xml" % random.randint(0, sys.maxint)
+ xml_results_file = os.path.join(testpy_output_dir, "results.xml")
f = open(xml_results_file, 'w')
f.write('<?xml version="1.0"?>\n')
f.write('<TestResults>\n')
@@ -929,7 +926,7 @@
job = Job()
job.set_is_example(False)
job.set_display_name(test)
- job.set_tmp_file_name(TMP_OUTPUT_DIR + "%d" % random.randint(0, sys.maxint))
+ job.set_tmp_file_name(os.path.join(testpy_output_dir, "%s.xml" % test))
job.set_cwd(os.getcwd())
job.set_basedir(os.getcwd())
if (options.multiple):
@@ -999,7 +996,7 @@
job.set_is_example(True)
job.set_display_name(test)
job.set_tmp_file_name("")
- job.set_cwd(TMP_TRACES_DIR)
+ job.set_cwd(testpy_output_dir)
job.set_basedir(os.getcwd())
job.set_shell_command("examples/%s" % test)
@@ -1022,7 +1019,7 @@
job.set_is_example(True)
job.set_display_name(options.example)
job.set_tmp_file_name("")
- job.set_cwd(TMP_TRACES_DIR)
+ job.set_cwd(testpy_output_dir)
job.set_basedir(os.getcwd())
job.set_shell_command("examples/%s" % options.example)
@@ -1192,11 +1189,6 @@
f.write("</TestSuite>\n")
f.close()
- try:
- os.remove(job.tmp_file_name)
- except:
- pass
-
#
# We have all of the tests run and the results written out. One final
# bit of housekeeping is to wait for all of the threads to close down
@@ -1233,14 +1225,21 @@
if len(options.xml):
shutil.copyfile(xml_results_file, options.xml)
+ #
+ # If we have been asked to retain all of the little temporary files, we
+ # don't delete tm. If we do delete the temporary files, delete only the
+ # directory we just created. We don't want to happily delete any retained
+ # directories, which will probably surprise the user.
+ #
+ if not options.retain:
+ shutil.rmtree(testpy_output_dir)
+
if passed_tests + skipped_tests == total_tests:
return 0 # success
else:
return 1 # catchall for general errors
def main(argv):
- random.seed()
-
parser = optparse.OptionParser()
parser.add_option("-c", "--constrain", action="store", type="string", dest="constrain", default="",
metavar="KIND",
@@ -1276,6 +1275,9 @@
metavar="HTML-FILE",
help="write detailed test results into HTML-FILE.html")
+ parser.add_option("-r", "--retain", action="store_true", dest="retain", default=False,
+ help="retain all temporary files (which are normally deleted)")
+
parser.add_option("-t", "--text", action="store", type="string", dest="text", default="",
metavar="TEXT-FILE",
help="write detailed test results into TEXT-FILE.txt")