Python: require new pybindgen and re-scan API to make the list of free functions and namespaces sorted.
This commit will change a lot the scanned API definitions, once, but should allow much more stable scanning in the future, as right now only types were being sorted, but free functions can jump up or down when different people on different machines scan the API. Well, no more will that happen in the future, I hope.
## -*- Mode: python; py-indent-offset: 4; indent-tabs-mode: nil; coding: utf-8; -*-
import re
import Params
import Configure
import Object
import Action
import os
import Task
import pproc as subprocess
from Params import fatal, warning
import shutil
import sys
## Adjust python path to look for our local copy of pybindgen
LOCAL_PYBINDGEN_PATH = os.path.join(os.getcwd(), "bindings", "python", "pybindgen")
#PYBINDGEN_BRANCH = 'lp:pybindgen'
PYBINDGEN_BRANCH = 'https://launchpad.net/pybindgen'
if os.environ.get('PYTHONPATH', ''):
os.environ['PYTHONPATH'] = LOCAL_PYBINDGEN_PATH + os.pathsep + os.environ.get('PYTHONPATH')
else:
os.environ['PYTHONPATH'] = LOCAL_PYBINDGEN_PATH
## https://launchpad.net/pybindgen/
REQUIRED_PYBINDGEN_VERSION = (0, 9, 0, 603)
REQUIRED_PYGCCXML_VERSION = (0, 9, 5)
def set_options(opt):
opt.tool_options('python')
opt.add_option('--disable-python',
help=("Don't build Python bindings."),
action="store_true", default=False,
dest='python_disable')
opt.add_option('--python-scan',
help=("Rescan Python bindings. Needs working GCCXML / pygccxml environment."),
action="store_true", default=False,
dest='python_scan')
opt.add_option('--pybindgen-checkout',
help=("During configure, force checkout of pybingen inside ns-3, "
"instead of using the system installed version."),
action="store_true", default=False,
dest='pybindgen_checkout')
def fetch_pybindgen(conf):
"""
Fetches pybindgen from launchpad as bindings/python/pybindgen.
Returns True if successful, False it not.
"""
bzr = conf.find_program("bzr")
if not bzr:
warning("the program 'bzr' is needed in order to fetch pybindgen")
return False
if len(REQUIRED_PYBINDGEN_VERSION) == 4:
rev = "-rrevno:%i" % REQUIRED_PYBINDGEN_VERSION[3]
else:
rev = "-rtag:%s" % '.'.join([str(x) for x in REQUIRED_PYBINDGEN_VERSION])
if os.path.exists(LOCAL_PYBINDGEN_PATH):
print "Trying to update pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip."
cmd = [bzr, "pull", rev, PYBINDGEN_BRANCH]
print " => ", ' '.join(cmd)
try:
if subprocess.Popen(cmd, cwd=LOCAL_PYBINDGEN_PATH).wait():
return False
except KeyboardInterrupt:
print "Interrupted; Python bindings will be disabled."
return False
print "Update was successful."
else:
print "Trying to fetch pybindgen; this will fail if no network connection is available. Hit Ctrl-C to skip."
cmd = [bzr, "checkout", rev, PYBINDGEN_BRANCH, LOCAL_PYBINDGEN_PATH]
print " => ", ' '.join(cmd)
try:
if subprocess.Popen(cmd).wait():
return False
except KeyboardInterrupt:
print "Interrupted; Python bindings will be disabled."
shutil.rmtree(LOCAL_PYBINDGEN_PATH, True)
return False
print "Fetch was successful."
## generate a fake version.py file in pybindgen it's safer this
## way, since the normal version generation process requires
## bazaar python bindings, which may not be available.
vfile = open(os.path.join(LOCAL_PYBINDGEN_PATH, "pybindgen", "version.py"), "wt")
vfile.write("""
# (fake version generated by ns-3)
__version__ = %r
""" % list(REQUIRED_PYBINDGEN_VERSION))
vfile.close()
return True
def configure(conf):
conf.env['ENABLE_PYTHON_BINDINGS'] = False
if Params.g_options.python_disable:
conf.report_optional_feature("python", "Python Bindings", False,
"disabled by user request")
return
conf.check_tool('misc')
if sys.platform == 'cygwin':
conf.report_optional_feature("python", "Python Bindings", False,
"unsupported platform 'cygwin'")
warning("Python is not supported in CygWin environment. Try MingW instead.")
return
## Check for Python
try:
conf.check_tool('python')
conf.check_python_version((2,3))
conf.check_python_headers()
except Configure.ConfigurationError, ex:
conf.report_optional_feature("python", "Python Bindings", False, str(ex))
return
# Fix a bug with WAF and g++ 4.3.2 (it does not include "(GCC") in
# the output of g++ --version, so the WAF python detection fails
# to recognize it is gcc)
gcc_version = os.popen("%s --version" % conf.env['CXX']).readline()
if '(GCC)' in gcc_version or 'g++' in gcc_version:
conf.env.append_value('CXXFLAGS_PYEMBED','-fno-strict-aliasing')
conf.env.append_value('CXXFLAGS_PYEXT','-fno-strict-aliasing')
## Check for pybindgen
if Params.g_options.pybindgen_checkout:
fetch_pybindgen(conf)
try:
conf.check_python_module('pybindgen')
except Configure.ConfigurationError:
warning("pybindgen missing")
if not fetch_pybindgen(conf):
conf.report_optional_feature("python", "Python Bindings", False,
"PyBindGen missing and could not be retrieved")
return
else:
out = subprocess.Popen([conf.env['PYTHON'], "-c",
"import pybindgen.version; "
"print '.'.join([str(x) for x in pybindgen.version.__version__])"],
stdout=subprocess.PIPE).communicate()[0]
pybindgen_version_str = out.strip()
pybindgen_version = tuple([int(x) for x in pybindgen_version_str.split('.')])
conf.check_message('pybindgen', 'version',
(pybindgen_version >= REQUIRED_PYBINDGEN_VERSION),
pybindgen_version_str)
if not (pybindgen_version >= REQUIRED_PYBINDGEN_VERSION):
warning("pybindgen (found %s) is too old (need %s)" %
(pybindgen_version_str,
'.'.join([str(x) for x in REQUIRED_PYBINDGEN_VERSION])))
if not fetch_pybindgen(conf):
conf.report_optional_feature("python", "Python Bindings", False,
"PyBindGen too old and newer version could not be retrieved")
return
## If all has gone well, we finally enable the Python bindings
conf.env['ENABLE_PYTHON_BINDINGS'] = True
conf.report_optional_feature("python", "Python Bindings", True, None)
## Check for pygccxml
try:
conf.check_python_module('pygccxml')
except Configure.ConfigurationError:
conf.report_optional_feature("pygccxml", "Python API Scanning Support", False,
"Missing 'pygccxml' Python module")
return
out = subprocess.Popen([conf.env['PYTHON'], "-c",
"import pygccxml; print pygccxml.__version__"],
stdout=subprocess.PIPE).communicate()[0]
pygccxml_version_str = out.strip()
pygccxml_version = tuple([int(x) for x in pygccxml_version_str.split('.')])
conf.check_message('pygccxml', 'version',
(pygccxml_version >= REQUIRED_PYGCCXML_VERSION),
pygccxml_version_str)
if not (pygccxml_version >= REQUIRED_PYGCCXML_VERSION):
warning("pygccxml (found %s) is too old (need %s) => "
"automatic scanning of API definitions will not be possible" %
(pygccxml_version_str,
'.'.join([str(x) for x in REQUIRED_PYGCCXML_VERSION])))
conf.report_optional_feature("pygccxml", "Python API Scanning Support", False,
"pygccxml too old")
return
## Check gccxml version
gccxml = conf.find_program('gccxml', var='GCCXML')
if not gccxml:
warning("gccxml missing; automatic scanning of API definitions will not be possible")
conf.report_optional_feature("pygccxml", "Python API Scanning Support", False,
"gccxml missing")
return
gccxml_version_line = os.popen(gccxml + " --version").readline().strip()
m = re.match( "^GCC-XML version (\d\.\d(\.\d)?)$", gccxml_version_line)
gccxml_version = m.group(1)
gccxml_version_ok = ([int(s) for s in gccxml_version.split('.')] >= [0, 9])
conf.check_message('gccxml', 'version', True, gccxml_version)
if not gccxml_version_ok:
warning("gccxml too old, need version >= 0.9; automatic scanning of API definitions will not be possible")
conf.report_optional_feature("pygccxml", "Python API Scanning Support", False,
"gccxml too old")
return
## If we reached
conf.env['ENABLE_PYTHON_SCANNING'] = True
conf.report_optional_feature("pygccxml", "Python API Scanning Support", True, None)
prio_headers = {
-2: (
"string.h", # work around http://www.gccxml.org/Bug/view.php?id=6682
),
-1: (
"propagation-delay-model.h",
"propagation-loss-model.h",
"net-device.h",
)
}
def get_header_prio(header):
for prio, headers in prio_headers.iteritems():
if header in headers:
return prio
return 1
def calc_header_include(path):
(head, tail) = os.path.split (path)
if tail == 'ns3':
return ''
else:
return os.path.join (calc_header_include (head), tail)
def gen_ns3_metaheader(task):
assert len(task.m_outputs) == 1
header_files = [calc_header_include(node.abspath(task.m_env)) for node in task.m_inputs]
outfile = file(task.m_outputs[0].bldpath(task.m_env), "w")
def sort_func(h1, h2):
return cmp((get_header_prio(h1), h1), (get_header_prio(h1), h2))
header_files.sort(sort_func)
for header in header_files:
print >> outfile, "#include \"ns3/%s\"" % (header,)
print >> outfile, """
namespace ns3 {
static inline Ptr<Object>
__dummy_function_to_force_template_instantiation (Ptr<Object> obj, TypeId typeId)
{
return obj->GetObject<Object> (typeId);
}
static inline void
__dummy_function_to_force_template_instantiation_v2 ()
{
Time t1, t2, t3;
t1 = t2 + t3;
t1 = t2 - t3;
TimeSquare tsq = t2*t3;
Time tsqdiv = tsq/Seconds(1);
Scalar scal = t2/t3;
TimeInvert inv = scal/t3;
t1 = scal*t1;
t1 = t1/scal;
t1 < t2;
t1 <= t2;
t1 == t2;
t1 != t2;
t1 >= t2;
t1 > t2;
}
}
"""
outfile.close()
return 0
class all_ns3_headers_taskgen(Object.task_gen):
"""Generates a 'everything.h' header file that includes some/all public ns3 headers.
This single header file is to be parsed only once by gccxml, for greater efficiency.
"""
def __init__(self, *features):
Object.task_gen.__init__(self, *features)
self.inst_var = 0#'INCLUDEDIR'
#self.inst_dir = 'ns3'
def apply(self):
## get all of the ns3 headers
ns3_dir_node = Params.g_build.m_srcnode.find_dir("ns3")
all_headers_inputs = []
for filename in self.to_list(self.source):
src_node = ns3_dir_node.find_build(filename)
if src_node is None:
Params.fatal("source ns3 header file %s not found" % (filename,))
all_headers_inputs.append(src_node)
## if self.source was empty, include all ns3 headers in enabled modules
if not all_headers_inputs:
for ns3headers in Object.g_allobjs:
if type(ns3headers).__name__ == 'ns3header_taskgen': # XXX: find less hackish way to compare
## skip headers not part of enabled modules
if self.env['NS3_ENABLED_MODULES']:
if ("ns3-%s" % ns3headers.module) not in self.env['NS3_ENABLED_MODULES']:
continue
for source in ns3headers.to_list(ns3headers.source):
#source = os.path.basename(source)
node = ns3_dir_node.find_build(source)
if node is None:
fatal("missing header file %s" % (source,))
all_headers_inputs.append(node)
assert all_headers_inputs
all_headers_outputs = [self.path.find_build("everything.h")]
task = self.create_task('gen-ns3-metaheader', self.env, 4)
task.set_inputs(all_headers_inputs)
task.set_outputs(all_headers_outputs)
def install(self):
pass
def get_modules_and_headers():
"""
Gets a dict of
module_name => ([module_dep1, module_dep2, ...], [module_header1, module_header2, ...])
tuples, one for each module.
"""
retval = {}
for module in Object.g_allobjs:
if not module.name.startswith('ns3-'):
continue
module_name = module.name[4:] # strip the ns3- prefix
## find the headers object for this module
headers = []
for ns3headers in Object.g_allobjs:
if type(ns3headers).__name__ != 'ns3header_taskgen': # XXX: find less hackish way to compare
continue
if ns3headers.module != module_name:
continue
for source in ns3headers.to_list(ns3headers.source):
headers.append(source)
retval[module_name] = (list(module.module_deps), headers)
return retval
class PythonScanTask(Task.TaskBase):
"""Uses gccxml to scan the file 'everything.h' and extract API definitions.
"""
def __init__(self, curdirnode, env):
self.m_display = 'python-scan'
self.prio = 5 # everything.h has prio 4
super(PythonScanTask, self).__init__()
self.curdirnode = curdirnode
self.env = env
def run(self):
#print "Rescanning the python bindings..."
argv = [
self.env['PYTHON'],
os.path.join(self.curdirnode.abspath(), 'ns3modulescan.py'), # scanning script
self.curdirnode.find_dir('../..').abspath(self.env), # include path (where the ns3 include dir is)
self.curdirnode.find_build('everything.h').abspath(self.env),
os.path.join(self.curdirnode.abspath(), 'ns3modulegen_generated.py'), # output file
]
scan = subprocess.Popen(argv, stdin=subprocess.PIPE)
scan.stdin.write(repr(get_modules_and_headers()))
scan.stdin.close()
if scan.wait():
raise SystemExit(1)
raise SystemExit(0)
def build(bld):
if Params.g_options.python_disable:
return
env = bld.env_of_name('default')
curdir = bld.m_curdirnode.abspath()
#Object.register('all-ns3-headers', AllNs3Headers)
Action.Action('gen-ns3-metaheader', func=gen_ns3_metaheader, color='BLUE')
if env['ENABLE_PYTHON_BINDINGS']:
obj = bld.create_obj('all_ns3_headers')
if Params.g_options.python_scan:
if not env['ENABLE_PYTHON_SCANNING']:
Params.fatal("Cannot re-scan python bindings: (py)gccxml not available")
PythonScanTask(bld.m_curdirnode, env)
## Get a list of scanned modules; the set of scanned modules
## may be smaller than the set of all modules, in case a new
## ns3 module is being developed which wasn't scanned yet.
scanned_modules = []
for filename in os.listdir(curdir):
m = re.match(r"^ns3_module_(.+)\.py$", filename)
if m is None:
continue
name = m.group(1)
if name.endswith("__local"):
continue
scanned_modules.append(name)
if env['ENABLE_PYTHON_BINDINGS']:
bindgen = bld.create_obj('command-output')
bindgen.name = 'pybindgen'
bindgen.command = env['PYTHON']
bindgen.command_is_external = True
bindgen.stderr = 'ns3modulegen.log'
bindgen.argv = [
#'-m', 'pdb',
bindgen.input_file("ns3modulegen.py"),
bindgen.output_file("ns3module.cc"),
]
bindgen.argv.extend(get_modules_and_headers().iterkeys())
bindgen.hidden_inputs = ['ns3modulegen_generated.py',
'ns3modulegen_core_customizations.py']
for module in scanned_modules:
bindgen.hidden_inputs.append("ns3_module_%s.py" % module)
local = "ns3_module_%s__local.py" % module
if os.path.exists(os.path.join(curdir, local)):
bindgen.hidden_inputs.append(local)
bindgen.hidden_outputs = ['ns3module.h']
for module in scanned_modules:
bindgen.hidden_outputs.append("ns3_module_%s.cc" % module)
bindgen.prio = 50
bindgen.os_env = dict(os.environ)
features = []
for (name, caption, was_enabled, reason_not_enabled) in env['NS3_OPTIONAL_FEATURES']:
if was_enabled:
features.append(name)
bindgen.os_env['NS3_ENABLED_FEATURES'] = ','.join(features)
## we build python bindings if either we have the tools to
## generate them or if the pregenerated source file is already
## present in the source dir.
if env['ENABLE_PYTHON_BINDINGS'] \
or os.path.exists(os.path.join(bld.m_curdirnode.abspath(), 'ns3module.cc')):
pymod = bld.create_obj('cpp', 'shlib', 'pyext')
pymod.source = ['ns3module.cc', 'ns3module_helpers.cc']
pymod.includes = '.'
for module in scanned_modules:
pymod.source.append("ns3_module_%s.cc" % module)
pymod.target = 'ns3/_ns3'
pymod.name = 'ns3module'
pymod.uselib_local = "ns3"
pymod.env.append_value('CXXDEFINES', ['NS_DEPRECATED=""', 'NS3_DEPRECATED_H'])
# copy the __init__.py file to the build dir waf can't handle
# this, it's against waf's principles to have build dir files
# with the same name as source dir files, apparently.
dirnode = bld.m_curdirnode.find_dir('ns3')
src = os.path.join(dirnode.abspath(), '__init__.py')
dst = os.path.join(dirnode.abspath(env), '__init__.py')
try:
need_copy = os.stat(src).st_mtime > os.stat(dst).st_mtime
except OSError:
need_copy = True
if need_copy:
try:
os.mkdir(os.path.dirname(dst))
except OSError:
pass
print "%r -> %r" % (src, dst)
shutil.copy2(src, dst)