Fix problem where API doc gen misses some files

SConstruct

@@ -335,7 +335,7 @@ python_scons = {
         'debian_deps'   : [
                             'debian/changelog',
                             'debian/compat',
-                            'debian/control',	    
+                            'debian/control',
                             'debian/copyright',
                             'debian/dirs',
                             'debian/docs',
@@ -499,8 +499,7 @@ for p in [ scons ]:
     # destination files.
     #
     manifest_in = File(os.path.join(src, 'MANIFEST.in')).rstr()
-    manifest_in_lines = open(manifest_in).readlines()
-    src_files = bootstrap.parseManifestLines(src, manifest_in_lines)
+    src_files = bootstrap.parseManifestLines(src, manifest_in)
     raw_files = src_files[:]
     dst_files = src_files[:]
 
@@ -520,12 +519,11 @@ for p in [ scons ]:
 
             MANIFEST_in = File(os.path.join(src, ssubdir, 'MANIFEST.in')).rstr()
             MANIFEST_in_list.append(MANIFEST_in)
-            files = bootstrap.parseManifestLines(os.path.join(src, ssubdir), open(MANIFEST_in).readlines())
+            files = bootstrap.parseManifestLines(os.path.join(src, ssubdir), MANIFEST_in)
 
             raw_files.extend(files)
             src_files.extend([os.path.join(ssubdir, x) for x in files])
 
-
             files = [os.path.join(isubdir, x) for x in files]
             dst_files.extend(files)
             for k, f in sp['filemap'].items():

bootstrap.py

@@ -21,6 +21,7 @@
 # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
+from __future__ import print_function
 
 import os
 import os.path
@@ -74,15 +75,24 @@
         "eaten" by the bootstrap.py script.
 """
 
-def parseManifestLines(basedir, lines):
-    """ Scans the single lines of a MANIFEST file,
-        and returns the list of source files.
-        Has basic support for recursive globs '**',
-        filename wildcards of the form '*.xml' and
-        comment lines, starting with a '#'.
+def parseManifestLines(basedir, manifest):
+    """
+    Scans a MANIFEST file, and returns the list of source files.
+
+    Has basic support for recursive globs '**',
+    filename wildcards of the form '*.xml' and
+    comment lines, starting with a '#'.
+
+    :param basedir: base path to find files in. Note this does not
+                    run in an SCons context so path must not need
+                    further processing (e.g. no '#' signs)
+    :param manifest: path to manifest file
+    :returns: list of files
     """
     sources = []
     basewd = os.path.abspath(basedir)
+    with open(manifest) as m:
+        lines = m.readlines()
     for l in lines:
         if l.startswith('#'):
             # Skip comments
@@ -107,16 +117,16 @@ def parseManifestLines(basedir, lines):
 
 def main():
     script_dir = os.path.abspath(os.path.dirname(__file__))
-    
+
     bootstrap_dir = os.path.join(script_dir, 'bootstrap')
-    
+
     pass_through_args = []
     update_only = None
-    
+
     requires_an_argument = 'bootstrap.py:  %s requires an argument\n'
-    
+
     search = [script_dir]
-    
+
     def find(filename, search=search):
         for dir_name in search:
             filepath = os.path.join(dir_name, filename)
@@ -125,20 +135,20 @@ def find(filename, search=search):
         sys.stderr.write("could not find `%s' in search path:\n" % filename)
         sys.stderr.write("\t" + "\n\t".join(search) + "\n")
         sys.exit(2)
-    
+
     def must_copy(dst, src):
         if not os.path.exists(dst):
             return 1
         return not filecmp.cmp(dst,src)
-    
+
     # Note:  We don't use the getopt module to process the command-line
     # arguments because we'd have to teach it about all of the SCons options.
-    
+
     command_line_args = sys.argv[1:]
-    
+
     while command_line_args:
         arg = command_line_args.pop(0)
-    
+
         if arg == '--bootstrap_dir':
             try:
                 bootstrap_dir = command_line_args.pop(0)
@@ -180,7 +190,7 @@ def must_copy(dst, src):
     MANIFEST_in = find(os.path.join(src_engine, 'MANIFEST.in'))
     manifest_files = [os.path.join(src_engine, x)
                       for x in parseManifestLines(os.path.join(script_dir, src_engine),
-                                                  open(MANIFEST_in).readlines())]
+                                                  MANIFEST_in)]
 
     files = [scons_py] + manifest_files
 

doc/SConscript

@@ -326,7 +326,7 @@ else:
     # get included by the document XML files in the subdirectories.
     #
     manifest = File('MANIFEST').rstr()
-    src_files = bootstrap.parseManifestLines('.', open(manifest).readlines())
+    src_files = bootstrap.parseManifestLines('.', manifest)
     for s in src_files:
         if not s:
             continue
@@ -351,7 +351,7 @@ else:
         if not os.path.exists(os.path.join(build, doc, 'titlepage')):
             env.Execute(Mkdir(os.path.join(build, doc, 'titlepage')))
         manifest = File(os.path.join(doc, 'MANIFEST')).rstr()
-        src_files = bootstrap.parseManifestLines(doc, open(manifest).readlines())
+        src_files = bootstrap.parseManifestLines(doc, manifest)
         for s in src_files:
             if not s:
                 continue
@@ -571,14 +571,15 @@ if not epydoc_cli and not epydoc:
 else:
     # XXX Should be in common with reading the same thing in
     # the SConstruct file.
-    e = os.path.join('#src', 'engine')
-    manifest_in = File(os.path.join(e, 'MANIFEST.in')).rstr()
-    sources = bootstrap.parseManifestLines(e, open(manifest_in).readlines())
-
-    # Don't omit this as we need Platform.virtualenv for the examples to be run
+    # bootstrap.py runs outside of SCons, so need to process the path
+    e = Dir(os.path.join('#src', 'engine')).rstr()
+    sources = bootstrap.parseManifestLines(e, os.path.join(e, 'MANIFEST.in'))
+
+    # Omit some files:
+    #
+    # Don't omit Platform as we need Platform.virtualenv for the examples to be run
     # sources = [x for x in sources if x.find('Platform') == -1]
     sources = [x for x in sources if x.find('Tool') == -1]
-    # XXX
     sources = [x for x in sources if x.find('Options') == -1]
 
     e = os.path.join(build, '..', 'scons', 'engine')