Expose package metadata from wheels in .par files

WORKSPACE

@@ -5,6 +5,14 @@ local_repository(
     name = "test_workspace",
     path = "tests/test_workspace",
 )
+local_repository(
+    name = "pypi__portpicker_1_2_0",
+    path = "third_party/pypi__portpicker_1_2_0",
+)
+local_repository(
+    name = "pypi__yapf_0_19_0",
+    path = "third_party/pypi__yapf_0_19_0",
+)
 
 # Not actually referenced anywhere, but must be marked as a separate
 # repository so that things like "bazel test //..." don't get confused

runtime/support.py

@@ -61,6 +61,83 @@ def _find_archive():
     return archive_path
 
 
+def _setup_pkg_resources():
+    try:
+        import pkg_resources
+        import zipimport
+    except ImportError:
+        # Skip setup
+        return
+
+    class DistInfoMetadata(pkg_resources.EggMetadata):
+        """Metadata provider for zip files containing .dist-info
+
+        In find_dist_info_in_zip(), we call
+        metadata.resource_listdir(directory_name).  However, it doesn't
+        work with EggMetadata, because _zipinfo_name() expects the
+        directory name to end with a /, but metadata._listdir() which
+        expects the directory to _not_ end with a /.
+
+        Therefore this class exists.
+        """
+
+        def _listdir(self, fspath):
+            """List of resource names in the directory (like ``os.listdir()``)
+
+            Overrides EggMetadata._listdir()
+            """
+
+            zipinfo_name = self._zipinfo_name(fspath)
+            while zipinfo_name.endswith('/'):
+                zipinfo_name = zipinfo_name[:-1]
+            result = self._index().get(zipinfo_name, ())
+            return list(result)
+
+
+    def find_dist_info_in_zip(importer, path_item, only=False):
+        """Find dist-info style metadata in zip files.
+
+        We ignore the `only` flag because it's not clear what it should
+        actually do in this case.
+        """
+        metadata = DistInfoMetadata(importer)
+        for subitem in metadata.resource_listdir('/'):
+            if subitem.lower().endswith('.dist-info'):
+                subpath = os.path.join(path_item, subitem)
+                submeta = pkg_resources.EggMetadata(zipimport.zipimporter(subpath))
+                submeta.egg_info = subpath
+                dist = pkg_resources.Distribution.from_location(path_item, subitem, submeta)
+                yield dist
+        return
+
+
+    def find_eggs_and_dist_info_in_zip(importer, path_item, only=False):
+        """Chain together our finder and the standard pkg_resources finder
+
+        For simplicity, and since pkg_resources doesn't provide a public
+        interface to do so, we hardcode the chaining (find_eggs_in_zip).
+        """
+        # Our finder
+        for dist in find_dist_info_in_zip(importer, path_item, only):
+            yield dist
+        # The standard pkg_resources finder
+        for dist in pkg_resources.find_eggs_in_zip(importer, path_item, only):
+            yield dist
+        return
+
+
+    # This overwrites the existing registered finder.
+    #
+    # Note that this also doesn't update the default WorkingSet created by
+    # pkg_resources when it is imported, since there is no public
+    # interface to do so that doesn't also have a "Don't use this"
+    # warning.
+    pkg_resources.register_finder(zipimport.zipimporter,
+                                  find_eggs_and_dist_info_in_zip)
+
+
+
+
 def setup(import_roots=None):
     """Initialize subpar run-time support"""
     # Add third-party library entries to sys.path
@@ -75,3 +152,6 @@ def setup(import_roots=None):
         new_path = os.path.join(archive_path, import_root)
         _log('# adding %s to sys.path' % new_path)
         sys.path.insert(1, new_path)
+
+    # Add hook for package metadata
+    _setup_pkg_resources()

tests/BUILD

@@ -104,6 +104,19 @@ par_binary(
     srcs_version = "PY2AND3",
 )
 
+par_binary(
+    name = "package_pkg_resources/main",
+    srcs = [
+        "package_pkg_resources/main.py",
+    ],
+    data = [
+        "@pypi__portpicker_1_2_0//:files",
+        "@pypi__yapf_0_19_0//:files",
+    ],
+    main = "package_pkg_resources/main.py",
+    srcs_version = "PY2AND3",
+)
+
 # Test targets
 [(
     # Run test without .par file as a control
@@ -147,4 +160,5 @@ par_binary(
     ("main_boilerplate", ":package_g/g", "/package_g/g"),
     ("shadow", ":package_shadow/main", "/package_shadow/main"),
     ("import_roots", ":package_import_roots/import_roots", "/package_import_roots/import_roots"),
+    ("pkg_resources", ":package_pkg_resources/main", "/package_pkg_resources/main"),
 ]]

tests/package_pkg_resources/main.py

@@ -0,0 +1,43 @@
+# Copyright 2017 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Integration test program for Subpar.
+
+Test that pkg_resources correctly identifies distribution packages
+inside a .par file.
+"""
+
+def main():
+    print('In pkg_resources test main()')
+    try:
+        import pkg_resources
+    except ImportError:
+        print('Skipping test, pkg_resources module is not available')
+        return
+
+    ws = pkg_resources.WorkingSet()
+
+    # Informational for debugging
+    distributions = list(ws)
+    print('Resources found: %s' % distributions)
+
+    # Check for the packages we provided metadata for.  There will
+    # also be metadata for whatever other packages happen to be
+    # installed in the current Python interpreter.
+    for spec in ['portpicker==1.2.0', 'yapf==0.19.0']:
+        dist = ws.find(pkg_resources.Requirement.parse(spec))
+        assert dist, (spec, distributions)
+
+if __name__ == '__main__':
+    main()

tests/package_pkg_resources/main_PY2_filelist.txt

@@ -0,0 +1,12 @@
+__main__.py
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json
+subpar/__init__.py
+subpar/runtime/__init__.py
+subpar/runtime/support.py
+subpar/tests/__init__.py
+subpar/tests/package_pkg_resources/__init__.py
+subpar/tests/package_pkg_resources/main
+subpar/tests/package_pkg_resources/main.py

tests/package_pkg_resources/main_PY3_filelist.txt

@@ -0,0 +1,12 @@
+__main__.py
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA
+pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/METADATA
+pypi__yapf_0_19_0/yapf-0.19.0.dist-info/metadata.json
+subpar/__init__.py
+subpar/runtime/__init__.py
+subpar/runtime/support.py
+subpar/tests/__init__.py
+subpar/tests/package_pkg_resources/__init__.py
+subpar/tests/package_pkg_resources/main
+subpar/tests/package_pkg_resources/main.py

third_party/README.md

@@ -0,0 +1,9 @@
+# Wheels
+
+This directory contains code, metadata files, and wheels, from other projects,
+used for testing.  The projects were chosen to match the licence and ownership
+of this project, i.e. Apache License 2.0, copyright owned by Google, from the
+Google organization repository.  Do not add any other type of thing here.
+
+- [python_portpicker](https://github.com/google/python_portpicker)
+- [yapf](https://github.com/google/yapf)

third_party/pypi__portpicker_1_2_0/BUILD

@@ -0,0 +1,10 @@
+# Test package for Subpar
+
+package(default_visibility = ["//visibility:public"])
+
+py_library(
+    name = "files",
+    srcs = [],
+    data = glob(["portpicker-1.2.0.dist-info/**"]),
+    imports = ["."],
+)

third_party/pypi__portpicker_1_2_0/WORKSPACE

@@ -0,0 +1 @@
+workspace(name = "pypi__portpicker_1_2_0")

third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/METADATA

@@ -0,0 +1,29 @@
+Metadata-Version: 2.0
+Name: portpicker
+Version: 1.2.0
+Summary: A library to choose unique available network ports.
+Home-page: https://github.com/google/python_portpicker
+Author: Google
+Author-email: [email protected]
+License: Apache 2.0
+Description-Content-Type: UNKNOWN
+Platform: POSIX
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: Jython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+
+Portpicker provides an API to find and return an available network
+port for an application to bind to.  Ideally suited for use from
+unittests or for test harnesses that launch local servers.
+

third_party/pypi__portpicker_1_2_0/portpicker-1.2.0.dist-info/metadata.json

@@ -0,0 +1 @@
+{"classifiers": ["Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", "Intended Audience :: Developers", "Programming Language :: Python", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.3", "Programming Language :: Python :: 3.4", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: Jython", "Programming Language :: Python :: Implementation :: PyPy"], "description_content_type": "UNKNOWN", "extensions": {"python.details": {"contacts": [{"email": "[email protected]", "name": "Google", "role": "author"}], "document_names": {"description": "DESCRIPTION.rst"}, "project_urls": {"Home": "https://github.com/google/python_portpicker"}}}, "generator": "bdist_wheel (0.30.0)", "license": "Apache 2.0", "metadata_version": "2.0", "name": "portpicker", "platform": "POSIX", "summary": "A library to choose unique available network ports.", "version": "1.2.0"}

third_party/pypi__yapf_0_19_0/BUILD

@@ -0,0 +1,10 @@
+# Test package for Subpar
+
+package(default_visibility = ["//visibility:public"])
+
+py_library(
+    name = "files",
+    srcs = [],
+    data = glob(["yapf-0.19.0.dist-info/**"]),
+    imports = ["."],
+)

third_party/pypi__yapf_0_19_0/WORKSPACE

@@ -0,0 +1 @@
+workspace(name = "pypi__yapf_0_19_0")