Make tox -evendor idempotent. (#651)

This affords adding a CI check to ensure we never commit vendored code
with modifications beyond import re-writes.

And for this first burn of the new CI check to go green, the edit of
vendored wheel code in 16cfd97 is undone by a fresh run of
`tox -evendor`.

Fixes #649
Fixes #650

.travis.yml

@@ -99,6 +99,11 @@ matrix:
       env: *x-py27
       script: tox -ve isort-check
 
+    - <<: *x-linux-shard
+      name: TOXENV=vendor-check
+      env: *x-py27
+      script: tox -ve vendor-check
+
     - <<: *x-linux-shard
       name: TOXENV=py27
       env: *x-py27

pex/environment.py

@@ -32,12 +32,12 @@
 
 def _import_pkg_resources():
   try:
-    import pkg_resources
+    import pkg_resources  # vendor:skip
     return pkg_resources, False
   except ImportError:
     from pex import third_party
     third_party.install(expose=['setuptools'])
-    import pkg_resources
+    import pkg_resources  # vendor:skip
     return pkg_resources, True
 
 

pex/vendor/__main__.py

@@ -10,7 +10,7 @@
 from collections import OrderedDict
 
 from colors import bold, green, yellow
-from redbaron import LiteralyEvaluable, NameNode, RedBaron
+from redbaron import CommentNode, LiteralyEvaluable, NameNode, RedBaron
 
 from pex import third_party
 from pex.common import safe_delete, safe_rmtree
@@ -32,6 +32,14 @@ def _parse(python_file):
       # losing formatting. See: https://github.com/PyCQA/redbaron
       return RedBaron(fp.read())
 
+  @staticmethod
+  def _skip(node):
+    next_node = node.next_recursive
+    if isinstance(next_node, CommentNode) and next_node.value.strip() == '# vendor:skip':
+      print('Skipping {} as directed by {}'.format(node, next_node))
+      return True
+    return False
+
   @staticmethod
   def _find_literal_node(statement, call_argument):
     # The list of identifiers is large and they represent disjoint types:
@@ -78,6 +86,9 @@ def rewrite(self, python_file):
   def _modify__import__calls(self, red_baron):  # noqa: We want __import__ as part of the name.
     for call_node in red_baron.find_all('CallNode'):
       if call_node.previous and call_node.previous.value == '__import__':
+        if self._skip(call_node):
+          continue
+
         parent = call_node.parent_find('AtomtrailersNode')
         original = parent.copy()
         first_argument = call_node[0]
@@ -91,6 +102,9 @@ def _modify__import__calls(self, red_baron):  # noqa: We want __import__ as part
 
   def _modify_import_statements(self, red_baron):
     for import_node in red_baron.find_all('ImportNode'):
+      if self._skip(import_node):
+        continue
+
       original = import_node.copy()
       for index, import_module in enumerate(import_node):
         root_package = import_module[0]
@@ -134,6 +148,9 @@ def prefixed_fullname():
 
   def _modify_from_import_statements(self, red_baron):
     for from_import_node in red_baron.find_all('FromImportNode'):
+      if self._skip(from_import_node):
+        continue
+
       if len(from_import_node) == 0:
         # NB: `from . import ...` has length 0, but we don't care about relative imports which will
         # point back into vendored code if the origin is within vendored code.
@@ -162,6 +179,13 @@ def vendorize(root_dir, vendor_specs, prefix):
     # We know we can get these as a by-product of a pip install but never need them.
     safe_rmtree(os.path.join(vendor_spec.target_dir, 'bin'))
     safe_delete(os.path.join(vendor_spec.target_dir, 'easy_install.py'))
+
+    # The RECORD contains file hashes of all installed files and is unfortunately unstable in the
+    # case of scripts which get a shebang added with a system-specific path to the python
+    # interpreter to execute.
+    safe_delete(os.path.join(vendor_spec.target_dir,
+                             '{}-{}.dist-info/RECORD'.format(vendor_spec.key, vendor_spec.version)))
+
     vendor_spec.create_packages()
 
   vendored_path = [vendor_spec.target_dir for vendor_spec in vendor_specs]

pex/vendor/_vendored/wheel/wheel/metadata.py

@@ -13,7 +13,7 @@
 
 # Wheel itself is probably the only program that uses non-extras markers
 # in METADATA/PKG-INFO. Support its syntax with the extra at the end only.
-EXTRA_RE = re.compile(r"^(?P<package>.*?)(;\s*(?P<condition>.*?)(extra == '(?P<extra>.*?)')?)$")
+EXTRA_RE = re.compile("""^(?P<package>.*?)(;\s*(?P<condition>.*?)(extra == '(?P<extra>.*?)')?)$""")
 
 MayRequiresKey = namedtuple('MayRequiresKey', ('condition', 'extra'))
 

tox.ini

@@ -25,6 +25,7 @@ deps =
 whitelist_externals =
     open
     bash
+    git
 
 [testenv:integration-tests]
 deps =
@@ -146,6 +147,13 @@ commands =
   python -m pex.vendor
   {[testenv:isort-run]commands}
 
+[testenv:vendor-check]
+deps =
+  tox
+commands =
+  tox -e vendor
+  git diff --quiet
+
 [testenv:docs]
 changedir = docs
 deps =