Support arbitrary interpreter constraint expressions.

pex/interpreter_constraints.py

@@ -7,6 +7,7 @@
 
 from pex.common import die
 from pex.interpreter import PythonIdentity
+from pex.third_party import boolean
 from pex.tracer import TRACER
 
 
@@ -26,13 +27,70 @@ def matched_interpreters_iter(interpreters_iter, constraints):
 
   :param interpreters_iter: A `PythonInterpreter` iterable for filtering.
   :param constraints: A sequence of strings that constrain the interpreter compatibility for this
-    pex. Each string uses the Requirement-style format, e.g. 'CPython>=3' or '>=2.7,<3' for
-    requirements agnostic to interpreter class. Multiple requirement strings may be combined
-    into a list to OR the constraints, such as ['CPython>=2.7,<3', 'CPython>=3.4'].
+    pex. Eeach string is an arbitrary boolean expression in which the atoms are Requirement-style
+    strings such as 'CPython>=3', or '>=2.7,<3' for requirements agnostic to interpreter class.
+    The infix boolean operators are |, & and ~, and parentheses are used for precedence.
+    Multiple requirement strings are OR-ed, e.g., ['CPython>=2.7,<3', 'CPython>=3.4'], is the same
+    as ['CPython>=2.7,<3 | CPython>=3.4'].
   :return interpreter: returns a generator that yields compatible interpreters
   """
+  # TODO: Deprecate specifying multiple constraints, and instead require the input to be a
+  #  single explicit boolean expression.
+  constraint_expr = '({})'.format(' | '.join(constraints))
   for interpreter in interpreters_iter:
-    if any(interpreter.identity.matches(filt) for filt in constraints):
+    if match_interpreter_constraint(interpreter.identity, constraint_expr):
       TRACER.log("Constraints on interpreters: %s, Matching Interpreter: %s"
                  % (constraints, interpreter.binary), V=3)
       yield interpreter
+
+
+class ConstraintAlgebra(boolean.BooleanAlgebra):
+  def __init__(self, identity):
+    super(ConstraintAlgebra, self).__init__()
+    self._identity = identity
+
+  def tokenize(self, s):
+    # Remove all spaces from the string. Doesn't change its semantics, but makes it much
+    # easier to tokenize.
+    s = ''.join(s.split())
+    if not s:
+      return
+    ops = {
+      '|': boolean.TOKEN_OR,
+      '&': boolean.TOKEN_AND,
+      '~': boolean.TOKEN_NOT,
+      '(': boolean.TOKEN_LPAR,
+      ')': boolean.TOKEN_RPAR,
+    }
+    s = '({})'.format(s)  # Wrap with parens, to simplify constraint tokenizing.
+    it = enumerate(s)
+    try:
+      i, c = next(it)
+      while True:
+        if c in ops:
+          yield ops[c], c, i
+          i, c = next(it)
+        else:
+          constraint_start = i
+          while not c in ops:
+            i, c = next(it)  # We wrapped with parens, so this cannot throw StopIteration.
+          constraint = s[constraint_start:i]
+          yield ((boolean.TOKEN_TRUE if self._identity.matches(constraint)
+                  else boolean.TOKEN_FALSE),
+                 constraint, constraint_start)
+    except StopIteration:
+      pass
+
+
+def match_interpreter_constraint(identity, constraint_expr):
+  """Return True iff the given identity matches the constraint expression.
+
+  The constraint expression is an arbitrary boolean expression in which the atoms are
+  Requirement-style strings such as 'CPython>=2.7,<3', the infix boolean operators are |, & and ~,
+  and parentheses are used for precedence.
+
+  :param identity: A `pex.interpreter.PythonIdentity` instance.
+  :param constraint_expr: A boolean interpreter constraint expression.
+  """
+  algebra = ConstraintAlgebra(identity)
+  return bool(algebra.parse(constraint_expr).simplify())

pex/pex_builder.py

@@ -370,7 +370,7 @@ def _prepare_bootstrap(self):
                           # NB: We use pip here in the builder, but that's only at buildtime and
                           # although we don't use pyparsing directly, packaging.markers, which we
                           # do use at runtime, does.
-                          root_module_names=['packaging', 'pkg_resources', 'pyparsing'])
+                          root_module_names=['boolean', 'packaging', 'pkg_resources', 'pyparsing'])
 
     source_name = 'pex'
     provider = get_provider(source_name)

pex/vendor/README.md

@@ -11,29 +11,7 @@ re-vendor when the fix is released.
 
 To update versions of vendored code or add new vendored code:
 
-1. Modify `pex.vendor.iter_vendor_specs` with updated versions or new distributions.
-   Today that function looks like:
-   ```python
-   def iter_vendor_specs():
-     """Iterate specifications for code vendored by pex.
-
-     :return: An iterator over specs of all vendored code.
-     :rtype: :class:`collection.Iterator` of :class:`VendorSpec`
-     """
-     # We use this via pex.third_party at runtime to check for compatible wheel tags.
-     yield VendorSpec.pinned('packaging', '19.2')
-
-     # We shell out to pip at buildtime to resolve and install dependencies.
-     # N.B.: This is pip 20.0.dev0 with a patch to support foreign download targets more fully.
-     yield VendorSpec.vcs('git+https://github.com/pantsbuild/pip@5eb9470c0c59#egg=pip', rewrite=False)
-
-     # We expose this to pip at buildtime for legacy builds, but we also use pkg_resources via
-     # pex.third_party at runtime in various ways.
-     yield VendorSpec.pinned('setuptools', '42.0.2')
-
-     # We expose this to pip at buildtime for legacy builds.
-     yield VendorSpec.pinned('wheel', '0.33.6', rewrite=False)
-   ```
+1. Modify [`pex.vendor.iter_vendor_specs`](./__init__.py#L91) with updated versions or new distributions.
    Simply edit an existing `VendorSpec` or `yield` a new one.
 2. Run `tox -e vendor`.
    This will replace all vendored code even if versions have not changed and then rewrite any

pex/vendor/__init__.py

@@ -97,6 +97,12 @@ def iter_vendor_specs():
   # We use this via pex.third_party at runtime to check for compatible wheel tags.
   yield VendorSpec.pinned('packaging', '19.2')
 
+  # We use this to evaluate interpreter compatibility expressions.
+  # TODO: Switch to a published release once https://github.com/bastikr/boolean.py/pull/95
+  #  is merged and released:
+  #yield VendorSpec.pinned('boolean.py', '3.8')
+  yield VendorSpec.vcs('git+https://github.com/benjyw/boolean.py@db41511ea311#egg=boolean')
+
   # We shell out to pip at buildtime to resolve and install dependencies.
   # N.B.: This is pip 20.0.dev0 with a patch to support foreign download targets more fully.
   yield VendorSpec.vcs('git+https://github.com/pantsbuild/pip@5eb9470c0c59#egg=pip', rewrite=False)

pex/vendor/__main__.py

@@ -82,17 +82,17 @@ def __init__(self, prefix, packages):
     self._packages = packages
 
   def rewrite(self, python_file):
-    modififications = OrderedDict()
+    modifications = OrderedDict()
 
     red_baron = self._parse(python_file)
-    modififications.update(self._modify__import__calls(red_baron))
-    modififications.update(self._modify_import_statements(red_baron))
-    modififications.update(self._modify_from_import_statements(red_baron))
+    modifications.update(self._modify__import__calls(red_baron))
+    modifications.update(self._modify_import_statements(red_baron))
+    modifications.update(self._modify_from_import_statements(red_baron))
 
-    if modififications:
+    if modifications:
       with open(python_file, 'w') as fp:
         fp.write(red_baron.dumps())
-      return modififications
+      return modifications
 
   def _modify__import__calls(self, red_baron):  # noqa: We want __import__ as part of the name.
     for call_node in red_baron.find_all('CallNode'):

pex/vendor/_vendored/boolean/boolean.py-3.7.dist-info/INSTALLER

@@ -0,0 +1 @@
+pip

pex/vendor/_vendored/boolean/boolean.py-3.7.dist-info/LICENSE.txt

@@ -0,0 +1,23 @@
+Copyright (c) 2009-2017 Sebastian Kraemer, [email protected]
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pex/vendor/_vendored/boolean/boolean.py-3.7.dist-info/METADATA

@@ -0,0 +1,39 @@
+Metadata-Version: 2.1
+Name: boolean.py
+Version: 3.7
+Summary: Define boolean algebras, create and parse boolean expressions and create custom boolean DSL.
+Home-page: https://github.com/bastikr/boolean.py
+Author: Sebastian Kraemer
+Author-email: [email protected]
+License: BSD-2-Clause
+Keywords: boolean expression,boolean algebra,logic,expression parser
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Classifier: Topic :: Software Development :: Compilers
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Utilities
+
+
+This library helps you deal with boolean expressions and algebra with variables
+and the boolean functions AND, OR, NOT.
+
+You can parse expressions from strings and simplify and compare expressions.
+You can also easily create your custom algreba and mini DSL and create custom
+tokenizers to handle custom expressions.
+
+For extensive documentation look either into the docs directory or view it online, at
+https://booleanpy.readthedocs.org/en/latest/
+
+https://github.com/bastikr/boolean.py
+
+Copyright (c) 2009-2017 Sebastian Kraemer, [email protected] and others
+
+Released under revised BSD license aka. BSD Simplified or BSD-2-Clause.
+
+

pex/vendor/_vendored/boolean/boolean.py-3.7.dist-info/WHEEL

@@ -0,0 +1,6 @@
+Wheel-Version: 1.0
+Generator: bdist_wheel (0.33.6)
+Root-Is-Purelib: true
+Tag: py2-none-any
+Tag: py3-none-any
+

pex/vendor/_vendored/boolean/boolean.py-3.7.dist-info/top_level.txt

@@ -0,0 +1 @@
+boolean

pex/vendor/_vendored/boolean/boolean/__init__.py

@@ -0,0 +1,101 @@
+"""
+Boolean Algebra.
+
+This module defines a Boolean Algebra over the set {TRUE, FALSE} with boolean
+variables and the boolean functions AND, OR, NOT. For extensive documentation
+look either into the docs directory or view it online, at
+https://booleanpy.readthedocs.org/en/latest/.
+
+Copyright (c) 2009-2017 Sebastian Kraemer, [email protected]
+Released under revised BSD license.
+"""
+
+from __future__ import absolute_import
+from __future__ import unicode_literals
+from __future__ import print_function
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import BooleanAlgebra  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import BooleanAlgebra
+
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import Expression  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import Expression
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import Symbol  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import Symbol
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import ParseError  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import ParseError
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import PARSE_ERRORS  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import PARSE_ERRORS
+
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import AND  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import AND
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import NOT  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import NOT
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import OR  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import OR
+
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import TOKEN_TRUE  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import TOKEN_TRUE
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import TOKEN_FALSE  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import TOKEN_FALSE
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import TOKEN_SYMBOL  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import TOKEN_SYMBOL
+
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import TOKEN_AND  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import TOKEN_AND
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import TOKEN_OR  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import TOKEN_OR
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import TOKEN_NOT  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import TOKEN_NOT
+
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import TOKEN_LPAR  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import TOKEN_LPAR
+
+if "__PEX_UNVENDORED__" in __import__("os").environ:
+  from boolean.boolean import TOKEN_RPAR  # vendor:skip
+else:
+  from pex.third_party.boolean.boolean import TOKEN_RPAR
+