Task to gather local python sources into a pex.

src/python/pants/backend/python/tasks2/gather_sources.py

@@ -0,0 +1,88 @@
+# coding=utf-8
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+import shutil
+
+from pex.interpreter import PythonInterpreter
+from pex.pex import PEX
+from pex.pex_builder import PEXBuilder
+
+from pants.backend.python.targets.python_target import PythonTarget
+from pants.base.build_environment import get_buildroot
+from pants.base.exceptions import TaskError
+from pants.build_graph.resources import Resources
+from pants.invalidation.cache_manager import VersionedTargetSet
+from pants.task.task import Task
+
+
+class GatherSources(Task):
+  """Gather local Python sources.
+
+  Creates an (unzipped) PEX on disk containing the local Python sources.
+  This PEX can be merged with a requirements PEX to create a unified Python environment
+  for running the relevant python code.
+  """
+
+  PYTHON_SOURCES = 'python_sources'
+
+  @classmethod
+  def product_types(cls):
+    return [cls.PYTHON_SOURCES]
+
+  @classmethod
+  def prepare(cls, options, round_manager):
+    round_manager.require_data(PythonInterpreter)
+
+  def execute(self):
+    targets = self.context.targets(lambda tgt: isinstance(tgt, (PythonTarget, Resources)))
+    with self.invalidated(targets) as invalidation_check:
+      # If there are no relevant targets, we still go through the motions of gathering
+      # an empty set of sources, to prevent downstream tasks from having to check
+      # for this special case.
+      if invalidation_check.all_vts:
+        target_set_id = VersionedTargetSet.from_versioned_targets(
+            invalidation_check.all_vts).cache_key.hash
+      else:
+        target_set_id = 'no_targets'
+
+      path = os.path.join(self.workdir, target_set_id)
+      path_tmp = path + '.tmp'
+
+      shutil.rmtree(path_tmp, ignore_errors=True)
+
+      interpreter = self.context.products.get_data(PythonInterpreter)
+      if not os.path.isdir(path):
+        self._build_pex(interpreter, path_tmp, invalidation_check.all_vts)
+        shutil.move(path_tmp, path)
+
+    pex = PEX(os.path.realpath(path), interpreter=interpreter)
+    self.context.products.get_data(self.PYTHON_SOURCES, lambda: pex)
+
+  def _build_pex(self, interpreter, path, vts):
+    builder = PEXBuilder(path=path, interpreter=interpreter, copy=True)
+    for vt in vts:
+      self._dump_sources(builder, vt.target)
+    builder.freeze()
+
+  def _dump_sources(self, builder, tgt):
+    buildroot = get_buildroot()
+    self.context.log.debug('  Dumping sources: {}'.format(tgt))
+    for relpath in tgt.sources_relative_to_source_root():
+      try:
+        src = os.path.join(buildroot, tgt.target_base, relpath)
+        builder.add_source(src, relpath)
+      except OSError:
+        self.context.log.error('Failed to copy {} for target {}'.format(
+            os.path.join(tgt.target_base, relpath), tgt.address.spec))
+        raise
+
+    if getattr(tgt, 'resources', None):
+      # No one should be on old-style resources any more.  And if they are,
+      # switching to the new python pipeline will be a great opportunity to fix that.
+      raise TaskError('Old-style resources not supported for target {}.  '
+                      'Depend on resources() targets instead.'.format(tgt.address.spec))

src/python/pants/backend/python/tasks2/resolve_requirements.py

@@ -56,6 +56,10 @@ def product_types(cls):
   def subsystem_dependencies(cls):
     return super(ResolveRequirements, cls).subsystem_dependencies() + (PythonSetup, PythonRepos)
 
+  @classmethod
+  def prepare(cls, options, round_manager):
+    round_manager.require_data(PythonInterpreter)
+
   def execute(self):
     req_libs = self.context.targets(lambda tgt: isinstance(tgt, PythonRequirementLibrary))
     fs = PythonRequirementFingerprintStrategy(task=self)

tests/python/pants_test/backend/python/tasks2/test_gather_sources.py

@@ -0,0 +1,67 @@
+# coding=utf-8
+# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import os
+
+from pex.interpreter import PythonInterpreter
+
+from pants.backend.python.interpreter_cache import PythonInterpreterCache
+from pants.backend.python.python_setup import PythonRepos
+from pants.backend.python.python_setup import PythonSetup
+from pants.backend.python.targets.python_library import PythonLibrary
+from pants.backend.python.tasks2.gather_sources import GatherSources
+from pants.build_graph.resources import Resources
+from pants_test.tasks.task_test_base import TaskTestBase
+
+
+class GatherSourcesTest(TaskTestBase):
+  @classmethod
+  def task_type(cls):
+    return GatherSources
+
+  def test_gather_sources(self):
+    filemap = {
+      'src/python/foo.py': 'foo_py_content',
+      'src/python/bar.py': 'bar_py_content',
+      'src/python/baz.py': 'baz_py_content',
+      'resources/qux/quux.txt': 'quux_txt_content',
+    }
+
+    for rel_path, content in filemap.items():
+      self.create_file(rel_path, content)
+
+    sources1 = self.make_target(spec='//:sources1_tgt', target_type=PythonLibrary,
+                                sources=['src/python/foo.py', 'src/python/bar.py'])
+    sources2 = self.make_target(spec='//:sources2_tgt', target_type=PythonLibrary,
+                                sources=['src/python/baz.py'])
+    resources = self.make_target(spec='//:resources_tgt', target_type=Resources,
+                                 sources=['resources/qux/quux.txt'])
+    pex = self._gather_sources([sources1, sources2, resources])
+    pex_root = pex.cmdline()[1]
+
+    for rel_path, expected_content in filemap.items():
+      with open(os.path.join(pex_root, rel_path)) as infile:
+        content = infile.read()
+      self.assertEquals(expected_content, content)
+
+  def _gather_sources(self, target_roots):
+    context = self.context(target_roots=target_roots, for_subsystems=[PythonSetup, PythonRepos])
+
+    # We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly,
+    # to ensure that the interpreter has setuptools and wheel support.
+    interpreter = PythonInterpreter.get()
+    interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(),
+                                               PythonRepos.global_instance(),
+                                               logger=context.log.debug)
+    interpreters = interpreter_cache.setup(paths=[os.path.dirname(interpreter.binary)],
+                                           filters=[str(interpreter.identity.requirement)])
+    context.products.get_data(PythonInterpreter, lambda: interpreters[0])
+
+    task = self.create_task(context)
+    task.execute()
+
+    return context.products.get_data(GatherSources.PYTHON_SOURCES)