diff --git a/src/python/pants/backend/python/tasks2/gather_sources.py b/src/python/pants/backend/python/tasks2/gather_sources.py new file mode 100644 index 00000000000..ca1fad41839 --- /dev/null +++ b/src/python/pants/backend/python/tasks2/gather_sources.py @@ -0,0 +1,88 @@ +# coding=utf-8 +# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os +import shutil + +from pex.interpreter import PythonInterpreter +from pex.pex import PEX +from pex.pex_builder import PEXBuilder + +from pants.backend.python.targets.python_target import PythonTarget +from pants.base.build_environment import get_buildroot +from pants.base.exceptions import TaskError +from pants.build_graph.resources import Resources +from pants.invalidation.cache_manager import VersionedTargetSet +from pants.task.task import Task + + +class GatherSources(Task): + """Gather local Python sources. + + Creates an (unzipped) PEX on disk containing the local Python sources. + This PEX can be merged with a requirements PEX to create a unified Python environment + for running the relevant python code. + """ + + PYTHON_SOURCES = 'python_sources' + + @classmethod + def product_types(cls): + return [cls.PYTHON_SOURCES] + + @classmethod + def prepare(cls, options, round_manager): + round_manager.require_data(PythonInterpreter) + + def execute(self): + targets = self.context.targets(lambda tgt: isinstance(tgt, (PythonTarget, Resources))) + with self.invalidated(targets) as invalidation_check: + # If there are no relevant targets, we still go through the motions of gathering + # an empty set of sources, to prevent downstream tasks from having to check + # for this special case. + if invalidation_check.all_vts: + target_set_id = VersionedTargetSet.from_versioned_targets( + invalidation_check.all_vts).cache_key.hash + else: + target_set_id = 'no_targets' + + path = os.path.join(self.workdir, target_set_id) + path_tmp = path + '.tmp' + + shutil.rmtree(path_tmp, ignore_errors=True) + + interpreter = self.context.products.get_data(PythonInterpreter) + if not os.path.isdir(path): + self._build_pex(interpreter, path_tmp, invalidation_check.all_vts) + shutil.move(path_tmp, path) + + pex = PEX(os.path.realpath(path), interpreter=interpreter) + self.context.products.get_data(self.PYTHON_SOURCES, lambda: pex) + + def _build_pex(self, interpreter, path, vts): + builder = PEXBuilder(path=path, interpreter=interpreter, copy=True) + for vt in vts: + self._dump_sources(builder, vt.target) + builder.freeze() + + def _dump_sources(self, builder, tgt): + buildroot = get_buildroot() + self.context.log.debug(' Dumping sources: {}'.format(tgt)) + for relpath in tgt.sources_relative_to_source_root(): + try: + src = os.path.join(buildroot, tgt.target_base, relpath) + builder.add_source(src, relpath) + except OSError: + self.context.log.error('Failed to copy {} for target {}'.format( + os.path.join(tgt.target_base, relpath), tgt.address.spec)) + raise + + if getattr(tgt, 'resources', None): + # No one should be on old-style resources any more. And if they are, + # switching to the new python pipeline will be a great opportunity to fix that. + raise TaskError('Old-style resources not supported for target {}. ' + 'Depend on resources() targets instead.'.format(tgt.address.spec)) diff --git a/src/python/pants/backend/python/tasks2/resolve_requirements.py b/src/python/pants/backend/python/tasks2/resolve_requirements.py index ade9a7d3c4b..5413303a52b 100644 --- a/src/python/pants/backend/python/tasks2/resolve_requirements.py +++ b/src/python/pants/backend/python/tasks2/resolve_requirements.py @@ -56,6 +56,10 @@ def product_types(cls): def subsystem_dependencies(cls): return super(ResolveRequirements, cls).subsystem_dependencies() + (PythonSetup, PythonRepos) + @classmethod + def prepare(cls, options, round_manager): + round_manager.require_data(PythonInterpreter) + def execute(self): req_libs = self.context.targets(lambda tgt: isinstance(tgt, PythonRequirementLibrary)) fs = PythonRequirementFingerprintStrategy(task=self) diff --git a/tests/python/pants_test/backend/python/tasks2/test_gather_sources.py b/tests/python/pants_test/backend/python/tasks2/test_gather_sources.py new file mode 100644 index 00000000000..5971cc90de3 --- /dev/null +++ b/tests/python/pants_test/backend/python/tasks2/test_gather_sources.py @@ -0,0 +1,67 @@ +# coding=utf-8 +# Copyright 2016 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import (absolute_import, division, generators, nested_scopes, print_function, + unicode_literals, with_statement) + +import os + +from pex.interpreter import PythonInterpreter + +from pants.backend.python.interpreter_cache import PythonInterpreterCache +from pants.backend.python.python_setup import PythonRepos +from pants.backend.python.python_setup import PythonSetup +from pants.backend.python.targets.python_library import PythonLibrary +from pants.backend.python.tasks2.gather_sources import GatherSources +from pants.build_graph.resources import Resources +from pants_test.tasks.task_test_base import TaskTestBase + + +class GatherSourcesTest(TaskTestBase): + @classmethod + def task_type(cls): + return GatherSources + + def test_gather_sources(self): + filemap = { + 'src/python/foo.py': 'foo_py_content', + 'src/python/bar.py': 'bar_py_content', + 'src/python/baz.py': 'baz_py_content', + 'resources/qux/quux.txt': 'quux_txt_content', + } + + for rel_path, content in filemap.items(): + self.create_file(rel_path, content) + + sources1 = self.make_target(spec='//:sources1_tgt', target_type=PythonLibrary, + sources=['src/python/foo.py', 'src/python/bar.py']) + sources2 = self.make_target(spec='//:sources2_tgt', target_type=PythonLibrary, + sources=['src/python/baz.py']) + resources = self.make_target(spec='//:resources_tgt', target_type=Resources, + sources=['resources/qux/quux.txt']) + pex = self._gather_sources([sources1, sources2, resources]) + pex_root = pex.cmdline()[1] + + for rel_path, expected_content in filemap.items(): + with open(os.path.join(pex_root, rel_path)) as infile: + content = infile.read() + self.assertEquals(expected_content, content) + + def _gather_sources(self, target_roots): + context = self.context(target_roots=target_roots, for_subsystems=[PythonSetup, PythonRepos]) + + # We must get an interpreter via the cache, instead of using PythonInterpreter.get() directly, + # to ensure that the interpreter has setuptools and wheel support. + interpreter = PythonInterpreter.get() + interpreter_cache = PythonInterpreterCache(PythonSetup.global_instance(), + PythonRepos.global_instance(), + logger=context.log.debug) + interpreters = interpreter_cache.setup(paths=[os.path.dirname(interpreter.binary)], + filters=[str(interpreter.identity.requirement)]) + context.products.get_data(PythonInterpreter, lambda: interpreters[0]) + + task = self.create_task(context) + task.execute() + + return context.products.get_data(GatherSources.PYTHON_SOURCES)