Zinc 1.0.0 upgrade: Python portion (#4729)

### Problem

(this change depends on #4728)

As described on #4728, we currently need internal knowledge of zinc analysis in order to make it portable, and to extract products that are used by other tasks. Since the zinc analysis format is changing significantly (moving to protobuf, gaining/losing fields, etc), we need to stop parsing it. 

### Solution

Incorporates the version of the zinc wrapper published in #4728, and uses it in a `analysis.zinc` task to generate the `product_deps_by_src` and `classes_by_source` products from a `zinc_analysis` product published by zinc.

Expands the extracted `Zinc` subsystem from #4720 to encapsulate compile dependency calculation, and expose the `FingerprintStrategy` and `DependencyContext` that `compile.zinc` was using.

### Result

Zinc analysis is now a black box from the perspective of the pants python code. Also, the `compile.jvm-dep-check` task moved to the `lint` goal, and gained the unused dependency checks that used to be inlined in `compile.zinc` (and which would have prevented separation of the analysis extraction task).

Fixes #4477, fixes #4513, fixes #4185, fixes #5444, and hopefully fixes a few other odd incremental compilation bugs.

build-support/ivy/ivysettings.xml

@@ -10,7 +10,7 @@ Licensed under the Apache License, Version 2.0 (see LICENSE).
   <property name="m2.repo.relpath" value="[organisation]/[module]/[revision]"/>
   <property name="m2.repo.pom" value="${m2.repo.relpath}/[module]-[revision].pom"/>
   <property name="m2.repo.artifact"
-            value="${m2.repo.relpath}/[artifact](-[classifier])-[revision].[ext]"/>
+            value="${m2.repo.relpath}/[artifact]-[revision](-[classifier]).[ext]"/>
   <property name="m2.repo.dir" value="${user.home}/.m2/repository" override="false"/>
   <property name="kythe.artifact" value="[organisation]/[artifact]/[revision]/[artifact]-[revision].[ext]"/>
   <!-- for retrieving jacoco snapshot, remove when a version containing cli is released -->

contrib/scrooge/tests/python/pants_test/contrib/scrooge/tasks/test_scrooge_gen_integration.py

@@ -20,7 +20,7 @@ def run_pants(self, command, config=None, stdin_data=None, extra_env=None, **kwa
         'pythonpath': ["%(buildroot)s/contrib/scrooge/src/python"],
         'backend_packages': ["pants.backend.codegen", "pants.backend.jvm", "pants.contrib.scrooge"]
       },
-      'scala-platform': { 'version': '2.11' },
+      'scala': { 'version': '2.11' },
       'gen.scrooge': {
         'service_deps': {
             'java': [

examples/src/scala/org/pantsbuild/example/README.md

@@ -56,11 +56,11 @@ Scala Version
 -------------
 
 You can override the default version of the entire Scala toolchain with the single
-`--scala-platform-version` option. You can set that option to one of the supported Scala versions
+`--scala-version` option. You can set that option to one of the supported Scala versions
 (currently "2.10" or "2.11"), or to the special value "custom".
 
-If you choose a custom version, you must use the `--scala-platform-runtime-spec`,
-`--scala-platform-repl-spec` and `--scala-platform-suffix-version` options to provide
+If you choose a custom version, you must use the `--scala-runtime-spec`,
+`--scala-repl-spec` and `--scala-suffix-version` options to provide
 information about your custom Scala version.  The first two of these default to the target
 addresses `//:scala-library` and `//:scala-repl` respectively, so you can simply define those
 targets (in the root `BUILD.tools` file by convention) to point to the relevant JARs.

pants.ini

@@ -180,8 +180,6 @@ exclude_patterns: [
 [compile.zinc]
 jvm_options: [
     '-Xmx4g', '-XX:+UseConcMarkSweepGC', '-XX:ParallelGCThreads=4',
-    # bigger cache size for our big projects (default is just 5)
-    '-Dzinc.analysis.cache.limit=1000',
   ]
 
 args: [
@@ -280,7 +278,7 @@ skip: True
 [pycheck-context-manager]
 skip: True
 
-[scala-platform]
+[scala]
 version: 2.11
 
 

src/python/pants/backend/jvm/register.py

@@ -30,6 +30,7 @@
 from pants.backend.jvm.targets.scala_library import ScalaLibrary
 from pants.backend.jvm.targets.scalac_plugin import ScalacPlugin
 from pants.backend.jvm.targets.unpacked_jars import UnpackedJars
+from pants.backend.jvm.tasks.analysis_extraction import AnalysisExtraction
 from pants.backend.jvm.tasks.benchmark_run import BenchmarkRun
 from pants.backend.jvm.tasks.binary_create import BinaryCreate
 from pants.backend.jvm.tasks.bootstrap_jvm_tools import BootstrapJvmTools
@@ -161,6 +162,9 @@ def register_goals():
   task(name='zinc', action=ZincCompile).install('compile')
   task(name='javac', action=JavacCompile).install('compile')
 
+  # Analysis extraction.
+  task(name='zinc', action=AnalysisExtraction).install('analysis')
+
   # Dependency resolution.
   task(name='ivy', action=IvyResolve).install('resolve', first=True)
   task(name='coursier', action=CoursierResolve).install('resolve')
@@ -173,7 +177,6 @@ def register_goals():
   task(name='services', action=PrepareServices).install('resources')
 
   task(name='export-classpath', action=RuntimeClasspathPublisher).install()
-  task(name='jvm-dep-check', action=JvmDependencyCheck).install('compile')
 
   task(name='jvm', action=JvmDependencyUsage).install('dep-usage')
 
@@ -210,6 +213,7 @@ def register_goals():
   task(name='scalafmt', action=ScalaFmtCheckFormat, serialize=False).install('lint')
   task(name='scalastyle', action=Scalastyle, serialize=False).install('lint')
   task(name='checkstyle', action=Checkstyle, serialize=False).install('lint')
+  task(name='jvm-dep-check', action=JvmDependencyCheck, serialize=False).install('lint')
 
   # Formatting.
   task(name='scalafmt', action=ScalaFmtFormat, serialize=False).install('fmt')

src/python/pants/backend/jvm/subsystems/BUILD

@@ -1,11 +1,25 @@
 # Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
+python_library(
+  name = 'dependency_context',
+  sources = ['dependency_context.py'],
+  dependencies = [
+    ':java',
+    ':scala_platform',
+    'src/python/pants/backend/codegen/thrift/java',
+    'src/python/pants/base:fingerprint_strategy',
+    'src/python/pants/build_graph',
+    'src/python/pants/java/jar',
+  ],
+)
+
 python_library(
   name = 'jvm_tool_mixin',
   sources = ['jvm_tool_mixin.py'],
   dependencies = [
     'src/python/pants/base:exceptions',
+    'src/python/pants/java/distribution',
     'src/python/pants/option',
   ],
 )
@@ -117,7 +131,9 @@ python_library(
   name = 'zinc',
   sources = ['zinc.py'],
   dependencies = [
+    ':dependency_context',
     ':jvm_tool_mixin',
+    ':shader',
     'src/python/pants/subsystem',
   ]
 )

src/python/pants/backend/jvm/subsystems/dependency_context.py

@@ -0,0 +1,118 @@
+# coding=utf-8
+# Copyright 2018 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
+                        unicode_literals, with_statement)
+
+import hashlib
+
+from pants.backend.codegen.protobuf.java.java_protobuf_library import JavaProtobufLibrary
+from pants.backend.codegen.thrift.java.java_thrift_library import JavaThriftLibrary
+from pants.backend.jvm.subsystems.java import Java
+from pants.backend.jvm.subsystems.scala_platform import ScalaPlatform
+from pants.backend.jvm.targets.annotation_processor import AnnotationProcessor
+from pants.backend.jvm.targets.jar_library import JarLibrary
+from pants.backend.jvm.targets.javac_plugin import JavacPlugin
+from pants.backend.jvm.targets.scalac_plugin import ScalacPlugin
+from pants.base.fingerprint_strategy import FingerprintStrategy
+from pants.build_graph.aliased_target import AliasTarget
+from pants.build_graph.resources import Resources
+from pants.build_graph.target import Target
+from pants.build_graph.target_scopes import Scopes
+from pants.subsystem.subsystem import Subsystem
+
+
+class SyntheticTargetNotFound(Exception):
+  """Exports were resolved for a thrift target which hasn't had a synthetic target generated yet."""
+
+
+class DependencyContext(Subsystem):
+  """Implements calculating `exports` and exception (compiler-plugin) aware dependencies.
+
+  This is a subsystem because in future the compiler plugin types should be injected
+  via subsystem or option dependencies rather than declared statically.
+  """
+
+  options_scope = 'jvm-dependency-context'
+
+  target_closure_kwargs = dict(include_scopes=Scopes.JVM_COMPILE_SCOPES, respect_intransitive=True)
+  compiler_plugin_types = (AnnotationProcessor, JavacPlugin, ScalacPlugin)
+  alias_types = (AliasTarget, Target)
+  codegen_types = (JavaThriftLibrary, JavaProtobufLibrary)
+
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(DependencyContext, cls).subsystem_dependencies() + (Java, ScalaPlatform)
+
+  def all_dependencies(self, target):
+    """All transitive dependencies of the context's target."""
+    for dep in target.closure(bfs=True, **self.target_closure_kwargs):
+      yield dep
+
+  def create_fingerprint_strategy(self, classpath_products):
+    return ResolvedJarAwareFingerprintStrategy(classpath_products, self)
+
+  def defaulted_property(self, target, selector):
+    """Computes a language property setting for the given JvmTarget.
+
+    :param selector A function that takes a target or platform and returns the boolean value of the
+                    property for that target or platform, or None if that target or platform does
+                    not directly define the property.
+
+    If the target does not override the language property, returns true iff the property
+    is true for any of the matched languages for the target.
+    """
+    if selector(target) is not None:
+      return selector(target)
+
+    prop = False
+    if target.has_sources('.java'):
+      prop |= selector(Java.global_instance())
+    if target.has_sources('.scala'):
+      prop |= selector(ScalaPlatform.global_instance())
+    return prop
+
+
+class ResolvedJarAwareFingerprintStrategy(FingerprintStrategy):
+  """Task fingerprint strategy that also includes the resolved coordinates of dependent jars."""
+
+  def __init__(self, classpath_products, dep_context):
+    super(ResolvedJarAwareFingerprintStrategy, self).__init__()
+    self._classpath_products = classpath_products
+    self._dep_context = dep_context
+
+  def compute_fingerprint(self, target):
+    if isinstance(target, Resources):
+      # Just do nothing, this kind of dependency shouldn't affect result's hash.
+      return None
+
+    hasher = hashlib.sha1()
+    hasher.update(target.payload.fingerprint())
+    if isinstance(target, JarLibrary):
+      # NB: Collects only the jars for the current jar_library, and hashes them to ensure that both
+      # the resolved coordinates, and the requested coordinates are used. This ensures that if a
+      # source file depends on a library with source compatible but binary incompatible signature
+      # changes between versions, that you won't get runtime errors due to using an artifact built
+      # against a binary incompatible version resolved for a previous compile.
+      classpath_entries = self._classpath_products.get_artifact_classpath_entries_for_targets(
+        [target])
+      for _, entry in classpath_entries:
+        hasher.update(str(entry.coordinate))
+    return hasher.hexdigest()
+
+  def direct(self, target):
+    return self._dep_context.defaulted_property(target, lambda x: x.strict_deps)
+
+  def dependencies(self, target):
+    if self.direct(target):
+      return target.strict_dependencies(self._dep_context)
+    return super(ResolvedJarAwareFingerprintStrategy, self).dependencies(target)
+
+  def __hash__(self):
+    # NB: FingerprintStrategy requires a useful override of eq/hash.
+    return hash(type(self))
+
+  def __eq__(self, other):
+    # NB: See __hash__.
+    return type(self) == type(other)

src/python/pants/backend/jvm/subsystems/java.py

@@ -39,6 +39,14 @@ def register_options(cls, register):
                           help='Java compiler to use.  If unspecified, we use the compiler '
                                'embedded in the Java distribution we run on.')
 
+    register('--javac-plugins', advanced=True, type=list, fingerprint=True,
+            help='Use these javac plugins.')
+    register('--javac-plugin-args', advanced=True, type=dict, default={}, fingerprint=True,
+            help='Map from javac plugin name to list of arguments for that plugin.')
+    cls.register_jvm_tool(register, 'javac-plugin-dep', classpath=[],
+                        help='Search for javac plugins here, as well as in any '
+                                'explicit dependencies.')
+
   def injectables(self, build_graph):
     tools_jar_address = Address.parse(self._tools_jar_spec)
     if not build_graph.contains_address(tools_jar_address):

src/python/pants/backend/jvm/subsystems/jvm_platform.py

@@ -57,7 +57,7 @@ def register_options(cls, register):
              help='Compile settings that can be referred to by name in jvm_targets.')
     register('--default-platform', advanced=True, type=str, default=None, fingerprint=True,
              help='Name of the default platform to use if none are specified.')
-    register('--compiler', choices=['zinc', 'javac'], default='zinc',
+    register('--compiler', advanced=True, choices=['zinc', 'javac'], default='zinc', fingerprint=True,
              help='Java compiler implementation to use.')
 
   @classmethod

src/python/pants/backend/jvm/subsystems/jvm_tool_mixin.py

@@ -9,6 +9,7 @@
 from textwrap import dedent
 
 from pants.base.exceptions import TaskError
+from pants.java.distribution.distribution import DistributionLocator
 from pants.option.custom_types import target_option
 
 
@@ -51,6 +52,10 @@ def is_default(self, options):
 
   _jvm_tools = []  # List of JvmTool objects.
 
+  @classmethod
+  def subsystem_dependencies(cls):
+    return super(JvmToolMixin, cls).subsystem_dependencies() + (DistributionLocator,)
+
   @classmethod
   def get_jvm_options_default(cls, bootstrap_option_values):
     """Subclasses may override to provide different defaults for their JVM options.
@@ -158,6 +163,20 @@ def reset_registered_tools():
     """Needed only for test isolation."""
     JvmToolMixin._jvm_tools = []
 
+  def set_distribution(self, minimum_version=None, maximum_version=None, jdk=False):
+    try:
+      self._dist = DistributionLocator.cached(minimum_version=minimum_version,
+                                              maximum_version=maximum_version, jdk=jdk)
+    except DistributionLocator.Error as e:
+      raise TaskError(e)
+
+  @property
+  def dist(self):
+    if getattr(self, '_dist', None) is None:
+      # Use default until told otherwise.
+      self.set_distribution()
+    return self._dist
+
   @classmethod
   def tool_jar_from_products(cls, products, key, scope):
     """Get the jar for the tool previously registered under key in the given scope.

src/python/pants/backend/jvm/subsystems/scala_platform.py

@@ -39,7 +39,10 @@ class ScalaPlatform(JvmToolMixin, ZincLanguageMixin, InjectablesMixin, Subsystem
 
   :API: public
   """
-  options_scope = 'scala-platform'
+  options_scope = 'scala'
+
+  deprecated_options_scope = 'scala-platform'
+  deprecated_options_scope_removal_version = '1.9.0.dev0'
 
   @classmethod
   def _create_jardep(cls, name, version):
@@ -88,6 +91,15 @@ def register_style_tool(version):
                             classpath=[scala_style_jar])
 
     super(ScalaPlatform, cls).register_options(register)
+
+    register('--scalac-plugins', advanced=True, type=list, fingerprint=True,
+            help='Use these scalac plugins.')
+    register('--scalac-plugin-args', advanced=True, type=dict, default={}, fingerprint=True,
+            help='Map from scalac plugin name to list of arguments for that plugin.')
+    cls.register_jvm_tool(register, 'scalac-plugin-dep', classpath=[],
+                        help='Search for scalac plugins here, as well as in any '
+                                'explicit dependencies.')
+
     register('--version', advanced=True, default='2.12',
              choices=['2.10', '2.11', '2.12', 'custom'], fingerprint=True,
              help='The scala platform version. If --version=custom, the targets '
@@ -153,7 +165,7 @@ def suffix_version(self, name):
         raise RuntimeError('Suffix version must be specified if using a custom scala version. '
                            'Suffix version is used for bootstrapping jars.  If a custom '
                            'scala version is not specified, then the version specified in '
-                           '--scala-platform-suffix-version is used.  For example for Scala '
+                           '--scala-suffix-version is used.  For example for Scala '
                            '2.10.7 you would use the suffix version "2.10".')
 
     elif name.endswith(self.version):