Add gradle thirdPartyAudit to precommit tasks

build.gradle

@@ -123,6 +123,16 @@ subprojects {
       }
     }
   }
+  // For reasons we don't fully understand yet, external dependencies are not picked up by Ant's optional tasks.
+  // But you can easily do it in another way.
+  // Only if your buildscript and Ant's optional task need the same library would you have to define it twice.
+  // https://docs.gradle.org/current/userguide/organizing_build_logic.html
+  configurations {
+    forbiddenApis
+  }
+  dependencies {
+    forbiddenApis 'de.thetaphi:forbiddenapis:2.0'
+  }
 }
 
 // Ensure similar tasks in dependent projects run first. The projectsEvaluated here is

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -34,7 +34,8 @@ class PrecommitTasks {
         List<Task> precommitTasks = [
             configureForbiddenApis(project),
             project.tasks.create('forbiddenPatterns', ForbiddenPatternsTask.class),
-            project.tasks.create('jarHell', JarHellTask.class)]
+            project.tasks.create('jarHell', JarHellTask.class),
+            project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)]
 
         // tasks with just tests don't need dependency licenses, so this flag makes adding
         // the task optional

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy

@@ -0,0 +1,158 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle.precommit
+
+import org.gradle.api.DefaultTask
+import org.gradle.api.artifacts.UnknownConfigurationException
+import org.gradle.api.file.FileCollection
+import org.gradle.api.tasks.TaskAction
+
+import org.apache.tools.ant.BuildLogger
+import org.apache.tools.ant.Project
+
+/**
+ * Basic static checking to keep tabs on third party JARs
+ */
+public class ThirdPartyAuditTask extends DefaultTask {
+
+    // true to be lenient about MISSING CLASSES
+    private boolean lenient;
+
+    // patterns for classes to exclude, because we understand their issues
+    private String[] excludes = new String[0];
+
+    ThirdPartyAuditTask() {
+        dependsOn(project.configurations.testCompile)
+        description = "Checks third party JAR bytecode for missing classes, use of internal APIs, and other horrors'"
+    }
+
+    /** 
+     * Set to true to be lenient with dependencies. By default this check will fail if it finds
+     * MISSING CLASSES. This means the set of jars is incomplete. However, in some cases
+     * this can be due to intentional exclusions that are well-tested and understood.
+     */      
+    public void setLenient(boolean value) {
+        lenient = value;
+    }
+
+    /**
+     * Returns true if leniency about missing classes is enabled.
+     */
+    public boolean isLenient() {
+        return lenient;
+    }
+
+    /**
+     * classes that should be excluded from the scan,
+     * e.g. because we know what sheisty stuff those particular classes are up to.
+     */
+    public void setExcludes(String[] classes) {
+        for (String s : classes) {
+            if (s.indexOf('*') != -1) {
+                throw new IllegalArgumentException("illegal third party audit exclusion: '" + s + "', wildcards are not permitted!")
+            }
+        }
+        excludes = classes;
+    }
+
+    /**
+     * Returns current list of exclusions.
+     */
+    public String[] getExcludes() {
+        return excludes;
+    }
+
+    @TaskAction
+    public void check() {
+        AntBuilder ant = new AntBuilder()
+
+        // we are noisy for many reasons, working around performance problems with forbidden-apis, dealing
+        // with warnings about missing classes, etc. so we use our own "quiet" AntBuilder
+        ant.project.buildListeners.each { listener ->
+            if (listener instanceof BuildLogger) {
+              listener.messageOutputLevel = Project.MSG_ERR;
+            }
+        };
+
+        // we only want third party dependencies.
+        FileCollection jars = project.configurations.testCompile.fileCollection({ dependency -> 
+            dependency.group != "org.elasticsearch"
+        })
+
+        // we don't want provided dependencies, which we have already scanned. e.g. don't
+        // scan ES core's dependencies for every single plugin
+        try {
+            jars -= project.configurations.getByName("provided")
+        } catch (UnknownConfigurationException ignored) {}
+
+        // no dependencies matched, we are done
+        if (jars.isEmpty()) {
+            return;
+        }
+
+        ant.taskdef(name:      "thirdPartyAudit",
+                    classname: "de.thetaphi.forbiddenapis.ant.AntTask",
+                    classpath: project.configurations.forbiddenApis.asPath)
+
+        // print which jars we are going to scan, always
+        // this is not the time to try to be succinct! Forbidden will print plenty on its own!
+        Set<String> names = new HashSet<>()
+        for (File jar : jars) {
+            names.add(jar.getName())
+        }
+        logger.error("[thirdPartyAudit] Scanning: " + names)
+
+        // warn that you won't see any forbidden apis warnings
+        if (lenient) {
+            logger.warn("[thirdPartyAudit] WARNING: leniency is enabled, will not fail if classes are missing!")
+        }
+
+        // TODO: forbidden-apis + zipfileset gives O(n^2) behavior unless we dump to a tmpdir first, 
+        // and then remove our temp dir afterwards. don't complain: try it yourself.
+        // we don't use gradle temp dir handling, just google it, or try it yourself.
+
+        File tmpDir = new File(project.buildDir, 'tmp/thirdPartyAudit')
+
+        // clean up any previous mess (if we failed), then unzip everything to one directory
+        ant.delete(dir: tmpDir.getAbsolutePath())
+        tmpDir.mkdirs()
+        for (File jar : jars) {
+            ant.unzip(src: jar.getAbsolutePath(), dest: tmpDir.getAbsolutePath())
+        }
+
+        // convert exclusion class names to binary file names
+        String[] excludedFiles = new String[excludes.length];
+        for (int i = 0; i < excludes.length; i++) {
+            excludedFiles[i] = excludes[i].replace('.', '/') + ".class"
+            // check if the excluded file exists, if not, sure sign things are outdated
+            if (! new File(tmpDir, excludedFiles[i]).exists()) {
+                throw new IllegalStateException("bogus thirdPartyAudit exclusion: '" + excludes[i] + "', not found in any dependency")
+            }
+        }
+
+        ant.thirdPartyAudit(internalRuntimeForbidden: true, 
+                            failOnUnsupportedJava: false, 
+                            failOnMissingClasses: !lenient,
+                            classpath: project.configurations.testCompile.asPath) {
+            fileset(dir: tmpDir, excludes: excludedFiles.join(','))
+        }
+        // clean up our mess (if we succeed)
+        ant.delete(dir: tmpDir.getAbsolutePath())
+    }
+}

core/build.gradle

@@ -111,6 +111,14 @@ forbiddenPatterns {
   exclude '**/org/elasticsearch/cluster/routing/shard_routes.txt'
 }
 
+// classes are missing, e.g. org.jboss.marshalling.Marshaller
+thirdPartyAudit.lenient = true
+// uses internal sun ssl classes!
+thirdPartyAudit.excludes = [
+    // uses internal java api: sun.security.x509 (X509CertInfo, X509CertImpl, X500Name)
+    'org.jboss.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
+]
+
 // dependency license are currently checked in distribution
 dependencyLicenses.enabled = false
 

modules/lang-expression/build.gradle

@@ -33,6 +33,10 @@ dependencyLicenses {
   mapping from: /lucene-.*/, to: 'lucene'
 }
 
+// do we or do we not depend on asm-tree, that is the question
+// classes are missing, e.g. org.objectweb.asm.tree.LabelNode
+thirdPartyAudit.lenient = true
+
 compileJava.options.compilerArgs << '-Xlint:-rawtypes'
 compileTestJava.options.compilerArgs << '-Xlint:-rawtypes'
 

modules/lang-groovy/build.gradle

@@ -35,3 +35,12 @@ integTest {
     systemProperty 'es.script.indexed', 'on'
   }
 }
+
+// classes are missing, e.g. jline.console.completer.Completer
+thirdPartyAudit.lenient = true
+thirdPartyAudit.excludes = [
+    // uses internal java api: sun.misc.Unsafe
+    'groovy.json.internal.FastStringUtils',
+    'groovy.json.internal.FastStringUtils$StringImplementation$1',
+    'groovy.json.internal.FastStringUtils$StringImplementation$2',
+]

plugins/discovery-azure/build.gradle

@@ -66,3 +66,14 @@ compileJava.options.compilerArgs << '-Xlint:-deprecation'
 // TODO: and why does this static not show up in maven...
 compileTestJava.options.compilerArgs << '-Xlint:-static'
 
+// classes are missing, e.g. org.osgi.framework.BundleActivator
+thirdPartyAudit.lenient = true
+// WE ARE JAR HELLING WITH THE JDK AND THAT IS WHY THIS HAPPENS
+// TODO: fix this!!!!!!!!!!!
+thirdPartyAudit.excludes = [
+    // uses internal java api: com.sun.xml.fastinfoset.stax.StAXDocumentParser
+    'com.sun.xml.bind.v2.runtime.unmarshaller.FastInfosetConnector',
+    'com.sun.xml.bind.v2.runtime.unmarshaller.FastInfosetConnector$CharSequenceImpl',
+    // uses internal java api: com.sun.xml.fastinfoset.stax.StAXDocumentSerializer
+    'com.sun.xml.bind.v2.runtime.output.FastInfosetStreamWriterOutput',
+]

plugins/discovery-ec2/build.gradle

@@ -48,3 +48,12 @@ test {
   // this is needed for insecure plugins, remove if possible!
   systemProperty 'tests.artifact', project.name 
 }
+
+// classes are missing, e.g. org.apache.avalon.framework.logger.Logger
+thirdPartyAudit.lenient = true
+thirdPartyAudit.excludes = [
+    // uses internal java api: com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl
+    // uses internal java api: com.sun.org.apache.xml.internal.dtm.ref.DTMManagerDefault
+    // uses internal java api: com.sun.org.apache.xpath.internal.XPathContext
+    'com.amazonaws.util.XpathUtils',
+]

plugins/discovery-gce/build.gradle

@@ -31,3 +31,6 @@ test {
   // this is needed for insecure plugins, remove if possible!
   systemProperty 'tests.artifact', project.name 
 }
+
+// classes are missing, e.g. org.apache.log.Logger
+thirdPartyAudit.lenient = true

plugins/lang-plan-a/build.gradle

@@ -33,6 +33,9 @@ dependencies {
 compileJava.options.compilerArgs << '-Xlint:-cast,-fallthrough,-rawtypes'
 compileTestJava.options.compilerArgs << '-Xlint:-unchecked'
 
+// classes are missing, e.g. org.objectweb.asm.tree.LabelNode
+thirdPartyAudit.lenient = true
+
 // regeneration logic, comes in via ant right now
 // don't port it to gradle, it works fine.
 

plugins/lang-python/build.gradle

@@ -36,3 +36,40 @@ integTest {
   }
 }
 
+// classes are missing, e.g. org.tukaani.xz.FilterOptions
+thirdPartyAudit.lenient = true
+thirdPartyAudit.excludes = [
+    // uses internal java api: sun.security.x509 (X509CertInfo, X509CertImpl, X500Name)
+    'org.python.netty.handler.ssl.util.OpenJdkSelfSignedCertGenerator',
+
+    // uses internal java api: sun.misc.Cleaner
+    'org.python.netty.util.internal.Cleaner0',
+
+    // uses internal java api: sun.misc.Signal
+    'jnr.posix.JavaPOSIX',
+    'jnr.posix.JavaPOSIX$SunMiscSignalHandler',
+
+    // uses internal java api: sun.misc.Unsafe
+    'com.kenai.jffi.MemoryIO$UnsafeImpl',
+    'com.kenai.jffi.MemoryIO$UnsafeImpl32',
+    'com.kenai.jffi.MemoryIO$UnsafeImpl64',
+    'org.python.google.common.cache.Striped64',
+    'org.python.google.common.cache.Striped64$1',
+    'org.python.google.common.cache.Striped64$Cell',
+    'org.python.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+    'org.python.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+    'org.python.netty.util.internal.chmv8.ForkJoinPool$2',
+    'org.python.netty.util.internal.PlatformDependent0',
+    'org.python.netty.util.internal.UnsafeAtomicIntegerFieldUpdater',
+    'org.python.netty.util.internal.UnsafeAtomicLongFieldUpdater',
+    'org.python.netty.util.internal.UnsafeAtomicReferenceFieldUpdater',
+    'org.python.netty.util.internal.chmv8.ConcurrentHashMapV8',
+    'org.python.netty.util.internal.chmv8.ConcurrentHashMapV8$1',
+    'org.python.netty.util.internal.chmv8.ConcurrentHashMapV8$TreeBin',
+    'org.python.netty.util.internal.chmv8.CountedCompleter',
+    'org.python.netty.util.internal.chmv8.CountedCompleter$1',
+    'org.python.netty.util.internal.chmv8.ForkJoinPool',
+    'org.python.netty.util.internal.chmv8.ForkJoinPool$WorkQueue',
+    'org.python.netty.util.internal.chmv8.ForkJoinTask',
+    'org.python.netty.util.internal.chmv8.ForkJoinTask$1',
+]

plugins/mapper-attachments/build.gradle

@@ -69,3 +69,10 @@ forbiddenPatterns {
   exclude '**/*.pdf'
   exclude '**/*.epub'
 }
+
+// classes are missing, e.g. org.openxmlformats.schemas.drawingml.x2006.chart.CTExtensionList
+thirdPartyAudit.lenient = true
+thirdPartyAudit.excludes = [
+    // uses internal java api: com.sun.syndication (SyndFeedInput, SyndFeed, SyndEntry, SyndContent)
+    'org.apache.tika.parser.feed.FeedParser',
+]

plugins/repository-hdfs/build.gradle

@@ -200,4 +200,7 @@ integTest {
     cluster {
         plugin(pluginProperties.extension.name, zipTree(distZipHadoop2.archivePath))
     }
-}
+}
+
+// classes are missing, e.g. org.mockito.Mockito
+thirdPartyAudit.lenient = true

plugins/repository-s3/build.gradle

@@ -49,3 +49,12 @@ test {
   // this is needed for insecure plugins, remove if possible!
   systemProperty 'tests.artifact', project.name 
 }
+
+// classes are missing, e.g. org.apache.log.Logger
+thirdPartyAudit.lenient = true
+thirdPartyAudit.excludes = [
+    // uses internal java api: com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl
+    // uses internal java api: com.sun.org.apache.xml.internal.dtm.ref.DTMManagerDefault
+    // uses internal java api: com.sun.org.apache.xpath.internal.XPathContext
+    'com.amazonaws.util.XpathUtils',
+]

qa/evil-tests/build.gradle

@@ -34,3 +34,14 @@ dependencies {
 test {
   systemProperty 'tests.security.manager', 'false'
 }
+
+// classes are missing, com.ibm.icu.lang.UCharacter
+thirdPartyAudit.lenient = true
+thirdPartyAudit.excludes = [
+    // uses internal java api: sun.misc.Unsafe
+    'com.google.common.cache.Striped64',
+    'com.google.common.cache.Striped64$1',
+    'com.google.common.cache.Striped64$Cell',
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+]

test-framework/build.gradle

@@ -47,3 +47,5 @@ forbiddenApisMain {
 // TODO: should we have licenses for our test deps?
 dependencyLicenses.enabled = false
 
+// we intentionally exclude the ant tasks because people were depending on them from their tests!!!!!!!
+thirdPartyAudit.lenient = true