diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy index 796b8ba23a462..7f182a96db864 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/ThirdPartyAuditTask.groovy @@ -78,6 +78,9 @@ public class ThirdPartyAuditTask extends AntTask { static final Pattern VIOLATION_PATTERN = Pattern.compile(/\s\sin ([a-zA-Z0-9\$\.]+) \(.*\)/); + static final Pattern INTERNAL_RUNTIME_PATTERN = + Pattern.compile(/Forbidden .* use:\s+(.*)\s+\[non-public internal runtime class\]/); + // we log everything (except missing classes warnings). Those we handle ourselves. static class EvilLogger extends DefaultLogger { final Set missingClasses = new TreeSet<>(); @@ -95,14 +98,20 @@ public class ThirdPartyAuditTask extends AntTask { } else if (event.getPriority() == Project.MSG_ERR) { Matcher m = VIOLATION_PATTERN.matcher(event.getMessage()); if (m.matches()) { - String violation = previousLine + '\n' + event.getMessage(); - String clazz = m.group(1).replace('.', '/') + ".class"; - List current = violations.get(clazz); - if (current == null) { - current = new ArrayList<>(); - violations.put(clazz, current); + // filter out false positives + Matcher m2 = INTERNAL_RUNTIME_PATTERN.matcher(previousLine); + if (m2.matches() && isReallyInternal(m2.group(1)) == false) { + // false positive + } else { + String violation = previousLine + '\n' + event.getMessage(); + String clazz = m.group(1).replace('.', '/') + ".class"; + List current = violations.get(clazz); + if (current == null) { + current = new ArrayList<>(); + violations.put(clazz, current); + } + current.add(violation); } - current.add(violation); } previousLine = event.getMessage(); } @@ -253,4 +262,68 @@ public class ThirdPartyAuditTask extends AntTask { }); return sheistySet; } + + // Forbidden apis has many false positives for internal apis: + // https://github.com/policeman-tools/forbidden-apis/issues/91 + // TODO: remove this when forbidden-apis is fixed! + // generated with Security.getProperty("package.access").split(",") from java 8 + // (this list can change in minor releases) + static final String[] INTERNAL_PACKAGES = [ + 'sun.', + 'com.sun.xml.internal.', + 'com.sun.imageio.', + 'com.sun.istack.internal.', + 'com.sun.jmx.', + 'com.sun.media.sound.', + 'com.sun.naming.internal.', + 'com.sun.proxy.', + 'com.sun.corba.se.', + 'com.sun.org.apache.bcel.internal.', + 'com.sun.org.apache.regexp.internal.', + 'com.sun.org.apache.xerces.internal.', + 'com.sun.org.apache.xpath.internal.', + 'com.sun.org.apache.xalan.internal.extensions.', + 'com.sun.org.apache.xalan.internal.lib.', + 'com.sun.org.apache.xalan.internal.res.', + 'com.sun.org.apache.xalan.internal.templates.', + 'com.sun.org.apache.xalan.internal.utils.', + 'com.sun.org.apache.xalan.internal.xslt.', + 'com.sun.org.apache.xalan.internal.xsltc.cmdline.', + 'com.sun.org.apache.xalan.internal.xsltc.compiler.', + 'com.sun.org.apache.xalan.internal.xsltc.trax.', + 'com.sun.org.apache.xalan.internal.xsltc.util.', + 'com.sun.org.apache.xml.internal.res.', + 'com.sun.org.apache.xml.internal.security.', + 'com.sun.org.apache.xml.internal.serializer.utils.', + 'com.sun.org.apache.xml.internal.utils.', + 'com.sun.org.glassfish.', + 'com.oracle.xmlns.internal.', + 'com.oracle.webservices.internal.', + 'oracle.jrockit.jfr.', + 'org.jcp.xml.dsig.internal.', + 'jdk.internal.', + 'jdk.nashorn.internal.', + 'jdk.nashorn.tools.', + 'com.sun.activation.registries.', + 'apple.', + 'com.sun.browser.', + 'com.sun.glass.', + 'com.sun.javafx.', + 'com.sun.media.', + 'com.sun.openpisces.', + 'com.sun.prism.', + 'com.sun.scenario.', + 'com.sun.t2k.', + 'com.sun.pisces.', + 'com.sun.webkit.', + ]; + + private static boolean isReallyInternal(String clazz) { + for (String pkg : INTERNAL_PACKAGES) { + if (clazz.startsWith(pkg)) { + return true; + } + } + return false; + } } diff --git a/plugins/discovery-azure/build.gradle b/plugins/discovery-azure/build.gradle index d922302afd075..d85d08794ea95 100644 --- a/plugins/discovery-azure/build.gradle +++ b/plugins/discovery-azure/build.gradle @@ -62,16 +62,7 @@ compileJava.options.compilerArgs << '-Xlint:-deprecation' // TODO: and why does this static not show up in maven... compileTestJava.options.compilerArgs << '-Xlint:-static' -// TODO: figure out what is happening and fix this!!!!!!!!!!! -// there might be still some undetected jar hell! -// we need to fix https://github.com/policeman-tools/forbidden-apis/issues/91 first thirdPartyAudit.excludes = [ - // uses internal java api: com.sun.xml.fastinfoset.stax.StAXDocumentParser - 'com.sun.xml.bind.v2.runtime.unmarshaller.FastInfosetConnector', - 'com.sun.xml.bind.v2.runtime.unmarshaller.FastInfosetConnector$CharSequenceImpl', - // uses internal java api: com.sun.xml.fastinfoset.stax.StAXDocumentSerializer - 'com.sun.xml.bind.v2.runtime.output.FastInfosetStreamWriterOutput', - // classes are missing 'javax.servlet.ServletContextEvent', 'javax.servlet.ServletContextListener', diff --git a/plugins/mapper-attachments/build.gradle b/plugins/mapper-attachments/build.gradle index c613076a06cf3..4b5b2d7fac2e4 100644 --- a/plugins/mapper-attachments/build.gradle +++ b/plugins/mapper-attachments/build.gradle @@ -70,9 +70,6 @@ forbiddenPatterns { } thirdPartyAudit.excludes = [ - // uses internal java api: com.sun.syndication (SyndFeedInput, SyndFeed, SyndEntry, SyndContent) - 'org.apache.tika.parser.feed.FeedParser', - // classes are missing: some due to our whitelisting of parsers 'com.coremedia.iso.IsoFile', 'com.coremedia.iso.boxes.Box', diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 608061ba00c95..7f1d59fa930ac 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -336,33 +336,6 @@ thirdPartyAudit.excludes = [ 'org.mortbay.util.ajax.JSON', 'org.znerd.xmlenc.XMLOutputter', - // note: the jersey ones may be bogus, see my bug report at forbidden-apis! - // internal java api: com.sun.jersey.server.impl.inject.AbstractHttpContextInjectable - // internal java api: com.sun.jersey.api.core.HttpContext - // internal java api: com.sun.jersey.core.spi.component.ComponentScope - // internal java api: com.sun.jersey.spi.inject.Injectable - // internal java api: com.sun.jersey.core.spi.component.ComponentContext - 'org.apache.hadoop.hdfs.web.resources.UserProvider', - - // internal java api: com.sun.jersey.spi.container.ResourceFilters - 'org.apache.hadoop.hdfs.server.namenode.web.resources.NamenodeWebHdfsMethods', - // internal java api: com.sun.jersey.spi.container.servlet.ServletContainer - 'org.apache.hadoop.http.HttpServer', - 'org.apache.hadoop.http.HttpServer2', - - // internal java api: com.sun.jersey.api.ParamException - 'org.apache.hadoop.hdfs.web.resources.ExceptionHandler', - 'org.apache.hadoop.hdfs.server.datanode.web.webhdfs.ExceptionHandler', - 'org.apache.hadoop.hdfs.web.ParamFilter', - - // internal java api: com.sun.jersey.spi.container.ContainerRequestFilter - // internal java api: com.sun.jersey.spi.container.ContainerRequest - 'org.apache.hadoop.hdfs.web.ParamFilter', - 'org.apache.hadoop.hdfs.web.ParamFilter$1', - - // internal java api: com.sun.jndi.ldap.LdapCtxFactory - 'org.apache.hadoop.security.LdapGroupsMapping', - // internal java api: sun.net.dns.ResolverConfiguration // internal java api: sun.net.util.IPAddressUtil 'org.apache.hadoop.security.SecurityUtil$QualifiedHostResolver',