From 98c1e31f38bb6e6790d99e930bb883beb2783233 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Thu, 5 Oct 2017 17:23:11 -0600 Subject: [PATCH 1/2] Shim a hook to modify ZipOutputStream entries This is motivated by wanting to zero timestamps on entries for byte-stable shadow jars, but generalized in case other uses are created. --- .../internal/DefaultZipCompressor.groovy | 15 +++++- .../shadow/internal/GradleVersionUtil.groovy | 8 ++-- .../plugins/shadow/tasks/ShadowJar.java | 46 ++++++++++++++++++- .../plugins/shadow/tasks/ShadowSpec.java | 5 ++ 4 files changed, 68 insertions(+), 6 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/DefaultZipCompressor.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/DefaultZipCompressor.groovy index d0b24f751..977451209 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/DefaultZipCompressor.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/DefaultZipCompressor.groovy @@ -16,21 +16,32 @@ package com.github.jengelman.gradle.plugins.shadow.internal import org.apache.tools.zip.Zip64Mode +import org.apache.tools.zip.ZipEntry import org.apache.tools.zip.ZipOutputStream +import org.gradle.api.Action import org.gradle.api.UncheckedIOException class DefaultZipCompressor implements ZipCompressor { private final int entryCompressionMethod private final Zip64Mode zip64Mode + private final List> actions - DefaultZipCompressor(boolean allowZip64Mode, int entryCompressionMethod) { + DefaultZipCompressor(boolean allowZip64Mode, int entryCompressionMethod, List> actions) { this.entryCompressionMethod = entryCompressionMethod zip64Mode = allowZip64Mode ? Zip64Mode.AsNeeded : Zip64Mode.Never + this.actions = actions } ZipOutputStream createArchiveOutputStream(File destination) { try { - ZipOutputStream zipOutputStream = new ZipOutputStream(destination) + // Shim an override into the outputstream so we can intercept all entries and act on them + ZipOutputStream zipOutputStream = new ZipOutputStream(destination) { + @Override + void putNextEntry(ZipEntry archiveEntry) throws IOException { + actions.each { it.execute(archiveEntry) } + super.putNextEntry(archiveEntry) + } + } zipOutputStream.setUseZip64(zip64Mode) zipOutputStream.setMethod(entryCompressionMethod) return zipOutputStream diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/GradleVersionUtil.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/GradleVersionUtil.groovy index 21b81939e..955a5e200 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/GradleVersionUtil.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/GradleVersionUtil.groovy @@ -1,6 +1,8 @@ package com.github.jengelman.gradle.plugins.shadow.internal +import org.apache.tools.zip.ZipEntry import org.apache.tools.zip.ZipOutputStream +import org.gradle.api.Action import org.gradle.api.internal.file.copy.CopySpecInternal import org.gradle.api.tasks.bundling.Jar import org.gradle.api.tasks.bundling.ZipEntryCompression @@ -19,12 +21,12 @@ class GradleVersionUtil { return mainSpec.buildRootResolver().getPatternSet() } - ZipCompressor getInternalCompressor(ZipEntryCompression entryCompression, Jar jar) { + ZipCompressor getInternalCompressor(ZipEntryCompression entryCompression, Jar jar, List> actions) { switch (entryCompression) { case ZipEntryCompression.DEFLATED: - return new DefaultZipCompressor(jar.zip64, ZipOutputStream.DEFLATED); + return new DefaultZipCompressor(jar.zip64, ZipOutputStream.DEFLATED, actions); case ZipEntryCompression.STORED: - return new DefaultZipCompressor(jar.zip64, ZipOutputStream.STORED); + return new DefaultZipCompressor(jar.zip64, ZipOutputStream.STORED, actions); default: throw new IllegalArgumentException(String.format("Unknown Compression type %s", entryCompression)); } diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java index 447be061d..7e7357a6b 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java @@ -9,6 +9,7 @@ import com.github.jengelman.gradle.plugins.shadow.transformers.ServiceFileTransformer; import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer; import groovy.lang.MetaClass; +import org.apache.tools.zip.ZipEntry; import org.codehaus.groovy.runtime.InvokerHelper; import org.gradle.api.Action; import org.gradle.api.artifacts.Configuration; @@ -28,6 +29,7 @@ public class ShadowJar extends Jar implements ShadowSpec { private List transformers; + private List> zipEntryActions; private List relocators; private List configurations; private DependencyFilter dependencyFilter; @@ -41,6 +43,7 @@ public ShadowJar() { dependencyFilter = new DefaultDependencyFilter(getProject()); setManifest(new DefaultInheritManifest(getServices().get(FileResolver.class))); transformers = new ArrayList<>(); + zipEntryActions = new ArrayList<>(); relocators = new ArrayList<>(); configurations = new ArrayList<>(); } @@ -63,7 +66,7 @@ protected CopyAction createCopyAction() { } protected ZipCompressor getInternalCompressor() { - return versionUtil.getInternalCompressor(getEntryCompression(), this); + return versionUtil.getInternalCompressor(getEntryCompression(), this, zipEntryActions); } @TaskAction @@ -100,6 +103,11 @@ public ShadowJar dependencies(Action c) { return this; } + public ShadowJar entryAction(Action action) { + zipEntryActions.add(action); + return this; + } + /** * Add a Transformer instance for modifying JAR resources and configure. * @@ -137,6 +145,42 @@ public ShadowJar transform(Transformer transformer) { return this; } + /** + * Act on all zipEntries being added to the final jar. + * + * Every {@link ZipEntry} to be added to the final jar will be presented + * to the {@code zipEntryAction}, which can inspect and modify it as desired. + * @param zipEntryAction the action instance to add + * @return this + */ + public ShadowSpec modifyZipEntries(Action zipEntryAction) { + zipEntryActions.add(zipEntryAction); + return this; + } + + private static final long DOS_TIMESTAMP_ZERO = 315558000000L; + /** + * Zero out timestamps on zip entries to Jan 1st 1980. (DOS epoch) + * + * Why would you want to do this? The timestamps usually are just set to the + * time the jar was built. However, those timestamps are repeated across every + * file inside the jar, which scatters byte-level changes throughout the + * shadow jar, which breaks any efforts to calculate efficient deltas between + * shadow jar builds (e.g. rsync). With static timestamps though, rsync just works. + * + * This is generally safe to turn on because nothing cares about internal timestamps, but it's off by default. + * @return this + */ + public ShadowJar zeroZipEntryTimestamps() { + zipEntryActions.add(new Action() { + @Override + public void execute(ZipEntry zipEntry) { + zipEntry.setTime(DOS_TIMESTAMP_ZERO); + } + }); + return this; + } + /** * Syntactic sugar for merging service files in JARs. * diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowSpec.java b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowSpec.java index c4aa001b9..693c0957d 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowSpec.java +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowSpec.java @@ -6,6 +6,7 @@ import com.github.jengelman.gradle.plugins.shadow.relocation.SimpleRelocator; import com.github.jengelman.gradle.plugins.shadow.transformers.ServiceFileTransformer; import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer; +import org.apache.tools.zip.ZipEntry; import org.gradle.api.Action; import org.gradle.api.file.CopySpec; @@ -19,6 +20,10 @@ interface ShadowSpec extends CopySpec { ShadowSpec transform(Transformer transformer); + ShadowSpec modifyZipEntries(Action zipEntryAction); + + ShadowSpec zeroZipEntryTimestamps(); + ShadowSpec mergeServiceFiles(); ShadowSpec mergeServiceFiles(String rootPath); From e20f3d8dd0f960c7bfcdfe190057b13ae26fad58 Mon Sep 17 00:00:00 2001 From: Jack Kelly Date: Wed, 25 Oct 2017 15:37:05 -0600 Subject: [PATCH 2/2] Test ZipEntry modification layer --- .../plugins/shadow/ShadowPluginSpec.groovy | 58 +++++++++++++++++++ .../shadow/util/PluginSpecification.groovy | 6 ++ 2 files changed, 64 insertions(+) diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy index 138b97ee3..52db0e2b0 100644 --- a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy @@ -663,6 +663,64 @@ class ShadowPluginSpec extends PluginSpecification { } + def 'zeroes zip entry timestamps if requested'() { + given: + file('src/main/java/shadow/Passed.java') << ''' + package shadow; + public class Passed {} + '''.stripIndent() + + buildFile << """ + dependencies { compile 'junit:junit:3.8.2' } + // tag::rename[] + shadowJar { + baseName = 'shadow' + classifier = null + version = null + zeroZipEntryTimestamps() + } + // end::rename[] + """.stripIndent() + + when: + runner.withArguments('-S', 'shadowJar').build() + + then: + getZipEntries(output("shadow.jar")).each { + assert it.getTime() == 315558000000L + } + } + + def 'calls provided ZipEntry Actions' () { + given: + file('src/main/java/shadow/Passed.java') << ''' + package shadow; + public class Passed {} + '''.stripIndent() + + buildFile << """ + dependencies { compile 'junit:junit:3.8.2' } + // tag::rename[] + shadowJar { + baseName = 'shadow' + classifier = null + version = null + modifyZipEntries { + ze -> ze.setName("overwritten") + } + } + // end::rename[] + """.stripIndent() + + when: + runner.withArguments('-S', 'shadowJar').build() + + then: + getZipEntries(output("shadow.jar")).each { + assert it.getName() == "overwritten" + } + } + private String escapedPath(File file) { file.path.replaceAll('\\\\', '\\\\\\\\') } diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/util/PluginSpecification.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/util/PluginSpecification.groovy index d331eae24..eaa3425da 100644 --- a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/util/PluginSpecification.groovy +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/util/PluginSpecification.groovy @@ -2,6 +2,8 @@ package com.github.jengelman.gradle.plugins.shadow.util import com.github.jengelman.gradle.plugins.shadow.util.file.TestFile import com.google.common.base.StandardSystemProperty +import org.apache.tools.zip.ZipEntry +import org.apache.tools.zip.ZipFile import org.codehaus.plexus.util.IOUtil import org.gradle.testkit.runner.GradleRunner import org.junit.Rule @@ -108,6 +110,10 @@ class PluginSpecification extends Specification { return sw.toString() } + List getZipEntries(File f) { + new ZipFile(f).entries.toList() + } + void contains(File f, List paths) { JarFile jar = new JarFile(f) paths.each { path ->