diff --git a/generator/protos/diagnostics.proto b/generator/protos/diagnostics.proto new file mode 100644 index 0000000..2f4c055 --- /dev/null +++ b/generator/protos/diagnostics.proto @@ -0,0 +1,62 @@ +syntax = "proto2"; + +package archive_patcher; + +option java_package = "com.google.archivepatcher"; +option java_multiple_files = true; + +enum EntryDeltaFormat { + DF_UNSPECIFIED = 0; + DF_BSDIFF = 1; + DF_FILE_BY_FILE = 2; +} + +enum EntryDeltaFormatExplanation { + DFE_UNSPECIFIED = 0; + DFE_DEFAULT = 1; + DFE_FILE_TYPE = 2; + DFE_UNSUITABLE = 3; + DFE_DEFLATE_UNSUITABLE = 4; + DFE_UNCHANGED = 5; + DFE_RESOURCE_CONSTRAINED = 6; +} + +enum UncompressionOption { + UO_UNSPECIFIED = 0; + UO_UNCOMPRESS_OLD = 1; + UO_UNCOMPRESS_NEW = 2; + UO_UNCOMPRESS_BOTH = 3; + UO_UNCOMPRESS_NEITHER = 4; +} + +enum UncompressionOptionExplanation { + UOE_UNSPECIFIED = 0; + UOE_DEFLATE_UNSUITABLE = 1; + UOE_UNSUITABLE = 2; + UOE_BOTH_ENTRIES_UNCOMPRESSED = 3; + UOE_UNCOMPRESSED_CHANGED_TO_COMPRESSED = 4; + UOE_COMPRESSED_CHANGED_TO_UNCOMPRESSED = 5; + UOE_COMPRESSED_BYTES_CHANGED = 6; + UOE_COMPRESSED_BYTES_IDENTICAL = 7; + UOE_RESOURCE_CONSTRAINED = 8; +} + +message DeltaEntryDiagnostics { + repeated File files = 1; + optional int64 total_patch_size = 2; + // Which patch generation algorithm was used. + optional EntryDeltaFormat delta_format = 3; + // Entries for a recursive archive, if relevant for the given delta_format. + repeated DeltaEntryDiagnostics children = 4; +} + +message File { + optional string original_filename = 1; + optional int64 original_file_size = 2; + optional string new_filename = 3; + optional int64 new_file_size = 4; + // How compression was handled (and why). + optional UncompressionOption uncompression_option = 7; + optional UncompressionOptionExplanation uncompression_option_explanation = 8; + optional EntryDeltaFormatExplanation delta_format_explanation = 9; +} \ No newline at end of file diff --git a/generator/src/main/java/com/google/archivepatcher/generator/DeltaEntries.java b/generator/src/main/java/com/google/archivepatcher/generator/DeltaEntries.java index f604dd2..7bc3d1c 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/DeltaEntries.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/DeltaEntries.java @@ -153,10 +153,17 @@ private static DeltaEntry combineEntry(DeltaEntry entry1, DeltaEntry entry2, Byt // Here we greedily diff the range against the entire old archive. // This will not make a difference in V1 as we will always be diffing the entire file. In V2, // if this turns out to be too expensive on the patch application, we can reduce this range. - return DeltaEntry.builder() - .deltaFormat(entry1.deltaFormat()) - .oldBlobRange(Range.of(0, oldBlob.length())) - .newBlobRange(Range.combine(entry1.newBlobRange(), entry2.newBlobRange())) - .build(); + DeltaEntry.Builder builder = + DeltaEntry.builder() + .deltaFormat(entry1.deltaFormat()) + .oldBlobRange(Range.of(0, oldBlob.length())) + .newBlobRange(Range.combine(entry1.newBlobRange(), entry2.newBlobRange())); + for (DiffPlanEntry diffPlanEntry : entry1.diffPlanEntries()) { + builder = builder.addDiffPlanEntry(diffPlanEntry); + } + for (DiffPlanEntry diffPlanEntry : entry2.diffPlanEntries()) { + builder = builder.addDiffPlanEntry(diffPlanEntry); + } + return builder.build(); } } diff --git a/generator/src/main/java/com/google/archivepatcher/generator/DeltaEntry.java b/generator/src/main/java/com/google/archivepatcher/generator/DeltaEntry.java index 04a316d..0b337e5 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/DeltaEntry.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/DeltaEntry.java @@ -17,6 +17,7 @@ import com.google.archivepatcher.shared.PatchConstants.DeltaFormat; import com.google.archivepatcher.shared.Range; import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableSet; /** * An encapsulation of delta entry in the patch generated. A {@link DeltaEntry} consists of both the @@ -34,6 +35,8 @@ public abstract class DeltaEntry { /** The {@link Range} inside delta-friendly new blob to compute delta. */ public abstract Range newBlobRange(); + public abstract ImmutableSet diffPlanEntries(); + /** Builder for {@link DeltaEntry}. */ @AutoValue.Builder public abstract static class Builder { @@ -46,6 +49,13 @@ public abstract static class Builder { /** See {@link #newBlobRange()}. */ public abstract Builder newBlobRange(Range newBlobRange); + abstract ImmutableSet.Builder diffPlanEntriesBuilder(); + + public final Builder addDiffPlanEntry(DiffPlanEntry diffPlanEntry) { + diffPlanEntriesBuilder().add(diffPlanEntry); + return this; + } + public abstract DeltaEntry build(); } diff --git a/generator/src/main/java/com/google/archivepatcher/generator/DeltaGenerator.java b/generator/src/main/java/com/google/archivepatcher/generator/DeltaGenerator.java index d60a3ea..534dbcd 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/DeltaGenerator.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/DeltaGenerator.java @@ -14,10 +14,12 @@ package com.google.archivepatcher.generator; +import com.google.archivepatcher.DeltaEntryDiagnostics; import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.File; import java.io.IOException; import java.io.OutputStream; +import java.util.List; /** An interface to be implemented by delta generators. */ public abstract class DeltaGenerator { @@ -51,4 +53,18 @@ public void generateDelta(File oldBlob, File newBlob, OutputStream deltaOut) */ public abstract void generateDelta(ByteSource oldBlob, ByteSource newBlob, OutputStream deltaOut) throws IOException, InterruptedException; + + /** + * Generates a delta in deltaOut that can be applied to oldBlob to produce newBlob. + * + * @param oldBlob the old blob + * @param newBlob the new blob + * @param deltaOut the stream to write the delta to + * @throws IOException in the event of an I/O error reading the input files or writing to the + * delta output stream + * @throws InterruptedException if any thread has interrupted the current thread + */ + public abstract List generateDeltaWithDiagnostics( + ByteSource oldBlob, ByteSource newBlob, OutputStream deltaOut) + throws IOException, InterruptedException; } diff --git a/generator/src/main/java/com/google/archivepatcher/generator/DiffPlanEntry.java b/generator/src/main/java/com/google/archivepatcher/generator/DiffPlanEntry.java index 4305e4a..0181f73 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/DiffPlanEntry.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/DiffPlanEntry.java @@ -48,6 +48,7 @@ public DeltaEntry asDeltaEntry() { .deltaFormat(preDiffPlanEntry().deltaFormat()) .oldBlobRange(oldDeltaFriendlyEntryRange()) .newBlobRange(newDeltaFriendlyEntryRange()) + .addDiffPlanEntry(this) .build(); } } diff --git a/generator/src/main/java/com/google/archivepatcher/generator/FileByFileDeltaGenerator.java b/generator/src/main/java/com/google/archivepatcher/generator/FileByFileDeltaGenerator.java index c914a3a..e4518f4 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/FileByFileDeltaGenerator.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/FileByFileDeltaGenerator.java @@ -18,6 +18,7 @@ import static com.google.archivepatcher.generator.DeltaEntries.fillGaps; import static com.google.archivepatcher.shared.PatchConstants.USE_NATIVE_BSDIFF_BY_DEFAULT; +import com.google.archivepatcher.DeltaEntryDiagnostics; import com.google.archivepatcher.shared.PatchConstants.DeltaFormat; import com.google.archivepatcher.shared.Range; import com.google.archivepatcher.shared.bytesource.ByteSource; @@ -127,7 +128,35 @@ public void generateDelta(ByteSource oldBlob, ByteSource newBlob, OutputStream p deltaFriendlyOldBlob, deltaFriendlyNewBlob, deltaGeneratorFactory); - patchWriter.writePatch(patchOut); + List unused = patchWriter.writePatch(patchOut); + } + } + } + + @Override + public List generateDeltaWithDiagnostics( + ByteSource oldBlob, ByteSource newBlob, OutputStream patchOut) + throws IOException, InterruptedException { + try (TempBlob deltaFriendlyOldFile = new TempBlob(); + TempBlob deltaFriendlyNewFile = new TempBlob()) { + PreDiffPlan preDiffPlan = + generatePreDiffPlanAndPrepareBlobs( + oldBlob, newBlob, deltaFriendlyOldFile, deltaFriendlyNewFile, supportedDeltaFormats); + + try (ByteSource deltaFriendlyOldBlob = deltaFriendlyOldFile.asByteSource(); + ByteSource deltaFriendlyNewBlob = deltaFriendlyNewFile.asByteSource()) { + List deltaEntries = + getDeltaEntries( + preDiffPlan.getPreDiffPlanEntries(), deltaFriendlyOldBlob, deltaFriendlyNewBlob); + PatchWriter patchWriter = + new PatchWriter( + preDiffPlan, + deltaFriendlyOldFile.length(), + deltaEntries, + deltaFriendlyOldBlob, + deltaFriendlyNewBlob, + deltaGeneratorFactory); + return patchWriter.writePatch(patchOut); } } } diff --git a/generator/src/main/java/com/google/archivepatcher/generator/PatchWriter.java b/generator/src/main/java/com/google/archivepatcher/generator/PatchWriter.java index 49e2df6..e28c375 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/PatchWriter.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/PatchWriter.java @@ -16,24 +16,27 @@ import static com.google.archivepatcher.shared.bytesource.ByteStreams.copy; -import com.google.archivepatcher.shared.JreDeflateParameters; -import com.google.archivepatcher.shared.PatchConstants; +import com.google.archivepatcher.DeltaEntryDiagnostics; +import com.google.archivepatcher.EntryDeltaFormat; +import com.google.archivepatcher.EntryDeltaFormatExplanation; +import com.google.archivepatcher.File; +import com.google.archivepatcher.UncompressionOption; +import com.google.archivepatcher.UncompressionOptionExplanation; +import com.google.archivepatcher.shared.PatchConstants.DeltaFormat; import com.google.archivepatcher.shared.Range; import com.google.archivepatcher.shared.TypedRange; import com.google.archivepatcher.shared.bytesource.ByteSource; +import com.google.common.annotations.VisibleForTesting; import java.io.DataOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; import java.util.List; -/** - * Writes patches. - */ +/** Writes patches. */ public class PatchWriter { - /** - * The patch plan. - */ + /** The patch plan. */ private final PreDiffPlan plan; /** @@ -85,7 +88,8 @@ public PatchWriter( * @param out the stream to write the patch to * @throws IOException if anything goes wrong */ - public void writePatch(OutputStream out) throws IOException, InterruptedException { + public List writePatch(OutputStream out) + throws IOException, InterruptedException { // Use DataOutputStream for ease of writing. This is deliberately left open, as closing it would // close the output stream that was passed in and that is not part of the method's documented // behavior. @@ -119,14 +123,18 @@ public void writePatch(OutputStream out) throws IOException, InterruptedExceptio // First write the number of deltas present in the patch. dataOut.writeInt(deltaEntries.size()); + List diagnosticsList = new ArrayList<>(); + for (DeltaEntry deltaEntry : deltaEntries) { - writeDeltaEntry(deltaEntry, oldBlob, newBlob, deltaGeneratorFactory, dataOut); + diagnosticsList.add( + writeDeltaEntry(deltaEntry, oldBlob, newBlob, deltaGeneratorFactory, dataOut)); } dataOut.flush(); + return diagnosticsList; } /** Writes the metadata and delta data associated with this entry into the output stream. */ - void writeDeltaEntry( + DeltaEntryDiagnostics writeDeltaEntry( DeltaEntry deltaEntry, ByteSource oldBlob, ByteSource newBlob, @@ -149,20 +157,118 @@ void writeDeltaEntry( outputStream.writeLong( deltaEntry.newBlobRange().length()); // i.e., length of the working range in new + DeltaEntryDiagnostics.Builder diagnostics = DeltaEntryDiagnostics.newBuilder(); + try (ByteSource inputBlobRange = oldBlob.slice(deltaEntry.oldBlobRange()); ByteSource destBlobRange = newBlob.slice(deltaEntry.newBlobRange()); TempBlob deltaFile = new TempBlob()) { try (OutputStream bufferedDeltaOut = deltaFile.openBufferedStream()) { DeltaGenerator deltaGenerator = deltaGeneratorFactory.create(deltaEntry.deltaFormat()); - deltaGenerator.generateDelta(inputBlobRange, destBlobRange, bufferedDeltaOut); + diagnostics.addAllChildren( + deltaGenerator.generateDeltaWithDiagnostics( + inputBlobRange, destBlobRange, bufferedDeltaOut)); } // Finally, the length of the delta and the delta itself. outputStream.writeLong(deltaFile.length()); + + diagnostics.setTotalPatchSize(deltaFile.length()); + getDiagnostics(deltaEntry, diagnostics); + try (ByteSource deltaSource = deltaFile.asByteSource(); InputStream deltaIn = deltaSource.openStream()) { copy(deltaIn, outputStream); } + return diagnostics.build(); + } + } + + public void getDiagnostics(DeltaEntry deltaEntry, DeltaEntryDiagnostics.Builder diagnostics) { + if (deltaEntry.deltaFormat() == DeltaFormat.BSDIFF) { + diagnostics.setDeltaFormat(EntryDeltaFormat.DF_BSDIFF); + } else if (deltaEntry.deltaFormat() == DeltaFormat.FILE_BY_FILE) { + diagnostics.setDeltaFormat(EntryDeltaFormat.DF_FILE_BY_FILE); + } + + for (DiffPlanEntry diffPlanEntry : deltaEntry.diffPlanEntries()) { + PreDiffPlanEntry preDiffPlanEntry = diffPlanEntry.preDiffPlanEntry(); + File.Builder file = File.newBuilder(); + file.setOriginalFilename(preDiffPlanEntry.oldEntry().getFileName()); + file.setNewFilename(preDiffPlanEntry.newEntry().getFileName()); + file.setOriginalFileSize(preDiffPlanEntry.oldEntry().uncompressedSize()); + file.setNewFileSize(preDiffPlanEntry.newEntry().uncompressedSize()); + + switch (preDiffPlanEntry.zipEntryUncompressionOption()) { + case UNCOMPRESS_OLD: + file.setUncompressionOption(UncompressionOption.UO_UNCOMPRESS_OLD); + break; + case UNCOMPRESS_NEW: + file.setUncompressionOption(UncompressionOption.UO_UNCOMPRESS_NEW); + break; + case UNCOMPRESS_BOTH: + file.setUncompressionOption(UncompressionOption.UO_UNCOMPRESS_BOTH); + break; + case UNCOMPRESS_NEITHER: + file.setUncompressionOption(UncompressionOption.UO_UNCOMPRESS_NEITHER); + break; + } + + switch (preDiffPlanEntry.uncompressionOptionExplanation()) { + case DEFLATE_UNSUITABLE: + file.setUncompressionOptionExplanation( + UncompressionOptionExplanation.UOE_DEFLATE_UNSUITABLE); + break; + case UNSUITABLE: + file.setUncompressionOptionExplanation(UncompressionOptionExplanation.UOE_UNSUITABLE); + break; + case BOTH_ENTRIES_UNCOMPRESSED: + file.setUncompressionOptionExplanation( + UncompressionOptionExplanation.UOE_BOTH_ENTRIES_UNCOMPRESSED); + break; + case UNCOMPRESSED_CHANGED_TO_COMPRESSED: + file.setUncompressionOptionExplanation( + UncompressionOptionExplanation.UOE_UNCOMPRESSED_CHANGED_TO_COMPRESSED); + break; + case COMPRESSED_CHANGED_TO_UNCOMPRESSED: + file.setUncompressionOptionExplanation( + UncompressionOptionExplanation.UOE_COMPRESSED_CHANGED_TO_UNCOMPRESSED); + break; + case COMPRESSED_BYTES_CHANGED: + file.setUncompressionOptionExplanation( + UncompressionOptionExplanation.UOE_COMPRESSED_BYTES_CHANGED); + break; + case COMPRESSED_BYTES_IDENTICAL: + file.setUncompressionOptionExplanation( + UncompressionOptionExplanation.UOE_COMPRESSED_BYTES_IDENTICAL); + break; + case RESOURCE_CONSTRAINED: + file.setUncompressionOptionExplanation( + UncompressionOptionExplanation.UOE_RESOURCE_CONSTRAINED); + break; + } + + switch (preDiffPlanEntry.deltaFormatExplanation()) { + case DEFAULT: + file.setDeltaFormatExplanation(EntryDeltaFormatExplanation.DFE_DEFAULT); + break; + case FILE_TYPE: + file.setDeltaFormatExplanation(EntryDeltaFormatExplanation.DFE_FILE_TYPE); + break; + case UNSUITABLE: + file.setDeltaFormatExplanation(EntryDeltaFormatExplanation.DFE_UNSUITABLE); + break; + case DEFLATE_UNSUITABLE: + file.setDeltaFormatExplanation(EntryDeltaFormatExplanation.DFE_DEFLATE_UNSUITABLE); + break; + case UNCHANGED: + file.setDeltaFormatExplanation(EntryDeltaFormatExplanation.DFE_UNCHANGED); + break; + case RESOURCE_CONSTRAINED: + file.setDeltaFormatExplanation(EntryDeltaFormatExplanation.DFE_RESOURCE_CONSTRAINED); + break; + } + + diagnostics.addFiles(file.build()); } } } diff --git a/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlan.java b/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlan.java index a0ba5c4..d9db35f 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlan.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/PreDiffPlan.java @@ -50,14 +50,10 @@ public class PreDiffPlan { /** The plan for uncompressing the old file, in file order. */ private final List oldFileUncompressionPlan; - /** - * The plan for uncompressing the new file, in file order. - */ + /** The plan for uncompressing the new file, in file order. */ private final List> newFileUncompressionPlan; - /** - * The plan for recompressing the delta-friendly new file, in file order. - */ + /** The plan for recompressing the delta-friendly new file, in file order. */ private final List> deltaFriendlyNewFileRecompressionPlan; /** The entries upon which the plans are based. */ @@ -130,6 +126,7 @@ public final List getOldFileUncompressionPlan() { /** * Returns the plan for uncompressing the new file to create the delta-friendly new file. + * * @return the plan */ public final List> getNewFileUncompressionPlan() { @@ -139,6 +136,7 @@ public final List> getNewFileUncompressionPlan( /** * Returns the plan for recompressing the delta-friendly new file to regenerate the original new * file. + * * @return the plan */ public final List> getDeltaFriendlyNewFileRecompressionPlan() { diff --git a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffDeltaGenerator.java b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffDeltaGenerator.java index 8264227..8d0495b 100644 --- a/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffDeltaGenerator.java +++ b/generator/src/main/java/com/google/archivepatcher/generator/bsdiff/BsDiffDeltaGenerator.java @@ -16,20 +16,21 @@ import static com.google.archivepatcher.shared.PatchConstants.USE_NATIVE_BSDIFF_BY_DEFAULT; +import com.google.archivepatcher.DeltaEntryDiagnostics; import com.google.archivepatcher.generator.DeltaGenerator; import com.google.archivepatcher.generator.bsdiff.wrapper.BsDiffNativePatchWriter; import com.google.archivepatcher.shared.bytesource.ByteSource; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; /** * An implementation of {@link DeltaGenerator} that uses {@link BsDiffPatchWriter} to write a bsdiff * patch that represents the delta between given inputs. */ public class BsDiffDeltaGenerator extends DeltaGenerator { - /** - * The minimum match length to use for bsdiff. - */ + /** The minimum match length to use for bsdiff. */ private static final int MATCH_LENGTH_BYTES = 16; /** Whether to use the native version of BsDiff for generating patches. */ @@ -53,6 +54,18 @@ public void generateDelta(ByteSource oldBlob, ByteSource newBlob, OutputStream d } } + @Override + public List generateDeltaWithDiagnostics( + ByteSource oldBlob, ByteSource newBlob, OutputStream deltaOut) + throws IOException, InterruptedException { + if (useNativeBsDiff) { + BsDiffNativePatchWriter.generatePatch(oldBlob, newBlob, deltaOut); + } else { + BsDiffPatchWriter.generatePatch(oldBlob, newBlob, deltaOut, MATCH_LENGTH_BYTES); + } + return new ArrayList<>(); + } + public static void generateDelta( byte[] oldData, byte[] newData, OutputStream deltaOut, boolean generateDeltaNatively) throws IOException, InterruptedException {