add PoC deflate64 support to ZOS

src/ICSharpCode.SharpZipLib/Zip/Compression/Deflater.cs

@@ -201,12 +201,12 @@ public Deflater(int level, bool noZlibHeaderOrFooter)
 		/// just created with the same compression level and strategy as it
 		/// had before.
 		/// </summary>
-		public void Reset()
+		public void Reset(bool deflate64 = false)
 		{
 			state = (noZlibHeaderOrFooter ? BUSY_STATE : INIT_STATE);
 			totalOut = 0;
 			pending.Reset();
-			engine.Reset();
+			engine.Reset(deflate64);
 		}
 
 		/// <summary>

src/ICSharpCode.SharpZipLib/Zip/Compression/DeflaterConstants.cs

@@ -142,5 +142,10 @@ public static class DeflaterConstants
 		/// Internal compression engine constant
 		/// </summary>
 		public static int[] COMPR_FUNC = { 0, 1, 1, 1, 1, 2, 2, 2, 2, 2 };
+
+
+		public static int WSIZE_64 = 262144;
+		public static int WMASK_64 = 262143;
+		public static int MAX_DIST_64 = 65538;
 	}
 }

src/ICSharpCode.SharpZipLib/Zip/Compression/DeflaterEngine.cs

@@ -209,10 +209,10 @@ public void SetDictionary(byte[] buffer, int offset, int length)
 				return;
 			}
 
-			if (length > DeflaterConstants.MAX_DIST)
+			if (length > maxDist)
 			{
-				offset += length - DeflaterConstants.MAX_DIST;
-				length = DeflaterConstants.MAX_DIST;
+				offset += length - maxDist;
+				length = maxDist;
 			}
 
 			System.Array.Copy(buffer, offset, window, strstart, length);
@@ -231,7 +231,7 @@ public void SetDictionary(byte[] buffer, int offset, int length)
 		/// <summary>
 		/// Reset internal state
 		/// </summary>
-		public void Reset()
+		public void Reset(bool deflate64 = false)
 		{
 			huffman.Reset();
 			adler?.Reset();
@@ -241,17 +241,40 @@ public void Reset()
 			prevAvailable = false;
 			matchLen = DeflaterConstants.MIN_MATCH - 1;
 
+			UpdateMode(deflate64);
+
 			for (int i = 0; i < DeflaterConstants.HASH_SIZE; i++)
 			{
 				head[i] = 0;
 			}
 
-			for (int i = 0; i < DeflaterConstants.WSIZE; i++)
+			for (int i = 0; i < windowSize; i++)
 			{
 				prev[i] = 0;
 			}
 		}
 
+		private void UpdateMode(bool deflate64)
+		{
+			if (deflate64)
+			{
+				windowSize = DeflaterConstants.WSIZE_64;
+				maxDist = DeflaterConstants.MAX_DIST_64;
+			}
+			else
+			{
+				windowSize = DeflaterConstants.WSIZE;
+				maxDist = DeflaterConstants.MAX_DIST;
+			}
+
+			var windowFullSize = windowSize * 2;
+			if (window.Length < windowFullSize)
+			{
+				Array.Resize(ref window, windowFullSize);
+				Array.Resize(ref prev, windowSize);
+			}
+		}
+
 		/// <summary>
 		/// Reset Adler checksum
 		/// </summary>
@@ -368,7 +391,7 @@ public void FillWindow()
 			/* If the window is almost full and there is insufficient lookahead,
 			 * move the upper half to the lower one to make room in the upper half.
 			 */
-			if (strstart >= DeflaterConstants.WSIZE + DeflaterConstants.MAX_DIST)
+			if (strstart >= windowSize + maxDist)
 			{
 				SlideWindow();
 			}
@@ -378,7 +401,7 @@ public void FillWindow()
 			 */
 			if (lookahead < DeflaterConstants.MIN_LOOKAHEAD && inputOff < inputEnd)
 			{
-				int more = 2 * DeflaterConstants.WSIZE - lookahead - strstart;
+				int more = 2 * windowSize - lookahead - strstart;
 
 				if (more > inputEnd - inputOff)
 				{
@@ -440,24 +463,24 @@ private int InsertString()
 
 		private void SlideWindow()
 		{
-			Array.Copy(window, DeflaterConstants.WSIZE, window, 0, DeflaterConstants.WSIZE);
-			matchStart -= DeflaterConstants.WSIZE;
-			strstart -= DeflaterConstants.WSIZE;
-			blockStart -= DeflaterConstants.WSIZE;
+			Array.Copy(window, windowSize, window, 0, windowSize);
+			matchStart -= windowSize;
+			strstart -= windowSize;
+			blockStart -= windowSize;
 
 			// Slide the hash table (could be avoided with 32 bit values
 			// at the expense of memory usage).
 			for (int i = 0; i < DeflaterConstants.HASH_SIZE; ++i)
 			{
 				int m = head[i] & 0xffff;
-				head[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0);
+				head[i] = (short)(m >= windowSize ? (m - windowSize) : 0);
 			}
 
 			// Slide the prev table.
-			for (int i = 0; i < DeflaterConstants.WSIZE; i++)
+			for (int i = 0; i < windowSize; i++)
 			{
 				int m = prev[i] & 0xffff;
-				prev[i] = (short)(m >= DeflaterConstants.WSIZE ? (m - DeflaterConstants.WSIZE) : 0);
+				prev[i] = (short)(m >= windowSize ? (m - windowSize) : 0);
 			}
 		}
 
@@ -477,7 +500,7 @@ private bool FindLongestMatch(int curMatch)
 			int scan = strstart;
 			// scanMax is the highest position that we can look at
 			int scanMax = scan + Math.Min(DeflaterConstants.MAX_MATCH, lookahead) - 1;
-			int limit = Math.Max(scan - DeflaterConstants.MAX_DIST, 0);
+			int limit = Math.Max(scan - maxDist, 0);
 
 			byte[] window = this.window;
 			short[] prev = this.prev;
@@ -624,7 +647,7 @@ private bool DeflateStored(bool flush, bool finish)
 			int storedLength = strstart - blockStart;
 
 			if ((storedLength >= DeflaterConstants.MAX_BLOCK_SIZE) || // Block is full
-				(blockStart < DeflaterConstants.WSIZE && storedLength >= DeflaterConstants.MAX_DIST) ||   // Block may move out of window
+				(blockStart < windowSize && storedLength >= maxDist) ||   // Block may move out of window
 				flush)
 			{
 				bool lastBlock = finish;
@@ -665,7 +688,7 @@ private bool DeflateFast(bool flush, bool finish)
 					return false;
 				}
 
-				if (strstart > 2 * DeflaterConstants.WSIZE - DeflaterConstants.MIN_LOOKAHEAD)
+				if (strstart > 2 * windowSize - DeflaterConstants.MIN_LOOKAHEAD)
 				{
 					/* slide window, as FindLongestMatch needs this.
 					 * This should only happen when flushing and the window
@@ -678,7 +701,7 @@ private bool DeflateFast(bool flush, bool finish)
 				if (lookahead >= DeflaterConstants.MIN_MATCH &&
 					(hashHead = InsertString()) != 0 &&
 					strategy != DeflateStrategy.HuffmanOnly &&
-					strstart - hashHead <= DeflaterConstants.MAX_DIST &&
+					strstart - hashHead <= maxDist &&
 					FindLongestMatch(hashHead))
 				{
 					// longestMatch sets matchStart and matchLen
@@ -768,7 +791,7 @@ private bool DeflateSlow(bool flush, bool finish)
 					return false;
 				}
 
-				if (strstart >= 2 * DeflaterConstants.WSIZE - DeflaterConstants.MIN_LOOKAHEAD)
+				if (strstart >= 2 * windowSize - DeflaterConstants.MIN_LOOKAHEAD)
 				{
 					/* slide window, as FindLongestMatch needs this.
 					 * This should only happen when flushing and the window
@@ -785,7 +808,7 @@ private bool DeflateSlow(bool flush, bool finish)
 
 					if (strategy != DeflateStrategy.HuffmanOnly &&
 						hashHead != 0 &&
-						strstart - hashHead <= DeflaterConstants.MAX_DIST &&
+						strstart - hashHead <= maxDist &&
 						FindLongestMatch(hashHead))
 					{
 						// longestMatch sets matchStart and matchLen
@@ -941,6 +964,9 @@ private bool DeflateSlow(bool flush, bool finish)
 		/// </summary>
 		private Adler32 adler;
 
+		private int windowSize;
+		private int maxDist;
+
 		#endregion Instance Fields
 	}
 }

src/ICSharpCode.SharpZipLib/Zip/ZipOutputStream.cs

@@ -377,7 +377,7 @@ internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0,
 			CompressionMethod method = entry.CompressionMethod;
 
 			// Check that the compression is one that we support
-			if (method != CompressionMethod.Deflated && method != CompressionMethod.Stored)
+			if (method != CompressionMethod.Deflated && method != CompressionMethod.Stored && method != CompressionMethod.Deflate64)
 			{
 				throw new NotImplementedException("Compression method not supported");
 			}
@@ -489,9 +489,9 @@ internal void PutNextEntry(Stream stream, ZipEntry entry, long streamOffset = 0,
 				return;
 
 			crc.Reset();
-			if (method == CompressionMethod.Deflated)
+			if (method == CompressionMethod.Deflated || method == CompressionMethod.Deflate64)
 			{
-				deflater_.Reset();
+				deflater_.Reset(method == CompressionMethod.Deflate64);
 				deflater_.SetLevel(compressionLevel);
 			}
 		}
@@ -572,7 +572,7 @@ private async Task FinishCompressionSyncOrAsync(CancellationToken? ct)
 			if (entryIsPassthrough) return;
 
 			// First finish the deflater, if appropriate
-			if (curMethod == CompressionMethod.Deflated)
+			if (curMethod == CompressionMethod.Deflated || curMethod == CompressionMethod.Deflate64)
 			{
 				if (size >= 0)
 				{
@@ -634,7 +634,7 @@ internal void WriteEntryFooter(Stream stream)
 
 			long csize = size;
 
-			if (curMethod == CompressionMethod.Deflated && size >= 0)
+			if ((curMethod == CompressionMethod.Deflated || curMethod == CompressionMethod.Deflate64) && size >= 0)
 			{
 				csize = deflater_.TotalOut;
 			}

test/ICSharpCode.SharpZipLib.Tests/TestSupport/SevenZip.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using NUnit.Framework;
@@ -108,5 +109,65 @@ internal static void VerifyZipWith7Zip(Stream zipStream, string password)
 				Assert.Warn("Skipping file verification since 7za is not in path");
 			}
 		}
+
+		internal static IEnumerable<(string, byte[])> GetZipContentsWith7Zip(Stream zipStream, string password)
+		{
+			if (TryGet7zBinPath(out string path7z))
+			{
+				Console.WriteLine($"Using 7z path: \"{path7z}\"");
+
+				var inputFile = Path.GetTempFileName();
+				using var outputDir = Utils.GetTempDir();
+
+				Console.WriteLine($"Extracting \"{inputFile}\" to \"{outputDir}\"");
+
+
+				try
+				{
+					using (var fs = File.OpenWrite(inputFile))
+					{
+						zipStream.Seek(0, SeekOrigin.Begin);
+						zipStream.CopyTo(fs);
+					}
+
+					var p = Process.Start(new ProcessStartInfo(path7z, $"e -bb3 -o\"{outputDir.FullName}\" -p{password} \"{inputFile}\"")
+					{
+						RedirectStandardOutput = true,
+						RedirectStandardError = true,
+						UseShellExecute = false,
+					});
+
+					if (p == null)
+					{
+						Assert.Inconclusive("Failed to start 7z process. Skipping!");
+					}
+					if (!p.WaitForExit(2000))
+					{
+						Assert.Warn("Timed out verifying zip file!");
+					}
+
+					TestContext.Out.Write(p.StandardOutput.ReadToEnd());
+					var errors = p.StandardError.ReadToEnd();
+					Assert.IsEmpty(errors, "7z reported errors");
+					Assert.AreEqual(0, p.ExitCode, "Extracting archive failed");
+
+					foreach(var outFile in Directory.EnumerateFiles(outputDir))
+					{
+						yield return (
+							Path.GetFileName(outFile), File.ReadAllBytes(outFile)
+						);
+					}
+
+				}
+				finally
+				{
+					File.Delete(inputFile);
+				}
+			}
+			else
+			{
+				Assert.Warn("Skipping extraction since 7za is not in path");
+			}
+		}
 	}
 }

test/ICSharpCode.SharpZipLib.Tests/Zip/ZipDeflate64Tests.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using ICSharpCode.SharpZipLib.Core;
+using ICSharpCode.SharpZipLib.Tests.TestSupport;
+using ICSharpCode.SharpZipLib.Zip;
+using NUnit.Framework;
+
+namespace ICSharpCode.SharpZipLib.Tests.Zip
+{
+	[TestFixture]
+	public class ZipDeflate64Tests
+	{
+		[Test]
+		[Category("Zip"), Category("Deflate64")]
+		public void WriteZipStreamWithDeflate64()
+		{
+			var contentLength = 2 * 1024 * 1024;
+
+			// Using different seeds so that we can verify that the contents have not been swapped
+			var seed = 5;
+			var seed64 = 6;
+
+			using var ms = new MemoryStream();
+
+			using (var zipOutputStream = new ZipOutputStream(ms) { IsStreamOwner = false })
+			{
+				zipOutputStream.PutNextEntry(new ZipEntry("deflate64.file")
+				{
+					CompressionMethod = CompressionMethod.Deflate64,
+				});
+
+				Utils.WriteDummyData(zipOutputStream, contentLength, seed64);
+
+				zipOutputStream.PutNextEntry(new ZipEntry("deflate.file")
+				{
+					CompressionMethod = CompressionMethod.Deflated,
+				});
+
+				Utils.WriteDummyData(zipOutputStream, contentLength, seed);
+			}
+
+			SevenZipHelper.VerifyZipWith7Zip(ms, null);
+			foreach (var (name, content) in SevenZipHelper.GetZipContentsWith7Zip(ms, null))
+			{
+				switch (name)
+				{
+					case "deflate.file":
+						Assert.That(content, Is.EqualTo(Utils.GetDummyBytes(contentLength, seed)));
+						break;
+					case "deflate64.file":
+						Assert.That(content, Is.EqualTo(Utils.GetDummyBytes(contentLength, seed64)));
+						break;
+					default:
+						Assert.Fail($"Unexpected file name {name}");
+						break;
+				};
+			}
+		}
+	}
+}