From c710bd2b6083e4ad337d4f20595631a7eb9a872a Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Wed, 17 Apr 2024 11:11:09 +0200 Subject: [PATCH 1/8] Tar: support GNU numeric format. The tar specification stores numeric fields using an octal representation. This limits the range of values that can be stored. To increase the supported range, a GNU extension defines that when the leading byte is 0xff/0x80 the remaining bytes are a negative/positive big endian formatted value. When writing under the PAX format, we continue to only use the only octal representation in the header fields. The values are overridden using extended attributes. --- .../src/Resources/Strings.resx | 3 - .../src/System/Formats/Tar/GnuTarEntry.cs | 2 - .../src/System/Formats/Tar/PosixTarEntry.cs | 2 - .../src/System/Formats/Tar/TarEntry.cs | 1 - .../src/System/Formats/Tar/TarHeader.Read.cs | 19 +- .../src/System/Formats/Tar/TarHeader.Write.cs | 127 ++++++++++--- .../src/System/Formats/Tar/TarHelpers.cs | 23 ++- .../tests/Manual/ManualTests.cs | 4 +- .../TarReader/TarReader.File.Async.Tests.cs | 1 - .../tests/TarReader/TarReader.File.Tests.cs | 1 - .../tests/TarTestsBase.Gnu.cs | 2 - .../tests/TarTestsBase.Posix.cs | 4 - .../TarWriter/TarWriter.WriteEntry.Base.cs | 54 ++++++ .../TarWriter.WriteEntry.Entry.Pax.Tests.cs | 88 ++------- .../TarWriter/TarWriter.WriteEntry.Tests.cs | 167 ++++++++++------- ...rWriter.WriteEntryAsync.Entry.Pax.Tests.cs | 88 ++------- .../TarWriter.WriteEntryAsync.Tests.cs | 168 ++++++++++-------- 17 files changed, 412 insertions(+), 342 deletions(-) diff --git a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx index a7b4cf8d53a379..8f452003be5697 100644 --- a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx +++ b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx @@ -193,9 +193,6 @@ The field '{0}' exceeds the maximum allowed length for this format. - - The value of the size field for the current entry of format '{0}' is greater than the format allows. - The extended attribute key '{0}' contains a disallowed '{1}' character. diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs index e8acadd9fcf4ba..ee3bad50c74506 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/GnuTarEntry.cs @@ -98,7 +98,6 @@ public DateTimeOffset AccessTime get => _header._aTime; set { - ArgumentOutOfRangeException.ThrowIfLessThan(value, DateTimeOffset.UnixEpoch); _header._aTime = value; } } @@ -112,7 +111,6 @@ public DateTimeOffset ChangeTime get => _header._cTime; set { - ArgumentOutOfRangeException.ThrowIfLessThan(value, DateTimeOffset.UnixEpoch); _header._cTime = value; } } diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs index 73d2f01b423356..f30013204cd89d 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs @@ -62,7 +62,6 @@ public int DeviceMajor } ArgumentOutOfRangeException.ThrowIfNegative(value); - ArgumentOutOfRangeException.ThrowIfGreaterThan(value, 0x1FFFFF); // 7777777 in octal _header._devMajor = value; } @@ -85,7 +84,6 @@ public int DeviceMinor } ArgumentOutOfRangeException.ThrowIfNegative(value); - ArgumentOutOfRangeException.ThrowIfGreaterThan(value, 0x1FFFFF); // 7777777 in octal _header._devMinor = value; } diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs index 83f915875d266e..e50776db039189 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs @@ -98,7 +98,6 @@ public DateTimeOffset ModificationTime get => _header._mTime; set { - ArgumentOutOfRangeException.ThrowIfLessThan(value, DateTimeOffset.UnixEpoch); _header._mTime = value; } } diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs index 5036a59334d85b..4d924c61056f13 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Read.cs @@ -374,8 +374,7 @@ private async Task ProcessDataBlockAsync(Stream archiveStream, bool copyData, Ca return null; } - long size = (long)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.Size, FieldLengths.Size)); - Debug.Assert(size <= TarHelpers.MaxSizeLength, "size exceeded the max value possible with 11 octal digits. Actual size " + size); + long size = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.Size, FieldLengths.Size)); if (size < 0) { throw new InvalidDataException(SR.Format(SR.TarSizeFieldNegative)); @@ -384,14 +383,14 @@ private async Task ProcessDataBlockAsync(Stream archiveStream, bool copyData, Ca // Continue with the rest of the fields that require no special checks TarHeader header = new(initialFormat, name: TarHelpers.GetTrimmedUtf8String(buffer.Slice(FieldLocations.Name, FieldLengths.Name)), - mode: (int)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.Mode, FieldLengths.Mode)), - mTime: TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch((long)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.MTime, FieldLengths.MTime))), + mode: TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.Mode, FieldLengths.Mode)), + mTime: TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.MTime, FieldLengths.MTime))), typeFlag: (TarEntryType)buffer[FieldLocations.TypeFlag]) { _checksum = checksum, _size = size, - _uid = (int)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.Uid, FieldLengths.Uid)), - _gid = (int)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.Gid, FieldLengths.Gid)), + _uid = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.Uid, FieldLengths.Uid)), + _gid = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.Gid, FieldLengths.Gid)), _linkName = TarHelpers.GetTrimmedUtf8String(buffer.Slice(FieldLocations.LinkName, FieldLengths.LinkName)) }; @@ -524,10 +523,10 @@ private void ReadPosixAndGnuSharedAttributes(Span buffer) if (_typeFlag is TarEntryType.CharacterDevice or TarEntryType.BlockDevice) { // Major number for a character device or block device entry. - _devMajor = (int)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.DevMajor, FieldLengths.DevMajor)); + _devMajor = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.DevMajor, FieldLengths.DevMajor)); // Minor number for a character device or block device entry. - _devMinor = (int)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.DevMinor, FieldLengths.DevMinor)); + _devMinor = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.DevMinor, FieldLengths.DevMinor)); } } @@ -536,10 +535,10 @@ private void ReadPosixAndGnuSharedAttributes(Span buffer) private void ReadGnuAttributes(Span buffer) { // Convert byte arrays - long aTime = (long)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.ATime, FieldLengths.ATime)); + long aTime = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.ATime, FieldLengths.ATime)); _aTime = TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(aTime); - long cTime = (long)TarHelpers.ParseOctal(buffer.Slice(FieldLocations.CTime, FieldLengths.CTime)); + long cTime = TarHelpers.ParseNumeric(buffer.Slice(FieldLocations.CTime, FieldLengths.CTime)); _cTime = TarHelpers.GetDateTimeOffsetFromSecondsSinceEpoch(cTime); // TODO: Read the bytes of the currently unsupported GNU fields, in case user wants to write this entry into another GNU archive, they need to be preserved. https://github.com/dotnet/runtime/issues/68230 diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs index a14c12443aea27..3088f117d4d851 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Buffers; +using System.Buffers.Binary; using System.Buffers.Text; using System.Collections.Generic; using System.Diagnostics; @@ -15,6 +16,9 @@ namespace System.Formats.Tar // Writes header attributes of a tar archive entry. internal sealed partial class TarHeader { + private const long Octal12ByteFieldMaxValue = (1L << (3 * 11)) - 1; // Max value of 11 octal digits. + private const int Octal8ByteFieldMaxValue = (1 << (3 * 7)) - 1; // Max value of 7 octal digits. + private static ReadOnlySpan UstarMagicBytes => "ustar\0"u8; private static ReadOnlySpan UstarVersionBytes => "00"u8; @@ -606,35 +610,22 @@ private int WriteCommonFields(Span buffer, TarEntryType actualEntryType) if (_mode > 0) { - checksum += FormatOctal(_mode, buffer.Slice(FieldLocations.Mode, FieldLengths.Mode)); + checksum += FormatNumeric(_mode, buffer.Slice(FieldLocations.Mode, FieldLengths.Mode)); } if (_uid > 0) { - checksum += FormatOctal(_uid, buffer.Slice(FieldLocations.Uid, FieldLengths.Uid)); + checksum += FormatNumeric(_uid, buffer.Slice(FieldLocations.Uid, FieldLengths.Uid)); } if (_gid > 0) { - checksum += FormatOctal(_gid, buffer.Slice(FieldLocations.Gid, FieldLengths.Gid)); + checksum += FormatNumeric(_gid, buffer.Slice(FieldLocations.Gid, FieldLengths.Gid)); } if (_size > 0) { - if (_size <= TarHelpers.MaxSizeLength) - { - checksum += FormatOctal(_size, buffer.Slice(FieldLocations.Size, FieldLengths.Size)); - } - else if (_format is not TarEntryFormat.Pax) - { - throw new ArgumentException(SR.Format(SR.TarSizeFieldTooLargeForEntryFormat, _format)); - } - else - { - // No writing, just verifications - Debug.Assert(_typeFlag is not TarEntryType.ExtendedAttributes and not TarEntryType.GlobalExtendedAttributes); - Debug.Assert(Convert.ToInt64(ExtendedAttributes[PaxEaSize]) > TarHelpers.MaxSizeLength); - } + checksum += FormatNumeric(_size, buffer.Slice(FieldLocations.Size, FieldLengths.Size)); } checksum += WriteAsTimestamp(_mTime, buffer.Slice(FieldLocations.MTime, FieldLengths.MTime)); @@ -739,12 +730,12 @@ private int WritePosixAndGnuSharedFields(Span buffer) if (_devMajor > 0) { - checksum += FormatOctal(_devMajor, buffer.Slice(FieldLocations.DevMajor, FieldLengths.DevMajor)); + checksum += FormatNumeric(_devMajor, buffer.Slice(FieldLocations.DevMajor, FieldLengths.DevMajor)); } if (_devMinor > 0) { - checksum += FormatOctal(_devMinor, buffer.Slice(FieldLocations.DevMinor, FieldLengths.DevMinor)); + checksum += FormatNumeric(_devMinor, buffer.Slice(FieldLocations.DevMinor, FieldLengths.DevMinor)); } return checksum; @@ -916,7 +907,7 @@ private void CollectExtendedAttributesFromStandardFieldsIfNeeded() ExtendedAttributes[PaxEaLinkName] = _linkName; } - if (_size > TarHelpers.MaxSizeLength) + if (_size > Octal12ByteFieldMaxValue) { ExtendedAttributes[PaxEaSize] = _size.ToString(); } @@ -925,6 +916,42 @@ private void CollectExtendedAttributesFromStandardFieldsIfNeeded() ExtendedAttributes.Remove(PaxEaSize); } + if (_uid > Octal8ByteFieldMaxValue) + { + ExtendedAttributes[PaxEaUid] = _uid.ToString(); + } + else + { + ExtendedAttributes.Remove(PaxEaUid); + } + + if (_gid > Octal8ByteFieldMaxValue) + { + ExtendedAttributes[PaxEaGid] = _gid.ToString(); + } + else + { + ExtendedAttributes.Remove(PaxEaGid); + } + + if (_devMajor > Octal8ByteFieldMaxValue) + { + ExtendedAttributes[PaxEaDevMajor] = _devMajor.ToString(); + } + else + { + ExtendedAttributes.Remove(PaxEaDevMajor); + } + + if (_devMinor > Octal8ByteFieldMaxValue) + { + ExtendedAttributes[PaxEaDevMinor] = _devMinor.ToString(); + } + else + { + ExtendedAttributes.Remove(PaxEaDevMinor); + } + // Sets the specified string to the dictionary if it's longer than the specified max byte length; otherwise, remove it. static void TryAddStringField(Dictionary extendedAttributes, string key, string? value, int maxLength) { @@ -1022,6 +1049,60 @@ private static int Checksum(ReadOnlySpan bytes) return checksum; } + private int FormatNumeric(int value, Span destination) + { + Debug.Assert(destination.Length == 8, "8 byte field expected."); + + // Prefer the octal format. For non-PAX, use GNU format to widen the range. + bool useOctal = (value >= 0 && value <= Octal8ByteFieldMaxValue) || _format is TarEntryFormat.Pax; + + if (useOctal) + { + return FormatOctal(value, destination); + } + else if (value < 0) + { + // GNU format: store negative numbers in big endian format with leading '0xff' byte. + BinaryPrimitives.WriteInt64BigEndian(destination, value); + return Checksum(destination); + } + else + { + // GNU format: store positive numbers in big endian format with leading '0x80' byte. + BinaryPrimitives.WriteUInt64BigEndian(destination, (1UL << 63) | (uint)value); + return Checksum(destination); + } + } + + private int FormatNumeric(long value, Span destination) + { + Debug.Assert(destination.Length == 12, "12 byte field expected."); + const int Offset = 4; // 4 bytes before the long. + + // Prefer the octal format. For non-PAX, use GNU format to widen the range. + bool useOctal = (value >= 0 && value <= Octal12ByteFieldMaxValue) || _format is TarEntryFormat.Pax; + + if (useOctal) + { + return FormatOctal(value, destination); + } + else if (value < 0) + { + // GNU format: store negative numbers in big endian format with leading '0xff' byte. + destination.Slice(0, Offset).Fill(0xff); + BinaryPrimitives.WriteInt64BigEndian(destination.Slice(Offset), value); + return Checksum(destination); + } + else + { + // GNU format: store positive numbers in big endian format with leading '0x80' byte. + destination.Slice(0, Offset).Fill(0); + destination[0] = 0x80; + BinaryPrimitives.WriteInt64BigEndian(destination.Slice(Offset), value); + return Checksum(destination); + } + } + // Writes the specified decimal number as a right-aligned octal number and returns its checksum. private static int FormatOctal(long value, Span destination) { @@ -1040,11 +1121,11 @@ private static int FormatOctal(long value, Span destination) return WriteRightAlignedBytesAndGetChecksum(digits.Slice(i), destination); } - // Writes the specified DateTimeOffset's Unix time seconds as a right-aligned octal number, and returns its checksum. - private static int WriteAsTimestamp(DateTimeOffset timestamp, Span destination) + // Writes the specified DateTimeOffset's Unix time seconds, and returns its checksum. + private int WriteAsTimestamp(DateTimeOffset timestamp, Span destination) { long unixTimeSeconds = timestamp.ToUnixTimeSeconds(); - return FormatOctal(unixTimeSeconds, destination); + return FormatNumeric(unixTimeSeconds, destination); } // Writes the specified text as an UTF8 string aligned to the left, and returns its checksum. diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs index 2d8f56a62eeb52..6db5286f8847b6 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs @@ -19,7 +19,6 @@ internal static partial class TarHelpers { internal const short RecordSize = 512; internal const int MaxBufferLength = 4096; - internal const long MaxSizeLength = (1L << 33) - 1; // Max value of 11 octal digits = 2^33 - 1 or 8 Gb. internal const UnixFileMode ValidUnixFileModes = UnixFileMode.UserRead | @@ -215,6 +214,28 @@ internal static TarEntryType GetCorrectTypeFlagForFormat(TarEntryFormat format, return entryType; } + /// Parses a numeric field. + internal static T ParseNumeric(ReadOnlySpan buffer) where T : struct, INumber, IBinaryInteger + { + // The tar standard specifies that numeric fields are stored using an octal representation. + // This limits the range of values that can be stored in the fields. + // To increase the supported range, a GNU extension defines that when the leading byte is + // '0xff'/'0x80' the remaining bytes are a negative/positive big formatted endian value. + byte leadingByte = buffer[0]; + if (leadingByte == 0xff) + { + return T.ReadBigEndian(buffer, isUnsigned: false); + } + else if (leadingByte == 0x80) + { + return T.ReadBigEndian(buffer.Slice(1), isUnsigned: true); + } + else + { + return ParseOctal(buffer); + } + } + /// Parses a byte span that represents an ASCII string containing a number in octal base. internal static T ParseOctal(ReadOnlySpan buffer) where T : struct, INumber { diff --git a/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs b/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs index 1fa1c686e40e84..6826ca42f98bdd 100644 --- a/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs +++ b/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs @@ -20,10 +20,8 @@ public static IEnumerable WriteEntry_LongFileSize_TheoryData() foreach (TarEntryFormat entryFormat in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Gnu, TarEntryFormat.Pax }) { yield return new object[] { entryFormat, LegacyMaxFileSize, unseekableStream }; + yield return new object[] { entryFormat, LegacyMaxFileSize + 1, unseekableStream }; } - - // Pax supports unlimited size files. - yield return new object[] { TarEntryFormat.Pax, LegacyMaxFileSize + 1, unseekableStream }; } } diff --git a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs index d86cfa4e34dd47..c17e3d769e0f81 100644 --- a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs @@ -263,7 +263,6 @@ public async Task AllowSpacesInOctalFieldsAsync(string folderName, string testCa [InlineData("invalid-go17")] // Many octal fields are all zero chars [InlineData("issue11169")] // Checksum with null in the middle [InlineData("issue10968")] // Garbage chars - [InlineData("writer-big")] // The size field contains an euro char public async Task Throw_ArchivesWithRandomCharsAsync(string testCaseName) { await using MemoryStream archiveStream = GetTarMemoryStream(CompressionMethod.Uncompressed, "golang_tar", testCaseName); diff --git a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs index 6b5dae94caf6bd..67670b4dec2197 100644 --- a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs @@ -271,7 +271,6 @@ public void AllowSpacesInOctalFields(string folderName, string testCaseName) [InlineData("invalid-go17")] // Many octal fields are all zero chars [InlineData("issue11169")] // Checksum with null in the middle [InlineData("issue10968")] // Garbage chars - [InlineData("writer-big")] // The size field contains an euro char public void Throw_ArchivesWithRandomChars(string testCaseName) { using MemoryStream archiveStream = GetTarMemoryStream(CompressionMethod.Uncompressed, "golang_tar", testCaseName); diff --git a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs index ae847e45ac2d3d..e3fda78813f515 100644 --- a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs +++ b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs @@ -60,12 +60,10 @@ protected void SetGnuProperties(GnuTarEntry entry) // ATime: Verify the default value was approximately "now" Assert.True(entry.AccessTime > approxNow); - Assert.Throws(() => entry.AccessTime = DateTimeOffset.MinValue); entry.AccessTime = TestAccessTime; // CTime: Verify the default value was approximately "now" Assert.True(entry.ChangeTime > approxNow); - Assert.Throws(() => entry.ChangeTime = DateTimeOffset.MinValue); entry.ChangeTime = TestChangeTime; } diff --git a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs index 498612de473d6e..fd850cb93b81c5 100644 --- a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs +++ b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs @@ -27,13 +27,11 @@ private void SetBlockDeviceProperties(PosixTarEntry device) // DeviceMajor Assert.Equal(DefaultDeviceMajor, device.DeviceMajor); Assert.Throws(() => device.DeviceMajor = -1); - Assert.Throws(() => device.DeviceMajor = 2097152); device.DeviceMajor = TestBlockDeviceMajor; // DeviceMinor Assert.Equal(DefaultDeviceMinor, device.DeviceMinor); Assert.Throws(() => device.DeviceMinor = -1); - Assert.Throws(() => device.DeviceMinor = 2097152); device.DeviceMinor = TestBlockDeviceMinor; } @@ -47,13 +45,11 @@ private void SetCharacterDeviceProperties(PosixTarEntry device) // DeviceMajor Assert.Equal(DefaultDeviceMajor, device.DeviceMajor); Assert.Throws(() => device.DeviceMajor = -1); - Assert.Throws(() => device.DeviceMajor = 2097152); device.DeviceMajor = TestCharacterDeviceMajor; // DeviceMinor Assert.Equal(DefaultDeviceMinor, device.DeviceMinor); Assert.Throws(() => device.DeviceMinor = -1); - Assert.Throws(() => device.DeviceMinor = 2097152); device.DeviceMinor = TestCharacterDeviceMinor; } diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs index 334a27e51fa92b..f4682dd838eb0f 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs @@ -48,5 +48,59 @@ protected void VerifyGlobalExtendedAttributesEntry(TarEntry entry, Dictionary WriteIntField_TheoryData() + { + foreach (TarEntryFormat format in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Pax, TarEntryFormat.Gnu }) + { + // Min value. + yield return new object[] { format, 0 }; + // Max value. + yield return new object[] { format, int.MaxValue }; // This doesn't fit an 8-byte field with octal representation. + + yield return new object[] { format, 1 }; + yield return new object[] { format, 42 }; + } + } + + public static IEnumerable WriteTimeStampsWithFormats_TheoryData() + { + foreach (TarEntryFormat entryFormat in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Gnu, TarEntryFormat.Pax }) + { + foreach (DateTimeOffset timestamp in GetWriteTimeStamps()) + { + yield return new object[] { entryFormat, timestamp }; + } + } + } + + public static IEnumerable WriteTimeStamps_TheoryData() + { + foreach (DateTimeOffset timestamp in GetWriteTimeStamps()) + { + yield return new object[] { timestamp }; + } + } + + private static IEnumerable GetWriteTimeStamps() + { + // One second past Y2K38 + yield return new DateTimeOffset(2038, 1, 19, 3, 14, 8, TimeSpan.Zero); + + // One second past what a 12-byte field can store with octal representation + yield return new DateTimeOffset(2242, 3, 16, 12, 56, 33, TimeSpan.Zero); + + // Min value + yield return DateTimeOffset.MinValue; + + // Max value. Everything below seconds is set to zero for test equality comparison. + yield return new DateTimeOffset(new DateTime(DateTime.MaxValue.Year, + DateTime.MaxValue.Month, + DateTime.MaxValue.Day, + DateTime.MaxValue.Hour, + DateTime.MaxValue.Minute, + DateTime.MaxValue.Second, + DateTime.MaxValue.Kind), TimeSpan.Zero); + } } } diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs index 1e81fb7b1e8a01..94ba4040100884 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs @@ -375,92 +375,30 @@ public void Add_Empty_GlobalExtendedAttributes() } } - [Fact] - // Y2K38 will happen one second after "2038/19/01 03:14:07 +00:00". This timestamp represents the seconds since the Unix epoch with a - // value of int.MaxValue: 2,147,483,647. - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // All our entry types should survive the Epochalypse because we internally use long to represent the seconds since Unix epoch, not int. - // So if the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, which - // is way past int MaxValue, but still within the long limits. That number represents the date "2242/16/03 12:56:32 +00:00". - public void WriteTimestampsBeyondEpochalypseInPax() - { - DateTimeOffset epochalypse = new DateTimeOffset(2038, 1, 19, 3, 14, 8, TimeSpan.Zero); - string strEpochalypse = GetTimestampStringFromDateTimeOffset(epochalypse); - - Dictionary ea = new Dictionary() - { - { PaxEaATime, strEpochalypse }, - { PaxEaCTime, strEpochalypse } - }; - - PaxTarEntry entry = new PaxTarEntry(TarEntryType.Directory, "dir", ea); - - entry.ModificationTime = epochalypse; - Assert.Equal(epochalypse, entry.ModificationTime); - - Assert.Contains(PaxEaATime, entry.ExtendedAttributes); - DateTimeOffset atime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaATime); - Assert.Equal(epochalypse, atime); - - Assert.Contains(PaxEaCTime, entry.ExtendedAttributes); - DateTimeOffset ctime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(epochalypse, ctime); - - using MemoryStream archiveStream = new MemoryStream(); - using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) - { - writer.WriteEntry(entry); - } - - archiveStream.Position = 0; - using (TarReader reader = new TarReader(archiveStream)) - { - PaxTarEntry readEntry = reader.GetNextEntry() as PaxTarEntry; - Assert.NotNull(readEntry); - - Assert.Equal(epochalypse, readEntry.ModificationTime); - - Assert.Contains(PaxEaATime, readEntry.ExtendedAttributes); - DateTimeOffset actualATime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaATime); - Assert.Equal(epochalypse, actualATime); - - Assert.Contains(PaxEaCTime, readEntry.ExtendedAttributes); - DateTimeOffset actualCTime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(epochalypse, actualCTime); - } - } - - [Fact] - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // We internally use long to represent the seconds since Unix epoch, not int. - // If the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, - // which represents the date "2242/03/16 12:56:32 +00:00". - // Pax should survive after this date because it stores the timestamps in the extended attributes dictionary - // without size restrictions. - public void WriteTimestampsBeyondOctalLimitInPax() + [Theory] + [MemberData(nameof(WriteTimeStamps_TheoryData))] + public void WriteTimestampsInPax(DateTimeOffset timestamp) { - DateTimeOffset overLimitTimestamp = new DateTimeOffset(2242, 3, 16, 12, 56, 33, TimeSpan.Zero); // One second past the octal limit - - string strOverLimitTimestamp = GetTimestampStringFromDateTimeOffset(overLimitTimestamp); + string strTimestamp = GetTimestampStringFromDateTimeOffset(timestamp); Dictionary ea = new Dictionary() { - { PaxEaATime, strOverLimitTimestamp }, - { PaxEaCTime, strOverLimitTimestamp } + { PaxEaATime, strTimestamp }, + { PaxEaCTime, strTimestamp } }; PaxTarEntry entry = new PaxTarEntry(TarEntryType.Directory, "dir", ea); - entry.ModificationTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, entry.ModificationTime); + entry.ModificationTime = timestamp; + Assert.Equal(timestamp, entry.ModificationTime); Assert.Contains(PaxEaATime, entry.ExtendedAttributes); DateTimeOffset atime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaATime); - Assert.Equal(overLimitTimestamp, atime); + Assert.Equal(timestamp, atime); Assert.Contains(PaxEaCTime, entry.ExtendedAttributes); DateTimeOffset ctime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(overLimitTimestamp, ctime); + Assert.Equal(timestamp, ctime); using MemoryStream archiveStream = new MemoryStream(); using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) @@ -474,15 +412,15 @@ public void WriteTimestampsBeyondOctalLimitInPax() PaxTarEntry readEntry = reader.GetNextEntry() as PaxTarEntry; Assert.NotNull(readEntry); - Assert.Equal(overLimitTimestamp, readEntry.ModificationTime); + Assert.Equal(timestamp, readEntry.ModificationTime); Assert.Contains(PaxEaATime, readEntry.ExtendedAttributes); DateTimeOffset actualATime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaATime); - Assert.Equal(overLimitTimestamp, actualATime); + Assert.Equal(timestamp, actualATime); Assert.Contains(PaxEaCTime, readEntry.ExtendedAttributes); DateTimeOffset actualCTime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(overLimitTimestamp, actualCTime); + Assert.Equal(timestamp, actualCTime); } } diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs index 487c6ab04dbcd5..e20ad1b9617bfc 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Tests.cs @@ -214,31 +214,22 @@ public void ReadAndWriteMultipleGlobalExtendedAttributesEntries(TarEntryFormat f } } - // Y2K38 will happen one second after "2038/19/01 03:14:07 +00:00". This timestamp represents the seconds since the Unix epoch with a - // value of int.MaxValue: 2,147,483,647. - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // All our entry types should survive the Epochalypse because we internally use long to represent the seconds since Unix epoch, not int. - // So if the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, which - // is way past int MaxValue, but still within the long limits. That number represents the date "2242/16/03 12:56:32 +00:00". [Theory] - [InlineData(TarEntryFormat.V7)] - [InlineData(TarEntryFormat.Ustar)] - [InlineData(TarEntryFormat.Gnu)] - public void WriteTimestampsBeyondEpochalypse(TarEntryFormat format) + [MemberData(nameof(WriteTimeStampsWithFormats_TheoryData))] + public void WriteTimeStamps(TarEntryFormat format, DateTimeOffset timestamp) { - DateTimeOffset epochalypse = new DateTimeOffset(2038, 1, 19, 3, 14, 8, TimeSpan.Zero); // One second past Y2K38 TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); - entry.ModificationTime = epochalypse; - Assert.Equal(epochalypse, entry.ModificationTime); + entry.ModificationTime = timestamp; + Assert.Equal(timestamp, entry.ModificationTime); if (entry is GnuTarEntry gnuEntry) { - gnuEntry.AccessTime = epochalypse; - Assert.Equal(epochalypse, gnuEntry.AccessTime); + gnuEntry.AccessTime = timestamp; + Assert.Equal(timestamp, gnuEntry.AccessTime); - gnuEntry.ChangeTime = epochalypse; - Assert.Equal(epochalypse, gnuEntry.ChangeTime); + gnuEntry.ChangeTime = timestamp; + Assert.Equal(timestamp, gnuEntry.ChangeTime); } using MemoryStream archiveStream = new MemoryStream(); @@ -253,43 +244,49 @@ public void WriteTimestampsBeyondEpochalypse(TarEntryFormat format) TarEntry readEntry = reader.GetNextEntry(); Assert.NotNull(readEntry); - Assert.Equal(epochalypse, readEntry.ModificationTime); + Assert.Equal(timestamp, readEntry.ModificationTime); if (readEntry is GnuTarEntry gnuReadEntry) { - Assert.Equal(epochalypse, gnuReadEntry.AccessTime); - Assert.Equal(epochalypse, gnuReadEntry.ChangeTime); + Assert.Equal(timestamp, gnuReadEntry.AccessTime); + Assert.Equal(timestamp, gnuReadEntry.ChangeTime); } } } - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // We internally use long to represent the seconds since Unix epoch, not int. - // If the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, - // which represents the date "2242/03/16 12:56:32 +00:00". - // V7, Ustar and GNU would not survive after this date because they only have the fixed size fields to store timestamps. [Theory] - [InlineData(TarEntryFormat.V7)] - [InlineData(TarEntryFormat.Ustar)] - [InlineData(TarEntryFormat.Gnu)] - public void WriteTimestampsBeyondOctalLimit(TarEntryFormat format) + [MemberData(nameof(WriteIntField_TheoryData))] + public void WriteUid(TarEntryFormat format, int value) { - DateTimeOffset overLimitTimestamp = new DateTimeOffset(2242, 3, 16, 12, 56, 33, TimeSpan.Zero); // One second past the octal limit - TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); - // Before writing the entry, the timestamps should have no issue - entry.ModificationTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, entry.ModificationTime); + entry.Uid = value; + Assert.Equal(value, entry.Uid); - if (entry is GnuTarEntry gnuEntry) + using MemoryStream archiveStream = new MemoryStream(); + using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) { - gnuEntry.AccessTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, gnuEntry.AccessTime); + writer.WriteEntry(entry); + } - gnuEntry.ChangeTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, gnuEntry.ChangeTime); + archiveStream.Position = 0; + using (TarReader reader = new TarReader(archiveStream)) + { + TarEntry readEntry = reader.GetNextEntry(); + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.Uid); } + } + + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public void WriteGid(TarEntryFormat format, int value) + { + TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); + + entry.Gid = value; + Assert.Equal(value, entry.Gid); using MemoryStream archiveStream = new MemoryStream(); using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) @@ -303,14 +300,69 @@ public void WriteTimestampsBeyondOctalLimit(TarEntryFormat format) TarEntry readEntry = reader.GetNextEntry(); Assert.NotNull(readEntry); - // The timestamps get stored as '{1970-01-01 12:00:00 AM +00:00}' due to the +1 overflow - Assert.NotEqual(overLimitTimestamp, readEntry.ModificationTime); + Assert.Equal(value, readEntry.Gid); + } + } - if (readEntry is GnuTarEntry gnuReadEntry) - { - Assert.NotEqual(overLimitTimestamp, gnuReadEntry.AccessTime); - Assert.NotEqual(overLimitTimestamp, gnuReadEntry.ChangeTime); - } + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public void WriteDeviceMajor(TarEntryFormat format, int value) + { + if (format == TarEntryFormat.V7) + { + return; // No DeviceMajor + } + + PosixTarEntry? entry = InvokeTarEntryCreationConstructor(format, TarEntryType.BlockDevice, "dir") as PosixTarEntry; + Assert.NotNull(entry); + + entry.DeviceMajor = value; + Assert.Equal(value, entry.DeviceMajor); + + using MemoryStream archiveStream = new MemoryStream(); + using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) + { + writer.WriteEntry(entry); + } + + archiveStream.Position = 0; + using (TarReader reader = new TarReader(archiveStream)) + { + PosixTarEntry? readEntry = reader.GetNextEntry() as PosixTarEntry; + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.DeviceMajor); + } + } + + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public void WriteDeviceMinor(TarEntryFormat format, int value) + { + if (format == TarEntryFormat.V7) + { + return; // No DeviceMinor + } + + PosixTarEntry? entry = InvokeTarEntryCreationConstructor(format, TarEntryType.BlockDevice, "dir") as PosixTarEntry; + Assert.NotNull(entry); + + entry.DeviceMinor = value; + Assert.Equal(value, entry.DeviceMinor); + + using MemoryStream archiveStream = new MemoryStream(); + using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) + { + writer.WriteEntry(entry); + } + + archiveStream.Position = 0; + using (TarReader reader = new TarReader(archiveStream)) + { + PosixTarEntry? readEntry = reader.GetNextEntry() as PosixTarEntry; + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.DeviceMinor); } } @@ -505,29 +557,6 @@ public void WriteEntry_UsingTarEntry_FromTarReader_IntoTarWriter(TarEntryFormat AssertExtensions.SequenceEqual(msSource.ToArray(), msDestination.ToArray()); } - [Theory] - [InlineData(TarEntryFormat.V7, false)] - [InlineData(TarEntryFormat.Ustar, false)] - [InlineData(TarEntryFormat.Gnu, false)] - [InlineData(TarEntryFormat.V7, true)] - [InlineData(TarEntryFormat.Ustar, true)] - [InlineData(TarEntryFormat.Gnu, true)] - public void WriteEntry_FileSizeOverLegacyLimit_Throws(TarEntryFormat entryFormat, bool unseekableStream) - { - const long FileSizeOverLimit = LegacyMaxFileSize + 1; - - using MemoryStream ms = new(); - using Stream s = unseekableStream ? new WrappedStream(ms, ms.CanRead, ms.CanWrite, canSeek: false) : ms; - - using TarWriter writer = new(s); - TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo"); - writeEntry.DataStream = new SimulatedDataStream(FileSizeOverLimit); - - Assert.Equal(FileSizeOverLimit, writeEntry.Length); - - Assert.Throws(() => writer.WriteEntry(writeEntry)); - } - [Theory] [InlineData(TarEntryFormat.V7)] [InlineData(TarEntryFormat.Ustar)] diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs index fc6445a3b3e971..fe06ff8f6954eb 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs @@ -397,92 +397,30 @@ public async Task Add_Empty_GlobalExtendedAttributes_Async() } } - [Fact] - // Y2K38 will happen one second after "2038/19/01 03:14:07 +00:00". This timestamp represents the seconds since the Unix epoch with a - // value of int.MaxValue: 2,147,483,647. - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // All our entry types should survive the Epochalypse because we internally use long to represent the seconds since Unix epoch, not int. - // So if the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, which - // is way past int MaxValue, but still within the long limits. That number represents the date "2242/16/03 12:56:32 +00:00". - public async Task WriteTimestampsBeyondEpochalypseInPax_Async() - { - DateTimeOffset epochalypse = new DateTimeOffset(2038, 1, 19, 3, 14, 8, TimeSpan.Zero); - string strEpochalypse = GetTimestampStringFromDateTimeOffset(epochalypse); - - Dictionary ea = new Dictionary() - { - { PaxEaATime, strEpochalypse }, - { PaxEaCTime, strEpochalypse } - }; - - PaxTarEntry entry = new PaxTarEntry(TarEntryType.Directory, "dir", ea); - - entry.ModificationTime = epochalypse; - Assert.Equal(epochalypse, entry.ModificationTime); - - Assert.Contains(PaxEaATime, entry.ExtendedAttributes); - DateTimeOffset atime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaATime); - Assert.Equal(epochalypse, atime); - - Assert.Contains(PaxEaCTime, entry.ExtendedAttributes); - DateTimeOffset ctime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(epochalypse, ctime); - - using MemoryStream archiveStream = new MemoryStream(); - await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) - { - await writer.WriteEntryAsync(entry); - } - - archiveStream.Position = 0; - await using (TarReader reader = new TarReader(archiveStream)) - { - PaxTarEntry readEntry = await reader.GetNextEntryAsync() as PaxTarEntry; - Assert.NotNull(readEntry); - - Assert.Equal(epochalypse, readEntry.ModificationTime); - - Assert.Contains(PaxEaATime, readEntry.ExtendedAttributes); - DateTimeOffset actualATime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaATime); - Assert.Equal(epochalypse, actualATime); - - Assert.Contains(PaxEaCTime, readEntry.ExtendedAttributes); - DateTimeOffset actualCTime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(epochalypse, actualCTime); - } - } - - [Fact] - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // We internally use long to represent the seconds since Unix epoch, not int. - // If the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, - // which represents the date "2242/03/16 12:56:32 +00:00". - // Pax should survive after this date because it stores the timestamps in the extended attributes dictionary - // without size restrictions. - public async Task WriteTimestampsBeyondOctalLimitInPax_Async() + [Theory] + [MemberData(nameof(WriteTimeStamps_TheoryData))] + public async Task WriteTimestampsInPax_Async(DateTimeOffset timestamp) { - DateTimeOffset overLimitTimestamp = new DateTimeOffset(2242, 3, 16, 12, 56, 33, TimeSpan.Zero); // One second past the octal limit - - string strOverLimitTimestamp = GetTimestampStringFromDateTimeOffset(overLimitTimestamp); + string strTimestamp = GetTimestampStringFromDateTimeOffset(timestamp); Dictionary ea = new Dictionary() { - { PaxEaATime, strOverLimitTimestamp }, - { PaxEaCTime, strOverLimitTimestamp } + { PaxEaATime, strTimestamp }, + { PaxEaCTime, strTimestamp } }; PaxTarEntry entry = new PaxTarEntry(TarEntryType.Directory, "dir", ea); - entry.ModificationTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, entry.ModificationTime); + entry.ModificationTime = timestamp; + Assert.Equal(timestamp, entry.ModificationTime); Assert.Contains(PaxEaATime, entry.ExtendedAttributes); DateTimeOffset atime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaATime); - Assert.Equal(overLimitTimestamp, atime); + Assert.Equal(timestamp, atime); Assert.Contains(PaxEaCTime, entry.ExtendedAttributes); DateTimeOffset ctime = GetDateTimeOffsetFromTimestampString(entry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(overLimitTimestamp, ctime); + Assert.Equal(timestamp, ctime); using MemoryStream archiveStream = new MemoryStream(); await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) @@ -496,15 +434,15 @@ public async Task WriteTimestampsBeyondOctalLimitInPax_Async() PaxTarEntry readEntry = await reader.GetNextEntryAsync() as PaxTarEntry; Assert.NotNull(readEntry); - Assert.Equal(overLimitTimestamp, readEntry.ModificationTime); + Assert.Equal(timestamp, readEntry.ModificationTime); Assert.Contains(PaxEaATime, readEntry.ExtendedAttributes); DateTimeOffset actualATime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaATime); - Assert.Equal(overLimitTimestamp, actualATime); + Assert.Equal(timestamp, actualATime); Assert.Contains(PaxEaCTime, readEntry.ExtendedAttributes); DateTimeOffset actualCTime = GetDateTimeOffsetFromTimestampString(readEntry.ExtendedAttributes, PaxEaCTime); - Assert.Equal(overLimitTimestamp, actualCTime); + Assert.Equal(timestamp, actualCTime); } } diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs index 45b26ad66e6005..f22b22dc63d1ce 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Tests.cs @@ -237,31 +237,22 @@ public async Task ReadAndWriteMultipleGlobalExtendedAttributesEntries_Async(TarE } } - // Y2K38 will happen one second after "2038/19/01 03:14:07 +00:00". This timestamp represents the seconds since the Unix epoch with a - // value of int.MaxValue: 2,147,483,647. - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // All our entry types should survive the Epochalypse because we internally use long to represent the seconds since Unix epoch, not int. - // So if the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, which - // is way past int MaxValue, but still within the long limits. That number represents the date "2242/16/03 12:56:32 +00:00". [Theory] - [InlineData(TarEntryFormat.V7)] - [InlineData(TarEntryFormat.Ustar)] - [InlineData(TarEntryFormat.Gnu)] - public async Task WriteTimestampsBeyondEpochalypse_Async(TarEntryFormat format) + [MemberData(nameof(WriteTimeStampsWithFormats_TheoryData))] + public async Task WriteTimeStamps_Async(TarEntryFormat format, DateTimeOffset timestamp) { - DateTimeOffset epochalypse = new DateTimeOffset(2038, 1, 19, 3, 14, 8, TimeSpan.Zero); // One second past Y2K38 TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); - entry.ModificationTime = epochalypse; - Assert.Equal(epochalypse, entry.ModificationTime); + entry.ModificationTime = timestamp; + Assert.Equal(timestamp, entry.ModificationTime); if (entry is GnuTarEntry gnuEntry) { - gnuEntry.AccessTime = epochalypse; - Assert.Equal(epochalypse, gnuEntry.AccessTime); + gnuEntry.AccessTime = timestamp; + Assert.Equal(timestamp, gnuEntry.AccessTime); - gnuEntry.ChangeTime = epochalypse; - Assert.Equal(epochalypse, gnuEntry.ChangeTime); + gnuEntry.ChangeTime = timestamp; + Assert.Equal(timestamp, gnuEntry.ChangeTime); } using MemoryStream archiveStream = new MemoryStream(); @@ -276,43 +267,49 @@ public async Task WriteTimestampsBeyondEpochalypse_Async(TarEntryFormat format) TarEntry readEntry = await reader.GetNextEntryAsync(); Assert.NotNull(readEntry); - Assert.Equal(epochalypse, readEntry.ModificationTime); + Assert.Equal(timestamp, readEntry.ModificationTime); if (readEntry is GnuTarEntry gnuReadEntry) { - Assert.Equal(epochalypse, gnuReadEntry.AccessTime); - Assert.Equal(epochalypse, gnuReadEntry.ChangeTime); + Assert.Equal(timestamp, gnuReadEntry.AccessTime); + Assert.Equal(timestamp, gnuReadEntry.ChangeTime); } } } - // The fixed size fields for mtime, atime and ctime can fit 12 ASCII characters, but the last character is reserved for an ASCII space. - // We internally use long to represent the seconds since Unix epoch, not int. - // If the max allowed value is 77,777,777,777 in octal, then the max allowed seconds since the Unix epoch are 8,589,934,591, - // which represents the date "2242/03/16 12:56:32 +00:00". - // V7, Ustar and GNU would not survive after this date because they only have the fixed size fields to store timestamps. [Theory] - [InlineData(TarEntryFormat.V7)] - [InlineData(TarEntryFormat.Ustar)] - [InlineData(TarEntryFormat.Gnu)] - public async Task WriteTimestampsBeyondOctalLimit_Async(TarEntryFormat format) + [MemberData(nameof(WriteIntField_TheoryData))] + public async Task WriteUid_Async(TarEntryFormat format, int value) { - DateTimeOffset overLimitTimestamp = new DateTimeOffset(2242, 3, 16, 12, 56, 33, TimeSpan.Zero); // One second past the octal limit - TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); - // Before writing the entry, the timestamps should have no issue - entry.ModificationTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, entry.ModificationTime); + entry.Uid = value; + Assert.Equal(value, entry.Uid); - if (entry is GnuTarEntry gnuEntry) + using MemoryStream archiveStream = new MemoryStream(); + await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) { - gnuEntry.AccessTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, gnuEntry.AccessTime); + await writer.WriteEntryAsync(entry); + } - gnuEntry.ChangeTime = overLimitTimestamp; - Assert.Equal(overLimitTimestamp, gnuEntry.ChangeTime); + archiveStream.Position = 0; + await using (TarReader reader = new TarReader(archiveStream)) + { + TarEntry readEntry = await reader.GetNextEntryAsync(); + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.Uid); } + } + + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public async Task WriteGid_Async(TarEntryFormat format, int value) + { + TarEntry entry = InvokeTarEntryCreationConstructor(format, TarEntryType.Directory, "dir"); + + entry.Gid = value; + Assert.Equal(value, entry.Gid); using MemoryStream archiveStream = new MemoryStream(); await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) @@ -326,14 +323,69 @@ public async Task WriteTimestampsBeyondOctalLimit_Async(TarEntryFormat format) TarEntry readEntry = await reader.GetNextEntryAsync(); Assert.NotNull(readEntry); - // The timestamps get stored as '{1970-01-01 12:00:00 AM +00:00}' due to the +1 overflow - Assert.NotEqual(overLimitTimestamp, readEntry.ModificationTime); + Assert.Equal(value, readEntry.Gid); + } + } - if (readEntry is GnuTarEntry gnuReadEntry) - { - Assert.NotEqual(overLimitTimestamp, gnuReadEntry.AccessTime); - Assert.NotEqual(overLimitTimestamp, gnuReadEntry.ChangeTime); - } + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public async Task WriteDeviceMajor_Async(TarEntryFormat format, int value) + { + if (format == TarEntryFormat.V7) + { + return; // No DeviceMajor + } + + PosixTarEntry? entry = InvokeTarEntryCreationConstructor(format, TarEntryType.BlockDevice, "dir") as PosixTarEntry; + Assert.NotNull(entry); + + entry.DeviceMajor = value; + Assert.Equal(value, entry.DeviceMajor); + + using MemoryStream archiveStream = new MemoryStream(); + await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) + { + await writer.WriteEntryAsync(entry); + } + + archiveStream.Position = 0; + await using (TarReader reader = new TarReader(archiveStream)) + { + PosixTarEntry? readEntry = await reader.GetNextEntryAsync() as PosixTarEntry; + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.DeviceMajor); + } + } + + [Theory] + [MemberData(nameof(WriteIntField_TheoryData))] + public async Task WriteDeviceMinor_Async(TarEntryFormat format, int value) + { + if (format == TarEntryFormat.V7) + { + return; // No DeviceMinor + } + + PosixTarEntry? entry = InvokeTarEntryCreationConstructor(format, TarEntryType.BlockDevice, "dir") as PosixTarEntry; + Assert.NotNull(entry); + + entry.DeviceMinor = value; + Assert.Equal(value, entry.DeviceMinor); + + using MemoryStream archiveStream = new MemoryStream(); + await using (TarWriter writer = new TarWriter(archiveStream, leaveOpen: true)) + { + await writer.WriteEntryAsync(entry); + } + + archiveStream.Position = 0; + await using (TarReader reader = new TarReader(archiveStream)) + { + PosixTarEntry? readEntry = await reader.GetNextEntryAsync() as PosixTarEntry; + Assert.NotNull(readEntry); + + Assert.Equal(value, readEntry.DeviceMinor); } } @@ -423,30 +475,6 @@ public async Task WriteEntry_UsingTarEntry_FromTarReader_IntoTarWriter_Async(Tar AssertExtensions.SequenceEqual(msSource.ToArray(), msDestination.ToArray()); } - [Theory] - [InlineData(TarEntryFormat.V7, false)] - [InlineData(TarEntryFormat.Ustar, false)] - [InlineData(TarEntryFormat.Gnu, false)] - [InlineData(TarEntryFormat.V7, true)] - [InlineData(TarEntryFormat.Ustar, true)] - [InlineData(TarEntryFormat.Gnu, true)] - public async Task WriteEntry_FileSizeOverLegacyLimit_Throws_Async(TarEntryFormat entryFormat, bool unseekableStream) - { - const long FileSizeOverLimit = LegacyMaxFileSize + 1; - - await using MemoryStream ms = new(); - await using Stream s = unseekableStream ? new WrappedStream(ms, ms.CanRead, ms.CanWrite, canSeek: false) : ms; - - string tarFilePath = GetTestFilePath(); - await using TarWriter writer = new(File.Create(tarFilePath)); - TarEntry writeEntry = InvokeTarEntryCreationConstructor(entryFormat, entryFormat is TarEntryFormat.V7 ? TarEntryType.V7RegularFile : TarEntryType.RegularFile, "foo"); - writeEntry.DataStream = new SimulatedDataStream(FileSizeOverLimit); - - Assert.Equal(FileSizeOverLimit, writeEntry.Length); - - await Assert.ThrowsAsync(() => writer.WriteEntryAsync(writeEntry)); - } - [Theory] [InlineData(TarEntryFormat.V7)] [InlineData(TarEntryFormat.Ustar)] From cf44af2ab62b80fc0e3ff8b7cdc1af077c2f1c00 Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Wed, 17 Apr 2024 12:49:08 +0200 Subject: [PATCH 2/8] Bit fiddling. --- .../src/System/Formats/Tar/TarHeader.Write.cs | 29 +++++++------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs index 3088f117d4d851..2a56757fcf2655 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs @@ -1060,16 +1060,16 @@ private int FormatNumeric(int value, Span destination) { return FormatOctal(value, destination); } - else if (value < 0) - { - // GNU format: store negative numbers in big endian format with leading '0xff' byte. - BinaryPrimitives.WriteInt64BigEndian(destination, value); - return Checksum(destination); - } else { - // GNU format: store positive numbers in big endian format with leading '0x80' byte. - BinaryPrimitives.WriteUInt64BigEndian(destination, (1UL << 63) | (uint)value); + // GNU format: store negative numbers in big endian format with leading '0xff' byte. + // store positive numbers in big endian format with leading '0x80' byte. + long destinationValue = value; + if (value >= 0) + { + destinationValue |= 1L << 63; + } + BinaryPrimitives.WriteInt64BigEndian(destination, destinationValue); return Checksum(destination); } } @@ -1086,18 +1086,11 @@ private int FormatNumeric(long value, Span destination) { return FormatOctal(value, destination); } - else if (value < 0) - { - // GNU format: store negative numbers in big endian format with leading '0xff' byte. - destination.Slice(0, Offset).Fill(0xff); - BinaryPrimitives.WriteInt64BigEndian(destination.Slice(Offset), value); - return Checksum(destination); - } else { - // GNU format: store positive numbers in big endian format with leading '0x80' byte. - destination.Slice(0, Offset).Fill(0); - destination[0] = 0x80; + // GNU format: store negative numbers in big endian format with leading '0xff' byte. + // store positive numbers in big endian format with leading '0x80' byte. + BinaryPrimitives.WriteUInt32BigEndian(destination, value < 0 ? 0xffffffff : 1U << 31); BinaryPrimitives.WriteInt64BigEndian(destination.Slice(Offset), value); return Checksum(destination); } From 4b1f5d5eb5df970f4d0003e2e0237548fb2936b2 Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Wed, 17 Apr 2024 13:49:43 +0200 Subject: [PATCH 3/8] More bit fiddling. --- .../src/System/Formats/Tar/TarHeader.Write.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs index 2a56757fcf2655..924f40f0edc3e1 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs @@ -1065,10 +1065,7 @@ private int FormatNumeric(int value, Span destination) // GNU format: store negative numbers in big endian format with leading '0xff' byte. // store positive numbers in big endian format with leading '0x80' byte. long destinationValue = value; - if (value >= 0) - { - destinationValue |= 1L << 63; - } + destinationValue |= 1L << 63; BinaryPrimitives.WriteInt64BigEndian(destination, destinationValue); return Checksum(destination); } From 64f7ef4198a5799e0535c3e4cedbf1d31aded16c Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Wed, 17 Apr 2024 14:43:57 +0200 Subject: [PATCH 4/8] Update test. --- src/libraries/System.Formats.Tar/tests/TarTestsBase.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs b/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs index 3b5d4ca93e0902..5b01bdd196b9e8 100644 --- a/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs +++ b/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs @@ -329,7 +329,6 @@ protected void SetCommonProperties(TarEntry entry, bool isDirectory = false) DateTimeOffset approxNow = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(6)); Assert.True(entry.ModificationTime > approxNow); - Assert.Throws(() => entry.ModificationTime = DateTime.MinValue); // Minimum allowed is UnixEpoch, not MinValue entry.ModificationTime = TestModificationTime; // Name From 58f651a6eeb2d39972c893bd34216ccba97785da Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Thu, 23 May 2024 10:20:29 +0200 Subject: [PATCH 5/8] Limit using the GNU numeric format to TarEntryFormat.Gnu. --- .../src/Resources/Strings.resx | 3 + .../src/System/Formats/Tar/PosixTarEntry.cs | 8 +++ .../src/System/Formats/Tar/TarEntry.cs | 7 +++ .../src/System/Formats/Tar/TarHeader.Write.cs | 28 +++++----- .../src/System/Formats/Tar/TarHelpers.cs | 1 + .../tests/Manual/ManualTests.cs | 5 +- .../tests/TarTestsBase.Gnu.cs | 19 +++++++ .../tests/TarTestsBase.Posix.cs | 38 +++++++++++++ .../System.Formats.Tar/tests/TarTestsBase.cs | 11 ++++ .../TarWriter/TarWriter.WriteEntry.Base.cs | 55 +++++++++++++------ 10 files changed, 143 insertions(+), 32 deletions(-) diff --git a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx index 8f452003be5697..a7b4cf8d53a379 100644 --- a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx +++ b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx @@ -193,6 +193,9 @@ The field '{0}' exceeds the maximum allowed length for this format. + + The value of the size field for the current entry of format '{0}' is greater than the format allows. + The extended attribute key '{0}' contains a disallowed '{1}' character. diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs index f30013204cd89d..ebd204091d646d 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs @@ -62,6 +62,10 @@ public int DeviceMajor } ArgumentOutOfRangeException.ThrowIfNegative(value); + if (FormatIsOctalOnly) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan(value, 0x1FFFFF); // 7777777 in octal + } _header._devMajor = value; } @@ -84,6 +88,10 @@ public int DeviceMinor } ArgumentOutOfRangeException.ThrowIfNegative(value); + if (FormatIsOctalOnly) + { + ArgumentOutOfRangeException.ThrowIfGreaterThan(value, 0x1FFFFF); // 7777777 in octal + } _header._devMinor = value; } diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs index e50776db039189..b415c30a93b52b 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs @@ -20,6 +20,9 @@ public abstract partial class TarEntry // Used to access the data section of this entry in an unseekable file private TarReader? _readerOfOrigin; + // These formats have a limited numeric range due to the octal number representation. + protected bool FormatIsOctalOnly => _header._format is TarEntryFormat.V7 or TarEntryFormat.Ustar; + // Constructor called when reading a TarEntry from a TarReader. internal TarEntry(TarHeader header, TarReader readerOfOrigin, TarEntryFormat format) { @@ -98,6 +101,10 @@ public DateTimeOffset ModificationTime get => _header._mTime; set { + if (FormatIsOctalOnly) + { + ArgumentOutOfRangeException.ThrowIfLessThan(value, DateTimeOffset.UnixEpoch); + } _header._mTime = value; } } diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs index 924f40f0edc3e1..01b2dfb354f85f 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs @@ -1053,14 +1053,10 @@ private int FormatNumeric(int value, Span destination) { Debug.Assert(destination.Length == 8, "8 byte field expected."); - // Prefer the octal format. For non-PAX, use GNU format to widen the range. - bool useOctal = (value >= 0 && value <= Octal8ByteFieldMaxValue) || _format is TarEntryFormat.Pax; + // Use GNU format to widen the range. + bool useGnuFormat = (value < 0 || value > Octal8ByteFieldMaxValue) && _format == TarEntryFormat.Gnu; - if (useOctal) - { - return FormatOctal(value, destination); - } - else + if (useGnuFormat) { // GNU format: store negative numbers in big endian format with leading '0xff' byte. // store positive numbers in big endian format with leading '0x80' byte. @@ -1069,6 +1065,10 @@ private int FormatNumeric(int value, Span destination) BinaryPrimitives.WriteInt64BigEndian(destination, destinationValue); return Checksum(destination); } + else + { + return FormatOctal(value, destination); + } } private int FormatNumeric(long value, Span destination) @@ -1076,14 +1076,10 @@ private int FormatNumeric(long value, Span destination) Debug.Assert(destination.Length == 12, "12 byte field expected."); const int Offset = 4; // 4 bytes before the long. - // Prefer the octal format. For non-PAX, use GNU format to widen the range. - bool useOctal = (value >= 0 && value <= Octal12ByteFieldMaxValue) || _format is TarEntryFormat.Pax; + // Use GNU format to widen the range. + bool useGnuFormat = (value < 0 || value > Octal12ByteFieldMaxValue) && _format == TarEntryFormat.Gnu; - if (useOctal) - { - return FormatOctal(value, destination); - } - else + if (useGnuFormat) { // GNU format: store negative numbers in big endian format with leading '0xff' byte. // store positive numbers in big endian format with leading '0x80' byte. @@ -1091,6 +1087,10 @@ private int FormatNumeric(long value, Span destination) BinaryPrimitives.WriteInt64BigEndian(destination.Slice(Offset), value); return Checksum(destination); } + else + { + return FormatOctal(value, destination); + } } // Writes the specified decimal number as a right-aligned octal number and returns its checksum. diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs index 6db5286f8847b6..639ad480f18a61 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHelpers.cs @@ -221,6 +221,7 @@ internal static T ParseNumeric(ReadOnlySpan buffer) where T : struct, I // This limits the range of values that can be stored in the fields. // To increase the supported range, a GNU extension defines that when the leading byte is // '0xff'/'0x80' the remaining bytes are a negative/positive big formatted endian value. + // Like the 'tar' tool we are permissive when encountering this representation in non GNU formats. byte leadingByte = buffer[0]; if (leadingByte == 0xff) { diff --git a/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs b/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs index 6826ca42f98bdd..34cc32183170f4 100644 --- a/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs +++ b/src/libraries/System.Formats.Tar/tests/Manual/ManualTests.cs @@ -20,8 +20,11 @@ public static IEnumerable WriteEntry_LongFileSize_TheoryData() foreach (TarEntryFormat entryFormat in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Gnu, TarEntryFormat.Pax }) { yield return new object[] { entryFormat, LegacyMaxFileSize, unseekableStream }; - yield return new object[] { entryFormat, LegacyMaxFileSize + 1, unseekableStream }; } + + // Pax and Gnu supports unlimited size files. + yield return new object[] { TarEntryFormat.Pax, LegacyMaxFileSize + 1, unseekableStream }; + yield return new object[] { TarEntryFormat.Gnu, LegacyMaxFileSize + 1, unseekableStream }; } } diff --git a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs index e3fda78813f515..325c6c916f6058 100644 --- a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs +++ b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Gnu.cs @@ -56,14 +56,33 @@ protected void SetFifo(GnuTarEntry fifo) protected void SetGnuProperties(GnuTarEntry entry) { + // The octal format limits the representable range. + bool formatIsOctalOnly = entry.Format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + DateTimeOffset approxNow = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(6)); // ATime: Verify the default value was approximately "now" Assert.True(entry.AccessTime > approxNow); + if (formatIsOctalOnly) + { + Assert.Throws(() => entry.AccessTime = DateTimeOffset.MinValue); + } + else + { + entry.AccessTime = DateTimeOffset.MinValue; + } entry.AccessTime = TestAccessTime; // CTime: Verify the default value was approximately "now" Assert.True(entry.ChangeTime > approxNow); + if (formatIsOctalOnly) + { + Assert.Throws(() => entry.ChangeTime = DateTimeOffset.MinValue); + } + else + { + entry.ChangeTime = DateTimeOffset.MinValue; + } entry.ChangeTime = TestChangeTime; } diff --git a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs index fd850cb93b81c5..3afc9a1746da12 100644 --- a/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs +++ b/src/libraries/System.Formats.Tar/tests/TarTestsBase.Posix.cs @@ -19,6 +19,9 @@ protected void SetPosixProperties(PosixTarEntry entry) private void SetBlockDeviceProperties(PosixTarEntry device) { + // The octal format limits the representable range. + bool formatIsOctalOnly = device.Format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + Assert.NotNull(device); Assert.Equal(TarEntryType.BlockDevice, device.EntryType); SetCommonProperties(device); @@ -27,16 +30,35 @@ private void SetBlockDeviceProperties(PosixTarEntry device) // DeviceMajor Assert.Equal(DefaultDeviceMajor, device.DeviceMajor); Assert.Throws(() => device.DeviceMajor = -1); + if (formatIsOctalOnly) + { + Assert.Throws(() => device.DeviceMajor = 2097152); + } + else + { + device.DeviceMajor = 2097152; + } device.DeviceMajor = TestBlockDeviceMajor; // DeviceMinor Assert.Equal(DefaultDeviceMinor, device.DeviceMinor); Assert.Throws(() => device.DeviceMinor = -1); + if (formatIsOctalOnly) + { + Assert.Throws(() => device.DeviceMinor = 2097152); + } + else + { + device.DeviceMinor = 2097152; + } device.DeviceMinor = TestBlockDeviceMinor; } private void SetCharacterDeviceProperties(PosixTarEntry device) { + // The octal format limits the representable range. + bool formatIsOctalOnly = device.Format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + Assert.NotNull(device); Assert.Equal(TarEntryType.CharacterDevice, device.EntryType); SetCommonProperties(device); @@ -45,11 +67,27 @@ private void SetCharacterDeviceProperties(PosixTarEntry device) // DeviceMajor Assert.Equal(DefaultDeviceMajor, device.DeviceMajor); Assert.Throws(() => device.DeviceMajor = -1); + if (formatIsOctalOnly) + { + Assert.Throws(() => device.DeviceMajor = 2097152); + } + else + { + device.DeviceMajor = 2097152; + } device.DeviceMajor = TestCharacterDeviceMajor; // DeviceMinor Assert.Equal(DefaultDeviceMinor, device.DeviceMinor); Assert.Throws(() => device.DeviceMinor = -1); + if (formatIsOctalOnly) + { + Assert.Throws(() => device.DeviceMinor = 2097152); + } + else + { + device.DeviceMinor = 2097152; + } device.DeviceMinor = TestCharacterDeviceMinor; } diff --git a/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs b/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs index 5b01bdd196b9e8..29a15217879eeb 100644 --- a/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs +++ b/src/libraries/System.Formats.Tar/tests/TarTestsBase.cs @@ -311,6 +311,9 @@ protected void SetCommonSymbolicLink(TarEntry symbolicLink) protected void SetCommonProperties(TarEntry entry, bool isDirectory = false) { + // The octal format limits the range. + bool formatIsOctalOnly = entry.Format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + // Length (Data is checked outside this method) Assert.Equal(0, entry.Length); @@ -329,6 +332,14 @@ protected void SetCommonProperties(TarEntry entry, bool isDirectory = false) DateTimeOffset approxNow = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(6)); Assert.True(entry.ModificationTime > approxNow); + if (formatIsOctalOnly) + { + Assert.Throws(() => entry.ModificationTime = DateTime.MinValue); // Minimum allowed is UnixEpoch, not MinValue + } + else + { + entry.ModificationTime = DateTimeOffset.MinValue; + } entry.ModificationTime = TestModificationTime; // Name diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs index f4682dd838eb0f..fdc30f9be77d7d 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs @@ -55,11 +55,21 @@ public static IEnumerable WriteIntField_TheoryData() { // Min value. yield return new object[] { format, 0 }; - // Max value. - yield return new object[] { format, int.MaxValue }; // This doesn't fit an 8-byte field with octal representation. yield return new object[] { format, 1 }; yield return new object[] { format, 42 }; + + // Max value octal. + yield return new object[] { format, 0x1FFFFF }; + + // These values do not fit the octal representation. + bool formatIsOctalOnly = format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + if (!formatIsOctalOnly) + { + // Max value property. + yield return new object[] { format, int.MaxValue }; + } + } } @@ -67,7 +77,7 @@ public static IEnumerable WriteTimeStampsWithFormats_TheoryData() { foreach (TarEntryFormat entryFormat in new[] { TarEntryFormat.V7, TarEntryFormat.Ustar, TarEntryFormat.Gnu, TarEntryFormat.Pax }) { - foreach (DateTimeOffset timestamp in GetWriteTimeStamps()) + foreach (DateTimeOffset timestamp in GetWriteTimeStamps(entryFormat)) { yield return new object[] { entryFormat, timestamp }; } @@ -76,31 +86,42 @@ public static IEnumerable WriteTimeStampsWithFormats_TheoryData() public static IEnumerable WriteTimeStamps_TheoryData() { - foreach (DateTimeOffset timestamp in GetWriteTimeStamps()) + foreach (DateTimeOffset timestamp in GetWriteTimeStamps(TarEntryFormat.Pax)) { yield return new object[] { timestamp }; } } - private static IEnumerable GetWriteTimeStamps() + private static IEnumerable GetWriteTimeStamps(TarEntryFormat format) { // One second past Y2K38 yield return new DateTimeOffset(2038, 1, 19, 3, 14, 8, TimeSpan.Zero); - // One second past what a 12-byte field can store with octal representation - yield return new DateTimeOffset(2242, 3, 16, 12, 56, 33, TimeSpan.Zero); + // Min value octal + yield return DateTimeOffset.UnixEpoch; - // Min value - yield return DateTimeOffset.MinValue; + // Max value 12-byte octal field. + yield return DateTimeOffset.UnixEpoch + new TimeSpan(0x1FFFFFFFF * TimeSpan.TicksPerSecond); - // Max value. Everything below seconds is set to zero for test equality comparison. - yield return new DateTimeOffset(new DateTime(DateTime.MaxValue.Year, - DateTime.MaxValue.Month, - DateTime.MaxValue.Day, - DateTime.MaxValue.Hour, - DateTime.MaxValue.Minute, - DateTime.MaxValue.Second, - DateTime.MaxValue.Kind), TimeSpan.Zero); + // These values do not fit the octal representation. + bool formatIsOctalOnly = format is not TarEntryFormat.Pax and not TarEntryFormat.Gnu; + if (!formatIsOctalOnly) + { + // Min value property. + yield return DateTimeOffset.MinValue; // This is not representable with the octal format. + + // One second past what a 12-byte field can store with octal representation + yield return DateTimeOffset.UnixEpoch + new TimeSpan((0x1FFFFFFFF + 1) * TimeSpan.TicksPerSecond); + + // Max value property. Everything below seconds is set to zero for test equality comparison. + yield return new DateTimeOffset(new DateTime(DateTime.MaxValue.Year, + DateTime.MaxValue.Month, + DateTime.MaxValue.Day, + DateTime.MaxValue.Hour, + DateTime.MaxValue.Minute, + DateTime.MaxValue.Second, + DateTime.MaxValue.Kind), TimeSpan.Zero); + } } } } From 432a667737e96ef4236b14dfa932416fb8fbd0fd Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Thu, 23 May 2024 11:01:05 +0200 Subject: [PATCH 6/8] Throw when formatting values that exceed the octal range. --- .../src/Resources/Strings.resx | 4 ++-- .../src/System/Formats/Tar/TarHeader.Write.cs | 22 ++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx index a7b4cf8d53a379..ac632535dfd180 100644 --- a/src/libraries/System.Formats.Tar/src/Resources/Strings.resx +++ b/src/libraries/System.Formats.Tar/src/Resources/Strings.resx @@ -193,8 +193,8 @@ The field '{0}' exceeds the maximum allowed length for this format. - - The value of the size field for the current entry of format '{0}' is greater than the format allows. + + The value of the field for the current entry of format '{0}' is greater than the format allows. The extended attribute key '{0}' contains a disallowed '{1}' character. diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs index 01b2dfb354f85f..df427319b78f55 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs @@ -1053,10 +1053,13 @@ private int FormatNumeric(int value, Span destination) { Debug.Assert(destination.Length == 8, "8 byte field expected."); - // Use GNU format to widen the range. - bool useGnuFormat = (value < 0 || value > Octal8ByteFieldMaxValue) && _format == TarEntryFormat.Gnu; + bool isOctalRange = value >= 0 && value <= Octal8ByteFieldMaxValue; - if (useGnuFormat) + if (isOctalRange || _format == TarEntryFormat.Pax) + { + return FormatOctal(value, destination); + } + else if (_format == TarEntryFormat.Gnu) { // GNU format: store negative numbers in big endian format with leading '0xff' byte. // store positive numbers in big endian format with leading '0x80' byte. @@ -1067,7 +1070,7 @@ private int FormatNumeric(int value, Span destination) } else { - return FormatOctal(value, destination); + throw new ArgumentException(SR.Format(SR.TarFieldTooLargeForEntryFormat, _format)); } } @@ -1076,10 +1079,13 @@ private int FormatNumeric(long value, Span destination) Debug.Assert(destination.Length == 12, "12 byte field expected."); const int Offset = 4; // 4 bytes before the long. - // Use GNU format to widen the range. - bool useGnuFormat = (value < 0 || value > Octal12ByteFieldMaxValue) && _format == TarEntryFormat.Gnu; + bool isOctalRange = value >= 0 && value <= Octal12ByteFieldMaxValue; - if (useGnuFormat) + if (isOctalRange || _format == TarEntryFormat.Pax) + { + return FormatOctal(value, destination); + } + else if (_format == TarEntryFormat.Gnu) { // GNU format: store negative numbers in big endian format with leading '0xff' byte. // store positive numbers in big endian format with leading '0x80' byte. @@ -1089,7 +1095,7 @@ private int FormatNumeric(long value, Span destination) } else { - return FormatOctal(value, destination); + throw new ArgumentException(SR.Format(SR.TarFieldTooLargeForEntryFormat, _format)); } } From 0d17598382c92c9b4f6d0be065b007649d232f11 Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Thu, 6 Jun 2024 15:06:13 +0200 Subject: [PATCH 7/8] PR feedback. --- .../src/System/Formats/Tar/PosixTarEntry.cs | 4 ++-- .../System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs | 2 +- .../src/System/Formats/Tar/TarHeader.Write.cs | 2 +- .../tests/TarWriter/TarWriter.WriteEntry.Base.cs | 2 +- .../tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs | 2 +- .../TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs index ebd204091d646d..72d1396f83beed 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/PosixTarEntry.cs @@ -50,7 +50,7 @@ internal PosixTarEntry(TarEntry other, TarEntryFormat format) /// /// Character and block devices are Unix-specific entry types. /// The entry does not represent a block device or a character device. - /// The value is negative, or larger than 2097151. + /// The value is negative, or larger than 2097151 when using or . public int DeviceMajor { get => _header._devMajor; @@ -76,7 +76,7 @@ public int DeviceMajor /// /// Character and block devices are Unix-specific entry types. /// The entry does not represent a block device or a character device. - /// The value is negative, or larger than 2097151. + /// The value is negative, or larger than 2097151 when using or . public int DeviceMinor { get => _header._devMinor; diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs index b415c30a93b52b..73b158b10957bf 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarEntry.cs @@ -95,7 +95,7 @@ public int Gid /// A timestamps that represents the last time the contents of the file represented by this entry were modified. /// /// In Unix platforms, this timestamp is commonly known as mtime. - /// The specified value is larger than . + /// The specified value is larger than when using or . public DateTimeOffset ModificationTime { get => _header._mTime; diff --git a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs index df427319b78f55..e0d003a657bc8e 100644 --- a/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs +++ b/src/libraries/System.Formats.Tar/src/System/Formats/Tar/TarHeader.Write.cs @@ -1089,7 +1089,7 @@ private int FormatNumeric(long value, Span destination) { // GNU format: store negative numbers in big endian format with leading '0xff' byte. // store positive numbers in big endian format with leading '0x80' byte. - BinaryPrimitives.WriteUInt32BigEndian(destination, value < 0 ? 0xffffffff : 1U << 31); + BinaryPrimitives.WriteUInt32BigEndian(destination, value < 0 ? 0xffffffff : 0x80000000); BinaryPrimitives.WriteInt64BigEndian(destination.Slice(Offset), value); return Checksum(destination); } diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs index fdc30f9be77d7d..d8d4018c40b017 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Base.cs @@ -84,7 +84,7 @@ public static IEnumerable WriteTimeStampsWithFormats_TheoryData() } } - public static IEnumerable WriteTimeStamps_TheoryData() + public static IEnumerable WriteTimeStamp_Pax_TheoryData() { foreach (DateTimeOffset timestamp in GetWriteTimeStamps(TarEntryFormat.Pax)) { diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs index 94ba4040100884..006aace683716c 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntry.Entry.Pax.Tests.cs @@ -376,7 +376,7 @@ public void Add_Empty_GlobalExtendedAttributes() } [Theory] - [MemberData(nameof(WriteTimeStamps_TheoryData))] + [MemberData(nameof(WriteTimeStamp_Pax_TheoryData))] public void WriteTimestampsInPax(DateTimeOffset timestamp) { string strTimestamp = GetTimestampStringFromDateTimeOffset(timestamp); diff --git a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs index fe06ff8f6954eb..b20d1c78e1aa06 100644 --- a/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarWriter/TarWriter.WriteEntryAsync.Entry.Pax.Tests.cs @@ -398,7 +398,7 @@ public async Task Add_Empty_GlobalExtendedAttributes_Async() } [Theory] - [MemberData(nameof(WriteTimeStamps_TheoryData))] + [MemberData(nameof(WriteTimeStamp_Pax_TheoryData))] public async Task WriteTimestampsInPax_Async(DateTimeOffset timestamp) { string strTimestamp = GetTimestampStringFromDateTimeOffset(timestamp); From 62a3ec14ddf461f255782995fd7dbd97ecb2652a Mon Sep 17 00:00:00 2001 From: Tom Deseyn Date: Mon, 17 Jun 2024 11:42:53 +0200 Subject: [PATCH 8/8] Add updated test for golang_tar/writer-big. --- .../tests/TarReader/TarReader.File.Async.Tests.cs | 10 ++++++++++ .../tests/TarReader/TarReader.File.Tests.cs | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs index c17e3d769e0f81..e603e6e9aa4691 100644 --- a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Async.Tests.cs @@ -270,6 +270,16 @@ public async Task Throw_ArchivesWithRandomCharsAsync(string testCaseName) await Assert.ThrowsAsync(async () => await reader.GetNextEntryAsync()); } + [Fact] + public async Task Throw_ArchiveIsShortAsync() + { + // writer-big has a header for a 16G file but not its contents. + using MemoryStream archiveStream = GetTarMemoryStream(CompressionMethod.Uncompressed, "golang_tar", "writer-big"); + using TarReader reader = new TarReader(archiveStream); + // MemoryStream throws when we try to change its Position past its Length. + await Assert.ThrowsAsync(async () => await reader.GetNextEntryAsync()); + } + [Fact] public async Task GarbageEntryChecksumZeroReturnNullAsync() { diff --git a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs index 67670b4dec2197..955f29c841849f 100644 --- a/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs +++ b/src/libraries/System.Formats.Tar/tests/TarReader/TarReader.File.Tests.cs @@ -278,6 +278,16 @@ public void Throw_ArchivesWithRandomChars(string testCaseName) Assert.Throws(() => reader.GetNextEntry()); } + [Fact] + public void Throw_ArchiveIsShort() + { + // writer-big has a header for a 16G file but not its contents. + using MemoryStream archiveStream = GetTarMemoryStream(CompressionMethod.Uncompressed, "golang_tar", "writer-big"); + using TarReader reader = new TarReader(archiveStream); + // MemoryStream throws when we try to change its Position past its Length. + Assert.Throws(() => reader.GetNextEntry()); + } + [Fact] public void GarbageEntryChecksumZeroReturnNull() {