From 6754e4f2b9c11c38a9701f5efcab3665ea82e3ff Mon Sep 17 00:00:00 2001 From: Jocelyn <41338290+jaschrep-msft@users.noreply.github.com> Date: Mon, 30 Oct 2023 11:51:12 -0400 Subject: [PATCH] Share checkpoint schema (#39556) * initial work * tests and fixes * fix csproj * fixes * csproj --- ...e.Storage.DataMovement.Files.Shares.csproj | 7 + .../src/DataMovementShareConstants.cs | 29 +++ .../ShareDirectoryStorageResourceContainer.cs | 2 +- .../src/ShareFileDestinationCheckpointData.cs | 182 +++++++++++++++++- .../src/ShareFileSourceCheckpointData.cs | 4 + .../src/ShareFileStorageResource.cs | 2 +- ...age.DataMovement.Files.Shares.Tests.csproj | 7 +- .../ShareDestinationCheckpointDataTests.cs | 133 +++++++++++++ .../tests/ShareSourceCheckpointDataTests.cs | 45 +++++ 9 files changed, 403 insertions(+), 8 deletions(-) create mode 100644 sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDestinationCheckpointDataTests.cs create mode 100644 sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareSourceCheckpointDataTests.cs diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/Azure.Storage.DataMovement.Files.Shares.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/Azure.Storage.DataMovement.Files.Shares.csproj index 8d4bed65f1754..3126c27f07636 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/Azure.Storage.DataMovement.Files.Shares.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/Azure.Storage.DataMovement.Files.Shares.csproj @@ -31,8 +31,15 @@ + + + + + + + diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementShareConstants.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementShareConstants.cs index a0c773c0901df..cb838b1214a3b 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementShareConstants.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementShareConstants.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Text; +using static Azure.Storage.DataMovement.DataMovementConstants; namespace Azure.Storage.DataMovement.Files.Shares { @@ -13,5 +14,33 @@ internal class DataMovementShareConstants public const int MB = KB * 1024; internal const int MaxRange = 4 * MB; + + internal class SourceCheckpointData + { + internal const int DataSize = 0; + } + + internal class DestinationCheckpointData + { + internal const int SchemaVersion = 1; + + internal const int VersionIndex = 0; + + internal const int ContentTypeOffsetIndex = VersionIndex + IntSizeInBytes; + internal const int ContentTypeLengthIndex = ContentTypeOffsetIndex + IntSizeInBytes; + internal const int ContentEncodingOffsetIndex = ContentTypeLengthIndex + IntSizeInBytes; + internal const int ContentEncodingLengthIndex = ContentEncodingOffsetIndex + IntSizeInBytes; + internal const int ContentLanguageOffsetIndex = ContentEncodingLengthIndex + IntSizeInBytes; + internal const int ContentLanguageLengthIndex = ContentLanguageOffsetIndex + IntSizeInBytes; + internal const int ContentDispositionOffsetIndex = ContentLanguageLengthIndex + IntSizeInBytes; + internal const int ContentDispositionLengthIndex = ContentDispositionOffsetIndex + IntSizeInBytes; + internal const int CacheControlOffsetIndex = ContentDispositionLengthIndex + IntSizeInBytes; + internal const int CacheControlLengthIndex = CacheControlOffsetIndex + IntSizeInBytes; + + internal const int MetadataOffsetIndex = CacheControlLengthIndex + IntSizeInBytes; + internal const int MetadataLengthIndex = MetadataOffsetIndex + IntSizeInBytes; + + internal const int VariableLengthStartIndex = MetadataLengthIndex + IntSizeInBytes; + } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs index 863839eeda4a9..7999acfcd0078 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs @@ -57,7 +57,7 @@ protected override StorageResourceCheckpointData GetSourceCheckpointData() protected override StorageResourceCheckpointData GetDestinationCheckpointData() { - return new ShareFileDestinationCheckpointData(); + return new ShareFileDestinationCheckpointData(null, null); } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs index da27081b0b78b..99d4a7ee9bc2c 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs @@ -1,16 +1,196 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +using System; using System.IO; +using System.Text; +using Azure.Core; +using Azure.Storage.Files.Shares.Models; +using Metadata = System.Collections.Generic.IDictionary; namespace Azure.Storage.DataMovement.Files.Shares { internal class ShareFileDestinationCheckpointData : StorageResourceCheckpointData { - public override int Length => 0; + private const char HeaderDelimiter = Constants.CommaChar; + + /// + /// Schema version. + /// + public int Version; + + /// + /// The content headers for the destination blob. + /// + public ShareFileHttpHeaders ContentHeaders; + private byte[] _contentTypeBytes; + private byte[] _contentEncodingBytes; + private byte[] _contentLanguageBytes; + private byte[] _contentDispositionBytes; + private byte[] _cacheControlBytes; + + /// + /// The metadata for the destination blob. + /// + public Metadata Metadata; + private byte[] _metadataBytes; + + public override int Length => CalculateLength(); + + public ShareFileDestinationCheckpointData( + ShareFileHttpHeaders contentHeaders, + Metadata metadata) + { + Version = DataMovementShareConstants.DestinationCheckpointData.SchemaVersion; + ContentHeaders = contentHeaders; + _contentTypeBytes = ContentHeaders?.ContentType != default ? Encoding.UTF8.GetBytes(ContentHeaders.ContentType) : Array.Empty(); + _contentEncodingBytes = ContentHeaders?.ContentEncoding != default ? Encoding.UTF8.GetBytes(string.Join(HeaderDelimiter.ToString(), ContentHeaders.ContentEncoding)) : Array.Empty(); + _contentLanguageBytes = ContentHeaders?.ContentLanguage != default ? Encoding.UTF8.GetBytes(string.Join(HeaderDelimiter.ToString(), ContentHeaders.ContentLanguage)) : Array.Empty(); + _contentDispositionBytes = ContentHeaders?.ContentDisposition != default ? Encoding.UTF8.GetBytes(ContentHeaders.ContentDisposition) : Array.Empty(); + _cacheControlBytes = ContentHeaders?.CacheControl != default ? Encoding.UTF8.GetBytes(ContentHeaders.CacheControl) : Array.Empty(); + Metadata = metadata; + _metadataBytes = Metadata != default ? Encoding.UTF8.GetBytes(Metadata.DictionaryToString()) : Array.Empty(); + } + + internal void SerializeInternal(Stream stream) => Serialize(stream); protected override void Serialize(Stream stream) { + Argument.AssertNotNull(stream, nameof(stream)); + + int currentVariableLengthIndex = DataMovementShareConstants.DestinationCheckpointData.VariableLengthStartIndex; + BinaryWriter writer = new(stream); + + // Version + writer.Write(Version); + + // Fixed position offset/lengths for variable length info + writer.WriteVariableLengthFieldInfo(_contentTypeBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_contentEncodingBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_contentLanguageBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_contentDispositionBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_cacheControlBytes.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(_metadataBytes.Length, ref currentVariableLengthIndex); + + // Variable length info + writer.Write(_contentTypeBytes); + writer.Write(_contentEncodingBytes); + writer.Write(_contentLanguageBytes); + writer.Write(_contentDispositionBytes); + writer.Write(_cacheControlBytes); + writer.Write(_metadataBytes); + } + + internal static ShareFileDestinationCheckpointData Deserialize(Stream stream) + { + Argument.AssertNotNull(stream, nameof(stream)); + + BinaryReader reader = new BinaryReader(stream); + + // Version + int version = reader.ReadInt32(); + if (version != DataMovementShareConstants.DestinationCheckpointData.SchemaVersion) + { + throw Storage.Errors.UnsupportedJobSchemaVersionHeader(version.ToString()); + } + + // ContentType offset/length + int contentTypeOffset = reader.ReadInt32(); + int contentTypeLength = reader.ReadInt32(); + + // ContentEncoding offset/length + int contentEncodingOffset = reader.ReadInt32(); + int contentEncodingLength = reader.ReadInt32(); + + // ContentLanguage offset/length + int contentLanguageOffset = reader.ReadInt32(); + int contentLanguageLength = reader.ReadInt32(); + + // ContentDisposition offset/length + int contentDispositionOffset = reader.ReadInt32(); + int contentDispositionLength = reader.ReadInt32(); + + // CacheControl offset/length + int cacheControlOffset = reader.ReadInt32(); + int cacheControlLength = reader.ReadInt32(); + + // Metadata offset/length + int metadataOffset = reader.ReadInt32(); + int metadataLength = reader.ReadInt32(); + + // ContentType + string contentType = null; + if (contentTypeOffset > 0) + { + reader.BaseStream.Position = contentTypeOffset; + contentType = Encoding.UTF8.GetString(reader.ReadBytes(contentTypeLength)); + } + + // ContentEncoding + string contentEncoding = null; + if (contentEncodingOffset > 0) + { + reader.BaseStream.Position = contentEncodingOffset; + contentEncoding = Encoding.UTF8.GetString(reader.ReadBytes(contentEncodingLength)); + } + + // ContentLanguage + string contentLanguage = null; + if (contentLanguageOffset > 0) + { + reader.BaseStream.Position = contentLanguageOffset; + contentLanguage = Encoding.UTF8.GetString(reader.ReadBytes(contentLanguageLength)); + } + + // ContentDisposition + string contentDisposition = null; + if (contentDispositionOffset > 0) + { + reader.BaseStream.Position = contentDispositionOffset; + contentDisposition = Encoding.UTF8.GetString(reader.ReadBytes(contentDispositionLength)); + } + + // CacheControl + string cacheControl = null; + if (cacheControlOffset > 0) + { + reader.BaseStream.Position = cacheControlOffset; + cacheControl = Encoding.UTF8.GetString(reader.ReadBytes(cacheControlLength)); + } + + // Metadata + string metadataString = string.Empty; + if (metadataOffset > 0) + { + reader.BaseStream.Position = metadataOffset; + metadataString = Encoding.UTF8.GetString(reader.ReadBytes(metadataLength)); + } + + ShareFileHttpHeaders contentHeaders = new() + { + ContentType = contentType, + ContentEncoding = contentEncoding.Split(HeaderDelimiter), + ContentLanguage = contentLanguage.Split(HeaderDelimiter), + ContentDisposition = contentDisposition, + CacheControl = cacheControl, + }; + + return new( + contentHeaders, + metadataString.ToDictionary(nameof(metadataString))); + } + + private int CalculateLength() + { + // Length is fixed size fields plus length of each variable length field + int length = DataMovementShareConstants.DestinationCheckpointData.VariableLengthStartIndex; + length += _contentTypeBytes.Length; + length += _contentEncodingBytes.Length; + length += _contentLanguageBytes.Length; + length += _contentDispositionBytes.Length; + length += _cacheControlBytes.Length; + length += _metadataBytes.Length; + return length; } } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs index 9b0bc38010add..29f104eedc1d2 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileSourceCheckpointData.cs @@ -9,8 +9,12 @@ internal class ShareFileSourceCheckpointData : StorageResourceCheckpointData { public override int Length => 0; + internal void SerializeInternal(Stream stream) => Serialize(stream); + protected override void Serialize(Stream stream) { } + + internal static ShareFileSourceCheckpointData Deserialize(Stream stream) => new(); } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs index 4a49a80b6a2ff..8ff96c57122e0 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs @@ -211,7 +211,7 @@ protected override StorageResourceCheckpointData GetSourceCheckpointData() protected override StorageResourceCheckpointData GetDestinationCheckpointData() { - return new ShareFileDestinationCheckpointData(); + return new ShareFileDestinationCheckpointData(null, null); } } diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj index de152cde090fa..8173b54d74520 100644 --- a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/Azure.Storage.DataMovement.Files.Shares.Tests.csproj @@ -14,9 +14,6 @@ - - - @@ -26,8 +23,8 @@ - - + + diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDestinationCheckpointDataTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDestinationCheckpointDataTests.cs new file mode 100644 index 0000000000000..25d20779a5264 --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareDestinationCheckpointDataTests.cs @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System.IO; +using System.Text; +using Azure.Storage.Files.Shares.Models; +using Azure.Storage.Test; +using NUnit.Framework; +using Metadata = System.Collections.Generic.IDictionary; + +namespace Azure.Storage.DataMovement.Files.Shares.Tests +{ + public class ShareDestinationCheckpointDataTests + { + private const string DefaultContentType = "text/plain"; + private readonly string[] DefaultContentEncoding = new string[] { "gzip" }; + private readonly string[] DefaultContentLanguage = new string[] { "en-US" }; + private const string DefaultContentDisposition = "inline"; + private const string DefaultCacheControl = "no-cache"; + private readonly Metadata DefaultMetadata = DataProvider.BuildMetadata(); + + private ShareFileDestinationCheckpointData CreateDefault() + { + return new ShareFileDestinationCheckpointData( + new ShareFileHttpHeaders() + { + ContentType = DefaultContentType, + ContentEncoding = DefaultContentEncoding, + ContentLanguage = DefaultContentLanguage, + ContentDisposition = DefaultContentDisposition, + CacheControl = DefaultCacheControl, + }, + DefaultMetadata); + } + + private void AssertEquals(ShareFileDestinationCheckpointData left, ShareFileDestinationCheckpointData right) + { + Assert.That(left.Version, Is.EqualTo(right.Version)); + Assert.That(left.ContentHeaders.ContentType, Is.EqualTo(right.ContentHeaders.ContentType)); + Assert.That(left.ContentHeaders.ContentEncoding, Is.EqualTo(right.ContentHeaders.ContentEncoding)); + Assert.That(left.ContentHeaders.ContentLanguage, Is.EqualTo(right.ContentHeaders.ContentLanguage)); + Assert.That(left.ContentHeaders.ContentDisposition, Is.EqualTo(right.ContentHeaders.ContentDisposition)); + Assert.That(left.ContentHeaders.CacheControl, Is.EqualTo(right.ContentHeaders.CacheControl)); + Assert.That(left.Metadata, Is.EqualTo(right.Metadata)); + } + + private byte[] CreateSerializedDefault() + { + using MemoryStream stream = new(); + using BinaryWriter writer = new(stream); + + byte[] contentType = Encoding.UTF8.GetBytes(DefaultContentType); + byte[] contentEncoding = Encoding.UTF8.GetBytes(string.Join(",", DefaultContentEncoding)); + byte[] contentLanguage = Encoding.UTF8.GetBytes(string.Join(",", DefaultContentLanguage)); + byte[] contentDisposition = Encoding.UTF8.GetBytes(DefaultContentDisposition); + byte[] cacheControl = Encoding.UTF8.GetBytes(DefaultCacheControl); + byte[] metadata = Encoding.UTF8.GetBytes(DefaultMetadata.DictionaryToString()); + + int currentVariableLengthIndex = DataMovementShareConstants.DestinationCheckpointData.VariableLengthStartIndex; + writer.Write(DataMovementShareConstants.DestinationCheckpointData.SchemaVersion); + writer.WriteVariableLengthFieldInfo(contentType.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(contentEncoding.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(contentLanguage.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(contentDisposition.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(cacheControl.Length, ref currentVariableLengthIndex); + writer.WriteVariableLengthFieldInfo(metadata.Length, ref currentVariableLengthIndex); + writer.Write(contentType); + writer.Write(contentEncoding); + writer.Write(contentLanguage); + writer.Write(contentDisposition); + writer.Write(cacheControl); + writer.Write(metadata); + + return stream.ToArray(); + } + + [Test] + public void Ctor() + { + ShareFileDestinationCheckpointData data = CreateDefault(); + + Assert.That(data.Version, Is.EqualTo(DataMovementShareConstants.DestinationCheckpointData.SchemaVersion)); + Assert.That(data.ContentHeaders.ContentType, Is.EqualTo(DefaultContentType)); + Assert.That(data.ContentHeaders.ContentEncoding, Is.EqualTo(DefaultContentEncoding)); + Assert.That(data.ContentHeaders.ContentLanguage, Is.EqualTo(DefaultContentLanguage)); + Assert.That(data.ContentHeaders.ContentDisposition, Is.EqualTo(DefaultContentDisposition)); + Assert.That(data.ContentHeaders.CacheControl, Is.EqualTo(DefaultCacheControl)); + Assert.That(data.Metadata, Is.EqualTo(DefaultMetadata)); + } + + [Test] + public void Serialize() + { + byte[] expected = CreateSerializedDefault(); + + ShareFileDestinationCheckpointData data = CreateDefault(); + byte[] actual; + using (MemoryStream stream = new()) + { + data.SerializeInternal(stream); + actual = stream.ToArray(); + } + + Assert.That(expected, Is.EqualTo(actual)); + } + + [Test] + public void Deserialize() + { + byte[] serialized = CreateSerializedDefault(); + ShareFileDestinationCheckpointData deserialized; + + using (MemoryStream stream = new(serialized)) + { + deserialized = ShareFileDestinationCheckpointData.Deserialize(stream); + } + + AssertEquals(deserialized, CreateDefault()); + } + + [Test] + public void RoundTrip() + { + ShareFileDestinationCheckpointData original = CreateDefault(); + using MemoryStream serialized = new(); + original.SerializeInternal(serialized); + serialized.Position = 0; + ShareFileDestinationCheckpointData deserialized = ShareFileDestinationCheckpointData.Deserialize(serialized); + + AssertEquals(original, deserialized); + } + } +} diff --git a/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareSourceCheckpointDataTests.cs b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareSourceCheckpointDataTests.cs new file mode 100644 index 0000000000000..3e3ab27823b9d --- /dev/null +++ b/sdk/storage/Azure.Storage.DataMovement.Files.Shares/tests/ShareSourceCheckpointDataTests.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +using System; +using System.IO; +using NUnit.Framework; + +namespace Azure.Storage.DataMovement.Files.Shares.Tests +{ + public class ShareSourceCheckpointDataTests + { + [Test] + public void Ctor() + { + ShareFileSourceCheckpointData data = new(); + } + + [Test] + public void Serialize() + { + byte[] expected = Array.Empty(); + + ShareFileSourceCheckpointData data = new(); + byte[] actual; + using (MemoryStream stream = new()) + { + data.SerializeInternal(stream); + actual = stream.ToArray(); + } + + Assert.That(expected, Is.EqualTo(actual)); + } + + [Test] + public void Deserialize() + { + ShareFileSourceCheckpointData deserialized; + + using (MemoryStream stream = new()) + { + deserialized = ShareFileSourceCheckpointData.Deserialize(Stream.Null); + } + } + } +}