From af68cae4c5497822572cfd4783600b4bd74d3b38 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Tue, 25 Apr 2023 12:09:16 -0400 Subject: [PATCH] PageBlob Client: Source Content Validation (#20616) * Adding tests for source content validation in pageblob apis * Fixing tests * Updating tests for UploadPagesFromURL * Update tests for UploadPages * Recorded tests and updated tests * removing spacing issue * Updating test * fixing linting issue --- sdk/storage/azblob/assets.json | 2 +- sdk/storage/azblob/pageblob/client_test.go | 313 ++++++++++++++------- 2 files changed, 216 insertions(+), 99 deletions(-) diff --git a/sdk/storage/azblob/assets.json b/sdk/storage/azblob/assets.json index cb758b7f29d7..e102edb50451 100644 --- a/sdk/storage/azblob/assets.json +++ b/sdk/storage/azblob/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azblob", - "Tag": "go/storage/azblob_37ff5d95f1" + "Tag": "go/storage/azblob_ff27f140f0" } diff --git a/sdk/storage/azblob/pageblob/client_test.go b/sdk/storage/azblob/pageblob/client_test.go index 9dc829de25f3..7b659b4bf4ee 100644 --- a/sdk/storage/azblob/pageblob/client_test.go +++ b/sdk/storage/azblob/pageblob/client_test.go @@ -226,86 +226,180 @@ func (s *PageBlobRecordedTestsSuite) TestPutGetPages() { // } // -// func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithMD5() { -// _require := require.New(s.T()) -// testName := s.T().Name() -// svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil) -// if err != nil { -// _require.Fail("Unable to fetch service client because " + err.Error()) -// } -// -// containerName := testcommon.GenerateContainerName(testName) -// containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient) -// defer testcommon.DeleteContainer(context.Background(), _require, containerClient) -// -// contentSize := 4 * 1024 * 1024 // 4MB -// r, sourceData := getRandomDataAndReader(contentSize) -// md5Value := md5.Sum(sourceData) -// contentMD5 := md5Value[:] -// ctx := ctx // Use default Background context -// srcBlob := createNewPageBlobWithSize(_require, "srcblob", containerClient, int64(contentSize)) -// destBlob := createNewPageBlobWithSize(_require, "dstblob", containerClient, int64(contentSize)) -// -// // Prepare source pbClient for copy. -// offset, _, count := int64(0), int64(contentSize-1), int64(contentSize) -// uploadPagesOptions := pageblob.UploadPagesOptions{Offset: to.Ptr(int64(offset)), Count: to.Ptr(int64(count)),} -// _, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), &uploadPagesOptions) -// _require.Nil(err) -// // _require.Equal(uploadSrcResp1.RawResponse.StatusCode, 201) -// -// // Get source pbClient URL with SAS for UploadPagesFromURL. -// credential, err := getGenericCredential(nil, testcommon.TestAccountDefault) -// _require.Nil(err) -// srcBlobParts, _ := NewBlobURLParts(srcBlob.URL()) -// -// srcBlobParts.SAS, err = azblob.BlobSASSignatureValues{ -// Protocol: SASProtocolHTTPS, // Users MUST use HTTPS (not HTTP) -// ExpiryTime: time.Now().UTC().Add(48 * time.Hour), // 48-hours before expiration -// ContainerName: srcBlobParts.ContainerName, -// BlobName: srcBlobParts.BlobName, -// Permissions: BlobSASPermissions{Read: true}.String(), -// }.Sign(credential) -// if err != nil { -// _require.Error(err) -// } -// -// srcBlobURLWithSAS := srcBlobParts.URL() -// -// // Upload page from URL with MD5. -// uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{ -// SourceContentMD5: contentMD5, -// } -// pResp1, err := destBlob.UploadPagesFromURL(ctx, srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions) -// _require.Nil(err) -// // _require.Equal(pResp1.RawResponse.StatusCode, 201) -// _require.NotNil(pResp1.ETag) -// _require.NotNil(pResp1.LastModified) -// _require.NotNil(pResp1.ContentMD5) -// _require.EqualValues(pResp1.ContentMD5, contentMD5) -// _require.NotNil(pResp1.RequestID) -// _require.NotNil(pResp1.Version) -// _require.NotNil(pResp1.Date) -// _require.Equal((*pResp1.Date).IsZero(), false) -// _require.Equal(*pResp1.BlobSequenceNumber, int64(0)) -// -// // Check data integrity through downloading. -// downloadResp, err := destBlob.Download(ctx, nil) -// _require.Nil(err) -// destData, err := io.ReadAll(downloadResp.BodyReader(&blob.RetryReaderOptions{})) -// _require.Nil(err) -// _require.EqualValues(destData, sourceData) -// -// // Upload page from URL with bad MD5 -// _, badMD5 := getRandomDataAndReader(16) -// badContentMD5 := badMD5[:] -// uploadPagesFromURLOptions = pageblob.UploadPagesFromURLOptions{ -// SourceContentMD5: badContentMD5, -// } -// _, err = destBlob.UploadPagesFromURL(ctx, srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions) -// _require.NotNil(err) -// -// testcommon.ValidateBlobErrorCode(_require, err, bloberror.MD5Mismatch) -// } +func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithMD5() { + _require := require.New(s.T()) + testName := s.T().Name() + svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil) + _require.NoError(err) + + containerName := testcommon.GenerateContainerName(testName) + containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient) + defer testcommon.DeleteContainer(context.Background(), _require, containerClient) + + contentSize := 4 * 1024 * 1024 // 4MB + r, sourceData := testcommon.GetDataAndReader(testName, contentSize) + md5Value := md5.Sum(sourceData) + contentMD5 := md5Value[:] + srcBlob := createNewPageBlobWithSize(context.Background(), _require, "srcblob"+testName, containerClient, int64(contentSize)) + destBlob := createNewPageBlobWithSize(context.Background(), _require, "dstblob"+testName, containerClient, int64(contentSize)) + + // Prepare source pbClient for copy. + offset, _, count := int64(0), int64(contentSize-1), int64(contentSize) + _, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), blob.HTTPRange{Offset: offset, Count: count}, nil) + _require.Nil(err) + + // Get source pbClient URL with SAS for UploadPagesFromURL. + credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault) + _require.Nil(err) + + srcBlobParts, _ := blob.ParseURL(srcBlob.URL()) + + srcBlobParts.SAS, err = sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) + ExpiryTime: time.Now().UTC().Add(15 * time.Minute), // 15 minutes before expiration + ContainerName: srcBlobParts.ContainerName, + BlobName: srcBlobParts.BlobName, + Permissions: to.Ptr(sas.BlobPermissions{Read: true}).String(), + }.SignWithSharedKey(credential) + _require.Nil(err) + + srcBlobURLWithSAS := srcBlobParts.String() + + // Upload page from URL with MD5. + uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{ + SourceContentValidation: blob.SourceContentValidationTypeMD5(contentMD5), + } + pResp1, err := destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions) + _require.Nil(err) + _require.EqualValues(pResp1.ContentMD5, contentMD5) + + // Download blob to do data integrity check. + downloadResp, err := destBlob.DownloadStream(context.Background(), nil) + _require.Nil(err) + destData, err := io.ReadAll(downloadResp.Body) + _require.Nil(err) + _require.EqualValues(destData, sourceData) + + // Upload page from URL with bad MD5 + _, badMD5 := testcommon.GetDataAndReader(testName+"bad-md5", contentSize) + badContentMD5 := badMD5[:] + uploadPagesFromURLOptions = pageblob.UploadPagesFromURLOptions{ + SourceContentValidation: blob.SourceContentValidationTypeMD5(badContentMD5), + } + _, err = destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions) + _require.NotNil(err) + testcommon.ValidateHTTPErrorCode(_require, err, 400) // Fails with 400 (Bad Request) +} + +func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithCRC64() { + _require := require.New(s.T()) + testName := s.T().Name() + svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil) + _require.NoError(err) + + containerName := testcommon.GenerateContainerName(testName) + containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient) + defer testcommon.DeleteContainer(context.Background(), _require, containerClient) + + contentSize := 4 * 1024 * 1024 // 4MB + r, sourceData := testcommon.GetDataAndReader(testName, contentSize) + crc64Value := crc64.Checksum(sourceData, shared.CRC64Table) + crc := make([]byte, 8) + binary.LittleEndian.PutUint64(crc, crc64Value) + srcBlob := createNewPageBlobWithSize(context.Background(), _require, "srcblob"+testName, containerClient, int64(contentSize)) + destBlob := createNewPageBlobWithSize(context.Background(), _require, "dstblob"+testName, containerClient, int64(contentSize)) + + // Prepare source pbClient for copy. + offset, _, count := int64(0), int64(contentSize-1), int64(contentSize) + _, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), blob.HTTPRange{Offset: offset, Count: count}, nil) + _require.Nil(err) + + // Get source pbClient URL with SAS for UploadPagesFromURL. + credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault) + _require.Nil(err) + + srcBlobParts, _ := blob.ParseURL(srcBlob.URL()) + + srcBlobParts.SAS, err = sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) + ExpiryTime: time.Now().UTC().Add(15 * time.Minute), // 15 minutes before expiration + ContainerName: srcBlobParts.ContainerName, + BlobName: srcBlobParts.BlobName, + Permissions: to.Ptr(sas.BlobPermissions{Read: true}).String(), + }.SignWithSharedKey(credential) + _require.Nil(err) + + srcBlobURLWithSAS := srcBlobParts.String() + + // Upload page from URL with CRC64. + uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{ + SourceContentValidation: blob.SourceContentValidationTypeCRC64(crc), + } + _, err = destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions) + _require.Nil(err) + // TODO: This does not work... ContentCRC64 is not returned. Fix this later. + // _require.EqualValues(pResp1.ContentCRC64, crc) + + // Download blob to do data integrity check. + downloadResp, err := destBlob.DownloadStream(context.Background(), nil) + _require.Nil(err) + destData, err := io.ReadAll(downloadResp.Body) + _require.Nil(err) + _require.EqualValues(destData, sourceData) +} + +func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithCRC64Negative() { + s.T().Skip("This test is skipped because of issues in the service.") + + _require := require.New(s.T()) + testName := s.T().Name() + svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil) + _require.NoError(err) + + containerName := testcommon.GenerateContainerName(testName) + containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient) + defer testcommon.DeleteContainer(context.Background(), _require, containerClient) + + contentSize := 4 * 1024 * 1024 // 4MB + r, sourceData := testcommon.GetDataAndReader(testName, contentSize) + crc64Value := crc64.Checksum(sourceData, shared.CRC64Table) + crc := make([]byte, 8) + binary.LittleEndian.PutUint64(crc, crc64Value) + srcBlob := createNewPageBlobWithSize(context.Background(), _require, "srcblob"+testName, containerClient, int64(contentSize)) + destBlob := createNewPageBlobWithSize(context.Background(), _require, "dstblob"+testName, containerClient, int64(contentSize)) + + // Prepare source pbClient for copy. + offset, _, count := int64(0), int64(contentSize-1), int64(contentSize) + _, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), blob.HTTPRange{Offset: offset, Count: count}, nil) + _require.Nil(err) + + // Get source pbClient URL with SAS for UploadPagesFromURL. + credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault) + _require.Nil(err) + + srcBlobParts, _ := blob.ParseURL(srcBlob.URL()) + + srcBlobParts.SAS, err = sas.BlobSignatureValues{ + Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP) + ExpiryTime: time.Now().UTC().Add(15 * time.Minute), // 15 minutes before expiration + ContainerName: srcBlobParts.ContainerName, + BlobName: srcBlobParts.BlobName, + Permissions: to.Ptr(sas.BlobPermissions{Read: true}).String(), + }.SignWithSharedKey(credential) + _require.Nil(err) + + srcBlobURLWithSAS := srcBlobParts.String() + + // Upload page from URL with bad CRC64 + badCRC64 := rand.Uint64() + badcrc := make([]byte, 8) + binary.LittleEndian.PutUint64(badcrc, badCRC64) + uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{ + SourceContentValidation: blob.SourceContentValidationTypeCRC64(badcrc), + } + _, err = destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions) + _require.NotNil(err) // TODO: UploadPagesFromURL should fail, but is currently not working due to service issue. +} func (s *PageBlobUnrecordedTestsSuite) TestClearDiffPages() { _require := require.New(s.T()) @@ -500,14 +594,50 @@ func (s *PageBlobRecordedTestsSuite) TestPageSequenceNumbers() { _require.Nil(err) } +func (s *PageBlobRecordedTestsSuite) TestPutPagesWithCRC64() { + _require := require.New(s.T()) + testName := s.T().Name() + svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil) + _require.NoError(err) + + containerName := testcommon.GenerateContainerName(testName) + containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient) + defer testcommon.DeleteContainer(context.Background(), _require, containerClient) + + blobName := testcommon.GenerateBlobName(testName) + pbClient := createNewPageBlob(context.Background(), _require, blobName, containerClient) + + // put page with valid auto-generated CRC64 + contentSize := 1024 + readerToBody, body := testcommon.GetDataAndReader(testName, contentSize) + offset, _, count := int64(0), int64(0)+int64(contentSize-1), int64(contentSize) + crc64Value := crc64.Checksum(body, shared.CRC64Table) + _ = body + + putResp, err := pbClient.UploadPages(context.Background(), streaming.NopCloser(readerToBody), blob.HTTPRange{Offset: offset, Count: count}, &pageblob.UploadPagesOptions{ + TransactionalValidation: blob.TransferValidationTypeCRC64(crc64Value), + }) + _require.Nil(err) + _require.NotNil(putResp.ContentCRC64) + _require.EqualValues(binary.LittleEndian.Uint64(putResp.ContentCRC64), crc64Value) + + // put page with bad CRC64 + readerToBody, _ = testcommon.GetDataAndReader(testName, 1024) + badCRC64 := rand.Uint64() + putResp, err = pbClient.UploadPages(context.Background(), streaming.NopCloser(readerToBody), blob.HTTPRange{Offset: offset, Count: count}, &pageblob.UploadPagesOptions{ + TransactionalValidation: blob.TransferValidationTypeCRC64(badCRC64), + }) + _require.NotNil(err) + + // testcommon.ValidateBlobErrorCode(_require, err, bloberror.CRC64Mismatch) +} + // nolint -func (s *PageBlobUnrecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() { +func (s *PageBlobRecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() { _require := require.New(s.T()) testName := s.T().Name() svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil) - if err != nil { - _require.Fail("Unable to fetch service client because " + err.Error()) - } + _require.NoError(err) containerName := testcommon.GenerateContainerName(testName) containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient) @@ -527,7 +657,6 @@ func (s *PageBlobUnrecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() { TransactionalValidation: blob.TransferValidationTypeComputeCRC64(), }) _require.Nil(err) - // _require.Equal(putResp.RawResponse.StatusCode, 201) _require.NotNil(putResp.LastModified) _require.Equal((*putResp.LastModified).IsZero(), false) _require.NotNil(putResp.ETag) @@ -538,16 +667,6 @@ func (s *PageBlobUnrecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() { _require.NotNil(*putResp.Version) _require.NotNil(putResp.Date) _require.Equal((*putResp.Date).IsZero(), false) - - // put page with bad MD5 - readerToBody, _ = testcommon.GetDataAndReader(testName, 1024) - badCRC64 := rand.Uint64() - putResp, err = pbClient.UploadPages(context.Background(), streaming.NopCloser(readerToBody), blob.HTTPRange{Offset: offset, Count: count}, &pageblob.UploadPagesOptions{ - TransactionalValidation: blob.TransferValidationTypeCRC64(badCRC64), - }) - _require.NotNil(err) - - testcommon.ValidateBlobErrorCode(_require, err, bloberror.CRC64Mismatch) } // nolint @@ -555,9 +674,7 @@ func (s *PageBlobRecordedTestsSuite) TestPutPagesWithMD5() { _require := require.New(s.T()) testName := s.T().Name() svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil) - if err != nil { - _require.Fail("Unable to fetch service client because " + err.Error()) - } + _require.NoError(err) containerName := testcommon.GenerateContainerName(testName) containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)