Skip to content

Commit

Permalink
PageBlob Client: Source Content Validation (Azure#20616)
Browse files Browse the repository at this point in the history 
* Adding tests for source content validation in pageblob apis

* Fixing tests

* Updating tests for UploadPagesFromURL

* Update tests for UploadPages

* Recorded tests and updated tests

* removing spacing issue

* Updating test

* fixing linting issue
  • Loading branch information
@siminsavani-msft
siminsavani-msft authored Apr 25, 2023
1 parent 526d841 commit af68cae
Show file tree
Hide file tree
Showing 2 changed files with 216 additions and 99 deletions.
2 changes: 1 addition & 1 deletion sdk/storage/azblob/assets.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,5 @@
"AssetsRepo": "Azure/azure-sdk-assets",
"AssetsRepoPrefixPath": "go",
"TagPrefix": "go/storage/azblob",
"Tag": "go/storage/azblob_37ff5d95f1"
"Tag": "go/storage/azblob_ff27f140f0"
}
313 changes: 215 additions & 98 deletions sdk/storage/azblob/pageblob/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,86 +226,180 @@ func (s *PageBlobRecordedTestsSuite) TestPutGetPages() {
// }
//

// func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithMD5() {
// _require := require.New(s.T())
// testName := s.T().Name()
// svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
// if err != nil {
// _require.Fail("Unable to fetch service client because " + err.Error())
// }
//
// containerName := testcommon.GenerateContainerName(testName)
// containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
// defer testcommon.DeleteContainer(context.Background(), _require, containerClient)
//
// contentSize := 4 * 1024 * 1024 // 4MB
// r, sourceData := getRandomDataAndReader(contentSize)
// md5Value := md5.Sum(sourceData)
// contentMD5 := md5Value[:]
// ctx := ctx // Use default Background context
// srcBlob := createNewPageBlobWithSize(_require, "srcblob", containerClient, int64(contentSize))
// destBlob := createNewPageBlobWithSize(_require, "dstblob", containerClient, int64(contentSize))
//
// // Prepare source pbClient for copy.
// offset, _, count := int64(0), int64(contentSize-1), int64(contentSize)
// uploadPagesOptions := pageblob.UploadPagesOptions{Offset: to.Ptr(int64(offset)), Count: to.Ptr(int64(count)),}
// _, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), &uploadPagesOptions)
// _require.Nil(err)
// // _require.Equal(uploadSrcResp1.RawResponse.StatusCode, 201)
//
// // Get source pbClient URL with SAS for UploadPagesFromURL.
// credential, err := getGenericCredential(nil, testcommon.TestAccountDefault)
// _require.Nil(err)
// srcBlobParts, _ := NewBlobURLParts(srcBlob.URL())
//
// srcBlobParts.SAS, err = azblob.BlobSASSignatureValues{
// Protocol: SASProtocolHTTPS, // Users MUST use HTTPS (not HTTP)
// ExpiryTime: time.Now().UTC().Add(48 * time.Hour), // 48-hours before expiration
// ContainerName: srcBlobParts.ContainerName,
// BlobName: srcBlobParts.BlobName,
// Permissions: BlobSASPermissions{Read: true}.String(),
// }.Sign(credential)
// if err != nil {
// _require.Error(err)
// }
//
// srcBlobURLWithSAS := srcBlobParts.URL()
//
// // Upload page from URL with MD5.
// uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{
// SourceContentMD5: contentMD5,
// }
// pResp1, err := destBlob.UploadPagesFromURL(ctx, srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
// _require.Nil(err)
// // _require.Equal(pResp1.RawResponse.StatusCode, 201)
// _require.NotNil(pResp1.ETag)
// _require.NotNil(pResp1.LastModified)
// _require.NotNil(pResp1.ContentMD5)
// _require.EqualValues(pResp1.ContentMD5, contentMD5)
// _require.NotNil(pResp1.RequestID)
// _require.NotNil(pResp1.Version)
// _require.NotNil(pResp1.Date)
// _require.Equal((*pResp1.Date).IsZero(), false)
// _require.Equal(*pResp1.BlobSequenceNumber, int64(0))
//
// // Check data integrity through downloading.
// downloadResp, err := destBlob.Download(ctx, nil)
// _require.Nil(err)
// destData, err := io.ReadAll(downloadResp.BodyReader(&blob.RetryReaderOptions{}))
// _require.Nil(err)
// _require.EqualValues(destData, sourceData)
//
// // Upload page from URL with bad MD5
// _, badMD5 := getRandomDataAndReader(16)
// badContentMD5 := badMD5[:]
// uploadPagesFromURLOptions = pageblob.UploadPagesFromURLOptions{
// SourceContentMD5: badContentMD5,
// }
// _, err = destBlob.UploadPagesFromURL(ctx, srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
// _require.NotNil(err)
//
// testcommon.ValidateBlobErrorCode(_require, err, bloberror.MD5Mismatch)
// }
func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithMD5() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

contentSize := 4 * 1024 * 1024 // 4MB
r, sourceData := testcommon.GetDataAndReader(testName, contentSize)
md5Value := md5.Sum(sourceData)
contentMD5 := md5Value[:]
srcBlob := createNewPageBlobWithSize(context.Background(), _require, "srcblob"+testName, containerClient, int64(contentSize))
destBlob := createNewPageBlobWithSize(context.Background(), _require, "dstblob"+testName, containerClient, int64(contentSize))

// Prepare source pbClient for copy.
offset, _, count := int64(0), int64(contentSize-1), int64(contentSize)
_, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), blob.HTTPRange{Offset: offset, Count: count}, nil)
_require.Nil(err)

// Get source pbClient URL with SAS for UploadPagesFromURL.
credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault)
_require.Nil(err)

srcBlobParts, _ := blob.ParseURL(srcBlob.URL())

srcBlobParts.SAS, err = sas.BlobSignatureValues{
Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP)
ExpiryTime: time.Now().UTC().Add(15 * time.Minute), // 15 minutes before expiration
ContainerName: srcBlobParts.ContainerName,
BlobName: srcBlobParts.BlobName,
Permissions: to.Ptr(sas.BlobPermissions{Read: true}).String(),
}.SignWithSharedKey(credential)
_require.Nil(err)

srcBlobURLWithSAS := srcBlobParts.String()

// Upload page from URL with MD5.
uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{
SourceContentValidation: blob.SourceContentValidationTypeMD5(contentMD5),
}
pResp1, err := destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
_require.Nil(err)
_require.EqualValues(pResp1.ContentMD5, contentMD5)

// Download blob to do data integrity check.
downloadResp, err := destBlob.DownloadStream(context.Background(), nil)
_require.Nil(err)
destData, err := io.ReadAll(downloadResp.Body)
_require.Nil(err)
_require.EqualValues(destData, sourceData)

// Upload page from URL with bad MD5
_, badMD5 := testcommon.GetDataAndReader(testName+"bad-md5", contentSize)
badContentMD5 := badMD5[:]
uploadPagesFromURLOptions = pageblob.UploadPagesFromURLOptions{
SourceContentValidation: blob.SourceContentValidationTypeMD5(badContentMD5),
}
_, err = destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
_require.NotNil(err)
testcommon.ValidateHTTPErrorCode(_require, err, 400) // Fails with 400 (Bad Request)
}

func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithCRC64() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

contentSize := 4 * 1024 * 1024 // 4MB
r, sourceData := testcommon.GetDataAndReader(testName, contentSize)
crc64Value := crc64.Checksum(sourceData, shared.CRC64Table)
crc := make([]byte, 8)
binary.LittleEndian.PutUint64(crc, crc64Value)
srcBlob := createNewPageBlobWithSize(context.Background(), _require, "srcblob"+testName, containerClient, int64(contentSize))
destBlob := createNewPageBlobWithSize(context.Background(), _require, "dstblob"+testName, containerClient, int64(contentSize))

// Prepare source pbClient for copy.
offset, _, count := int64(0), int64(contentSize-1), int64(contentSize)
_, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), blob.HTTPRange{Offset: offset, Count: count}, nil)
_require.Nil(err)

// Get source pbClient URL with SAS for UploadPagesFromURL.
credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault)
_require.Nil(err)

srcBlobParts, _ := blob.ParseURL(srcBlob.URL())

srcBlobParts.SAS, err = sas.BlobSignatureValues{
Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP)
ExpiryTime: time.Now().UTC().Add(15 * time.Minute), // 15 minutes before expiration
ContainerName: srcBlobParts.ContainerName,
BlobName: srcBlobParts.BlobName,
Permissions: to.Ptr(sas.BlobPermissions{Read: true}).String(),
}.SignWithSharedKey(credential)
_require.Nil(err)

srcBlobURLWithSAS := srcBlobParts.String()

// Upload page from URL with CRC64.
uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{
SourceContentValidation: blob.SourceContentValidationTypeCRC64(crc),
}
_, err = destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
_require.Nil(err)
// TODO: This does not work... ContentCRC64 is not returned. Fix this later.
// _require.EqualValues(pResp1.ContentCRC64, crc)

// Download blob to do data integrity check.
downloadResp, err := destBlob.DownloadStream(context.Background(), nil)
_require.Nil(err)
destData, err := io.ReadAll(downloadResp.Body)
_require.Nil(err)
_require.EqualValues(destData, sourceData)
}

func (s *PageBlobUnrecordedTestsSuite) TestUploadPagesFromURLWithCRC64Negative() {
s.T().Skip("This test is skipped because of issues in the service.")

_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

contentSize := 4 * 1024 * 1024 // 4MB
r, sourceData := testcommon.GetDataAndReader(testName, contentSize)
crc64Value := crc64.Checksum(sourceData, shared.CRC64Table)
crc := make([]byte, 8)
binary.LittleEndian.PutUint64(crc, crc64Value)
srcBlob := createNewPageBlobWithSize(context.Background(), _require, "srcblob"+testName, containerClient, int64(contentSize))
destBlob := createNewPageBlobWithSize(context.Background(), _require, "dstblob"+testName, containerClient, int64(contentSize))

// Prepare source pbClient for copy.
offset, _, count := int64(0), int64(contentSize-1), int64(contentSize)
_, err = srcBlob.UploadPages(context.Background(), streaming.NopCloser(r), blob.HTTPRange{Offset: offset, Count: count}, nil)
_require.Nil(err)

// Get source pbClient URL with SAS for UploadPagesFromURL.
credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDefault)
_require.Nil(err)

srcBlobParts, _ := blob.ParseURL(srcBlob.URL())

srcBlobParts.SAS, err = sas.BlobSignatureValues{
Protocol: sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP)
ExpiryTime: time.Now().UTC().Add(15 * time.Minute), // 15 minutes before expiration
ContainerName: srcBlobParts.ContainerName,
BlobName: srcBlobParts.BlobName,
Permissions: to.Ptr(sas.BlobPermissions{Read: true}).String(),
}.SignWithSharedKey(credential)
_require.Nil(err)

srcBlobURLWithSAS := srcBlobParts.String()

// Upload page from URL with bad CRC64
badCRC64 := rand.Uint64()
badcrc := make([]byte, 8)
binary.LittleEndian.PutUint64(badcrc, badCRC64)
uploadPagesFromURLOptions := pageblob.UploadPagesFromURLOptions{
SourceContentValidation: blob.SourceContentValidationTypeCRC64(badcrc),
}
_, err = destBlob.UploadPagesFromURL(context.Background(), srcBlobURLWithSAS, 0, 0, int64(contentSize), &uploadPagesFromURLOptions)
_require.NotNil(err) // TODO: UploadPagesFromURL should fail, but is currently not working due to service issue.
}

func (s *PageBlobUnrecordedTestsSuite) TestClearDiffPages() {
_require := require.New(s.T())
Expand Down Expand Up @@ -500,14 +594,50 @@ func (s *PageBlobRecordedTestsSuite) TestPageSequenceNumbers() {
_require.Nil(err)
}

func (s *PageBlobRecordedTestsSuite) TestPutPagesWithCRC64() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
defer testcommon.DeleteContainer(context.Background(), _require, containerClient)

blobName := testcommon.GenerateBlobName(testName)
pbClient := createNewPageBlob(context.Background(), _require, blobName, containerClient)

// put page with valid auto-generated CRC64
contentSize := 1024
readerToBody, body := testcommon.GetDataAndReader(testName, contentSize)
offset, _, count := int64(0), int64(0)+int64(contentSize-1), int64(contentSize)
crc64Value := crc64.Checksum(body, shared.CRC64Table)
_ = body

putResp, err := pbClient.UploadPages(context.Background(), streaming.NopCloser(readerToBody), blob.HTTPRange{Offset: offset, Count: count}, &pageblob.UploadPagesOptions{
TransactionalValidation: blob.TransferValidationTypeCRC64(crc64Value),
})
_require.Nil(err)
_require.NotNil(putResp.ContentCRC64)
_require.EqualValues(binary.LittleEndian.Uint64(putResp.ContentCRC64), crc64Value)

// put page with bad CRC64
readerToBody, _ = testcommon.GetDataAndReader(testName, 1024)
badCRC64 := rand.Uint64()
putResp, err = pbClient.UploadPages(context.Background(), streaming.NopCloser(readerToBody), blob.HTTPRange{Offset: offset, Count: count}, &pageblob.UploadPagesOptions{
TransactionalValidation: blob.TransferValidationTypeCRC64(badCRC64),
})
_require.NotNil(err)

// testcommon.ValidateBlobErrorCode(_require, err, bloberror.CRC64Mismatch)
}

// nolint
func (s *PageBlobUnrecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() {
func (s *PageBlobRecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
if err != nil {
_require.Fail("Unable to fetch service client because " + err.Error())
}
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
Expand All @@ -527,7 +657,6 @@ func (s *PageBlobUnrecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() {
TransactionalValidation: blob.TransferValidationTypeComputeCRC64(),
})
_require.Nil(err)
// _require.Equal(putResp.RawResponse.StatusCode, 201)
_require.NotNil(putResp.LastModified)
_require.Equal((*putResp.LastModified).IsZero(), false)
_require.NotNil(putResp.ETag)
Expand All @@ -538,26 +667,14 @@ func (s *PageBlobUnrecordedTestsSuite) TestPutPagesWithAutoGeneratedCRC64() {
_require.NotNil(*putResp.Version)
_require.NotNil(putResp.Date)
_require.Equal((*putResp.Date).IsZero(), false)

// put page with bad MD5
readerToBody, _ = testcommon.GetDataAndReader(testName, 1024)
badCRC64 := rand.Uint64()
putResp, err = pbClient.UploadPages(context.Background(), streaming.NopCloser(readerToBody), blob.HTTPRange{Offset: offset, Count: count}, &pageblob.UploadPagesOptions{
TransactionalValidation: blob.TransferValidationTypeCRC64(badCRC64),
})
_require.NotNil(err)

testcommon.ValidateBlobErrorCode(_require, err, bloberror.CRC64Mismatch)
}

// nolint
func (s *PageBlobRecordedTestsSuite) TestPutPagesWithMD5() {
_require := require.New(s.T())
testName := s.T().Name()
svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDefault, nil)
if err != nil {
_require.Fail("Unable to fetch service client because " + err.Error())
}
_require.NoError(err)

containerName := testcommon.GenerateContainerName(testName)
containerClient := testcommon.CreateNewContainer(context.Background(), _require, containerName, svcClient)
Expand Down

0 comments on commit af68cae

Please sign in to comment.