diff --git a/sdk/storage/azdatalake/CHANGELOG.md b/sdk/storage/azdatalake/CHANGELOG.md index 62598c61d6e2..bc7f4bc6c481 100644 --- a/sdk/storage/azdatalake/CHANGELOG.md +++ b/sdk/storage/azdatalake/CHANGELOG.md @@ -3,6 +3,8 @@ ## 1.0.1 (Unreleased) ### Features Added +* Encryption Scope For SAS +* CPK for Datalake ### Breaking Changes diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index 78c1d69d4497..a3472b483893 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_1ee92f87a9" + "Tag": "go/storage/azdatalake_36b1978c0d" } \ No newline at end of file diff --git a/sdk/storage/azdatalake/directory/client.go b/sdk/storage/azdatalake/directory/client.go index 45f4f0b31c94..97e907e11f7c 100644 --- a/sdk/storage/azdatalake/directory/client.go +++ b/sdk/storage/azdatalake/directory/client.go @@ -295,7 +295,7 @@ func (d *Client) Rename(ctx context.Context, destinationPath string, options *Re newPathURL = strings.Split(newPathURL, "?")[0] + "?" + newDestQuery } newBlobURL, _ := shared.GetURLs(newPathURL) - lac, mac, smac, createOpts := path.FormatRenameOptions(options, newSrcPath) + lac, mac, smac, createOpts, cpkOpts := path.FormatRenameOptions(options, newSrcPath) if d.identityCredential() != nil { newBlobClient, err = blockblob.NewClient(newBlobURL, *d.identityCredential(), nil) @@ -310,7 +310,7 @@ func (d *Client) Rename(ctx context.Context, destinationPath string, options *Re return RenameResponse{}, exported.ConvertToDFSError(err) } newDirClient := (*Client)(base.NewPathClient(newPathURL, newBlobURL, newBlobClient, d.generatedDirClientWithDFS().InternalClient().WithClientName(shared.DirectoryClient), d.sharedKey(), d.identityCredential(), d.getClientOptions())) - resp, err := newDirClient.generatedDirClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, nil) + resp, err := newDirClient.generatedDirClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, cpkOpts) //return RenameResponse{ // Response: resp, // NewDirectoryClient: newDirClient, diff --git a/sdk/storage/azdatalake/directory/client_test.go b/sdk/storage/azdatalake/directory/client_test.go index cc2f7bab8ee0..9a43fa362733 100644 --- a/sdk/storage/azdatalake/directory/client_test.go +++ b/sdk/storage/azdatalake/directory/client_test.go @@ -147,6 +147,31 @@ func (s *RecordedTestSuite) TestCreateDirAndDelete() { _require.NotNil(resp) } +func (s *RecordedTestSuite) TestCreateDirUsingCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + dirOpts := &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue} + resp, err := dirClient.Create(context.Background(), dirOpts) + _require.NoError(err) + _require.NotNil(resp) + + _require.Equal(true, *(resp.IsServerEncrypted)) + _require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, resp.EncryptionKeySHA256) +} + func (s *RecordedTestSuite) TestGetAndCreateFileClient() { _require := require.New(s.T()) testName := s.T().Name() @@ -1912,6 +1937,65 @@ func (s *RecordedTestSuite) TestDirSetMetadataWithAccessConditions() { _require.NoError(err) } +func (s *RecordedTestSuite) TestDirSetMetadataWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + defer testcommon.DeleteDir(context.Background(), _require, dirClient) + + resp, err := dirClient.Create(context.Background(), &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.NotNil(resp) + + opts := &directory.SetMetadataOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + _, err = dirClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts) + _require.NoError(err) +} + +func (s *RecordedTestSuite) TestDirSetMetadataWithCPKNegative() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + defer testcommon.DeleteDir(context.Background(), _require, dirClient) + + resp, err := dirClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp) + + opts := &directory.SetMetadataOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + _, err = dirClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts) + _require.Error(err) + _require.ErrorContains(err, "PathDoesNotUseCustomerSpecifiedEncryption") +} + func validatePropertiesSet(_require *require.Assertions, dirClient *directory.Client, disposition string) { resp, err := dirClient.GetProperties(context.Background(), nil) _require.NoError(err) @@ -2243,6 +2327,38 @@ func (s *RecordedTestSuite) TestDirRenameNoOptions() { //_require.Contains(resp1.NewDirectoryClient.DFSURL(), "newName") } +func (s *RecordedTestSuite) TestDirRenameRequestWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createOpts := &directory.CreateOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + resp, err := dirClient.Create(context.Background(), createOpts) + _require.NoError(err) + _require.NotNil(resp) + + renameFileOpts := &directory.RenameOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + _, err = dirClient.Rename(context.Background(), "newName", renameFileOpts) + _require.NoError(err) +} + func (s *RecordedTestSuite) TestRenameDirWithNilAccessConditions() { _require := require.New(s.T()) testName := s.T().Name() @@ -2548,6 +2664,35 @@ func (s *RecordedTestSuite) TestDirGetPropertiesResponseCapture() { _require.Equal("directory", respFromCtxService.Header.Get("x-ms-resource-type")) } +func (s *RecordedTestSuite) TestDirGetPropertiesWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + dirOpts := &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue} + resp, err := dirClient.Create(context.Background(), dirOpts) + _require.NoError(err) + _require.NotNil(resp) + + getPropertiesOpts := &directory.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue} + response, err := dirClient.GetProperties(context.Background(), getPropertiesOpts) + _require.NoError(err) + _require.NotNil(response) + _require.Equal(*(resp.IsServerEncrypted), true) + _require.Equal(resp.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) +} + func (s *UnrecordedTestSuite) TestDirCreateDeleteUsingOAuth() { _require := require.New(s.T()) testName := s.T().Name() diff --git a/sdk/storage/azdatalake/file/client.go b/sdk/storage/azdatalake/file/client.go index 58b97cd5d342..be8c5a3ec84f 100644 --- a/sdk/storage/azdatalake/file/client.go +++ b/sdk/storage/azdatalake/file/client.go @@ -278,7 +278,7 @@ func (f *Client) Rename(ctx context.Context, destinationPath string, options *Re newPathURL = strings.Split(newPathURL, "?")[0] + "?" + newDestQuery } newBlobURL, _ := shared.GetURLs(newPathURL) - lac, mac, smac, createOpts := path.FormatRenameOptions(options, newSrcPath) + lac, mac, smac, createOpts, cpkOpts := path.FormatRenameOptions(options, newSrcPath) if f.identityCredential() != nil { newBlobClient, err = blockblob.NewClient(newBlobURL, *f.identityCredential(), nil) @@ -293,7 +293,7 @@ func (f *Client) Rename(ctx context.Context, destinationPath string, options *Re return RenameResponse{}, exported.ConvertToDFSError(err) } newFileClient := (*Client)(base.NewPathClient(newPathURL, newBlobURL, newBlobClient, f.generatedFileClientWithDFS().InternalClient().WithClientName(shared.FileClient), f.sharedKey(), f.identityCredential(), f.getClientOptions())) - resp, err := newFileClient.generatedFileClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, nil) + resp, err := newFileClient.generatedFileClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, cpkOpts) //return RenameResponse{ // Response: resp, diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index 2716968c3ea9..33794a13a3ba 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -170,6 +170,35 @@ func (s *RecordedTestSuite) TestCreateFileWithNilAccessConditions() { _require.NotNil(resp) } +func (s *RecordedTestSuite) TestCreateFileWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + defer testcommon.DeleteFile(context.Background(), _require, fClient) + + createFileOpts := &file.CreateOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + resp, err := fClient.Create(context.Background(), createFileOpts) + _require.NoError(err) + _require.NotNil(resp) + _require.Equal(*(resp.IsServerEncrypted), true) + _require.Equal(resp.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) +} + func (s *RecordedTestSuite) TestCreateFileIfModifiedSinceTrue() { _require := require.New(s.T()) testName := s.T().Name() @@ -1893,6 +1922,37 @@ func (s *RecordedTestSuite) TestFileSetMetadataWithAccessConditions() { _require.NoError(err) } +func (s *RecordedTestSuite) TestFileSetMetadataWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + defer testcommon.DeleteFile(context.Background(), _require, fClient) + + resp, err := fClient.Create(context.Background(), &file.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.NotNil(resp) + + opts := &file.SetMetadataOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + res, err := fClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts) + _require.NoError(err) + _require.Equal(*(res.IsServerEncrypted), true) + _require.Equal(res.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) +} + func validatePropertiesSet(_require *require.Assertions, fileClient *file.Client, disposition string) { resp, err := fileClient.GetProperties(context.Background(), nil) _require.NoError(err) @@ -2217,11 +2277,36 @@ func (s *RecordedTestSuite) TestRenameNoOptions() { _require.NoError(err) _require.NotNil(resp) - //resp1, err := fClient.Rename(context.Background(), "newName", renameFileOpts) _, err = fClient.Rename(context.Background(), "newName", nil) _require.NoError(err) - //_require.NotNil(resp1) - //_require.Contains(resp1.NewFileClient.DFSURL(), "newName") +} + +func (s *RecordedTestSuite) TestRenameFileWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := fClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp) + + renameFileOpts := &file.RenameOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + _, err = fClient.Rename(context.Background(), "newName", renameFileOpts) + _require.NoError(err) } func (s *RecordedTestSuite) TestRenameFileWithNilAccessConditions() { @@ -2620,6 +2705,95 @@ func (s *RecordedTestSuite) TestFileUploadTinyStream() { _require.EqualValues(downloadedContentMD5, contentMD5) } +func (s *UnrecordedTestSuite) TestFileUploadDownloadStreamWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 1 * 1024 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := fClient.Create(context.Background(), &file.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.NotNil(resp) + + content := make([]byte, fileSize) + _, err = rand.Read(content) + _require.NoError(err) + md5Value := md5.Sum(content) + contentMD5 := md5Value[:] + + err = fClient.UploadStream(context.Background(), streaming.NopCloser(bytes.NewReader(content)), &file.UploadStreamOptions{ + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + + gResp2, err := fClient.GetProperties(context.Background(), &file.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, fileSize) + + dResp, err := fClient.DownloadStream(context.Background(), &file.DownloadStreamOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + + data, err := io.ReadAll(dResp.Body) + _require.NoError(err) + + downloadedMD5Value := md5.Sum(data) + downloadedContentMD5 := downloadedMD5Value[:] + + _require.EqualValues(downloadedContentMD5, contentMD5) + _require.Equal(true, *(dResp.IsServerEncrypted)) + _require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, dResp.EncryptionKeySHA256) +} + +func (s *UnrecordedTestSuite) TestFileUploadDownloadStreamWithCPKNegative() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 1 * 1024 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := fClient.Create(context.Background(), &file.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.NotNil(resp) + + content := make([]byte, fileSize) + _, err = rand.Read(content) + _require.NoError(err) + + err = fClient.UploadStream(context.Background(), streaming.NopCloser(bytes.NewReader(content)), &file.UploadStreamOptions{ + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + + gResp2, err := fClient.GetProperties(context.Background(), &file.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, fileSize) + + _, err = fClient.DownloadStream(context.Background(), &file.DownloadStreamOptions{}) + _require.Error(err) + _require.ErrorContains(err, "PathUsesCustomerSpecifiedEncryption") +} + func (s *UnrecordedTestSuite) TestFileUploadFile() { _require := require.New(s.T()) testName := s.T().Name() @@ -3283,6 +3457,47 @@ func (s *RecordedTestSuite) TestFileAppendAndFlushDataWithEmptyOpts() { _require.Equal(*gResp2.ContentLength, int64(contentSize)) } +func (s *RecordedTestSuite) TestFileAppendAndFlushDataWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + srcFileName := "src" + testcommon.GenerateFileName(testName) + + srcFClient, err := testcommon.GetFileClient(filesystemName, srcFileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createOptions := &file.CreateOptions{CPKInfo: &testcommon.TestCPKByValue} + resp, err := srcFClient.Create(context.Background(), createOptions) + _require.NoError(err) + _require.NotNil(resp) + + contentSize := 1024 * 8 // 8KB + rsc, _ := testcommon.GenerateData(contentSize) + + opts := &file.AppendDataOptions{CPKInfo: &testcommon.TestCPKByValue} + opts1 := &file.FlushDataOptions{CPKInfo: &testcommon.TestCPKByValue} + + _, err = srcFClient.AppendData(context.Background(), 0, rsc, opts) + _require.NoError(err) + + _, err = srcFClient.FlushData(context.Background(), int64(contentSize), opts1) + _require.NoError(err) + getPropertiesOptions := &file.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue} + gResp2, err := srcFClient.GetProperties(context.Background(), getPropertiesOptions) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, int64(contentSize)) + _require.Equal(true, *(gResp2.IsServerEncrypted)) + _require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, gResp2.EncryptionKeySHA256) +} + func (s *RecordedTestSuite) TestFileAppendAndFlushDataWithLeasedFile() { _require := require.New(s.T()) testName := s.T().Name() @@ -4087,6 +4302,95 @@ func (s *RecordedTestSuite) TestFileUploadDownloadSmallFileWithAccessConditions( _require.Equal(*gResp2.ContentLength, fileSize) } +func (s *RecordedTestSuite) TestFileUploadDownloadSmallFileWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 10 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := fClient.Create(context.Background(), &file.CreateOptions{ + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + _require.NotNil(resp) + + // create local file + _, content := testcommon.GenerateData(int(fileSize)) + srcFileName := "testFileUpload" + err = os.WriteFile(srcFileName, content, 0644) + _require.NoError(err) + defer func() { + err = os.Remove(srcFileName) + _require.NoError(err) + }() + fh, err := os.Open(srcFileName) + _require.NoError(err) + defer func(fh *os.File) { + err := fh.Close() + _require.NoError(err) + }(fh) + + srcHash := md5.New() + _, err = io.Copy(srcHash, fh) + _require.NoError(err) + contentMD5 := srcHash.Sum(nil) + + err = fClient.UploadFile(context.Background(), fh, &file.UploadFileOptions{ + Concurrency: 5, + ChunkSize: 2 * 1024, + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + + destFileName := "SmallFile-downloaded.bin" + destFile, err := os.Create(destFileName) + _require.NoError(err) + defer func(name string) { + err = os.Remove(name) + _require.NoError(err) + }(destFileName) + defer func(destFile *os.File) { + err = destFile.Close() + _require.NoError(err) + }(destFile) + + cnt, err := fClient.DownloadFile(context.Background(), destFile, &file.DownloadFileOptions{ + ChunkSize: 2 * 1024, + Concurrency: 5, + Range: &file.HTTPRange{ + Offset: 0, + Count: 10 * 1024, + }, + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + _require.Equal(cnt, fileSize) + + destHash := md5.New() + _, err = io.Copy(destHash, destFile) + _require.NoError(err) + downloadedContentMD5 := destHash.Sum(nil) + + _require.EqualValues(downloadedContentMD5, contentMD5) + + gResp2, err := fClient.GetProperties(context.Background(), &file.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, fileSize) + _require.Equal(*(gResp2.IsServerEncrypted), true) + _require.Equal(gResp2.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) +} + func (s *RecordedTestSuite) TestFileUploadDownloadWithProgress() { _require := require.New(s.T()) testName := s.T().Name() @@ -4361,6 +4665,55 @@ func (s *RecordedTestSuite) TestFileDownloadSmallBufferWithAccessConditions() { _require.Equal(*gResp2.ContentLength, fileSize) } +func (s *RecordedTestSuite) TestFileDownloadBufferWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 10 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + fileCreateOpts := &file.CreateOptions{CPKInfo: &testcommon.TestCPKByValue} + resp, err := fClient.Create(context.Background(), fileCreateOpts) + _require.NoError(err) + _require.NotNil(resp) + + _, content := testcommon.GenerateData(int(fileSize)) + md5Value := md5.Sum(content[0:fileSize]) + contentMD5 := md5Value[:] + + err = fClient.UploadBuffer(context.Background(), content, &file.UploadBufferOptions{ + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + + destBuffer := make([]byte, fileSize) + _, err = fClient.DownloadBuffer(context.Background(), destBuffer, &file.DownloadBufferOptions{ + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + + downloadedMD5Value := md5.Sum(destBuffer) + downloadedContentMD5 := downloadedMD5Value[:] + + _require.EqualValues(downloadedContentMD5, contentMD5) + + gResp2, err := fClient.GetProperties(context.Background(), &file.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, fileSize) + _require.Equal(*(gResp2.IsServerEncrypted), true) + _require.Equal(gResp2.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) +} + func (s *RecordedTestSuite) TestFileGetPropertiesResponseCapture() { _require := require.New(s.T()) testName := s.T().Name() @@ -4437,6 +4790,52 @@ func (s *RecordedTestSuite) TestFileGetPropertiesResponseCapture() { _require.Equal("file", respFromCtxDir.Header.Get("x-ms-resource-type")) } +func (s *RecordedTestSuite) TestFileGetPropertiesWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := dirClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp) + + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, dirName+"/"+fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createFileOpts := &file.CreateOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + resp, err = fClient.Create(context.Background(), createFileOpts) + _require.NoError(err) + _require.NotNil(resp) + + GetPropertiesOpts := &file.GetPropertiesOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + // This tests file.NewClient + var respFromCtxFile *http.Response + ctxWithRespFile := runtime.WithCaptureResponse(context.Background(), &respFromCtxFile) + response, err := fClient.GetProperties(ctxWithRespFile, GetPropertiesOpts) + _require.NoError(err) + _require.NotNil(response) + _require.NotNil(respFromCtxFile.Header.Get("x-ms-encryption-key-sha256")) // validate that the x-ms-encryption-key-sha256 is actually populated + _require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, response.EncryptionKeySHA256) +} + func (s *UnrecordedTestSuite) TestFileCreateDeleteUsingOAuth() { _require := require.New(s.T()) testName := s.T().Name() diff --git a/sdk/storage/azdatalake/internal/path/models.go b/sdk/storage/azdatalake/internal/path/models.go index 3a25acce5a27..64cb04cc1aa5 100644 --- a/sdk/storage/azdatalake/internal/path/models.go +++ b/sdk/storage/azdatalake/internal/path/models.go @@ -38,9 +38,11 @@ type RenameOptions struct { SourceAccessConditions *SourceAccessConditions // AccessConditions contains parameters for accessing the path. AccessConditions *AccessConditions + // CPKInfo contains CPK related information. + CPKInfo *CPKInfo } -func FormatRenameOptions(o *RenameOptions, path string) (*generated.LeaseAccessConditions, *generated.ModifiedAccessConditions, *generated.SourceModifiedAccessConditions, *generated.PathClientCreateOptions) { +func FormatRenameOptions(o *RenameOptions, path string) (*generated.LeaseAccessConditions, *generated.ModifiedAccessConditions, *generated.SourceModifiedAccessConditions, *generated.PathClientCreateOptions, *generated.CPKInfo) { // we don't need sourceModAccCond since this is not rename mode := generated.PathRenameModeLegacy createOpts := &generated.PathClientCreateOptions{ @@ -48,7 +50,15 @@ func FormatRenameOptions(o *RenameOptions, path string) (*generated.LeaseAccessC RenameSource: &path, } if o == nil { - return nil, nil, nil, createOpts + return nil, nil, nil, createOpts, nil + } + var cpkOpts *generated.CPKInfo + if o.CPKInfo != nil { + cpkOpts = &generated.CPKInfo{ + EncryptionAlgorithm: o.CPKInfo.EncryptionAlgorithm, + EncryptionKey: o.CPKInfo.EncryptionKey, + EncryptionKeySHA256: o.CPKInfo.EncryptionKeySHA256, + } } leaseAccessConditions, modifiedAccessConditions := exported.FormatPathAccessConditions(o.AccessConditions) if o.SourceAccessConditions != nil { @@ -62,10 +72,10 @@ func FormatRenameOptions(o *RenameOptions, path string) (*generated.LeaseAccessC SourceIfNoneMatch: o.SourceAccessConditions.SourceModifiedAccessConditions.SourceIfNoneMatch, SourceIfUnmodifiedSince: o.SourceAccessConditions.SourceModifiedAccessConditions.SourceIfUnmodifiedSince, } - return leaseAccessConditions, modifiedAccessConditions, sourceModifiedAccessConditions, createOpts + return leaseAccessConditions, modifiedAccessConditions, sourceModifiedAccessConditions, createOpts, cpkOpts } } - return leaseAccessConditions, modifiedAccessConditions, nil, createOpts + return leaseAccessConditions, modifiedAccessConditions, nil, createOpts, cpkOpts } // GetPropertiesOptions contains the optional parameters for the Client.GetProperties method. diff --git a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go index 87e1c434809e..7a45a06c5385 100644 --- a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go +++ b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go @@ -54,6 +54,17 @@ var ( DatalakeContentEncoding = "my_encoding" ) +var ( + testEncryptedKey = "MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=" + testEncryptedHash = "3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=" + testEncryptionAlgorithm = file.EncryptionAlgorithmTypeAES256 + TestCPKByValue = file.CPKInfo{ + EncryptionKey: &testEncryptedKey, + EncryptionKeySHA256: &testEncryptedHash, + EncryptionAlgorithm: &testEncryptionAlgorithm, + } +) + var BasicHeaders = file.HTTPHeaders{ ContentType: &DatalakeContentType, ContentDisposition: &DatalakeContentDisposition,