Encryption scope SAS (#22099)

* corrected release date in changelog * STG84 swagger and azcore, azidentity, azblob dependency updates * dependancy updates * Initial changes for encryption scope sas * indirect dependancy * update code generator version * using STG 82 swagger, and revert code gen version * adding transforms for missing header files * re-generate zz_options.go file * reverting to azblob 1.1.0 * reverting to azblob 1.0.0 * Updated input file from main to latest commit id * added tests for encryption scope sas * add test for user delegation sas * lint error fix * Fix lint error in test * created new datalake encryption scope var * removed blob account encryption scope * push recording * add encryption scope in user delegation * recordings * linter error * minor comments resolve * arm template changes * arm template changes * test fix --------- Co-authored-by: Sourav Gupta <[email protected]>
Azure · Dec 13, 2023 · 4f79633 · 4f79633
1 parent fa9d146
commit 4f79633
Show file tree

Hide file tree

Showing 7 changed files with 248 additions and 54 deletions.
diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json
@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "go",
   "TagPrefix": "go/storage/azdatalake",
-  "Tag": "go/storage/azdatalake_7c0fb050cb"
+  "Tag": "go/storage/azdatalake_b5323f920e"
 }
diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go
@@ -11,6 +11,7 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/binary"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake/service"
 	"hash/crc64"
 	"io"
 	"math/rand"
@@ -1337,6 +1338,179 @@ func (s *UnrecordedTestSuite) TestFileDeleteWithSAS() {
 	_require.NoError(err)
 }
 
+func (s *UnrecordedTestSuite) TestFileEncryptionScopeSAS() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient := testcommon.CreateNewFileSystem(context.Background(), _require, filesystemName, svcClient)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	encryptionScope, err := testcommon.GetRequiredEnv(testcommon.DataLakeEncryptionScopeEnvVar)
+	_require.Nil(err)
+
+	cred, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDatalake)
+	_require.NoError(err)
+
+	perms := sas.FilePermissions{Read: true, Create: true, Write: true, Move: true, Delete: true, List: true}
+	sasQueryParams, err := sas.DatalakeSignatureValues{
+		Protocol:        sas.ProtocolHTTPS,                    // Users MUST use HTTPS (not HTTP)
+		ExpiryTime:      time.Now().UTC().Add(48 * time.Hour), // 48-hours before expiration
+		FileSystemName:  filesystemName,
+		Permissions:     perms.String(),
+		EncryptionScope: encryptionScope,
+	}.SignWithSharedKey(cred)
+	_require.NoError(err)
+
+	sasToken := sasQueryParams.Encode()
+
+	srcFileClient, err := file.NewClientWithNoCredential(fsClient.DFSURL()+"/file?"+sasToken, nil)
+	_require.NoError(err)
+	_require.NotNil(srcFileClient)
+
+	_, err = srcFileClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	response, err := srcFileClient.SetMetadata(context.Background(), testcommon.BasicMetadata, nil)
+	_require.NoError(err)
+	_require.Equal(encryptionScope, *response.EncryptionScope)
+
+}
+
+func (s *UnrecordedTestSuite) TestAccountEncryptionScopeSAS() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+	svcClient, err := testcommon.GetServiceClient(s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient := testcommon.CreateNewFileSystem(context.Background(), _require, filesystemName, svcClient)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	encryptionScope, err := testcommon.GetRequiredEnv(testcommon.DataLakeEncryptionScopeEnvVar)
+	_require.Nil(err)
+
+	credential, err := testcommon.GetGenericSharedKeyCredential(testcommon.TestAccountDatalake)
+	_require.Nil(err)
+
+	sasQueryParams, err := sas.AccountSignatureValues{
+		Protocol:        sas.ProtocolHTTPS,                    // Users MUST use HTTPS (not HTTP)
+		ExpiryTime:      time.Now().UTC().Add(48 * time.Hour), // 48-hours before expiration
+		Permissions:     to.Ptr(sas.AccountPermissions{Read: true, Create: true, Write: true, Delete: true}).String(),
+		ResourceTypes:   to.Ptr(sas.AccountResourceTypes{Service: true, Container: true, Object: true}).String(),
+		EncryptionScope: encryptionScope,
+	}.SignWithSharedKey(credential)
+	_require.NoError(err)
+
+	sasToken := sasQueryParams.Encode()
+
+	srcFileClient, err := file.NewClientWithNoCredential(fsClient.DFSURL()+"/file?"+sasToken, nil)
+	_require.NoError(err)
+	_require.NotNil(srcFileClient)
+
+	resp, err := srcFileClient.Create(context.Background(), nil)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	// create local file
+	_, content := generateData(10 * 1024)
+	err = os.WriteFile("testFile", content, 0644)
+	_require.NoError(err)
+
+	defer func() {
+		err = os.Remove("testFile")
+		_require.NoError(err)
+	}()
+
+	fh, err := os.Open("testFile")
+	_require.NoError(err)
+
+	defer func(fh *os.File) {
+		err := fh.Close()
+		_require.NoError(err)
+	}(fh)
+
+	// upload the file
+	err = srcFileClient.UploadFile(context.Background(), fh, &file.UploadFileOptions{
+		Concurrency: 5,
+		ChunkSize:   2 * 1024,
+	})
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	response, err := srcFileClient.DownloadStream(context.Background(), nil)
+	_require.NoError(err)
+	testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+	_require.Equal(encryptionScope, *response.EncryptionScope)
+
+	// validate the data downloaded
+	downloadedData, err := io.ReadAll(response.Body)
+	_require.NoError(err)
+	_require.Equal(len(content), len(downloadedData))
+	_require.EqualValues(content, downloadedData)
+}
+
+func (s *UnrecordedTestSuite) TestGetUserDelegationEncryptionScopeSAS() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	accountName, _ := testcommon.GetGenericAccountInfo(testcommon.TestAccountDatalake)
+	_require.Greater(len(accountName), 0)
+
+	cred, err := testcommon.GetGenericTokenCredential()
+	_require.NoError(err)
+
+	svcClient, err := service.NewClient("https://"+accountName+".dfs.core.windows.net/", cred, nil)
+	_require.NoError(err)
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient := testcommon.CreateNewFileSystem(context.Background(), _require, filesystemName, svcClient)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	// Set current and past time and create key
+	currentTime := time.Now().UTC().Add(-10 * time.Second)
+	pastTime := currentTime.Add(48 * time.Hour)
+	info := service.KeyInfo{
+		Start:  to.Ptr(currentTime.UTC().Format(sas.TimeFormat)),
+		Expiry: to.Ptr(pastTime.UTC().Format(sas.TimeFormat)),
+	}
+
+	udc, err := svcClient.GetUserDelegationCredential(context.Background(), info, nil)
+	_require.NoError(err)
+
+	// get permissions and details for sas
+	encryptionScope, err := testcommon.GetRequiredEnv(testcommon.DataLakeEncryptionScopeEnvVar)
+	_require.Nil(err)
+
+	// Create Blob Signature Values with desired permissions and sign with user delegation credential
+	perms := sas.FilePermissions{Read: true, Create: true, Write: true, Move: true, Delete: true, List: true}
+	sasQueryParams, err := sas.DatalakeSignatureValues{
+		Protocol:        sas.ProtocolHTTPS, // Users MUST use HTTPS (not HTTP)
+		StartTime:       time.Now().UTC().Add(time.Second * -10),
+		ExpiryTime:      time.Now().UTC().Add(15 * time.Minute), // 15 minutes before expiration
+		FileSystemName:  filesystemName,
+		Permissions:     perms.String(),
+		EncryptionScope: encryptionScope,
+	}.SignWithUserDelegation(udc)
+	_require.Nil(err)
+
+	sasURL := fsClient.DFSURL() + "/file?" + sasQueryParams.Encode()
+	// This URL can be used to authenticate requests now
+	srcFileClient, err := file.NewClientWithNoCredential(sasURL, nil)
+	_require.NoError(err)
+
+	_, err = srcFileClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	response, err := srcFileClient.SetMetadata(context.Background(), testcommon.BasicMetadata, nil)
+	_require.NoError(err)
+	_require.Equal(encryptionScope, *response.EncryptionScope)
+}
+
 func (s *RecordedTestSuite) TestFileGetAccessControlWithNilAccessConditions() {
 	_require := require.New(s.T())
 	testName := s.T().Name()

diff --git a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go
@@ -27,13 +27,14 @@ import (
 )
 
 const (
-	DefaultEndpointSuffix       = "core.windows.net/"
-	DefaultBlobEndpointSuffix   = "blob.core.windows.net/"
-	AccountNameEnvVar           = "AZURE_STORAGE_ACCOUNT_NAME"
-	AccountKeyEnvVar            = "AZURE_STORAGE_ACCOUNT_KEY"
-	DefaultEndpointSuffixEnvVar = "AZURE_STORAGE_ENDPOINT_SUFFIX"
-	SubscriptionID              = "SUBSCRIPTION_ID"
-	ResourceGroupName           = "RESOURCE_GROUP_NAME"
+	DefaultEndpointSuffix         = "core.windows.net/"
+	DefaultBlobEndpointSuffix     = "blob.core.windows.net/"
+	AccountNameEnvVar             = "AZURE_STORAGE_ACCOUNT_NAME"
+	AccountKeyEnvVar              = "AZURE_STORAGE_ACCOUNT_KEY"
+	DefaultEndpointSuffixEnvVar   = "AZURE_STORAGE_ENDPOINT_SUFFIX"
+	DataLakeEncryptionScopeEnvVar = "DATALAKE_AZURE_STORAGE_ENCRYPTION_SCOPE"
+	SubscriptionID                = "SUBSCRIPTION_ID"
+	ResourceGroupName             = "RESOURCE_GROUP_NAME"
 )
 
 const (

diff --git a/sdk/storage/azdatalake/sas/account.go b/sdk/storage/azdatalake/sas/account.go
@@ -25,13 +25,14 @@ type UserDelegationCredential = exported.UserDelegationCredential
 // AccountSignatureValues is used to generate a Shared Access Signature (SAS) for an Azure Storage account.
 // For more information, see https://docs.microsoft.com/rest/api/storageservices/constructing-an-account-sas
 type AccountSignatureValues struct {
-	Version       string    `param:"sv"`  // If not specified, this format to SASVersion
-	Protocol      Protocol  `param:"spr"` // See the SASProtocol* constants
-	StartTime     time.Time `param:"st"`  // Not specified if IsZero
-	ExpiryTime    time.Time `param:"se"`  // Not specified if IsZero
-	Permissions   string    `param:"sp"`  // Create by initializing AccountPermissions and then call String()
-	IPRange       IPRange   `param:"sip"`
-	ResourceTypes string    `param:"srt"` // Create by initializing AccountResourceTypes and then call String()
+	Version         string    `param:"sv"`  // If not specified, this format to SASVersion
+	Protocol        Protocol  `param:"spr"` // See the SASProtocol* constants
+	StartTime       time.Time `param:"st"`  // Not specified if IsZero
+	ExpiryTime      time.Time `param:"se"`  // Not specified if IsZero
+	Permissions     string    `param:"sp"`  // Create by initializing AccountPermissions and then call String()
+	IPRange         IPRange   `param:"sip"`
+	ResourceTypes   string    `param:"srt"` // Create by initializing AccountResourceTypes and then call String()
+	EncryptionScope string    `param:"ses"`
 }
 
 // SignWithSharedKey uses an account's shared key credential to sign this signature values to produce
@@ -68,6 +69,7 @@ func (v AccountSignatureValues) SignWithSharedKey(sharedKeyCredential *SharedKey
 		v.IPRange.String(),
 		string(v.Protocol),
 		v.Version,
+		v.EncryptionScope,
 		""}, // That is right, the account SAS requires a terminating extra newline
 		"\n")
 
@@ -77,12 +79,13 @@ func (v AccountSignatureValues) SignWithSharedKey(sharedKeyCredential *SharedKey
 	}
 	p := QueryParameters{
 		// Common SAS parameters
-		version:     v.Version,
-		protocol:    v.Protocol,
-		startTime:   v.StartTime,
-		expiryTime:  v.ExpiryTime,
-		permissions: v.Permissions,
-		ipRange:     v.IPRange,
+		version:         v.Version,
+		protocol:        v.Protocol,
+		startTime:       v.StartTime,
+		expiryTime:      v.ExpiryTime,
+		permissions:     v.Permissions,
+		ipRange:         v.IPRange,
+		encryptionScope: v.EncryptionScope,
 
 		// Account-specific SAS parameters
 		services:      "b", // will always be "b"

diff --git a/sdk/storage/azdatalake/sas/query_params.go b/sdk/storage/azdatalake/sas/query_params.go
@@ -21,7 +21,7 @@ const (
 
 var (
 	// Version is the default version encoded in the SAS token.
-	Version = "2020-02-10"
+	Version = "2020-12-06"
 )
 
 // TimeFormats ISO 8601 format.
@@ -137,6 +137,7 @@ type QueryParameters struct {
 	authorizedObjectID   string    `param:"saoid"`
 	unauthorizedObjectID string    `param:"suoid"`
 	correlationID        string    `param:"scid"`
+	encryptionScope      string    `param:"ses"`
 	// private member used for startTime and expiryTime formatting.
 	stTimeFormat string
 	seTimeFormat string
@@ -277,6 +278,11 @@ func (p *QueryParameters) SignedDirectoryDepth() string {
 	return p.signedDirectoryDepth
 }
 
+// SignedEncryptionScope returns encryptionScope.
+func (p *QueryParameters) SignedEncryptionScope() string {
+	return p.encryptionScope
+}
+
 // Encode encodes the SAS query parameters into URL encoded form sorted by key.
 func (p *QueryParameters) Encode() string {
 	v := url.Values{}
@@ -349,6 +355,9 @@ func (p *QueryParameters) Encode() string {
 	if p.correlationID != "" {
 		v.Add("scid", p.correlationID)
 	}
+	if p.encryptionScope != "" {
+		v.Add("ses", p.encryptionScope)
+	}
 
 	return v.Encode()
 }
@@ -418,6 +427,8 @@ func NewQueryParameters(values url.Values) QueryParameters {
 			p.unauthorizedObjectID = val
 		case "scid":
 			p.correlationID = val
+		case "ses":
+			p.encryptionScope = val
 		default:
 			continue // query param didn't get recognized
 		}
@@ -495,6 +506,8 @@ func newQueryParameters(values url.Values, deleteSASParametersFromValues bool) Q
 			p.unauthorizedObjectID = val
 		case "scid":
 			p.correlationID = val
+		case "ses":
+			p.encryptionScope = val
 		default:
 			isSASKey = false // We didn't recognize the query parameter
 		}

diff --git a/sdk/storage/azdatalake/sas/service.go b/sdk/storage/azdatalake/sas/service.go
@@ -42,6 +42,7 @@ type DatalakeSignatureValues struct {
 	AuthorizedObjectID   string // saoid
 	UnauthorizedObjectID string // suoid
 	CorrelationID        string // scid
+	EncryptionScope      string `param:"ses"`
 }
 
 //TODO: add snapshot and versioning support in the future
@@ -94,7 +95,8 @@ func (v DatalakeSignatureValues) SignWithSharedKey(sharedKeyCredential *SharedKe
 		string(v.Protocol),
 		v.Version,
 		resource,
-		"",                   //snapshot not supported
+		"", //snapshot not supported
+		v.EncryptionScope,
 		v.CacheControl,       // rscc
 		v.ContentDisposition, // rscd
 		v.ContentEncoding,    // rsce
@@ -109,13 +111,13 @@ func (v DatalakeSignatureValues) SignWithSharedKey(sharedKeyCredential *SharedKe
 
 	p := QueryParameters{
 		// Common SAS parameters
-		version:     v.Version,
-		protocol:    v.Protocol,
-		startTime:   v.StartTime,
-		expiryTime:  v.ExpiryTime,
-		permissions: v.Permissions,
-		ipRange:     v.IPRange,
-
+		version:         v.Version,
+		protocol:        v.Protocol,
+		startTime:       v.StartTime,
+		expiryTime:      v.ExpiryTime,
+		permissions:     v.Permissions,
+		ipRange:         v.IPRange,
+		encryptionScope: v.EncryptionScope,
 		// Container/Blob-specific SAS parameters
 		resource:             resource,
 		cacheControl:         v.CacheControl,
@@ -197,7 +199,8 @@ func (v DatalakeSignatureValues) SignWithUserDelegation(userDelegationCredential
 		string(v.Protocol),
 		v.Version,
 		resource,
-		"",                   //snapshot not supported
+		"", //snapshot not supported
+		v.EncryptionScope,
 		v.CacheControl,       // rscc
 		v.ContentDisposition, // rscd
 		v.ContentEncoding,    // rsce
@@ -212,12 +215,13 @@ func (v DatalakeSignatureValues) SignWithUserDelegation(userDelegationCredential
 
 	p := QueryParameters{
 		// Common SAS parameters
-		version:     v.Version,
-		protocol:    v.Protocol,
-		startTime:   v.StartTime,
-		expiryTime:  v.ExpiryTime,
-		permissions: v.Permissions,
-		ipRange:     v.IPRange,
+		version:         v.Version,
+		protocol:        v.Protocol,
+		startTime:       v.StartTime,
+		expiryTime:      v.ExpiryTime,
+		permissions:     v.Permissions,
+		ipRange:         v.IPRange,
+		encryptionScope: v.EncryptionScope,
 
 		// Container/Blob-specific SAS parameters
 		resource:             resource,