Skip to content

Commit

Permalink
[Storage] Fix Bug 10483492: Remove-AzDataLakeGen2Item fails with read…
Browse files Browse the repository at this point in the history 
…only sas (#15582)

* [Storage] Fix Bug 10483492: Remove-AzDataLakeGen2Item fails with readonly sas

* Revised destination existing check in move datalakegen2 item
  • Loading branch information
@blueww
blueww authored Aug 11, 2021
1 parent 061923b commit 4ad1464
Show file tree
Hide file tree
Showing 5 changed files with 90 additions and 78 deletions.
8 changes: 5 additions & 3 deletions src/Storage/Storage.Management.Test/ScenarioTests/StorageDataPlaneTests.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -900,9 +900,11 @@ function Test-DatalakeGen2
Assert-AreEqual 2 $result.TotalFilesSuccessfulCount
Assert-AreEqual 3 $result.TotalDirectoriesSuccessfulCount

# Remove Items
Remove-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $filePath1 -Force
Remove-AzDataLakeGen2Item -Context $storageContext -FileSystem $filesystemName -Path $directoryPath1 -Force
# Remove Items with delete only SAS
$sas = New-AzStorageContainerSASToken -Name $filesystemName -Permission d -Context $storageContext
$storageContextSas = New-AzStorageContext -StorageAccountName $storageContext.StorageAccountName -SasToken $sas
Remove-AzDataLakeGen2Item -Context $storageContextSas -FileSystem $filesystemName -Path $filePath1 -Force
Remove-AzDataLakeGen2Item -Context $storageContextSas -FileSystem $filesystemName -Path $directoryPath1 -Force

# Clean Storage Account
Get-AzDataLakeGen2ChildItem -Context $storageContext -FileSystem $filesystemName | Remove-AzDataLakeGen2Item -Force
Expand Down
4 changes: 4 additions & 0 deletions src/Storage/Storage.Management/ChangeLog.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
- Additional information about change #1
-->
## Upcoming Release
* Fixed remove datalakegen2 item fail with readonly SAS token
- `Remove-AzDataLakeGen2Item`
* Revised destination existing check in move datalakegen2 item
- `Move-AzDataLakeGen2Item`

## Version 3.10.0
* Supported Blob Last Access Time
Expand Down
71 changes: 52 additions & 19 deletions src/Storage/Storage/Common/AzureDataLakeGen2Item.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public class AzureDataLakeGen2Item : AzureStorageBase
/// </summary>
[Ps1Xml(Label = "Group", Target = ViewControl.Table, Position = 7, TableColumnWidth = 10)]
public string Group { get; set; }

/// <summary>
/// Azure DataLakeGen2 Item constructor
/// </summary>
Expand All @@ -112,16 +112,30 @@ public AzureDataLakeGen2Item(DataLakeFileClient fileClient)
Name = fileClient.Name;
Path = fileClient.Path;
File = fileClient;
Properties = fileClient.GetProperties();
AccessControl = File.GetAccessControl();
Length = Properties.ContentLength;
ContentType = Properties.ContentType;
LastModified = Properties.LastModified;
IsDirectory = false;
Permissions = AccessControl.Permissions;
ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList);
Owner = AccessControl.Owner;
Group = AccessControl.Group;
try
{
Properties = fileClient.GetProperties();
Length = Properties.ContentLength;
ContentType = Properties.ContentType;
LastModified = Properties.LastModified;
}
catch (global::Azure.RequestFailedException e) when (e.Status == 403 || e.Status == 404)
{
// skip get file properties if don't have read permission
}
try
{
AccessControl = File.GetAccessControl();
Permissions = AccessControl.Permissions;
ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList);
Owner = AccessControl.Owner;
Group = AccessControl.Group;
}
catch (global::Azure.RequestFailedException e) when (e.Status == 403 || e.Status == 404)
{
// skip get file ACL if don't have read permission
}
}

/// <summary>
Expand All @@ -136,16 +150,35 @@ public AzureDataLakeGen2Item(DataLakeDirectoryClient directoryClient)
IsDirectory = true;
if (directoryClient.Path != "/" || string.IsNullOrEmpty(directoryClient.Path)) //if root directory, GetProperties() will fail. Skip until this is fixed.
{
Properties = directoryClient.GetProperties();
Length = Properties.ContentLength;
ContentType = Properties.ContentType;
LastModified = Properties.LastModified;
try
{


Properties = directoryClient.GetProperties();
Length = Properties.ContentLength;
ContentType = Properties.ContentType;
LastModified = Properties.LastModified;
}
catch (global::Azure.RequestFailedException e) when (e.Status == 403 || e.Status == 404)
{
// skip get dir properties if don't have read permission
}
}

try
{

AccessControl = directoryClient.GetAccessControl();
Permissions = AccessControl.Permissions;
ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList);
Owner = AccessControl.Owner;
Group = AccessControl.Group;
}
catch (global::Azure.RequestFailedException e) when (e.Status == 403 || e.Status == 404)
{
// skip get dir ACL if don't have read permission

}
AccessControl = directoryClient.GetAccessControl();
Permissions = AccessControl.Permissions;
ACL = PSPathAccessControlEntry.ParsePSPathAccessControlEntrys(AccessControl.AccessControlList);
Owner = AccessControl.Owner;
Group = AccessControl.Group;
}


Expand Down
25 changes: 2 additions & 23 deletions src/Storage/Storage/DatalakeGen2/Cmdlet/MoveAzDataLakeGen2Item.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,6 @@ public MoveAzDataLakeGen2ItemCommand(IStorageBlobManagement channel)
public override void ExecuteCmdlet()
{
IStorageBlobManagement localChannel = Channel;
BlobRequestOptions requestOptions = RequestOptions;

bool foundAFolder = false;
DataLakeFileClient srcBlob = null;
Expand All @@ -124,20 +123,10 @@ public override void ExecuteCmdlet()
{
if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlobDir), "Move Directory: "))
{
// check dest exist
bool destExist = true;
DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem);
DataLakeDirectoryClient destBlobDir = destFileSystem.GetDirectoryClient(this.DestPath);
try
{
destBlobDir.GetProperties();
}
catch (RequestFailedException e) when (e.Status == 404)
{
destExist = false;
}

if (this.Force || !destExist || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destBlobDir)), ""))
if (this.Force || !destBlobDir.Exists() || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destBlobDir)), ""))
{
destBlobDir = srcBlobDir.Rename(this.DestPath, this.DestFileSystem).Value;
WriteDataLakeGen2Item(localChannel, destBlobDir);
Expand All @@ -148,20 +137,10 @@ public override void ExecuteCmdlet()
{
if (ShouldProcess(GetDataLakeItemUriWithoutSas(srcBlob), "Move File: "))
{
// check dest exist
bool destExist = true;
DataLakeFileSystemClient destFileSystem = GetFileSystemClientByName(localChannel, this.DestFileSystem != null ? this.DestFileSystem : this.FileSystem);
DataLakeFileClient destFile = destFileSystem.GetFileClient(this.DestPath);
try
{
destFile.GetProperties();
}
catch (RequestFailedException e) when (e.Status == 404)
{
destExist = false;
}

if (this.Force || !destExist || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destFile)), ""))
if (this.Force || !destFile.Exists() || ShouldContinue(string.Format("Overwrite destination {0}", GetDataLakeItemUriWithoutSas(destFile)), ""))
{
destFile = srcBlob.Rename(this.DestPath, this.DestFileSystem).Value;
WriteDataLakeGen2Item(localChannel, destFile);
Expand Down
60 changes: 27 additions & 33 deletions src/Storage/Storage/DatalakeGen2/Cmdlet/RemoveAzDataLakeGen2Item.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,49 +103,43 @@ protected override void BeginProcessing()
public override void ExecuteCmdlet()
{
IStorageBlobManagement localChannel = Channel;
BlobRequestOptions requestOptions = RequestOptions;

bool foundAFolder = false;

DataLakeFileClient fileClient = null;
DataLakeDirectoryClient dirClient = null;
if (ParameterSetName == ManualParameterSet)
{
DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem);
foundAFolder = GetExistDataLakeGen2Item(fileSystem, this.Path, out fileClient, out dirClient);
}
else //BlobParameterSet
if (ShouldProcess(ParameterSetName == ManualParameterSet ? this.Path : InputObject.Path, "remove"))
{
if (!InputObject.IsDirectory)
if (ParameterSetName == ManualParameterSet)
{
fileClient = InputObject.File;
DataLakeFileSystemClient fileSystem = GetFileSystemClientByName(localChannel, this.FileSystem);
DataLakePathClient pathClient = new DataLakePathClient(fileSystem, this.Path);
if (force || ShouldContinue(string.Format("Remove DatalakeGen2 Item: {0}", GetDataLakeItemUriWithoutSas(pathClient)), ""))
{
pathClient.Delete(true, cancellationToken: this.CmdletCancellationToken);
}
}
else
else //ItemPipeline
{
dirClient = InputObject.Directory;
foundAFolder = true;
if (!InputObject.IsDirectory)
{
DataLakeFileClient fileClient = InputObject.File;
if (force || ShouldContinue(string.Format("Remove File: {0}", GetDataLakeItemUriWithoutSas(fileClient)), ""))
{
fileClient.Delete(cancellationToken: this.CmdletCancellationToken);
}
}
else
{
DataLakeDirectoryClient dirClient = InputObject.Directory;
if (force || ShouldContinue(string.Format("Remove Directory: {0}", GetDataLakeItemUriWithoutSas(dirClient)), ""))
{
dirClient.Delete(true, cancellationToken: this.CmdletCancellationToken);
}
}
}
}

if (foundAFolder)
{
if (force || ShouldContinue(string.Format("Remove Directory: {0}", GetDataLakeItemUriWithoutSas(dirClient)), ""))
{
dirClient.Delete(true);
}
}
else
{
if (force || ShouldContinue(string.Format("Remove File: {0}", GetDataLakeItemUriWithoutSas(fileClient)), ""))
if (PassThru)
{
fileClient.Delete();
WriteObject(true);
}
}

if (PassThru)
{
WriteObject(true);
}
}
}
}

0 comments on commit 4ad1464

Please sign in to comment.