-
Notifications
You must be signed in to change notification settings - Fork 4.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Initial creation of filesystem scanner for DMLib (#21492)
- Loading branch information
Showing
11 changed files
with
350 additions
and
22 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
17 changes: 8 additions & 9 deletions
17
...rage/Azure.Storage.Blobs.DataMovement/tests/Azure.Storage.Blobs.DataMovement.Tests.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,20 +1,19 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFrameworks>$(RequiredTargetFrameworks)</TargetFrameworks> | ||
| </PropertyGroup> | ||
| <PropertyGroup> | ||
| <AssemblyTitle>Microsoft Azure.Storage.Common.DataMovement client library tests</AssemblyTitle> | ||
| <IsPackable>false</IsPackable> | ||
| <RootNamespace>Azure.Storage.Tests</RootNamespace> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\src\Azure.Storage.Blobs.DataMovement.csproj" /> | ||
| <ProjectReference Include="..\..\Azure.Storage.Common.DataMovement\tests\Azure.Storage.Common.DataMovement.Tests.csproj" /> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Common.DataMovement\tests\Azure.Storage.Common.DataMovement.Tests.csproj" /> | ||
| </ItemGroup> | ||
| <!-- Ensure an empty TestConfigurations.xml is always present so the build can copy it --> | ||
| <Target Name="TouchTestConfigurationsFile" BeforeTargets="PreBuildEvent"> | ||
| <Touch Files="$(MSBuildThisFileDirectory)Shared\TestConfigurations.xml" AlwaysCreate="True" ContinueOnError="WarnAndContinue" /> | ||
| </Target> | ||
| </Project> | ||
| <ItemGroup> | ||
| <Folder Include="SessionRecords\" /> | ||
| <None Include="$(AzureStorageSharedTestSources)\*.xml"> | ||
| <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
| </None> | ||
| </ItemGroup> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
114 changes: 114 additions & 0 deletions
114
sdk/storage/Azure.Storage.Common.DataMovement/src/FilesystemScanner.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Text; | ||
|
|
||
| namespace Azure.Storage.Common.DataMovement | ||
| { | ||
| /// <summary> | ||
| /// FilesystemScanner class. | ||
| /// </summary> | ||
| public static class FilesystemScanner | ||
| { | ||
| /// <summary> | ||
| /// Enumerates all files pointed to by the provided path, including those in subdirectories (if path is a directory). | ||
| /// </summary> | ||
| /// <param name="path">Filesystem location.</param> | ||
| /// <returns>Enumerable list of absolute paths containing all relevant files the user has permission to access.</returns> | ||
| public static IEnumerable<string> ScanLocation(string path) | ||
| { | ||
| // Path type is ambiguous at start | ||
| bool isDirectory = false; | ||
|
|
||
| try | ||
| { | ||
| // Make sure we're dealing with absolute, well-formatted path | ||
| path = Path.GetFullPath(path); | ||
|
|
||
| // Check if path points to a directory | ||
| if ((File.GetAttributes(path) & FileAttributes.Directory) == FileAttributes.Directory) | ||
| { | ||
| isDirectory = true; | ||
| } | ||
| } | ||
| catch (Exception) | ||
| { | ||
| // If there's an error here, there aren't any valid entries to scan at the given path; | ||
| // the path is either invalid or nonexistant. In this case, throw the resulting exception. | ||
| // | ||
| // TODO: Logging for invalid path exceptions | ||
| throw; | ||
| } | ||
|
|
||
| // If we're given a directory, parse its children recursively | ||
| if (isDirectory) | ||
| { | ||
| // Create a queue of folders to enumerate files from, starting with provided path | ||
| Queue<string> folders = new(); | ||
| folders.Enqueue(path); | ||
|
|
||
| while (folders.Count > 0) | ||
| { | ||
| // Grab a folder from the queue | ||
| string dir = folders.Dequeue(); | ||
|
|
||
| // Try to enumerate and queue all subdirectories of the current folder | ||
| try | ||
| { | ||
| foreach (string subdir in Directory.EnumerateDirectories(dir)) | ||
| { | ||
| folders.Enqueue(subdir); | ||
| } | ||
| } | ||
| // If we lack permissions to enumerate, skip the folder and continue processing | ||
| // the rest of the queue | ||
| catch (Exception) | ||
| { | ||
| // TODO: Logging for missing permissions to enumerate folder | ||
| if (dir == path) | ||
| { | ||
| // If we can't even enumerate the path supplied by the user, throw | ||
| // the error | ||
| throw; | ||
| } | ||
|
|
||
| // Otherwise, just log the failed subdirectory and continue to list as many | ||
| // files as accessible. Maybe let users decide whether to always throw here? | ||
| continue; | ||
| } | ||
|
|
||
| // Add all files in the directory to be returned | ||
| foreach (string file in Directory.EnumerateFiles(dir)) | ||
| { | ||
| yield return file; | ||
| } | ||
| } | ||
| } | ||
| // Otherwise we can just return the original path | ||
| else | ||
| { | ||
| yield return path; | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Enumerates files pointed to by several paths. | ||
| /// </summary> | ||
| /// <param name="paths">Filesystem locations.</param> | ||
| /// <returns>Enumerable list of absolute paths containing all relevant files the user has permission to access.</returns> | ||
| public static IEnumerable<string> ScanLocations(string[] paths) | ||
| { | ||
| // Redirect all paths provided to ScanLocation(), and collect all results together | ||
| foreach (string path in paths) | ||
| { | ||
| foreach (string file in ScanLocation(path)) | ||
| { | ||
| yield return file; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
17 changes: 9 additions & 8 deletions
17
...ge/Azure.Storage.Common.DataMovement/tests/Azure.Storage.Common.DataMovement.Tests.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,20 +1,21 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFrameworks>$(RequiredTargetFrameworks)</TargetFrameworks> | ||
| </PropertyGroup> | ||
| <PropertyGroup> | ||
| <AssemblyTitle>Microsoft Azure.Storage.Common.DataMovement client library tests</AssemblyTitle> | ||
| <IsPackable>false</IsPackable> | ||
| <RootNamespace>Azure.Storage.Tests</RootNamespace> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Mono.Posix" /> | ||
| <PackageReference Include="System.IO.FileSystem.AccessControl" /> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\src\Azure.Storage.Common.DataMovement.csproj" /> | ||
| <ProjectReference Include="..\..\Azure.Storage.Common\tests\Azure.Storage.Common.Tests.csproj" /> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Common\tests\Azure.Storage.Common.Tests.csproj" /> | ||
| </ItemGroup> | ||
| <ItemGroup> | ||
| <Folder Include="SessionRecords\" /> | ||
| <None Include="$(AzureStorageSharedTestSources)\*.xml"> | ||
| <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
| </None> | ||
| </ItemGroup> | ||
| <!-- Ensure an empty TestConfigurations.xml is always present so the build can copy it --> | ||
| <Target Name="TouchTestConfigurationsFile" BeforeTargets="PreBuildEvent"> | ||
| <Touch Files="$(MSBuildThisFileDirectory)Shared\TestConfigurations.xml" AlwaysCreate="True" ContinueOnError="WarnAndContinue" /> | ||
| </Target> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.