Initial creation of filesystem scanner for DMLib #21492
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,20 +1,19 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <PropertyGroup> | ||
| <TargetFrameworks>$(RequiredTargetFrameworks)</TargetFrameworks> | ||
| </PropertyGroup> | ||
| <PropertyGroup> | ||
| <AssemblyTitle>Microsoft Azure.Storage.Common.DataMovement client library tests</AssemblyTitle> | ||
| <IsPackable>false</IsPackable> | ||
| </PropertyGroup> | ||
| <ItemGroup> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\src\Azure.Storage.Blobs.DataMovement.csproj" /> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Common.DataMovement\tests\Azure.Storage.Common.DataMovement.Tests.csproj" /> | ||
| </ItemGroup> | ||
| <ItemGroup> | ||
| <Folder Include="SessionRecords\" /> | ||
| <None Include="$(AzureStorageSharedTestSources)\*.xml"> | ||
| <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
| </None> | ||
| </ItemGroup> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| // Copyright (c) Microsoft Corporation. All rights reserved. | ||
| // Licensed under the MIT License. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Text; | ||
|
|
||
| namespace Azure.Storage.Common.DataMovement | ||
| { | ||
| /// <summary> | ||
| /// FilesystemScanner class. | ||
| /// </summary> | ||
| public static class FilesystemScanner | ||
| { | ||
| /// <summary> | ||
| /// Enumerates all files pointed to by the provided path, including those in subdirectories (if path is a directory). | ||
| /// </summary> | ||
| /// <param name="path">Filesystem location.</param> | ||
| /// <returns>Enumerable list of absolute paths containing all relevant files the user has permission to access.</returns> | ||
| public static IEnumerable<string> ScanLocation(string path) | ||
|
There was a problem hiding this comment. There was a problem hiding this comment. #21715 I refactored the scanner to make it non-static and shift a couple things around, let me know if there's any issues with this implementation. |
||
| { | ||
| // Path type is ambiguous at start | ||
| bool isDirectory = false; | ||
|
|
||
| try | ||
| { | ||
| // Make sure we're dealing with absolute, well-formatted path | ||
| path = Path.GetFullPath(path); | ||
|
|
||
| // Check if path points to a directory | ||
| if ((File.GetAttributes(path) & FileAttributes.Directory) == FileAttributes.Directory) | ||
| { | ||
| isDirectory = true; | ||
| } | ||
| } | ||
| catch (Exception) | ||
| { | ||
| // If there's an error here, there aren't any valid entries to scan at the given path; | ||
| // the path is either invalid or nonexistant. In this case, throw the resulting exception. | ||
| // | ||
| // TODO: Logging for invalid path exceptions | ||
| throw; | ||
| } | ||
|
|
||
| // If we're given a directory, parse its children recursively | ||
| if (isDirectory) | ||
| { | ||
| // Create a queue of folders to enumerate files from, starting with provided path | ||
| Queue<string> folders = new(); | ||
| folders.Enqueue(path); | ||
|
|
||
| while (folders.Count > 0) | ||
| { | ||
| // Grab a folder from the queue | ||
| string dir = folders.Dequeue(); | ||
|
|
||
| // Try to enumerate and queue all subdirectories of the current folder | ||
| try | ||
| { | ||
| foreach (string subdir in Directory.EnumerateDirectories(dir)) | ||
| { | ||
| folders.Enqueue(subdir); | ||
| } | ||
| } | ||
| // If we lack permissions to enumerate, skip the folder and continue processing | ||
| // the rest of the queue | ||
| catch (Exception) | ||
| { | ||
| // TODO: Logging for missing permissions to enumerate folder | ||
| if (dir == path) | ||
| { | ||
| // If we can't even enumerate the path supplied by the user, throw | ||
| // the error | ||
| throw; | ||
| } | ||
|
|
||
| // Otherwise, just log the failed subdirectory and continue to list as many | ||
| // files as accessible. Maybe let users decide whether to always throw here? | ||
| continue; | ||
amnguye marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| } | ||
|
|
||
| // Add all files in the directory to be returned | ||
| foreach (string file in Directory.EnumerateFiles(dir)) | ||
| { | ||
| yield return file; | ||
| } | ||
| } | ||
| } | ||
| // Otherwise we can just return the original path | ||
| else | ||
| { | ||
| yield return path; | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Enumerates files pointed to by several paths. | ||
| /// </summary> | ||
| /// <param name="paths">Filesystem locations.</param> | ||
| /// <returns>Enumerable list of absolute paths containing all relevant files the user has permission to access.</returns> | ||
| public static IEnumerable<string> ScanLocations(string[] paths) | ||
| { | ||
| // Redirect all paths provided to ScanLocation(), and collect all results together | ||
| foreach (string path in paths) | ||
| { | ||
| foreach (string file in ScanLocation(path)) | ||
| { | ||
| yield return file; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,20 +1,21 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
| <PropertyGroup> | ||
| <TargetFrameworks>$(RequiredTargetFrameworks)</TargetFrameworks> | ||
| </PropertyGroup> | ||
| <PropertyGroup> | ||
| <AssemblyTitle>Microsoft Azure.Storage.Common.DataMovement client library tests</AssemblyTitle> | ||
| <IsPackable>false</IsPackable> | ||
| </PropertyGroup> | ||
| <ItemGroup> | ||
| <PackageReference Include="Mono.Posix" /> | ||
| <PackageReference Include="System.IO.FileSystem.AccessControl" /> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\src\Azure.Storage.Common.DataMovement.csproj" /> | ||
| <ProjectReference Include="$(MSBuildThisFileDirectory)..\..\Azure.Storage.Common\tests\Azure.Storage.Common.Tests.csproj" /> | ||
| </ItemGroup> | ||
| <ItemGroup> | ||
| <Folder Include="SessionRecords\" /> | ||
| <None Include="$(AzureStorageSharedTestSources)\*.xml"> | ||
| <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> | ||
| </None> | ||
| </ItemGroup> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should check if we can introduce these dependencies first.
My guess is that we can't as they appear to be 3rd party.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If these are only for testing we can add them in test project file directly.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
#21715 I took the two dependencies I introduced for testing out of this file and used VersionOverride in the .csproj. The error spooked me and override sounded scary.
I can't tell if the Mono.Posix is/isn't 3rd party, since Microsoft is listed as an owner alongside some other users. There was another package called Mono.Posix.NETStandard with only Microsoft as an owner, but for some reason, that package was searching for the assembly "Mono.Posix". I couldn't think of anything to get that one to properly find the assembly without potentially causing other issues.
Do you think I should drop the dependency and maybe just execute bash + call chmod manually? I fiddled with manually loading the c library for chmod, but I couldn't do it with .NET Core 2.1 as a target.