From a8542c56756c57270bbf9421f83c37a9b21356f2 Mon Sep 17 00:00:00 2001 From: Koen Zomers Date: Mon, 8 Apr 2024 11:17:35 +0200 Subject: [PATCH] Adding upload entire local folder to SharePoint Online into `Copy-PnPFolder` (#3850) * Adding functionality to allow a local folder with all its files and optionally recursed subfolders to be uploaded to SharePoint Online * Added PR reference * Typo fix * Adding verbose parameter * Fixing syntax issue * Added that empty folders will also be removed when providing -RemoveAfterCopy * Updated help text to reflect folders being deleted now as well --------- Co-authored-by: Gautam Sheth --- CHANGELOG.md | 3 +- documentation/Copy-PnPFolder.md | 74 +++++++- src/Commands/Files/CopyFile.cs | 1 - src/Commands/Files/CopyFolder.cs | 293 +++++++++++++++++++++++++++++++ 4 files changed, 362 insertions(+), 9 deletions(-) create mode 100644 src/Commands/Files/CopyFolder.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 699a72b15..fd457c25e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Added `-UseVersionExpirationReport` parameter to `Get-PnPFileVersion` cmdlet to get the version expiration report for a single file. [#3799](https://github.com/pnp/powershell/pull/3799) - Added `-DelayDenyAddAndCustomizePagesEnforcement` parameter to `Set-PnPTenant` cmdlet which allows delay of the change to custom script set on the Tenant until mid-November 2024. [#3815](https://github.com/pnp/powershell/pull/3815) - Added additional permissions for Graph application permission validate sets. [#3835](https://github.com/pnp/powershell/issues/3835) -- Added `LoopDefaultSharingLinkRole`, `DefaultShareLinkScope`, `DefaultShareLinkRole`, `LoopDefaultSharingLinkScope` and `DefaultLinkToExistingAccessReset` parameters to `Set-PnPTenant` cmdlet +- Added the ability to upload entire local folders with files and optionally subfolders to SharePoint Online into 'Copy-PnPFolder' [#3850](https://github.com/pnp/powershell/pull/3850) +- Added `LoopDefaultSharingLinkRole`, `DefaultShareLinkScope`, `DefaultShareLinkRole`, `LoopDefaultSharingLinkScope` and `DefaultLinkToExistingAccessReset` parameters to `Set-PnPTenant` cmdlet. [#3874](https://github.com/pnp/powershell/pull/3874) ### Fixed diff --git a/documentation/Copy-PnPFolder.md b/documentation/Copy-PnPFolder.md index dc41eba01..007129fae 100644 --- a/documentation/Copy-PnPFolder.md +++ b/documentation/Copy-PnPFolder.md @@ -10,21 +10,32 @@ title: Copy-PnPFolder # Copy-PnPFolder ## SYNOPSIS -Copies a folder or file to a different location +Copies a folder or file to a different location within SharePoint Online or allows uploading of an entire local folder with optionally subfolders to SharePoint Online. ## SYNTAX +### Copy files within Microsoft 365 + +```powershell +Copy-PnPFolder -SourceUrl -TargetUrl [-Overwrite] [-Force] [-IgnoreVersionHistory] [-NoWait] [-Connection ] [-Verbose] + +``` + +### Copy files from local to Microsoft 365 + ```powershell -Copy-PnPFolder [-SourceUrl] [-TargetUrl] [-Overwrite] [-Force] [-IgnoreVersionHistory] [-NoWait] [-Connection ] +Copy-PnPFolder -LocalPath -TargetUrl [-Overwrite] [-Recurse] [-RemoveAfterCopy] [-Connection ] [-Verbose] ``` ## DESCRIPTION -Copies a folder or file to a different location. This location can be within the same document library, same site, same site collection or even to another site collection on the same tenant. Notice that if copying between sites or to a subsite you cannot specify a target filename, only a folder name. +Copies a folder or file to a different location within SharePoiint. This location can be within the same document library, same site, same site collection or even to another site collection on the same tenant. Notice that if copying between sites or to a subsite you cannot specify a target filename, only a folder name. Copying files and folders is bound to some restrictions. You can find more on it here: https://learn.microsoft.com/office365/servicedescriptions/sharepoint-online-service-description/sharepoint-online-limits#moving-and-copying-across-sites +It can also accommodate copying an entire folder with all its files and optionally even subfolders and files from a local path onto SharePoint Online. + ## EXAMPLES ### EXAMPLE 1 @@ -109,6 +120,13 @@ if($jobStatus.JobState == 0) Copies a file named company.docx from the current document library to the documents library in SubSite2. It will not wait for the action to return but returns job information instead. The Receive-PnPCopyMoveJobStatus cmdlet will return the job status. +### EXAMPLE 12 +```powershell +Copy-PnPFolder -LocalPath "c:\temp" -TargetUrl "Subsite1/Shared Documents" -Recurse -Overwrite +``` + +Copies all the files and underlying folders from the local folder c:\temp to the document library Shared Documents in Subsite1. If a file already exists, it will be overwritten. + ## PARAMETERS ### -Force @@ -116,7 +134,7 @@ If provided, no confirmation will be requested and the action will be performed ```yaml Type: SwitchParameter -Parameter Sets: (All) +Parameter Sets: WITHINM365 Required: False Position: Named @@ -130,7 +148,7 @@ If provided, only the latest version of the document will be copied and its hist ```yaml Type: SwitchParameter -Parameter Sets: (All) +Parameter Sets: WITHINM365 Required: False Position: Named @@ -140,7 +158,7 @@ Accept wildcard characters: False ``` ### -Overwrite -If provided, if a file already exists at the TargetUrl, it will be overwritten. If omitted, the copy operation will be canceled if the file already exists at the TargetUrl location. +If provided, if a file already exists at the TargetUrl, it will be overwritten. If omitted, the copy operation will be canceled if the file already exists at the TargetUrl location when copying between two locations on SharePoint Online. If copying files from a local path to SharePoint Online, it will skip any file that already exists and still continue with the next one. ```yaml Type: SwitchParameter @@ -158,7 +176,7 @@ Site or server relative URL specifying the file or folder to copy. Must include ```yaml Type: String -Parameter Sets: (All) +Parameter Sets: WITHINM365 Aliases: SiteRelativeUrl, ServerRelativeUrl Required: True @@ -186,6 +204,48 @@ Accept wildcard characters: False ### -NoWait If specified the task will return immediately after creating the copy job. The cmdlet will return a job object which can be used with Receive-PnPCopyMoveJobStatus to retrieve the status of the job. +```yaml +Type: SwitchParameter +Parameter Sets: WITHINM365 + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Recurse +When copying files from a local folder to SharePoint Online, this parameter will copy all files and folders within the local folder and all of its subfolders as well. + +```yaml +Type: SwitchParameter +Parameter Sets: FROMLOCAL + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -RemoveAfterCopy +When copying files from a local folder to SharePoint Online, this parameter will remove all files locally that have successfully been uploaded to SharePoint Online. If a file fails, it will not be removed locally. Local folders will be removed after all files have been uploaded and the folder is empty. + +```yaml +Type: SwitchParameter +Parameter Sets: FROMLOCAL + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + +### -Verbose +When provided, additional debug statements will be shown while executing the cmdlet. + ```yaml Type: SwitchParameter Parameter Sets: (All) diff --git a/src/Commands/Files/CopyFile.cs b/src/Commands/Files/CopyFile.cs index 12dc64ca6..bc2e6d9b6 100644 --- a/src/Commands/Files/CopyFile.cs +++ b/src/Commands/Files/CopyFile.cs @@ -7,7 +7,6 @@ namespace PnP.PowerShell.Commands.Files { - [Alias("Copy-PnPFolder")] [Cmdlet(VerbsCommon.Copy, "PnPFile")] public class CopyFile : PnPWebCmdlet { diff --git a/src/Commands/Files/CopyFolder.cs b/src/Commands/Files/CopyFolder.cs new file mode 100644 index 000000000..d94ffc3d5 --- /dev/null +++ b/src/Commands/Files/CopyFolder.cs @@ -0,0 +1,293 @@ +using System; +using System.Linq; +using System.Management.Automation; +using Microsoft.SharePoint.Client; +using Resources = PnP.PowerShell.Commands.Properties.Resources; +using PnP.Framework.Utilities; +using System.Collections.Generic; + +namespace PnP.PowerShell.Commands.Files +{ + [Cmdlet(VerbsCommon.Copy, "PnPFolder", DefaultParameterSetName = ParameterSet_WITHINM365)] + public class CopyFolder : PnPWebCmdlet + { + private const string ParameterSet_WITHINM365 = "Copy files within Microsoft 365"; + private const string ParameterSet_FROMLOCAL = "Copy files from local to Microsoft 365"; + + [Parameter(Mandatory = true, Position = 0, ValueFromPipeline = true, ParameterSetName = ParameterSet_WITHINM365)] + [Alias("ServerRelativeUrl")] + public string SourceUrl = string.Empty; + + [Parameter(Mandatory = true, Position = 0, ParameterSetName = ParameterSet_FROMLOCAL)] + public string LocalPath = string.Empty; + + [Parameter(Mandatory = true, Position = 1, ParameterSetName = ParameterSet_WITHINM365)] + [Parameter(Mandatory = true, Position = 1, ParameterSetName = ParameterSet_FROMLOCAL)] + [Alias("TargetServerRelativeUrl")] + public string TargetUrl = string.Empty; + + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_WITHINM365)] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FROMLOCAL)] + [Alias("OverwriteIfAlreadyExists")] + public SwitchParameter Overwrite; + + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_WITHINM365)] + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FROMLOCAL)] + public SwitchParameter Force; + + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_WITHINM365)] + public SwitchParameter IgnoreVersionHistory; + + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_WITHINM365)] + public SwitchParameter AllowSchemaMismatch; + + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_WITHINM365)] + public SwitchParameter NoWait; + + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FROMLOCAL)] + public SwitchParameter Recurse; + + [Parameter(Mandatory = false, ParameterSetName = ParameterSet_FROMLOCAL)] + public SwitchParameter RemoveAfterCopy; + + protected override void ExecuteCmdlet() + { + var webServerRelativeUrl = CurrentWeb.EnsureProperty(w => w.ServerRelativeUrl); + + switch(ParameterSetName) + { + case ParameterSet_FROMLOCAL: + // Copy a folder from local to Microsoft 365 + CopyFromLocalToMicrosoft365(); + break; + + case ParameterSet_WITHINM365: + // Copy a folder within Microsoft 365 + CopyWithinMicrosoft365(); + break; + } + } + + /// + /// Copies a folder from local to Microsoft 365 + /// + private void CopyFromLocalToMicrosoft365() + { + if (TargetUrl.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase)) + { + // Remove the server part from the full FQDN making it server relative + TargetUrl = TargetUrl[8..]; + TargetUrl = TargetUrl[TargetUrl.IndexOf('/')..]; + } + + if (!TargetUrl.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) && TargetUrl.StartsWith(CurrentWeb.ServerRelativeUrl, StringComparison.InvariantCultureIgnoreCase)) + { + // Remove the server relative path making it web relative + TargetUrl = TargetUrl[CurrentWeb.ServerRelativeUrl.Length..]; + } + + if (!System.IO.Path.IsPathRooted(LocalPath)) + { + LocalPath = System.IO.Path.Combine(SessionState.Path.CurrentFileSystemLocation.Path, LocalPath); + } + + // Normalize the path, taking out relative links and such to make it more readable + LocalPath = System.IO.Path.GetFullPath(new Uri(LocalPath).LocalPath); + + if (!System.IO.Directory.Exists(LocalPath)) + { + throw new PSArgumentException($"{nameof(LocalPath)} does not exist", nameof(LocalPath)); + } + + WriteVerbose($"Copying folder from local path {LocalPath} to Microsoft 365 location {UrlUtility.Combine(CurrentWeb.ServerRelativeUrl, TargetUrl)}"); + WriteVerbose($"Retrieving local files {(Recurse.ToBool() ? "recursively " : "")}to upload from {LocalPath}"); + + var filesToCopy = System.IO.Directory.GetFiles(LocalPath, string.Empty, Recurse.ToBool() ? System.IO.SearchOption.AllDirectories : System.IO.SearchOption.TopDirectoryOnly); + + WriteVerbose($"Uploading {filesToCopy.Length} file{(filesToCopy.Length != 1 ? "s" : "")}"); + + // Start with the root + string currentRemotePath = null; + Folder folder = null; + var folderPathsToRemove = new List(); + + foreach(var fileToCopy in filesToCopy) + { + var fileName = System.IO.Path.GetFileName(fileToCopy); + var relativeLocalPathWithFileName = System.IO.Path.GetRelativePath(LocalPath, fileToCopy); + var relativePath = relativeLocalPathWithFileName.Remove(relativeLocalPathWithFileName.Length - fileName.Length); + + // Check if we're dealing with a different subfolder now as we did during the previous cycle + if(relativePath != currentRemotePath) + { + // Check files should be removed after uploading and if so, if the previous folder is empty and should be removed. Skip the root folder, that should never be removed. + if(RemoveAfterCopy.ToBool() && currentRemotePath != null) + { + // Add the folder to be examined at the end of the upload session. If we would do it now, we could still have subfolders of which the files have not been uploaded yet, so we cannot delete the folder yet + var localFolder = System.IO.Path.Combine(LocalPath, relativePath); + folderPathsToRemove.Add(localFolder); + } + + // New subfolder, ensure the folder exists remotely as well + currentRemotePath = relativePath; + + var newRemotePath = UrlUtility.Combine(TargetUrl, relativePath); + + WriteVerbose($"* Ensuring remote folder {newRemotePath}"); + + folder = CurrentWeb.EnsureFolderPath(newRemotePath); + } + + // Upload the file from local to remote + WriteVerbose($" * Uploading {fileToCopy} => {relativeLocalPathWithFileName}"); + try + { + folder.UploadFile(fileName, fileToCopy, Overwrite.ToBool()); + + if(RemoveAfterCopy.ToBool()) + { + WriteVerbose($" * Removing {fileToCopy}"); + System.IO.File.Delete(fileToCopy); + } + } + catch(Exception ex) + { + WriteWarning($"* Upload failed: {ex.Message}"); + } + } + + // Check if we should and can clean up folders no longer containing files + if (RemoveAfterCopy.ToBool() && folderPathsToRemove.Count > 0) + { + WriteVerbose($"Checking if {folderPathsToRemove.Count} folder{(folderPathsToRemove.Count != 1 ? "s" : "")} are empty and can be removed"); + + // Reverse the list so we start with the deepest nested folder first + folderPathsToRemove.Reverse(); + + foreach (var folderPathToRemove in folderPathsToRemove) + { + if (System.IO.Directory.GetFiles(folderPathToRemove).Length == 0) + { + WriteVerbose($"* Removing empty folder {folderPathToRemove}"); + try + { + System.IO.Directory.Delete(folderPathToRemove); + } + catch(Exception ex) + { + WriteWarning($"* Failed to remove empty folder {folderPathToRemove}: {ex.Message}"); + } + } + else + { + WriteVerbose($"* Folder {folderPathToRemove} is not empty and thus will not be removed"); + } + } + } + + } + + /// + /// Copies a folder within Microsoft 365 + /// + private void CopyWithinMicrosoft365() + { + if (!TargetUrl.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) && !TargetUrl.StartsWith("/")) + { + TargetUrl = UrlUtility.Combine(CurrentWeb.ServerRelativeUrl, TargetUrl); + } + + if (!SourceUrl.StartsWith("/")) + { + SourceUrl = UrlUtility.Combine(CurrentWeb.ServerRelativeUrl, SourceUrl); + } + + WriteVerbose($"Copying folder within Microsoft 365 from {SourceUrl} to {TargetUrl}"); + + string sourceFolder = SourceUrl.Substring(0, SourceUrl.LastIndexOf('/')); + string targetFolder = TargetUrl; + if (System.IO.Path.HasExtension(TargetUrl)) + { + targetFolder = TargetUrl[..TargetUrl.LastIndexOf('/')]; + } + Uri currentContextUri = new(ClientContext.Url); + Uri sourceUri = new(currentContextUri, EncodePath(sourceFolder)); + Uri sourceWebUri = Web.WebUrlFromFolderUrlDirect(ClientContext, sourceUri); + Uri targetUri = new(currentContextUri, EncodePath(targetFolder)); + Uri targetWebUri; + if (TargetUrl.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase)) + { + targetUri = new Uri(TargetUrl); + targetWebUri = targetUri; + } + else + { + targetWebUri = Microsoft.SharePoint.Client.Web.WebUrlFromFolderUrlDirect(ClientContext, targetUri); + } + + if (Force || ShouldContinue(string.Format(Resources.CopyFile0To1, SourceUrl, TargetUrl), Resources.Confirm)) + { + if (sourceWebUri != targetWebUri) + { + Copy(currentContextUri, sourceUri, targetUri, SourceUrl, TargetUrl, false, NoWait); + } + else + { + var isFolder = false; + try + { + var folder = CurrentWeb.GetFolderByServerRelativePath(ResourcePath.FromDecodedUrl(TargetUrl)); + var folderServerRelativePath = folder.EnsureProperty(f => f.ServerRelativePath); + isFolder = folderServerRelativePath.DecodedUrl == ResourcePath.FromDecodedUrl(TargetUrl).DecodedUrl; + } + catch + { + } + if (isFolder) + { + Copy(currentContextUri, sourceUri, targetUri, SourceUrl, TargetUrl, true, NoWait); + } + else + { + var file = CurrentWeb.GetFileByServerRelativePath(ResourcePath.FromDecodedUrl(SourceUrl)); + file.CopyToUsingPath(ResourcePath.FromDecodedUrl(TargetUrl), Overwrite); + ClientContext.ExecuteQueryRetry(); + } + } + } + } + + private string EncodePath(string path) + { + var parts = path.Split("/"); + return string.Join("/", parts.Select(p => Uri.EscapeDataString(p))); + } + + private void Copy(Uri currentContextUri, Uri source, Uri destination, string sourceUrl, string targetUrl, bool sameWebCopyMoveOptimization, bool noWait) + { + if (!sourceUrl.StartsWith(source.ToString())) + { + sourceUrl = $"{source.Scheme}://{source.Host}/{sourceUrl.TrimStart('/')}"; + } + if (!targetUrl.StartsWith("https://", StringComparison.InvariantCultureIgnoreCase) && !targetUrl.StartsWith(destination.ToString())) + { + targetUrl = $"{destination.Scheme}://{destination.Host}/{targetUrl.TrimStart('/')}"; + } + var results = Utilities.CopyMover.CopyAsync(HttpClient, ClientContext, currentContextUri, sourceUrl, targetUrl, IgnoreVersionHistory, Overwrite, AllowSchemaMismatch, sameWebCopyMoveOptimization, false, noWait).GetAwaiter().GetResult(); + if (NoWait) + { + WriteObject(results.jobInfo); + } + else + { + foreach (var log in results.logs) + { + if (log.Event == "JobError") + { + WriteObject(log); + } + } + } + } + } +}