From 152793d78f2230048d1d75ba41b56ed0137d92bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 15:35:29 +0100 Subject: [PATCH 1/4] Bump HIC.FAnsiSql from 3.2.3 to 3.2.4 (#1850) Bumps [HIC.FAnsiSql](https://github.com/HicServices/FAnsiSql) from 3.2.3 to 3.2.4. - [Release notes](https://github.com/HicServices/FAnsiSql/releases) - [Changelog](https://github.com/HicServices/FAnsiSql/blob/main/CHANGELOG.md) - [Commits](https://github.com/HicServices/FAnsiSql/compare/v3.2.3...v3.2.4) --- updated-dependencies: - dependency-name: HIC.FAnsiSql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 00a7797c86..c77cd10f6d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,7 +5,7 @@ - + all From 88446767969820de819614ad1957920ce88624d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 15:39:44 +0100 Subject: [PATCH 2/4] Bump HIC.FAnsiSql from 3.2.4 to 3.2.5 (#1852) Bumps [HIC.FAnsiSql](https://github.com/HicServices/FAnsiSql) from 3.2.4 to 3.2.5. - [Release notes](https://github.com/HicServices/FAnsiSql/releases) - [Changelog](https://github.com/HicServices/FAnsiSql/blob/main/CHANGELOG.md) - [Commits](https://github.com/HicServices/FAnsiSql/compare/v3.2.4...v3.2.5) --- updated-dependencies: - dependency-name: HIC.FAnsiSql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index c77cd10f6d..802be23915 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,7 +5,7 @@ - + all From 87eba65376ae6dbe707d018200d311104a8bb3d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 01:11:57 +0000 Subject: [PATCH 3/4] Bump Autoupdater.NET.Official from 1.8.6 to 1.9.0 Bumps [Autoupdater.NET.Official](https://github.com/ravibpatel/AutoUpdater.NET) from 1.8.6 to 1.9.0. - [Release notes](https://github.com/ravibpatel/AutoUpdater.NET/releases) - [Commits](https://github.com/ravibpatel/AutoUpdater.NET/compare/v1.8.6...v1.9.0) --- updated-dependencies: - dependency-name: Autoupdater.NET.Official dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Packages.props b/Directory.Packages.props index 802be23915..202b6dbad1 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -22,7 +22,7 @@ - + From f739d6d048e36e8f601a271ebf28f0cedbdc1143 Mon Sep 17 00:00:00 2001 From: James A Sutherland Date: Fri, 14 Jun 2024 14:19:08 -0500 Subject: [PATCH 4/4] FTP, SFTP timeouts and liveness checks --- CHANGELOG.md | 4 ++ .../DataLoad/Modules/FTP/FTPDownloader.cs | 42 +++++++++++-------- .../DataLoad/Modules/FTP/SFTPDownloader.cs | 7 +++- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a92cb4b639..046631813a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +- Implement keepalive and liveness checks for FTP, SFTP fetches before deletion attempt + ## [8.1.6] - 2024-05-27 ## Changed diff --git a/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs b/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs index c90631e536..1f9246a7b4 100644 --- a/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs +++ b/Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs @@ -66,18 +66,18 @@ public FTPDownloader() [DemandsInitialization("The directory on the FTP server that you want to download files from")] public string? RemoteDirectory { get; set; } - [DemandsInitialization("True to set keep alive", DefaultValue = true)] + [DemandsInitialization("True to set keep alive",DefaultValue = true)] public bool KeepAlive { get; set; } - public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo) + public void Initialize(ILoadDirectory directory,DiscoveredDatabase dbInfo) { _directory = directory; } - public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken) + public ExitCodeType Fetch(IDataLoadJob job,GracefulCancellationToken cancellationToken) { - return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job); + return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"),job); } private FtpClient SetupFtp() @@ -86,32 +86,35 @@ private FtpClient SetupFtp() var username = FTPServer.Username ?? "anonymous"; var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword(); var c = new FtpClient(host, username, password); + + // Enable periodic NOOP keepalive operations to keep connection active until we're done + c.Config.Noop = true; c.AutoConnect(); return c; } - private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener) + private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination,IDataLoadEventListener listener) { var files = GetFileList().ToArray(); - listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, + listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information, $"Identified the following files on the FTP server:{string.Join(',',files)}")); var forLoadingContainedCachedFiles = false; foreach (var file in files) { - var action = GetSkipActionForFile(file, destination); + var action = GetSkipActionForFile(file,destination); - listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information, + listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information, $"File {file} was evaluated as {action}")); switch (action) { case SkipReason.DoNotSkip: listener.OnNotify(this, - new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}")); - Download(file, destination); + new NotifyEventArgs(ProgressEventType.Information,$"About to download {file}")); + Download(file,destination); break; case SkipReason.InForLoading: forLoadingContainedCachedFiles = true; @@ -120,7 +123,7 @@ private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEve } // it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound - if (forLoadingContainedCachedFiles || _filesRetrieved.Any() || !SendLoadNotRequiredIfFileNotFound) + if (forLoadingContainedCachedFiles || _filesRetrieved.Count != 0 || !SendLoadNotRequiredIfFileNotFound) return ExitCodeType.Success; // if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so @@ -138,7 +141,7 @@ protected enum SkipReason IsImaginaryFile } - protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination) + protected SkipReason GetSkipActionForFile(string file,ILoadDirectory destination) { if (file.StartsWith(".",StringComparison.Ordinal)) return SkipReason.IsImaginaryFile; @@ -152,7 +155,7 @@ protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destinatio } - private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3, + private static bool ValidateServerCertificate(object _1,X509Certificate _2,X509Chain _3, SslPolicyErrors _4) => true; //any cert will do! yay @@ -161,21 +164,24 @@ protected virtual IEnumerable GetFileList() return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists); } - protected virtual void Download(string file, ILoadDirectory destination) + protected virtual void Download(string file,ILoadDirectory destination) { var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory) ? $"{RemoteDirectory}/{file}" : file; - var destinationFileName = Path.Combine(destination.ForLoading.FullName, file); - _connection.Value.DownloadFile(destinationFileName, remotePath); + var destinationFileName = Path.Combine(destination.ForLoading.FullName,file); + _connection.Value.DownloadFile(destinationFileName,remotePath); _filesRetrieved.Add(remotePath); } - public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener) + public virtual void LoadCompletedSoDispose(ExitCodeType exitCode,IDataLoadEventListener postLoadEventListener) { if (exitCode != ExitCodeType.Success || !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return; + // Force a reconnection attempt if we got cut off + if (!_connection.Value.IsStillConnected()) + _connection.Value.Connect(true); foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file); } @@ -188,7 +194,7 @@ public void Check(ICheckNotifier notifier) } catch (Exception e) { - notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e)); + notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP",CheckResult.Fail,e)); } } } \ No newline at end of file diff --git a/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs b/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs index 8c489cf276..a33972c609 100644 --- a/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs +++ b/Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs @@ -8,7 +8,6 @@ using System.IO; using System.Linq; using System.Threading; -using FluentFTP; using Rdmp.Core.Curation; using Rdmp.Core.Curation.Data; using Rdmp.Core.ReusableLibraryCode.Progress; @@ -46,6 +45,8 @@ private SftpClient SetupSftp() var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword(); var c = new SftpClient(host, username, password); c.Connect(); + if (KeepAlive) + c.KeepAliveInterval = TimeSpan.FromMilliseconds(KeepAliveIntervalMilliseconds); return c; } @@ -70,6 +71,10 @@ public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEven { if (exitCode != ExitCodeType.Success) return; + // Reconnect if we got cut off, for example due to idle timers + if (!_connection.Value.IsConnected) + _connection.Value.Connect(); + foreach (var retrievedFiles in _filesRetrieved) try {