FTP, SFTP timeouts and liveness checks

CHANGELOG.md

@@ -14,7 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add ability to update an extraction's cohort from the command line using `SetExtractionConfigurationCohort ExtractionConfiguration:{id} ExtractableCohort:{id}`
 - Fix issue with non-default named PostgreSQL Table Info not being checkable
 - Improve default timeouts on database lookups
-
+- Implement keepalive and liveness checks for FTP, SFTP fetches before deletion attempt
 
 ## [8.1.6] - 2024-05-27
 

Directory.Packages.props

@@ -22,7 +22,7 @@
 		<PackageVersion Include="Terminal.Gui" Version="1.17.0"/>
 		<PackageVersion Include="YamlDotNet" Version="15.1.6"/>
 		<PackageVersion Include="ConsoleControl" Version="1.3.0"/>
-		<PackageVersion Include="Autoupdater.NET.Official" Version="1.8.6"/>
+		<PackageVersion Include="Autoupdater.NET.Official" Version="1.9.0"/>
 		<PackageVersion Include="DockPanelSuite.ThemeVS2015" Version="3.1.0"/>
 		<PackageVersion Include="System.Threading.ThreadPool" Version="4.3.0"/>
 		<PackageVersion Include="WeCantSpell.Hunspell" Version="5.0.0"/>

Rdmp.Core/DataLoad/Modules/FTP/FTPDownloader.cs

@@ -66,18 +66,18 @@ public FTPDownloader()
     [DemandsInitialization("The directory on the FTP server that you want to download files from")]
     public string? RemoteDirectory { get; set; }
 
-    [DemandsInitialization("True to set keep alive", DefaultValue = true)]
+    [DemandsInitialization("True to set keep alive",DefaultValue = true)]
     public bool KeepAlive { get; set; }
 
 
-    public void Initialize(ILoadDirectory directory, DiscoveredDatabase dbInfo)
+    public void Initialize(ILoadDirectory directory,DiscoveredDatabase dbInfo)
     {
         _directory = directory;
     }
 
-    public ExitCodeType Fetch(IDataLoadJob job, GracefulCancellationToken cancellationToken)
+    public ExitCodeType Fetch(IDataLoadJob job,GracefulCancellationToken cancellationToken)
     {
-        return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"), job);
+        return DownloadFilesOnFTP(_directory ?? throw new InvalidOperationException("No output directory set"),job);
     }
 
     private FtpClient SetupFtp()
@@ -86,32 +86,35 @@ private FtpClient SetupFtp()
         var username = FTPServer.Username ?? "anonymous";
         var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
         var c = new FtpClient(host, username, password);
+
+        // Enable periodic NOOP keepalive operations to keep connection active until we're done
+        c.Config.Noop = true;
         c.AutoConnect();
         return c;
     }
 
-    private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEventListener listener)
+    private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination,IDataLoadEventListener listener)
     {
         var files = GetFileList().ToArray();
 
-        listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
+        listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,
             $"Identified the following files on the FTP server:{string.Join(',',files)}"));
 
         var forLoadingContainedCachedFiles = false;
 
         foreach (var file in files)
         {
-            var action = GetSkipActionForFile(file, destination);
+            var action = GetSkipActionForFile(file,destination);
 
-            listener.OnNotify(this, new NotifyEventArgs(ProgressEventType.Information,
+            listener.OnNotify(this,new NotifyEventArgs(ProgressEventType.Information,
                 $"File {file} was evaluated as {action}"));
 
             switch (action)
             {
                 case SkipReason.DoNotSkip:
                     listener.OnNotify(this,
-                        new NotifyEventArgs(ProgressEventType.Information, $"About to download {file}"));
-                    Download(file, destination);
+                        new NotifyEventArgs(ProgressEventType.Information,$"About to download {file}"));
+                    Download(file,destination);
                     break;
                 case SkipReason.InForLoading:
                     forLoadingContainedCachedFiles = true;
@@ -120,7 +123,7 @@ private ExitCodeType DownloadFilesOnFTP(ILoadDirectory destination, IDataLoadEve
         }
 
         // it was a success - even if no files were actually retrieved... hey that's what the user said, otherwise he would have set SendLoadNotRequiredIfFileNotFound
-        if (forLoadingContainedCachedFiles || _filesRetrieved.Any() || !SendLoadNotRequiredIfFileNotFound)
+        if (forLoadingContainedCachedFiles || _filesRetrieved.Count != 0 || !SendLoadNotRequiredIfFileNotFound)
             return ExitCodeType.Success;
 
         // if no files were downloaded (and there were none skipped because they were in forLoading) and in that eventuality we have our flag set to return LoadNotRequired then do so
@@ -138,7 +141,7 @@ protected enum SkipReason
         IsImaginaryFile
     }
 
-    protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destination)
+    protected SkipReason GetSkipActionForFile(string file,ILoadDirectory destination)
     {
         if (file.StartsWith(".",StringComparison.Ordinal))
             return SkipReason.IsImaginaryFile;
@@ -152,7 +155,7 @@ protected SkipReason GetSkipActionForFile(string file, ILoadDirectory destinatio
     }
 
 
-    private static bool ValidateServerCertificate(object _1, X509Certificate _2, X509Chain _3,
+    private static bool ValidateServerCertificate(object _1,X509Certificate _2,X509Chain _3,
         SslPolicyErrors _4) => true; //any cert will do! yay
 
 
@@ -161,21 +164,24 @@ protected virtual IEnumerable<string> GetFileList()
         return _connection.Value.GetNameListing().ToList().Where(_connection.Value.FileExists);
     }
 
-    protected virtual void Download(string file, ILoadDirectory destination)
+    protected virtual void Download(string file,ILoadDirectory destination)
     {
         var remotePath = !string.IsNullOrWhiteSpace(RemoteDirectory)
             ? $"{RemoteDirectory}/{file}"
             : file;
 
-        var destinationFileName = Path.Combine(destination.ForLoading.FullName, file);
-        _connection.Value.DownloadFile(destinationFileName, remotePath);
+        var destinationFileName = Path.Combine(destination.ForLoading.FullName,file);
+        _connection.Value.DownloadFile(destinationFileName,remotePath);
         _filesRetrieved.Add(remotePath);
     }
 
-    public virtual void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEventListener postLoadEventListener)
+    public virtual void LoadCompletedSoDispose(ExitCodeType exitCode,IDataLoadEventListener postLoadEventListener)
     {
         if (exitCode != ExitCodeType.Success || !DeleteFilesOffFTPServerAfterSuccesfulDataLoad) return;
 
+        // Force a reconnection attempt if we got cut off
+        if (!_connection.Value.IsStillConnected())
+            _connection.Value.Connect(true);
         foreach (var file in _filesRetrieved) _connection.Value.DeleteFile(file);
     }
 
@@ -188,7 +194,7 @@ public void Check(ICheckNotifier notifier)
         }
         catch (Exception e)
         {
-            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP", CheckResult.Fail, e));
+            notifier.OnCheckPerformed(new CheckEventArgs("Failed to SetupFTP",CheckResult.Fail,e));
         }
     }
 }

Rdmp.Core/DataLoad/Modules/FTP/SFTPDownloader.cs

@@ -8,7 +8,6 @@
 using System.IO;
 using System.Linq;
 using System.Threading;
-using FluentFTP;
 using Rdmp.Core.Curation;
 using Rdmp.Core.Curation.Data;
 using Rdmp.Core.ReusableLibraryCode.Progress;
@@ -46,6 +45,8 @@ private SftpClient SetupSftp()
         var password = string.IsNullOrWhiteSpace(FTPServer.Password) ? "guest" : FTPServer.GetDecryptedPassword();
         var c = new SftpClient(host, username, password);
         c.Connect();
+        if (KeepAlive)
+            c.KeepAliveInterval = TimeSpan.FromMilliseconds(KeepAliveIntervalMilliseconds);
         return c;
     }
 
@@ -70,6 +71,10 @@ public override void LoadCompletedSoDispose(ExitCodeType exitCode, IDataLoadEven
     {
         if (exitCode != ExitCodeType.Success) return;
 
+        // Reconnect if we got cut off, for example due to idle timers
+        if (!_connection.Value.IsConnected)
+            _connection.Value.Connect();
+
         foreach (var retrievedFiles in _filesRetrieved)
             try
             {