Add FTP import support

ARKBreedingStats/ARKBreedingStats.csproj

@@ -56,6 +56,7 @@
     <Reference Include="System.Configuration" />
     <Reference Include="System.Core" />
     <Reference Include="System.Runtime.Serialization" />
+    <Reference Include="System.Security" />
     <Reference Include="System.Speech" />
     <Reference Include="System.Windows.Forms.DataVisualization" />
     <Reference Include="System.Xml.Linq" />
@@ -121,6 +122,8 @@
       <DesignTime>True</DesignTime>
       <DependentUpon>strings.resx</DependentUpon>
     </Compile>
+    <Compile Include="miscClasses\Encryption.cs" />
+    <Compile Include="miscClasses\FtpCredentials.cs" />
     <Compile Include="miscClasses\IssueNotes.cs" />
     <Compile Include="mods\HandleUnknownMods.cs" />
     <Compile Include="mods\ModInfo.cs" />
@@ -151,8 +154,20 @@
     <Compile Include="settings\ATImportFileLocationDialog.Designer.cs">
       <DependentUpon>ATImportFileLocationDialog.cs</DependentUpon>
     </Compile>
+    <Compile Include="settings\FtpProgress.cs">
+      <SubType>Form</SubType>
+    </Compile>
+    <Compile Include="settings\FtpProgress.Designer.cs">
+      <DependentUpon>FtpProgress.cs</DependentUpon>
+    </Compile>
     <Compile Include="species\ARKColor.cs" />
     <Compile Include="species\ARKColors.cs" />
+    <Compile Include="settings\FtpCredentials.cs">
+      <SubType>Form</SubType>
+    </Compile>
+    <Compile Include="settings\FtpCredentials.Designer.cs">
+      <DependentUpon>FtpCredentials.cs</DependentUpon>
+    </Compile>
     <Compile Include="species\CreatureColors.cs" />
     <Compile Include="multiplierTesting\StatsMultiplierTesting.cs">
       <SubType>UserControl</SubType>
@@ -591,6 +606,12 @@
     <EmbeddedResource Include="settings\customSoundChooser.resx">
       <DependentUpon>customSoundChooser.cs</DependentUpon>
     </EmbeddedResource>
+    <EmbeddedResource Include="settings\FtpProgress.resx">
+      <DependentUpon>FtpProgress.cs</DependentUpon>
+    </EmbeddedResource>
+    <EmbeddedResource Include="settings\FtpCredentials.resx">
+      <DependentUpon>FtpCredentials.cs</DependentUpon>
+    </EmbeddedResource>
     <EmbeddedResource Include="SpeciesSelector.resx">
       <DependentUpon>SpeciesSelector.cs</DependentUpon>
     </EmbeddedResource>
@@ -756,6 +777,9 @@
     </ProjectReference>
   </ItemGroup>
   <ItemGroup>
+    <PackageReference Include="FluentFTP">
+      <Version>27.1.2</Version>
+    </PackageReference>
     <PackageReference Include="Newtonsoft.Json">
       <Version>11.0.2</Version>
     </PackageReference>

ARKBreedingStats/Form1.cs

@@ -1,20 +1,26 @@
 using ARKBreedingStats.duplicates;
 using ARKBreedingStats.Library;
+using ARKBreedingStats.miscClasses;
 using ARKBreedingStats.ocr;
 using ARKBreedingStats.settings;
 using ARKBreedingStats.species;
 using ARKBreedingStats.uiControls;
 using ARKBreedingStats.values;
 
+using FluentFTP;
+using Newtonsoft.Json;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Drawing;
 using System.IO;
+using System.IO.Compression;
 using System.Linq;
+using System.Net;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
+using System.Threading.Tasks;
 using System.Windows.Forms;
 using System.Xml.Serialization;
 
@@ -806,9 +812,27 @@ private async void RunSavegameImport(object sender, EventArgs e)
                 {
                     workingCopyfilename = Path.GetTempPath();
                 }
-                workingCopyfilename = Path.Combine(workingCopyfilename, Path.GetFileName(atImportFileLocation.FileLocation));
 
-                File.Copy(atImportFileLocation.FileLocation, workingCopyfilename, true);
+
+                if (Uri.TryCreate(atImportFileLocation.FileLocation, UriKind.Absolute, out var uri))
+                {
+                    switch(uri.Scheme)
+                    {
+                        case "ftp":
+                            workingCopyfilename = await CopyFtpFileAsync(uri, atImportFileLocation.ConvenientName, workingCopyfilename);
+                            if (workingCopyfilename == null)
+                                // the user didn't enter credentials
+                                return;
+                            break;
+                        default:
+                            throw new Exception($"Unsuppoerted uri scheme: {uri.Scheme}");
+                    }
+                }
+                else
+                {
+                    workingCopyfilename = Path.Combine(workingCopyfilename, Path.GetFileName(atImportFileLocation.FileLocation));
+                    File.Copy(atImportFileLocation.FileLocation, workingCopyfilename, true);
+                }
 
                 await ImportSavegame.ImportCollectionFromSavegame(creatureCollection, workingCopyfilename, atImportFileLocation.ServerName);
 
@@ -845,6 +869,156 @@ private async void RunSavegameImport(object sender, EventArgs e)
             }
         }
 
+        private async Task<string> CopyFtpFileAsync(Uri ftpUri, string serverName, string workingCopyFolder)
+        {
+            var credentialsByServerName = LoadSavedCredentials();
+            credentialsByServerName.TryGetValue(serverName, out var credentials);
+
+            var dialogText = $"Ftp Credentials for {serverName}";
+
+            while (true)
+            {
+                if (credentials == null)
+                {
+                    // get new credentials
+                    using (var dialog = new FtpCredentialsForm { Text = dialogText })
+                    {
+                        if (dialog.ShowDialog(this) == DialogResult.Cancel)
+                        {
+                            return null;
+                        }
+
+                        credentials = dialog.Credentials;
+
+                        if (dialog.SaveCredentials)
+                        {
+                            credentialsByServerName[serverName] = credentials;
+                            Properties.Settings.Default.SavedFtpCredentials = Encryption.Protect(JsonConvert.SerializeObject(credentialsByServerName));
+                            Properties.Settings.Default.Save();
+                        }
+                    }
+                }
+
+                var client = new FtpClient(ftpUri.Host, ftpUri.Port, credentials.Username, credentials.Password);
+
+                var cancellationTokenSource = new CancellationTokenSource();
+                using (var progressDialog = new FtpProgressForm(cancellationTokenSource))
+                {
+                    try
+                    {
+                        progressDialog.StatusText = $"Authenticating";
+                        progressDialog.Show(this);
+
+                        await client.ConnectAsync();
+
+                        progressDialog.StatusText = $"Finding most recent file";
+                        await Task.Yield();
+
+                        var ftpPath = ftpUri.AbsolutePath;
+                        var lastSegment = ftpUri.Segments.Last();
+                        if (lastSegment.Contains("*"))
+                        {
+                            var mostRecentlyModifiedMatch = await GetLastModifiedFileAsync(client, ftpUri, cancellationTokenSource.Token);
+                            if (mostRecentlyModifiedMatch == null)
+                            {
+                                throw new Exception($"No file found matching pattern '{lastSegment}'");
+                            }
+
+                            ftpPath = mostRecentlyModifiedMatch.FullName;
+                        }
+
+                        var fileName = Path.GetFileName(ftpPath);
+
+                        progressDialog.FileName = fileName;
+                        progressDialog.StatusText = $"Downloading {fileName}";
+                        await Task.Yield();
+
+                        var filePath = Path.Combine(workingCopyFolder, Path.GetFileName(ftpPath));
+                        await client.DownloadFileAsync(filePath, ftpPath, FtpLocalExists.Overwrite, FtpVerify.Retry, progressDialog, token: cancellationTokenSource.Token);
+                        await Task.Delay(500);
+
+                        if (filePath.EndsWith(".gz"))
+                        {
+                            progressDialog.StatusText = $"Decompressing {fileName}";
+                            await Task.Yield();
+
+                            filePath = await DecompressGZippedFileAsync(filePath, cancellationTokenSource.Token);
+                        }
+
+                        return filePath;
+                    }
+                    catch (FtpAuthenticationException ex)
+                    {
+                        // if auth fails, clear credentials, alert the user and loop until the either auth succeeds or the user cancels
+                        progressDialog.StatusText = $"Authentication failed: {ex.Message}";
+                        credentials = null;
+                        await Task.Delay(1000);
+                    }
+                    catch(OperationCanceledException)
+                    {
+                        return null;
+                    }
+                    catch(Exception ex)
+                    {
+                        progressDialog.StatusText = $"Unexpected error: {ex.Message}";
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Loads the encrypted ftp crednetials from settings, decrypts them, then returns them as a hostname to credentials dictionary
+        /// </summary>
+        private static Dictionary<string, FtpCredentials> LoadSavedCredentials()
+        {
+            try
+            {
+                var savedCredentials = Encryption.Unprotect(Properties.Settings.Default.SavedFtpCredentials);
+
+                if (!string.IsNullOrEmpty(savedCredentials))
+                {
+                    var savedDictionary = JsonConvert.DeserializeObject<Dictionary<string, FtpCredentials>>(savedCredentials);
+
+                    // Ensure that the resulting dictionary is case insensitive on hostname
+                    return new Dictionary<string, FtpCredentials>(savedDictionary, StringComparer.OrdinalIgnoreCase);
+                }
+            }
+            catch (Exception ex)
+            {
+                MessageBox.Show($"An error occured while loading saved ftp credentials. Message: \n\n{ex.Message}",
+                        "Settings Error", MessageBoxButtons.OK);
+            }
+
+            return new Dictionary<string, FtpCredentials>(StringComparer.OrdinalIgnoreCase);
+        }
+
+        private async Task<string> DecompressGZippedFileAsync(string filePath, CancellationToken cancellationToken)
+        {
+            var newFileName = filePath.Remove(filePath.Length - 3);
+
+            using (var originalFileStream = File.OpenRead(filePath))
+            using (var decompressedFileStream = File.Create(newFileName))
+            using (var decompressionStream = new GZipStream(originalFileStream, CompressionMode.Decompress))
+            {
+                await decompressionStream.CopyToAsync(decompressedFileStream, 81920, cancellationToken);
+            }
+
+            return newFileName;
+        }
+
+        public async Task<FtpListItem> GetLastModifiedFileAsync(FtpClient client, Uri ftpUri, CancellationToken cancellationToken)
+        {
+            var folderUri = new Uri(ftpUri, ".");
+            var listItems = await client.GetListingAsync(folderUri.AbsolutePath, cancellationToken);
+
+            //  Turn the wildcard into a regex pattern   "super*.foo" ->  "^super.*?\.foo$"
+            var nameRegex = new Regex("^" + Regex.Escape(ftpUri.Segments.Last()).Replace(@"\*", ".*?") + "$");
+
+            return listItems
+                .OrderByDescending(x => x.Modified)
+                .FirstOrDefault(x => nameRegex.IsMatch(x.Name));
+        }
+
         /// <summary>
         /// Checks each creature for tags and saves them in a list.
         /// </summary>

ARKBreedingStats/ImportSavegame.cs

@@ -116,7 +116,7 @@ private static void importCollection(CreatureCollection creatureCollection, List
             if (creatureCollection.changeCreatureStatusOnSavegameImport)
             {
                 // mark creatures that are no longer present as unavailable
-                var removedCreatures = creatureCollection.creatures.Where(c => c.status == CreatureStatus.Available).Except(newCreatures);
+                var removedCreatures = creatureCollection.creatures.Where(c => c.status == CreatureStatus.Available && c.server == serverName).Except(newCreatures);
                 foreach (var c in removedCreatures)
                     c.status = CreatureStatus.Unavailable;
 
@@ -130,6 +130,7 @@ private static void importCollection(CreatureCollection creatureCollection, List
             {
                 creature.server = serverName;
             });
+
             creatureCollection.mergeCreatureList(newCreatures, true);
         }
 

ARKBreedingStats/Properties/Settings.settings

@@ -169,6 +169,8 @@
     </Setting>
     <Setting Name="WarnWhenImportingMoreCreaturesThan" Type="System.Int32" Scope="User">
       <Value Profile="(Default)">50</Value>
+    <Setting Name="SavedFtpCredentials" Type="System.String" Scope="User">
+      <Value Profile="(Default)" />
     </Setting>
     <Setting Name="prettifyCollectionJson" Type="System.Boolean" Scope="User">
       <Value Profile="(Default)">False</Value>

ARKBreedingStats/library/CreatureCollection.cs

@@ -218,6 +218,12 @@ public bool mergeCreatureList(List<Creature> creaturesToMerge, bool update = fal
                             creaturesWereAdded = true;
                         }
 
+                        if (old.server != creature.server)
+                        {
+                            old.server = creature.server;
+                            creaturesWereAdded = true;
+                        }
+
                         if (!old.levelsWild.SequenceEqual(creature.levelsWild))
                         {
                             old.levelsWild = creature.levelsWild;