Auto-epoch based on queue message attribute

Core/Versioning/ModuleVersion.cs

@@ -59,6 +59,14 @@ public ModuleVersion(string version)
             _string = version;
         }
 
+        /// <returns>
+        /// New module version with same version as 'this' but with one greater epoch
+        /// </returns>
+        public ModuleVersion IncrementEpoch()
+        {
+            return new ModuleVersion($"{_epoch + 1}:{_version}");
+        }
+
         /// <summary>
         /// Converts the value of the current <see cref="ModuleVersion"/> object to its equivalent
         /// <see cref="String"/> representation.

Netkan/Processors/Inflator.cs

@@ -5,6 +5,7 @@
 using Autofac;
 using log4net;
 using CKAN.Configuration;
+using CKAN.Versioning;
 using CKAN.NetKAN.Model;
 using CKAN.NetKAN.Services;
 using CKAN.NetKAN.Transformers;
@@ -30,7 +31,7 @@ public Inflator(string cacheDir, bool overwriteCache, string githubToken, bool p
             transformer   = new NetkanTransformer(http, fileService, moduleService, githubToken, prerelease);
         }
 
-        internal IEnumerable<Metadata> Inflate(string filename, Metadata netkan, int? releases)
+        internal IEnumerable<Metadata> Inflate(string filename, Metadata netkan, TransformOptions opts)
         {
             log.DebugFormat("Inflating {0}", filename);
             try
@@ -42,7 +43,7 @@ internal IEnumerable<Metadata> Inflate(string filename, Metadata netkan, int? re
                 log.Info("Input successfully passed pre-validation");
 
                 IEnumerable<Metadata> ckans = transformer
-                    .Transform(netkan, new TransformOptions(releases))
+                    .Transform(netkan, opts)
                     .ToList();
                 log.Info("Finished transformation");
 

Netkan/Processors/QueueHandler.cs

@@ -9,6 +9,8 @@
 using log4net;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
+using CKAN.Versioning;
+using CKAN.NetKAN.Transformers;
 using CKAN.NetKAN.Model;
 
 namespace CKAN.NetKAN.Processors
@@ -103,13 +105,21 @@ private IEnumerable<SendMessageBatchRequestEntry> Inflate(Message msg)
                 releases = int.Parse(releasesAttr.StringValue);
             }
 
+            ModuleVersion highVer = null;
+            MessageAttributeValue highVerAttr;
+            if (msg.MessageAttributes.TryGetValue("HighestVersion", out highVerAttr))
+            {
+                highVer = new ModuleVersion(highVerAttr.StringValue);
+            }
+
             log.InfoFormat("Inflating {0}", netkan.Identifier);
             IEnumerable<Metadata> ckans = null;
             bool   caught        = false;
             string caughtMessage = null;
+            var    opts          = new TransformOptions(releases, highVer);
             try
             {
-                ckans = inflator.Inflate($"{netkan.Identifier}.netkan", netkan, releases);
+                ckans = inflator.Inflate($"{netkan.Identifier}.netkan", netkan, opts);
             }
             catch (Exception e)
             {
@@ -122,20 +132,25 @@ private IEnumerable<SendMessageBatchRequestEntry> Inflate(Message msg)
             }
             if (caught)
             {
-                yield return inflationMessage(null, netkan, false, caughtMessage);
+                yield return inflationMessage(null, netkan, opts, false, caughtMessage);
             }
             if (ckans != null)
             {
                 foreach (Metadata ckan in ckans)
                 {
                     log.InfoFormat("Sending {0}-{1}", ckan.Identifier, ckan.Version);
-                    yield return inflationMessage(ckan, netkan, true);
+                    yield return inflationMessage(ckan, netkan, opts, true);
                 }
             }
         }
 
-        private SendMessageBatchRequestEntry inflationMessage(Metadata ckan, Metadata netkan, bool success, string err = null)
+        private SendMessageBatchRequestEntry inflationMessage(Metadata ckan, Metadata netkan, TransformOptions opts, bool success, string err = null)
         {
+            bool staged = netkan.Staged || opts.Staged;
+            string stagingReason = 
+                  !string.IsNullOrEmpty(netkan.StagingReason) ? netkan.StagingReason
+                : !string.IsNullOrEmpty(opts.StagingReason)   ? opts.StagingReason
+                : null;
             var attribs = new Dictionary<string, MessageAttributeValue>()
             {
                 {
@@ -151,7 +166,7 @@ private SendMessageBatchRequestEntry inflationMessage(Metadata ckan, Metadata ne
                     new MessageAttributeValue()
                     {
                         DataType    = "String",
-                        StringValue = netkan.Staged.ToString()
+                        StringValue = staged.ToString()
                     }
                 },
                 {
@@ -193,14 +208,14 @@ private SendMessageBatchRequestEntry inflationMessage(Metadata ckan, Metadata ne
                     }
                 );
             }
-            if (netkan.Staged && !string.IsNullOrEmpty(netkan.StagingReason))
+            if (staged && stagingReason != null)
             {
                 attribs.Add(
                     "StagingReason",
                     new MessageAttributeValue()
                     {
                         DataType    = "String",
-                        StringValue = netkan.StagingReason
+                        StringValue = stagingReason,
                     }
                 );
             }

Netkan/Program.cs

@@ -87,7 +87,7 @@ public static int Main(string[] args)
                         Options.GitHubToken,
                         Options.PreRelease
                     );
-                    var ckans = inf.Inflate(Options.File, netkan, ParseReleases(Options.Releases));
+                    var ckans = inf.Inflate(Options.File, netkan, new TransformOptions(ParseReleases(Options.Releases), null));
                     foreach (Metadata ckan in ckans)
                     {
                         WriteCkan(ckan);

Netkan/Transformers/EpochTransformer.cs

@@ -1,8 +1,9 @@
 using System;
 using System.Collections.Generic;
-using CKAN.NetKAN.Model;
 using log4net;
 using Newtonsoft.Json.Linq;
+using CKAN.Versioning;
+using CKAN.NetKAN.Model;
 
 namespace CKAN.NetKAN.Transformers
 {
@@ -38,8 +39,30 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
                 }
                 else
                 {
-                    throw new BadMetadataKraken(null, "Invalid epoch: " + epoch + "In " + json["identifier"]);
+                    throw new BadMetadataKraken(null, "Invalid epoch: " + epoch + " In " + json["identifier"]);
+                }
+            }
+
+            JToken allowOOO;
+            if (json.TryGetValue("x_netkan_allow_out_of_order", out allowOOO) && (bool)allowOOO)
+            {
+                Log.Debug("Out of order versions enabled in netkan, skipping OOO check");
+            }
+            else if (opts.HighestVersion != null)
+            {
+                // Ensure we are greater or equal to the previous max
+                ModuleVersion currentV = new ModuleVersion((string)json["version"]);
+                while (currentV < opts.HighestVersion)
+                {
+                    Log.DebugFormat("Auto-epoching out of order version: {0} < {1}",
+                        currentV, opts.HighestVersion);
+                    // Tell the Indexer to be careful
+                    opts.Staged = true;
+                    opts.StagingReason = $"Auto-epoching out of order version: {currentV} < {opts.HighestVersion}";
+                    // Increment epoch if too small
+                    currentV = currentV.IncrementEpoch();
                 }
+                json["version"] = currentV.ToString();
             }
 
             yield return new Metadata(json);

Netkan/Transformers/ITransformer.cs

@@ -1,16 +1,21 @@
 using System.Collections.Generic;
 using CKAN.NetKAN.Model;
+using CKAN.Versioning;
 
 namespace CKAN.NetKAN.Transformers
 {
     internal class TransformOptions
     {
-        public TransformOptions(int? releases)
+        public TransformOptions(int? releases, ModuleVersion highVer)
         {
-            Releases = releases;
+            Releases       = releases;
+            HighestVersion = highVer;
         }
 
-        public readonly int? Releases;
+        public readonly int?          Releases;
+        public readonly ModuleVersion HighestVersion;
+        public          bool          Staged;
+        public          string        StagingReason;
     }
 
     /// <summary>

Spec.md

@@ -855,16 +855,35 @@ If (and only if) no mod version number has been identified (eg a `#/ckan/http/:u
 
 ##### `x_netkan_epoch`
 
-The `x_netkan_epoch` field is used to specify a particular `epoch` number in the `version` field. Its value should be
+The `x_netkan_epoch` field is used to specify a minimum `epoch` number manually in the `version` field. Its value should be
 an unsigned 32-bit integer.
 
+Note that an epoch can be added without this property or incremented automatically, if the auto-indexer detects an out-of-order
+release.
+
 An example `.netkan` excerpt:
 ```json
 {
     "x_netkan_epoch": 1
 }
 ```
 
+##### `x_netkan_allow_out_of_order`
+
+If true, then this module allows a freshly released version to have a version number smaller than previously existing releases.
+
+This should be used for modules that intentionally release "backport" versions, to disable automatic incrementing of epochs for
+out of order releases.
+Typical mods should not use this property, to allow their version epochs to be automatically incremented if they release an out
+of order version.
+
+An example `.netkan` excerpt:
+```json
+{
+    "x_netkan_allow_out_of_order": true
+}
+```
+
 ##### `x_netkan_force_v`
 
 The `x_netkan_force_v` field is used to specify that a `v` should be prepended to the `version` field. It is a

Tests/NetKAN/MainClass.cs

@@ -12,7 +12,7 @@ namespace Tests.NetKAN
     [TestFixture]
     public class MainClassTests
     {
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         [Test]
         public void FixVersionStringsUnharmed()

Tests/NetKAN/NetkanOverride.cs

@@ -11,7 +11,7 @@ namespace Tests.NetKAN
     public class NetkanOverride
     {
         JObject such_metadata;
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         [SetUp]
         public void Setup()

Tests/NetKAN/Transformers/AvcKrefTransformerTests.cs

@@ -12,7 +12,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class AvcKrefTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         [Test,
             TestCase(

Tests/NetKAN/Transformers/AvcTransformerTests.cs

@@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class AvcTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         [Test]
         public void AddsMissingVersionInfo()

Tests/NetKAN/Transformers/CurseTransformerTests.cs

@@ -12,7 +12,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class CurseTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         // GH #199: Don't pre-fill KSP version fields if we see a ksp_min/max
         [Test]

Tests/NetKAN/Transformers/DownloadAttributeTransformerTests.cs

@@ -13,7 +13,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class DownloadAttributeTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         [Test]
         public void AddsDownloadAttributes()

Tests/NetKAN/Transformers/EpochTransformerTests.cs

@@ -0,0 +1,49 @@
+using System.Linq;
+using Newtonsoft.Json.Linq;
+using NUnit.Framework;
+using CKAN.Versioning;
+using CKAN.NetKAN.Model;
+using CKAN.NetKAN.Transformers;
+
+namespace Tests.NetKAN.Transformers
+{
+    [TestFixture]
+    public sealed class EpochTransformerTests
+    {
+        [Test,
+            TestCase("1.1",     null,    "1.1"),
+            TestCase("1.1",    "1.1",    "1.1"),
+            TestCase("1.1",    "1.2",  "1:1.1"),
+            TestCase("1.1",  "5:1.1",  "5:1.1"),
+            TestCase("1.1",  "5:1.2",  "6:1.1"),
+            TestCase("0.7",   "0.65",  "1:0.7"),
+            TestCase("2.5",   "v2.4",  "1:2.5"),
+            TestCase("2.5",   "V2.4",  "1:2.5"),
+            TestCase("v2.5", "vV2.4", "1:v2.5"),
+        ]
+        public void Transform_WithHighVersionParam_MatchesExpected(string version, string highVer, string expected)
+        {
+            // Arrange
+            var json = new JObject()
+            {
+                { "spec_version", "v1.4"       },
+                { "identifier",   "AwesomeMod" },
+                { "version",      version      },
+            };
+            ITransformer     sut  = new EpochTransformer();
+            TransformOptions opts = new TransformOptions(
+                1,
+                string.IsNullOrEmpty(highVer)
+                    ? null
+                    : new ModuleVersion(highVer)
+            );
+
+            // Act
+            var result = sut.Transform(new Metadata(json), opts).First();
+            var transformedJson = result.Json();
+
+            // Assert
+            Assert.AreEqual(expected, (string)transformedJson["version"]);
+        }
+    }
+}

Tests/NetKAN/Transformers/GeneratedByTransformerTests.cs

@@ -9,7 +9,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class GeneratedByTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         [Test]
         public void AddsGeneratedByProperty()

Tests/NetKAN/Transformers/GithubTransformerTests.cs

@@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class GithubTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         [Test]
         public void CalculatesRepositoryUrlCorrectly()

Tests/NetKAN/Transformers/HttpTransformerTests.cs

@@ -9,7 +9,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class HttpTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         [TestCase("#/ckan/github/foo/bar")]
         [TestCase("#/ckan/netkan/http://awesomemod.example/awesomemod.netkan")]

Tests/NetKAN/Transformers/InternalCkanTransformerTests.cs

@@ -12,7 +12,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class InternalCkanTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         [Test]
         public void AddsMiddingProperties()

Tests/NetKAN/Transformers/MetaNetkanTransformerTests.cs

@@ -13,7 +13,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class MetaNetkanTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         [Test]
         public void DoesNothingWhenNoMatch()

Tests/NetKAN/Transformers/SpacedockTransformerTests.cs

@@ -14,7 +14,7 @@ namespace Tests.NetKAN.Transformers
     [TestFixture]
     public sealed class SpacedockTransformerTests
     {
-        private TransformOptions opts = new TransformOptions(1);
+        private TransformOptions opts = new TransformOptions(1, null);
 
         // GH #199: Don't pre-fill KSP version fields if we see a ksp_min/max
         [Test]