Addition of a Reproducibility Framework

Classification/Core/src/main/java/org/tribuo/classification/baseline/DummyClassifierTrainer.java

@@ -86,9 +86,17 @@ public void postConfig() {
 
     @Override
     public Model<Label> train(Dataset<Label> examples, Map<String, Provenance> instanceProvenance) {
+        return(train(examples, instanceProvenance, INCREMENT_INVOCATION_COUNT));
+    }
+
+    @Override
+    public Model<Label> train(Dataset<Label> examples, Map<String, Provenance> instanceProvenance, int invocationCount) {
+        if(invocationCount != INCREMENT_INVOCATION_COUNT) {
+            this.invocationCount = invocationCount;
+        }
         ModelProvenance provenance = new ModelProvenance(DummyClassifierModel.class.getName(), OffsetDateTime.now(), examples.getProvenance(), getProvenance(), instanceProvenance);
         ImmutableFeatureMap featureMap = examples.getFeatureIDMap();
-        invocationCount++;
+        this.invocationCount++;
         switch (dummyType) {
             case CONSTANT:
                 MutableOutputInfo<Label> labelInfo = examples.getOutputInfo().generateMutableOutputInfo();
@@ -114,6 +122,15 @@ public int getInvocationCount() {
         return invocationCount;
     }
 
+    @Override
+    public synchronized void setInvocationCount(int invocationCount){
+        if(invocationCount < 0){
+            throw new IllegalArgumentException("The supplied invocationCount is less than zero.");
+        }
+
+        this.invocationCount = invocationCount;
+    }
+
     @Override
     public String toString() {
         switch (dummyType) {

Classification/Core/src/main/java/org/tribuo/classification/ensemble/AdaBoostTrainer.java

@@ -137,13 +137,21 @@ public String toString() {
      */
     @Override
     public Model<Label> train(Dataset<Label> examples, Map<String, Provenance> runProvenance) {
+        return(train(examples, runProvenance, INCREMENT_INVOCATION_COUNT));
+    }
+
+    @Override
+    public Model<Label> train(Dataset<Label> examples, Map<String, Provenance> runProvenance, int invocationCount) {
         if (examples.getOutputInfo().getUnknownCount() > 0) {
             throw new IllegalArgumentException("The supplied Dataset contained unknown Outputs, and this Trainer is supervised.");
         }
         // Creates a new RNG, adds one to the invocation count.
         SplittableRandom localRNG;
         TrainerProvenance trainerProvenance;
         synchronized(this) {
+            if(invocationCount != INCREMENT_INVOCATION_COUNT) {
+                setInvocationCount(invocationCount);
+            }
             localRNG = rng.split();
             trainerProvenance = getProvenance();
             trainInvocationCounter++;
@@ -195,7 +203,7 @@ public Model<Label> train(Dataset<Label> examples, Map<String, Provenance> runPr
                 EnsembleModelProvenance provenance = new EnsembleModelProvenance(WeightedEnsembleModel.class.getName(), OffsetDateTime.now(), examples.getProvenance(), trainerProvenance, runProvenance, ListProvenance.createListProvenance(models));
                 return new WeightedEnsembleModel<>("boosted-ensemble",provenance,featureIDs,labelIDs,models,new VotingCombiner(),newModelWeights);
             }
-            
+
             //
             // Update the weights
             for (int j = 0; j < predictions.size(); j++) {
@@ -221,6 +229,22 @@ public int getInvocationCount() {
         return trainInvocationCounter;
     }
 
+    @Override
+    public synchronized void setInvocationCount(int invocationCount){
+        if(invocationCount < 0){
+            throw new IllegalArgumentException("The supplied invocationCount is less than zero.");
+        }
+
+        rng = new SplittableRandom(seed);
+        SplittableRandom localRNG;
+        trainInvocationCounter = 0;
+        for (int invocationCounter = 0; invocationCounter < invocationCount; invocationCounter++){
+            localRNG = rng.split();
+            trainInvocationCounter++;
+        }
+
+    }
+
     private float accuracy(List<Prediction<Label>> predictions, Dataset<Label> examples, float[] weights) {
         float correctSum = 0;
         float total = 0;

Classification/DecisionTree/src/test/java/org/tribuo/classification/dtree/TestCART.java

@@ -161,4 +161,46 @@ public void testEmptyExample() {
     public void testRandomEmptyExample() {
         runEmptyExample(randomt);
     }
+
+    @Test
+    public void testSetInvocationCount() {
+        // Create new trainer and dataset so as not to mess with the other tests
+        CARTClassificationTrainer originalTrainer = new CARTClassificationTrainer(5,2, 0.0f,1.0f, true,
+                new GiniIndex(), Trainer.DEFAULT_SEED);
+        Pair<Dataset<Label>,Dataset<Label>> data = LabelledDataGenerator.denseTrainTest();
+        DatasetView<Label> trainingData = DatasetView.createView(data.getA(),(Example<Label> e) -> e.getOutput().getLabel().equals("Foo"), "Foo selector");
+
+        // The number of times to call train before final training.
+        // Original trainer will be trained numOfInvocations + 1 times
+        // New trainer will have it's invocation count set to numOfInvocations then trained once
+        int numOfInvocations = 2;
+
+        // Create the first model and train it numOfInvocations + 1 times
+        TreeModel<Label> originalModel = null;
+        for(int invocationCounter = 0; invocationCounter < numOfInvocations + 1; invocationCounter++){
+            originalModel = originalTrainer.train(trainingData);
+        }
+
+        // Create a new model with same configuration, but set the invocation count to numOfInvocations
+        // Assert that this succeeded, this means RNG will be at state where originalTrainer was before
+        // it performed its last train.
+        CARTClassificationTrainer newTrainer = new CARTClassificationTrainer(5,2, 0.0f,1.0f, true,
+                new GiniIndex(), Trainer.DEFAULT_SEED);
+        newTrainer.setInvocationCount(numOfInvocations);
+        assertEquals(numOfInvocations,newTrainer.getInvocationCount());
+
+        // Training newTrainer should now have the same result as if it
+        // had trained numOfInvocations times previously even though it hasn't
+        TreeModel<Label> newModel = newTrainer.train(trainingData);
+        assertEquals(originalTrainer.getInvocationCount(),newTrainer.getInvocationCount());
+    }
+
+    @Test
+    public void testNegativeInvocationCount(){
+        assertThrows(IllegalArgumentException.class, () -> {
+            CARTClassificationTrainer t = new CARTClassificationTrainer(5,2, 0.0f,1.0f, true,
+                    new GiniIndex(), Trainer.DEFAULT_SEED);
+            t.setInvocationCount(-1);
+        });
+    }
 }

Classification/MultinomialNaiveBayes/src/main/java/org/tribuo/classification/mnb/MultinomialNaiveBayesTrainer.java

@@ -78,6 +78,11 @@ public MultinomialNaiveBayesTrainer(double alpha) {
 
     @Override
     public Model<Label> train(Dataset<Label> examples, Map<String, Provenance> runProvenance) {
+        return train(examples, runProvenance, INCREMENT_INVOCATION_COUNT);
+    }
+
+    @Override
+    public Model<Label> train(Dataset<Label> examples, Map<String, Provenance> runProvenance, int invocationCount) {
         if (examples.getOutputInfo().getUnknownCount() > 0) {
             throw new IllegalArgumentException("The supplied Dataset contained unknown Outputs, and this Trainer is supervised.");
         }
@@ -101,7 +106,9 @@ public Model<Label> train(Dataset<Label> examples, Map<String, Provenance> runPr
                 featureMap.merge(featureInfos.getID(feat.getName()), curWeight*feat.getValue(), Double::sum);
             }
         }
-
+        if(invocationCount != INCREMENT_INVOCATION_COUNT) {
+            setInvocationCount(invocationCount);
+        }
         TrainerProvenance trainerProvenance = getProvenance();
         ModelProvenance provenance = new ModelProvenance(MultinomialNaiveBayesModel.class.getName(), OffsetDateTime.now(), examples.getProvenance(), trainerProvenance, runProvenance);
         invocationCount++;
@@ -125,6 +132,15 @@ public int getInvocationCount() {
         return invocationCount;
     }
 
+    @Override
+    public void setInvocationCount(int invocationCount) {
+        if(invocationCount < 0){
+            throw new IllegalArgumentException("The supplied invocationCount is less than zero.");
+        }
+
+        this.invocationCount = invocationCount;
+    }
+
     @Override
     public String toString() {
         return "MultinomialNaiveBayesTrainer(alpha=" + alpha + ")";

Classification/SGD/src/main/java/org/tribuo/classification/sgd/kernel/KernelSVMTrainer.java

@@ -134,13 +134,21 @@ public void setShuffle(boolean shuffle) {
 
     @Override
     public KernelSVMModel train(Dataset<Label> examples, Map<String, Provenance> runProvenance) {
+        return(train(examples, runProvenance, INCREMENT_INVOCATION_COUNT));
+    }
+
+    @Override
+    public KernelSVMModel train(Dataset<Label> examples, Map<String, Provenance> runProvenance, int invocationCount) {
         if (examples.getOutputInfo().getUnknownCount() > 0) {
             throw new IllegalArgumentException("The supplied Dataset contained unknown Outputs, and this Trainer is supervised.");
         }
         // Creates a new RNG, adds one to the invocation count.
         TrainerProvenance trainerProvenance;
         SplittableRandom localRNG;
         synchronized(this) {
+            if(invocationCount != INCREMENT_INVOCATION_COUNT) {
+                setInvocationCount(invocationCount);
+            }
             localRNG = rng.split();
             trainerProvenance = getProvenance();
             trainInvocationCounter++;
@@ -223,6 +231,23 @@ public int getInvocationCount() {
         return trainInvocationCounter;
     }
 
+    @Override
+    public synchronized void setInvocationCount(int invocationCount){
+        if(invocationCount < 0){
+            throw new IllegalArgumentException("The supplied invocationCount is less than zero.");
+        }
+
+        rng = new SplittableRandom(seed);
+        SplittableRandom localRNG;
+        trainInvocationCounter = 0;
+
+        for (int invocationCounter = 0; invocationCounter < invocationCount; invocationCounter++){
+            localRNG = rng.split();
+            trainInvocationCounter++;
+        }
+
+    }
+
     @Override
     public String toString() {
         return "KernelSVMTrainer(kernel="+kernel.toString()+",lambda="+lambda+",epochs="+epochs+",seed="+seed+")";

Classification/SGD/src/test/java/org/tribuo/classification/sgd/kernel/TestSGDKernel.java

@@ -24,17 +24,21 @@
 import org.tribuo.classification.evaluation.LabelEvaluation;
 import org.tribuo.classification.evaluation.LabelEvaluator;
 import org.tribuo.classification.example.LabelledDataGenerator;
+import org.tribuo.common.sgd.AbstractLinearSGDModel;
 import org.tribuo.math.kernel.RBF;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
+import org.tribuo.math.la.Tensor;
+import org.tribuo.math.optimisers.AdaGrad;
 import org.tribuo.test.Helpers;
 
 import java.util.List;
 import java.util.Map;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
 public class TestSGDKernel {
@@ -97,4 +101,42 @@ public void testEmptyExample() {
         });
     }
 
+    @Test
+    public void testSetInvocationCount() {
+        // Create new trainer and dataset so as not to mess with the other tests
+        KernelSVMTrainer originalTrainer = new KernelSVMTrainer(new RBF(1.0),1,5,1000, Trainer.DEFAULT_SEED);
+        Pair<Dataset<Label>,Dataset<Label>> p = LabelledDataGenerator.denseTrainTest();
+
+        // The number of times to call train before final training.
+        // Original trainer will be trained numOfInvocations + 1 times
+        // New trainer will have it's invocation count set to numOfInvocations then trained once
+        int numOfInvocations = 2;
+
+        // Create the first model and train it numOfInvocations + 1 times
+        Model<Label> originalModel = null;
+        for(int invocationCounter = 0; invocationCounter < numOfInvocations + 1; invocationCounter++){
+            originalModel = originalTrainer.train(p.getA());
+        }
+
+        // Create a new model with same configuration, but set the invocation count to numOfInvocations
+        // Assert that this succeeded, this means RNG will be at state where originalTrainer was before
+        // it performed its last train.
+        KernelSVMTrainer newTrainer = new KernelSVMTrainer(new RBF(1.0),1,5,1000, Trainer.DEFAULT_SEED);
+        newTrainer.setInvocationCount(numOfInvocations);
+        assertEquals(numOfInvocations,newTrainer.getInvocationCount());
+
+        // Training newTrainer should now have the same result as if it
+        // had trained numOfInvocations times previously even though it hasn't
+        Model<Label> newModel = newTrainer.train(p.getA());
+        assertEquals(originalTrainer.getInvocationCount(),newTrainer.getInvocationCount());
+    }
+
+    @Test
+    public void testNegativeInvocationCount(){
+        assertThrows(IllegalArgumentException.class, () -> {
+            KernelSVMTrainer t = new KernelSVMTrainer(new RBF(1.0),1,5,1000, Trainer.DEFAULT_SEED);
+            t.setInvocationCount(-1);
+        });
+    }
+
 }

Classification/XGBoost/src/main/java/org/tribuo/classification/xgboost/XGBoostClassificationTrainer.java

@@ -176,11 +176,19 @@ public void postConfig() {
 
     @Override
     public synchronized XGBoostModel<Label> train(Dataset<Label> examples, Map<String, Provenance> runProvenance) {
+        return (train(examples, runProvenance, INCREMENT_INVOCATION_COUNT));
+    }
+
+    @Override
+    public synchronized XGBoostModel<Label> train(Dataset<Label> examples, Map<String, Provenance> runProvenance, int invocationCount) {
         if (examples.getOutputInfo().getUnknownCount() > 0) {
             throw new IllegalArgumentException("The supplied Dataset contained unknown Outputs, and this Trainer is supervised.");
         }
         ImmutableFeatureMap featureMap = examples.getFeatureIDMap();
         ImmutableOutputInfo<Label> outputInfo = examples.getOutputIDInfo();
+        if(invocationCount != INCREMENT_INVOCATION_COUNT) {
+            setInvocationCount(invocationCount);
+        }
         TrainerProvenance trainerProvenance = getProvenance();
         trainInvocationCounter++;
         parameters.put("num_class", outputInfo.size());

Clustering/KMeans/src/main/java/org/tribuo/clustering/kmeans/KMeansTrainer.java

@@ -184,10 +184,18 @@ public synchronized void postConfig() {
 
     @Override
     public KMeansModel train(Dataset<ClusterID> examples, Map<String, Provenance> runProvenance) {
+        return train(examples, runProvenance, INCREMENT_INVOCATION_COUNT);
+    }
+
+    @Override
+    public KMeansModel train(Dataset<ClusterID> examples, Map<String, Provenance> runProvenance, int invocationCount) {
         // Creates a new local RNG and adds one to the invocation count.
         TrainerProvenance trainerProvenance;
         SplittableRandom localRNG;
         synchronized (this) {
+            if(invocationCount != INCREMENT_INVOCATION_COUNT) {
+                setInvocationCount(invocationCount);
+            }
             localRNG = rng.split();
             trainerProvenance = getProvenance();
             trainInvocationCounter++;
@@ -303,6 +311,23 @@ public int getInvocationCount() {
         return trainInvocationCounter;
     }
 
+    @Override
+    public synchronized void setInvocationCount(int invocationCount){
+        if(invocationCount < 0){
+            throw new IllegalArgumentException("The supplied invocationCount is less than zero.");
+        }
+
+        rng = new SplittableRandom(seed);
+        SplittableRandom localRNG;
+        trainInvocationCounter = 0;
+
+        for (int invocationCounter = 0; invocationCounter < invocationCount; invocationCounter++){
+            localRNG = rng.split();
+            trainInvocationCounter++;
+        }
+
+    }
+
     /**
      * Initialisation method called at the start of each train call when using the default centroid initialisation.
      * Centroids are initialised using a uniform random sample from the feature domain.