refactor + impute (#282)

* refactor + impute * fix consistency test * fixes * cleanup and fixes of ImputePreprocessor * renaming fields and introducing compute state * renaming fields and introducing compute state * cleanup * name changes and fixes * consistency fix * refactor and introducing Point * javadocs and cleanup * templates and interfaces * fix:adding IPreprocessor * enabling anomaly detection for partial input * cleanup * changes * changes
aws · Nov 4, 2021 · 9296855 · 9296855
1 parent c8b5039
commit 9296855
Show file tree

Hide file tree

Showing 26 changed files with 2,534 additions and 1,196 deletions.
diff --git a/Java/core/src/main/java/com/amazon/randomcutforest/config/ImputationMethod.java b/Java/core/src/main/java/com/amazon/randomcutforest/config/ImputationMethod.java
@@ -32,13 +32,17 @@ public enum ImputationMethod {
      * last known value in each input dimension
      */
     PREVIOUS,
+    /**
+     * next seen value in each input dimension
+     */
+    NEXT,
     /**
      * linear interpolation
      */
     LINEAR,
     /**
      * use the RCF imputation; but would often require a minimum number of
-     * observations and would uses a default (often LINEAR) till that point
+     * observations and would use defaults (often LINEAR) till that point
      */
     RCF;
 }
diff --git a/Java/core/src/main/java/com/amazon/randomcutforest/tree/BoxCacheDouble.java b/Java/core/src/main/java/com/amazon/randomcutforest/tree/BoxCacheDouble.java
@@ -49,6 +49,8 @@ void initialize() {
             }
         } else if (cacheFraction == 1.0) {
             cachedBoxes = new BoundingBox[maxSize];
+        } else if (cacheFraction == 0) {
+            cachedBoxes = null;
         }
     }
 

diff --git a/Java/core/src/main/java/com/amazon/randomcutforest/tree/BoxCacheFloat.java b/Java/core/src/main/java/com/amazon/randomcutforest/tree/BoxCacheFloat.java
@@ -49,6 +49,8 @@ void initialize() {
             }
         } else if (cacheFraction == 1.0) {
             cachedBoxes = new BoundingBoxFloat[maxSize];
+        } else if (cacheFraction == 0) {
+            cachedBoxes = null;
         }
     }
 

diff --git a/.../main/java/com/amazon/randomcutforest/examples/parkservices/Thresholded1DGaussianMix.java b/.../main/java/com/amazon/randomcutforest/examples/parkservices/Thresholded1DGaussianMix.java
@@ -77,8 +77,8 @@ public void run() throws Exception {
             AnomalyDescriptor result = forest.process(point, count);
 
             if (keyCounter < dataWithKeys.changeIndices.length
-                    && result.getTimestamp() == dataWithKeys.changeIndices[keyCounter]) {
-                System.out.println("timestamp " + (result.getTimestamp()) + " CHANGE");
+                    && result.getInternalTimeStamp() == dataWithKeys.changeIndices[keyCounter]) {
+                System.out.println("timestamp " + (result.getInputTimestamp()) + " CHANGE");
                 ++keyCounter;
             }
 
@@ -90,32 +90,31 @@ public void run() throws Exception {
             if (result.getAnomalyGrade() != 0) {
                 System.out.print("timestamp " + (count) + " RESULT value ");
                 for (int i = 0; i < baseDimensions; i++) {
-                    System.out.print(result.getCurrentValues()[i] + ", ");
+                    System.out.print(result.getCurrentInput()[i] + ", ");
                 }
-                System.out.print("score " + result.getRcfScore() + ", grade " + result.getAnomalyGrade() + ", ");
+                System.out.print("score " + result.getRCFScore() + ", grade " + result.getAnomalyGrade() + ", ");
 
                 if (result.isExpectedValuesPresent()) {
                     if (result.getRelativeIndex() != 0 && result.isStartOfAnomaly()) {
                         System.out.print(-result.getRelativeIndex() + " steps ago, instead of ");
                         for (int i = 0; i < baseDimensions; i++) {
-                            System.out.print(result.getOldValues()[i] + ", ");
+                            System.out.print(result.getPastValues()[i] + ", ");
                         }
                         System.out.print("expected ");
                         for (int i = 0; i < baseDimensions; i++) {
                             System.out.print(result.getExpectedValuesList()[0][i] + ", ");
-                            if (result.getOldValues()[i] != result.getExpectedValuesList()[0][i]) {
+                            if (result.getPastValues()[i] != result.getExpectedValuesList()[0][i]) {
                                 System.out.print("( "
-                                        + (result.getOldValues()[i] - result.getExpectedValuesList()[0][i]) + " ) ");
+                                        + (result.getPastValues()[i] - result.getExpectedValuesList()[0][i]) + " ) ");
                             }
                         }
                     } else {
                         System.out.print("expected ");
                         for (int i = 0; i < baseDimensions; i++) {
                             System.out.print(result.getExpectedValuesList()[0][i] + ", ");
-                            if (result.getCurrentValues()[i] != result.getExpectedValuesList()[0][i]) {
-                                System.out.print(
-                                        "( " + (result.getCurrentValues()[i] - result.getExpectedValuesList()[0][i])
-                                                + " ) ");
+                            if (result.getCurrentInput()[i] != result.getExpectedValuesList()[0][i]) {
+                                System.out.print("( "
+                                        + (result.getCurrentInput()[i] - result.getExpectedValuesList()[0][i]) + " ) ");
                             }
                         }
                     }

diff --git a/...com/amazon/randomcutforest/examples/parkservices/ThresholdedInternalShinglingExample.java b/...com/amazon/randomcutforest/examples/parkservices/ThresholdedInternalShinglingExample.java
@@ -15,11 +15,14 @@
 
 package com.amazon.randomcutforest.examples.parkservices;
 
+import static com.amazon.randomcutforest.CommonUtils.checkArgument;
+
 import java.util.Arrays;
 import java.util.Random;
 
 import com.amazon.randomcutforest.config.ForestMode;
 import com.amazon.randomcutforest.config.Precision;
+import com.amazon.randomcutforest.config.TransformMethod;
 import com.amazon.randomcutforest.examples.Example;
 import com.amazon.randomcutforest.parkservices.AnomalyDescriptor;
 import com.amazon.randomcutforest.parkservices.ThresholdedRandomCutForest;
@@ -46,7 +49,7 @@ public String description() {
     public void run() throws Exception {
         // Create and populate a random cut forest
 
-        int shingleSize = 8;
+        int shingleSize = 4;
         int numberOfTrees = 50;
         int sampleSize = 256;
         Precision precision = Precision.FLOAT_32;
@@ -57,15 +60,29 @@ public void run() throws Exception {
         int baseDimensions = 1;
 
         long count = 0;
-
         int dimensions = baseDimensions * shingleSize;
+        TransformMethod transformMethod = TransformMethod.NONE;
         ThresholdedRandomCutForest forest = ThresholdedRandomCutForest.builder().compact(true).dimensions(dimensions)
                 .randomSeed(0).numberOfTrees(numberOfTrees).shingleSize(shingleSize).sampleSize(sampleSize)
                 .internalShinglingEnabled(true).precision(precision).anomalyRate(0.01).forestMode(ForestMode.STANDARD)
-                .outputAfter(32).initialAcceptFraction(0.125).adjustThreshold(true).build();
+                .weightTime(0).transformMethod(transformMethod).normalizeTime(true).outputAfter(32)
+                .initialAcceptFraction(0.125).build();
+        ThresholdedRandomCutForest second = ThresholdedRandomCutForest.builder().compact(true).dimensions(dimensions)
+                .randomSeed(0).numberOfTrees(numberOfTrees).shingleSize(shingleSize).sampleSize(sampleSize)
+                .internalShinglingEnabled(true).precision(precision).anomalyRate(0.01)
+                .forestMode(ForestMode.TIME_AUGMENTED).weightTime(0).transformMethod(transformMethod)
+                .normalizeTime(true).outputAfter(32).initialAcceptFraction(0.125).build();
+
+        // ensuring that the parameters are the same; otherwise the grades/scores cannot
+        // be the same
+        // weighTime has to be 0
+        forest.setLowerThreshold(1.1);
+        second.setLowerThreshold(1.1);
+        forest.setHorizon(0.75);
+        second.setHorizon(0.75);
 
         long seed = new Random().nextLong();
-        Random noise = new Random();
+        Random noise = new Random(0);
 
         System.out.println("seed = " + seed);
         // change the last argument seed for a different run
@@ -80,7 +97,11 @@ public void run() throws Exception {
             // then the noise corresponds to a jitter; one can try TIME_AUGMENTED and
             // .normalizeTime(true)
 
-            AnomalyDescriptor result = forest.process(point, 100 * count + noise.nextInt(10) - 5);
+            long timestamp = 100 * count + noise.nextInt(10) - 5;
+            AnomalyDescriptor result = forest.process(point, timestamp);
+            AnomalyDescriptor test = second.process(point, timestamp);
+            checkArgument(Math.abs(result.getRCFScore() - test.getRCFScore()) < 1e-10, " error");
+            checkArgument(Math.abs(result.getAnomalyGrade() - test.getAnomalyGrade()) < 1e-10, " error");
 
             if (keyCounter < dataWithKeys.changeIndices.length && count == dataWithKeys.changeIndices[keyCounter]) {
                 System.out
@@ -89,36 +110,35 @@ public void run() throws Exception {
             }
 
             if (result.getAnomalyGrade() != 0) {
-                System.out.print("timestamp " + count + " RESULT value ");
+                System.out.print("timestamp " + count + " RESULT value " + result.getInternalTimeStamp() + " ");
                 for (int i = 0; i < baseDimensions; i++) {
-                    System.out.print(result.getCurrentValues()[i] + ", ");
+                    System.out.print(result.getCurrentInput()[i] + ", ");
                 }
-                System.out.print("score " + result.getRcfScore() + ", grade " + result.getAnomalyGrade() + ", ");
+                System.out.print("score " + result.getRCFScore() + ", grade " + result.getAnomalyGrade() + ", ");
                 if (result.getRelativeIndex() != 0 && result.isStartOfAnomaly()) {
                     System.out.print(-result.getRelativeIndex() + " steps ago, ");
                 }
                 if (result.isExpectedValuesPresent()) {
                     if (result.getRelativeIndex() != 0 && result.isStartOfAnomaly()) {
                         System.out.print("instead of ");
                         for (int i = 0; i < baseDimensions; i++) {
-                            System.out.print(result.getOldValues()[i] + ", ");
+                            System.out.print(result.getPastValues()[i] + ", ");
                         }
                         System.out.print("expected ");
                         for (int i = 0; i < baseDimensions; i++) {
                             System.out.print(result.getExpectedValuesList()[0][i] + ", ");
-                            if (result.getOldValues()[i] != result.getExpectedValuesList()[0][i]) {
+                            if (result.getPastValues()[i] != result.getExpectedValuesList()[0][i]) {
                                 System.out.print("( "
-                                        + (result.getOldValues()[i] - result.getExpectedValuesList()[0][i]) + " ) ");
+                                        + (result.getPastValues()[i] - result.getExpectedValuesList()[0][i]) + " ) ");
                             }
                         }
                     } else {
                         System.out.print("expected ");
                         for (int i = 0; i < baseDimensions; i++) {
                             System.out.print(result.getExpectedValuesList()[0][i] + ", ");
-                            if (result.getCurrentValues()[i] != result.getExpectedValuesList()[0][i]) {
-                                System.out.print(
-                                        "( " + (result.getCurrentValues()[i] - result.getExpectedValuesList()[0][i])
-                                                + " ) ");
+                            if (result.getCurrentInput()[i] != result.getExpectedValuesList()[0][i]) {
+                                System.out.print("( "
+                                        + (result.getCurrentInput()[i] - result.getExpectedValuesList()[0][i]) + " ) ");
                             }
                         }
                     }

diff --git a/.../com/amazon/randomcutforest/examples/parkservices/ThresholdedMultiDimensionalExample.java b/.../com/amazon/randomcutforest/examples/parkservices/ThresholdedMultiDimensionalExample.java
@@ -82,32 +82,32 @@ public void run() throws Exception {
             if (result.getAnomalyGrade() != 0) {
                 System.out.print("timestamp " + (count + shingleSize - 1) + " RESULT value ");
                 for (int i = (shingleSize - 1) * baseDimensions; i < shingleSize * baseDimensions; i++) {
-                    System.out.print(result.getCurrentValues()[i] + ", ");
+                    System.out.print(result.getCurrentInput()[i] + ", ");
                 }
-                System.out.print("score " + result.getRcfScore() + ", grade " + result.getAnomalyGrade() + ", ");
+                System.out.print("score " + result.getRCFScore() + ", grade " + result.getAnomalyGrade() + ", ");
 
                 if (result.isExpectedValuesPresent()) {
                     if (result.getRelativeIndex() != 0 && result.isStartOfAnomaly()) {
                         System.out.print(-result.getRelativeIndex() + " steps ago, instead of ");
                         for (int i = 0; i < baseDimensions; i++) {
-                            System.out.print(result.getOldValues()[i] + ", ");
+                            System.out.print(result.getPastValues()[i] + ", ");
                         }
                         System.out.print("expected ");
                         for (int i = 0; i < baseDimensions; i++) {
                             System.out.print(result.getExpectedValuesList()[0][i] + ", ");
-                            if (result.getOldValues()[i] != result.getExpectedValuesList()[0][i]) {
+                            if (result.getPastValues()[i] != result.getExpectedValuesList()[0][i]) {
                                 System.out.print("( "
-                                        + (result.getOldValues()[i] - result.getExpectedValuesList()[0][i]) + " ) ");
+                                        + (result.getPastValues()[i] - result.getExpectedValuesList()[0][i]) + " ) ");
                             }
                         }
                     } else {
                         System.out.print("expected ");
                         for (int i = 0; i < baseDimensions; i++) {
                             System.out.print(result.getExpectedValuesList()[0][i] + ", ");
-                            if (result.getCurrentValues()[(shingleSize - 1) * baseDimensions
+                            if (result.getCurrentInput()[(shingleSize - 1) * baseDimensions
                                     + i] != result.getExpectedValuesList()[0][i]) {
                                 System.out
-                                        .print("( " + (result.getCurrentValues()[(shingleSize - 1) * baseDimensions + i]
+                                        .print("( " + (result.getCurrentInput()[(shingleSize - 1) * baseDimensions + i]
                                                 - result.getExpectedValuesList()[0][i]) + " ) ");
                             }
                         }

diff --git a/...mples/src/main/java/com/amazon/randomcutforest/examples/parkservices/ThresholdedTime.java b/...mples/src/main/java/com/amazon/randomcutforest/examples/parkservices/ThresholdedTime.java
@@ -83,7 +83,7 @@ public void run() throws Exception {
             AnomalyDescriptor result = forest.process(data, time);
 
             if (keyCounter < dataWithKeys.changeIndices.length && count == dataWithKeys.changeIndices[keyCounter]) {
-                System.out.print("Sequence " + count + " stamp " + (result.getTimestamp()) + " CHANGE ");
+                System.out.print("Sequence " + count + " stamp " + (result.getInternalTimeStamp()) + " CHANGE ");
                 if (!anomalyState) {
                     System.out.println(" to Distribution 1 ");
                 } else {
@@ -94,18 +94,18 @@ public void run() throws Exception {
             }
 
             if (result.getAnomalyGrade() != 0) {
-                System.out.print("Sequence " + count + " stamp " + (result.getTimestamp()) + " RESULT ");
-                System.out.print("score " + result.getRcfScore() + ", grade " + result.getAnomalyGrade() + ", ");
+                System.out.print("Sequence " + count + " stamp " + (result.getInternalTimeStamp()) + " RESULT ");
+                System.out.print("score " + result.getRCFScore() + ", grade " + result.getAnomalyGrade() + ", ");
 
                 if (result.isExpectedValuesPresent()) {
                     if (result.getRelativeIndex() != 0 && result.isStartOfAnomaly()) {
                         System.out.print(-result.getRelativeIndex() + " steps ago, instead of stamp "
-                                + result.getOldTimeStamp());
+                                + result.getPastTimeStamp());
                         System.out.print(", expected timestamp " + result.getExpectedTimeStamp() + " ( "
-                                + (result.getOldTimeStamp() - result.getExpectedTimeStamp() + ")"));
+                                + (result.getPastTimeStamp() - result.getExpectedTimeStamp() + ")"));
                     } else {
                         System.out.print("expected " + result.getExpectedTimeStamp() + " ( "
-                                + (result.getTimestamp() - result.getExpectedTimeStamp() + ")"));
+                                + (result.getInternalTimeStamp() - result.getExpectedTimeStamp() + ")"));
                     }
                 }
                 System.out.println();