[Backport to 1.x] backport multiple PRs

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: "[BUG]"
-labels: 'bug, untriaged, Beta'
+labels: 'bug, untriaged'
 assignees: ''
 
 ---

.github/workflows/CI.yml

@@ -29,28 +29,28 @@ jobs:
 
       - name: Assemble anomaly-detection
         run: |
-          ./gradlew assemble -Dopensearch.version=1.2.0-SNAPSHOT
-          echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT ..."
-          mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT
-          echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT ..."
+          ./gradlew assemble -Dopensearch.version=1.3.0-SNAPSHOT
+          echo "Creating ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT ..."
+          mkdir -p ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT
+          echo "Copying ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT ..."
           ls ./build/distributions/
-          cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT
-          echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT ..."
-          ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.2.0.0-SNAPSHOT
+          cp ./build/distributions/*.zip ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT
+          echo "Copied ./build/distributions/*.zip to ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT ..."
+          ls ./src/test/resources/org/opensearch/ad/bwc/anomaly-detection/1.3.0.0-SNAPSHOT
       - name: Build and Run Tests
         run: |
-          ./gradlew build -Dopensearch.version=1.2.0-SNAPSHOT
+          ./gradlew build -Dopensearch.version=1.3.0-SNAPSHOT
       - name: Publish to Maven Local
         run: |
-          ./gradlew publishToMavenLocal -Dopensearch.version=1.2.0-SNAPSHOT
+          ./gradlew publishToMavenLocal -Dopensearch.version=1.3.0-SNAPSHOT
       - name: Multi Nodes Integration Testing
         run: |
           ./gradlew integTest -PnumNodes=3
       - name: Pull and Run Docker
         run: |
           plugin=`ls build/distributions/*.zip`
-          version=1.2.0-SNAPSHOT
-          plugin_version=1.2.0.0-SNAPSHOT
+          version=1.3.0-SNAPSHOT
+          plugin_version=1.3.0.0-SNAPSHOT
           echo Using OpenSearch $version with AD $plugin_version
           cd ..
           if docker pull opensearchstaging/opensearch:$version

.github/workflows/link-check-workflow.yml

@@ -15,7 +15,7 @@ jobs:
         id: lychee
         uses: lycheeverse/lychee-action@master
         with:
-          args: --accept=200,403,429 --exclude=localhost "**/*.html" "**/*.md" "**/*.txt" "**/*.json"
+          args: --accept=200,403,429 --exclude=localhost **/*.html **/*.md **/*.txt **/*.json
         env:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
       - name: Fail if there were link errors

.gitignore

@@ -14,4 +14,3 @@ out/
 .classpath
 .vscode
 bin/
-src/test/resources/job-scheduler

.whitesource

@@ -0,0 +1,15 @@
+{
+  "scanSettings": {
+    "configMode": "AUTO",
+    "configExternalURL": "",
+    "projectToken": "",
+    "baseBranches": []
+  },
+  "checkRunSettings": {
+    "vulnerableCheckRunConclusionLevel": "failure",
+    "displayMode": "diff"
+  },
+  "issueSettings": {
+    "minSeverityLevel": "LOW"
+  }
+}

build.gradle

@@ -16,7 +16,7 @@ import org.opensearch.gradle.testclusters.StandaloneRestIntegTestTask
 buildscript {
     ext {
         opensearch_group = "org.opensearch"
-        opensearch_version = System.getProperty("opensearch.version", "1.2.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "1.3.0-SNAPSHOT")
         // 1.2.0 -> 1.2.0.0, and 1.2.0-SNAPSHOT -> 1.2.0.0-SNAPSHOT
         opensearch_build = opensearch_version.replaceAll(/(\.\d)([^\d]*)$/, '$1.0$2')
         common_utils_version = System.getProperty("common_utils.version", opensearch_build)
@@ -112,7 +112,7 @@ configurations.all {
         force "org.objenesis:objenesis:3.0.1"
         force "net.bytebuddy:byte-buddy:1.9.15"
         force "net.bytebuddy:byte-buddy-agent:1.9.15"
-        force "com.google.code.gson:gson:2.8.6"
+        force "com.google.code.gson:gson:2.8.9"
         force "junit:junit:4.12"
     }
 }
@@ -300,7 +300,7 @@ String bwcFilePath = "src/test/resources/org/opensearch/ad/bwc/"
     testClusters {
         "${baseName}$i" {
             testDistribution = "ARCHIVE"
-            versions = ["7.10.2","1.2.0-SNAPSHOT"]
+            versions = ["7.10.2","1.3.0-SNAPSHOT"]
             numberOfNodes = 3
             plugin(provider(new Callable<RegularFile>(){
                 @Override
@@ -522,11 +522,7 @@ List<String> jacocoExclusions = [
         // TODO: fix unstable code coverage caused by null NodeClient issue
         // https://github.com/opensearch-project/anomaly-detection/issues/241
         'org.opensearch.ad.task.ADBatchTaskRunner',
-
-        // TODO: add tests for multi-category API
-        'org.opensearch.ad.transport.SearchTopAnomalyResult*',
-        'org.opensearch.ad.rest.RestSearchTopAnomalyResultAction',
-        'org.opensearch.ad.model.AnomalyResultBucket',
+
         //TODO: custom result index caused coverage drop
         'org.opensearch.ad.indices.AnomalyDetectionIndices',
         'org.opensearch.ad.transport.handler.AnomalyResultBulkIndexHandler'
@@ -572,7 +568,7 @@ dependencies {
     compile "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
     compile group: 'com.google.guava', name: 'guava', version:'29.0-jre'
     compile group: 'org.apache.commons', name: 'commons-math3', version: '3.6.1'
-    compile group: 'com.google.code.gson', name: 'gson', version: '2.8.6'
+    compile group: 'com.google.code.gson', name: 'gson', version: '2.8.9'
     compile group: 'com.yahoo.datasketches', name: 'sketches-core', version: '0.13.4'
     compile group: 'com.yahoo.datasketches', name: 'memory', version: '0.12.2'
     compile group: 'commons-lang', name: 'commons-lang', version: '2.6'
@@ -589,6 +585,7 @@ dependencies {
     // used for serializing/deserializing rcf models.
     compile group: 'io.protostuff', name: 'protostuff-core', version: '1.7.4'
     compile group: 'io.protostuff', name: 'protostuff-runtime', version: '1.7.4'
+    compileOnly group: 'org.apache.commons', name: 'commons-lang3', version: '3.12.0'
 
     compile "org.jacoco:org.jacoco.agent:0.8.5"
     compile ("org.jacoco:org.jacoco.ant:0.8.5") {
@@ -606,7 +603,6 @@ dependencies {
     testImplementation group: 'org.powermock', name: 'powermock-api-support', version: '2.0.2'
     testImplementation group: 'org.powermock', name: 'powermock-reflect', version: '2.0.7'
     testImplementation group: 'org.objenesis', name: 'objenesis', version: '3.0.1'
-    testImplementation group: 'org.javassist', name: 'javassist', version: '3.27.0-GA'
     testCompile group: 'net.bytebuddy', name: 'byte-buddy', version: '1.9.15'
     testCompile group: 'net.bytebuddy', name: 'byte-buddy-agent', version: '1.9.15'
     testCompileOnly 'org.apiguardian:apiguardian-api:1.1.0'

src/main/java/org/opensearch/ad/constant/CommonErrorMessages.java

@@ -13,6 +13,7 @@
 
 import static org.opensearch.ad.constant.CommonName.CUSTOM_RESULT_INDEX_PREFIX;
 import static org.opensearch.ad.model.AnomalyDetector.MAX_RESULT_INDEX_NAME_SIZE;
+import static org.opensearch.ad.rest.handler.AbstractAnomalyDetectorActionHandler.MAX_DETECTOR_NAME_SIZE;
 
 import java.util.Locale;
 
@@ -51,6 +52,7 @@ public class CommonErrorMessages {
     public static final String BUG_RESPONSE = "We might have bugs.";
     public static final String INDEX_NOT_FOUND = "index does not exist";
     public static final String NOT_EXISTENT_VALIDATION_TYPE = "The given validation type doesn't exist";
+    public static final String UNSUPPORTED_PROFILE_TYPE = "Unsupported profile types";
 
     private static final String TOO_MANY_CATEGORICAL_FIELD_ERR_MSG_FORMAT = "We can have only %d categorical field/s.";
 
@@ -73,8 +75,10 @@ public static String getTooManyCategoricalFieldErr(int limit) {
     public static String HISTORICAL_ANALYSIS_CANCELLED = "Historical analysis cancelled by user";
     public static String HC_DETECTOR_TASK_IS_UPDATING = "HC detector task is updating";
     public static String NEGATIVE_TIME_CONFIGURATION = "should be non-negative";
+    public static String INVALID_TIME_CONFIGURATION_UNITS = "Time unit %s is not supported";
     public static String INVALID_DETECTOR_NAME =
         "Valid characters for detector name are a-z, A-Z, 0-9, -(hyphen), _(underscore) and .(period)";
+    public static String DUPLICATE_FEATURE_AGGREGATION_NAMES = "Detector has duplicate feature aggregation query names: ";
 
     public static String FAIL_TO_GET_DETECTOR = "Fail to get detector";
     public static String FAIL_TO_GET_DETECTOR_INFO = "Fail to get detector info";
@@ -96,4 +100,7 @@ public static String getTooManyCategoricalFieldErr(int limit) {
     public static String INVALID_CHAR_IN_RESULT_INDEX_NAME =
         "Result index name has invalid character. Valid characters are a-z, 0-9, -(hyphen) and _(underscore)";
     public static String INVALID_RESULT_INDEX_MAPPING = "Result index mapping is not correct for index: ";
+    public static String INVALID_DETECTOR_NAME_SIZE = "Name should be shortened. The maximum limit is "
+        + MAX_DETECTOR_NAME_SIZE
+        + " characters.";
 }

src/main/java/org/opensearch/ad/model/ADEntityTaskProfile.java

@@ -14,6 +14,7 @@
 import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 import java.io.IOException;
+import java.util.Objects;
 
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
@@ -269,4 +270,38 @@ public String getAdTaskType() {
     public void setAdTaskType(String adTaskType) {
         this.adTaskType = adTaskType;
     }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        ADEntityTaskProfile that = (ADEntityTaskProfile) o;
+        return Objects.equals(shingleSize, that.shingleSize)
+            && Objects.equals(rcfTotalUpdates, that.rcfTotalUpdates)
+            && Objects.equals(thresholdModelTrained, that.thresholdModelTrained)
+            && Objects.equals(thresholdModelTrainingDataSize, that.thresholdModelTrainingDataSize)
+            && Objects.equals(modelSizeInBytes, that.modelSizeInBytes)
+            && Objects.equals(nodeId, that.nodeId)
+            && Objects.equals(taskId, that.taskId)
+            && Objects.equals(adTaskType, that.adTaskType)
+            && Objects.equals(entity, that.entity);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects
+            .hash(
+                shingleSize,
+                rcfTotalUpdates,
+                thresholdModelTrained,
+                thresholdModelTrainingDataSize,
+                modelSizeInBytes,
+                nodeId,
+                entity,
+                taskId,
+                adTaskType
+            );
+    }
 }

src/main/java/org/opensearch/ad/model/AnomalyResultBucket.java

@@ -14,6 +14,8 @@
 import java.io.IOException;
 import java.util.Map;
 
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.opensearch.ad.annotation.Generated;
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
 import org.opensearch.common.io.stream.Writeable;
@@ -22,6 +24,8 @@
 import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation.Bucket;
 import org.opensearch.search.aggregations.metrics.InternalMax;
 
+import com.google.common.base.Objects;
+
 /**
  * Represents a single bucket when retrieving top anomaly results for HC detectors
  */
@@ -72,6 +76,34 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeDouble(maxAnomalyGrade);
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o)
+            return true;
+        if (o == null || getClass() != o.getClass())
+            return false;
+        AnomalyResultBucket that = (AnomalyResultBucket) o;
+        return Objects.equal(getKey(), that.getKey())
+            && Objects.equal(getDocCount(), that.getDocCount())
+            && Objects.equal(getMaxAnomalyGrade(), that.getMaxAnomalyGrade());
+    }
+
+    @Generated
+    @Override
+    public int hashCode() {
+        return Objects.hashCode(getKey(), getDocCount(), getMaxAnomalyGrade());
+    }
+
+    @Generated
+    @Override
+    public String toString() {
+        return new ToStringBuilder(this)
+            .append("key", key)
+            .append("docCount", docCount)
+            .append("maxAnomalyGrade", maxAnomalyGrade)
+            .toString();
+    }
+
     public Map<String, Object> getKey() {
         return key;
     }

src/main/java/org/opensearch/ad/model/DetectorProfileName.java

@@ -15,6 +15,7 @@
 import java.util.Set;
 
 import org.opensearch.ad.Name;
+import org.opensearch.ad.constant.CommonErrorMessages;
 import org.opensearch.ad.constant.CommonName;
 
 public enum DetectorProfileName implements Name {
@@ -68,7 +69,7 @@ public static DetectorProfileName getName(String name) {
             case CommonName.AD_TASK:
                 return AD_TASK;
             default:
-                throw new IllegalArgumentException("Unsupported profile types");
+                throw new IllegalArgumentException(CommonErrorMessages.UNSUPPORTED_PROFILE_TYPE);
         }
     }
 

src/main/java/org/opensearch/ad/model/IntervalTimeConfiguration.java

@@ -46,7 +46,7 @@ public IntervalTimeConfiguration(long interval, ChronoUnit unit) {
             );
         }
         if (!SUPPORTED_UNITS.contains(unit)) {
-            throw new IllegalArgumentException(String.format(Locale.ROOT, "Timezone %s is not supported", unit));
+            throw new IllegalArgumentException(String.format(Locale.ROOT, CommonErrorMessages.INVALID_TIME_CONFIGURATION_UNITS, unit));
         }
         this.interval = interval;
         this.unit = unit;

src/main/java/org/opensearch/ad/rest/handler/AbstractAnomalyDetectorActionHandler.java

@@ -119,7 +119,7 @@ public abstract class AbstractAnomalyDetectorActionHandler<T extends ActionRespo
     public static final String FEATURE_WITH_INVALID_QUERY_MSG = FEATURE_INVALID_MSG_PREFIX + " causing a runtime exception: ";
     public static final String UNKNOWN_SEARCH_QUERY_EXCEPTION_MSG =
         "Feature has an unknown exception caught while executing the feature query: ";
-
+    public static final String VALIDATION_FEATURE_FAILURE = "Validation failed for feature(s) of detector %s";
     public static final String NAME_REGEX = "[a-zA-Z0-9._-]+";
     public static final Integer MAX_DETECTOR_NAME_SIZE = 64;
 
@@ -303,7 +303,7 @@ protected void validateDetectorName(boolean indexingDryRun) {
             listener
                 .onFailure(
                     new ADValidationException(
-                        "Name should be shortened. The maximum limit is " + MAX_DETECTOR_NAME_SIZE + " characters.",
+                        CommonErrorMessages.INVALID_DETECTOR_NAME_SIZE,
                         DetectorValidationIssueType.NAME,
                         ValidationAspect.DETECTOR
                     )
@@ -405,7 +405,6 @@ protected void validateAgainstExistingMultiEntityAnomalyDetector(String detector
             SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(query).size(0).timeout(requestTimeout);
 
             SearchRequest searchRequest = new SearchRequest(ANOMALY_DETECTORS_INDEX).source(searchSourceBuilder);
-
             client
                 .search(
                     searchRequest,
@@ -817,7 +816,7 @@ protected void validateAnomalyDetectorFeatures(String detectorId, boolean indexi
             new MultiResponsesDelegateActionListener<MergeableList<Optional<double[]>>>(
                 validateFeatureQueriesListener,
                 anomalyDetector.getFeatureAttributes().size(),
-                String.format(Locale.ROOT, "Validation failed for feature(s) of detector %s", anomalyDetector.getName()),
+                String.format(Locale.ROOT, VALIDATION_FEATURE_FAILURE, anomalyDetector.getName()),
                 false
             );
 

src/main/java/org/opensearch/ad/transport/SearchTopAnomalyResultRequest.java

@@ -48,6 +48,7 @@ public class SearchTopAnomalyResultRequest extends ActionRequest {
     private Instant endTime;
 
     public SearchTopAnomalyResultRequest(StreamInput in) throws IOException {
+        super(in);
         detectorId = in.readOptionalString();
         taskId = in.readOptionalString();
         historical = in.readBoolean();

src/main/java/org/opensearch/ad/transport/SearchTopAnomalyResultTransportAction.java

@@ -182,12 +182,12 @@ public class SearchTopAnomalyResultTransportAction extends
     private static final String defaultIndex = ALL_AD_RESULTS_INDEX_PATTERN;
     private static final String COUNT_FIELD = "_count";
     private static final String BUCKET_SORT_FIELD = "bucket_sort";
-    private static final String MULTI_BUCKETS_FIELD = "multi_buckets";
+    public static final String MULTI_BUCKETS_FIELD = "multi_buckets";
     private static final Logger logger = LogManager.getLogger(SearchTopAnomalyResultTransportAction.class);
     private final Client client;
     private Clock clock;
 
-    private enum OrderType {
+    public enum OrderType {
         SEVERITY("severity"),
         OCCURRENCE("occurrence");
 

src/main/java/org/opensearch/ad/util/RestHandlerUtils.java

@@ -27,6 +27,7 @@
 import org.opensearch.action.search.ShardSearchFailure;
 import org.opensearch.ad.common.exception.AnomalyDetectionException;
 import org.opensearch.ad.common.exception.ResourceNotFoundException;
+import org.opensearch.ad.constant.CommonErrorMessages;
 import org.opensearch.ad.model.AnomalyDetector;
 import org.opensearch.ad.model.Feature;
 import org.opensearch.common.Strings;
@@ -151,7 +152,7 @@ private static String validateFeaturesConfig(List<Feature> features) {
             errorMsgBuilder.append(". ");
         }
         if (duplicateFeatureAggNames.size() > 0) {
-            errorMsgBuilder.append("Detector has duplicate feature aggregation query names: ");
+            errorMsgBuilder.append(CommonErrorMessages.DUPLICATE_FEATURE_AGGREGATION_NAMES);
             errorMsgBuilder.append(String.join(", ", duplicateFeatureAggNames));
         }
         return errorMsgBuilder.toString();