From 0bae0301b023cb13827d53482eb6c370d67865a3 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Tue, 5 May 2020 11:31:40 +0300 Subject: [PATCH] [7.x][ML] Adjust DF Analytics process phases (#56107) As of elastic/ml-cpp#1179, the analytics process reports phases depending on the analysis type. This commit adjusts the phases of current analyses from `analyzing` to the following: - outlier_detection: [`computing_outlier`] - regression/classification: [`feature_selection`, `coarse_parameter_search`, `fine_tuning_parameters`, `final_training`] Backport of #56107 --- .../org/elasticsearch/client/MachineLearningIT.java | 2 +- .../core/ml/dataframe/analyses/Classification.java | 12 +++++++++++- .../core/ml/dataframe/analyses/OutlierDetection.java | 4 +++- .../xpack/core/ml/dataframe/analyses/Regression.java | 12 +++++++++++- .../test/ml/data_frame_analytics_cat_apis.yml | 6 +++--- 5 files changed, 29 insertions(+), 7 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java index 04673f635815..adfebb65fb8b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/MachineLearningIT.java @@ -1562,7 +1562,7 @@ public void testGetDataFrameAnalyticsStats() throws Exception { assertThat(progress.size(), equalTo(4)); assertThat(progress.get(0), equalTo(new PhaseProgress("reindexing", 0))); assertThat(progress.get(1), equalTo(new PhaseProgress("loading_data", 0))); - assertThat(progress.get(2), equalTo(new PhaseProgress("analyzing", 0))); + assertThat(progress.get(2), equalTo(new PhaseProgress("computing_outliers", 0))); assertThat(progress.get(3), equalTo(new PhaseProgress("writing_results", 0))); assertThat(stats.getMemoryUsage().getPeakUsageBytes(), equalTo(0L)); assertThat(stats.getDataCounts(), equalTo(new DataCounts(0, 0, 0))); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java index 83264c667463..0a1fa707a338 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Classification.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -106,6 +107,15 @@ public static Classification fromXContent(XContentParser parser, boolean ignoreU */ private static final int DEFAULT_NUM_TOP_CLASSES = 2; + private static final List PROGRESS_PHASES = Collections.unmodifiableList( + Arrays.asList( + "feature_selection", + "coarse_parameter_search", + "fine_tuning_parameters", + "final_training" + ) + ); + private final String dependentVariable; private final BoostedTreeParams boostedTreeParams; private final String predictionFieldName; @@ -350,7 +360,7 @@ public String getStateDocId(String jobId) { @Override public List getProgressPhases() { - return Collections.singletonList("analyzing"); + return PROGRESS_PHASES; } public static String extractJobIdFromStateDoc(String stateDocId) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java index 2d955c9c7548..b1f5c7e48bfe 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/OutlierDetection.java @@ -58,6 +58,8 @@ public static OutlierDetection fromXContent(XContentParser parser, boolean ignor return ignoreUnknownFields ? LENIENT_PARSER.apply(parser, null).build() : STRICT_PARSER.apply(parser, null).build(); } + private static final List PROGRESS_PHASES = Collections.singletonList("computing_outliers"); + /** * The number of neighbors. Leave unspecified for dynamic detection. */ @@ -251,7 +253,7 @@ public String getStateDocId(String jobId) { @Override public List getProgressPhases() { - return Collections.singletonList("analyzing"); + return PROGRESS_PHASES; } public enum Method { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java index 75a1e83da834..2196c7223291 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/dataframe/analyses/Regression.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -64,6 +65,15 @@ public static Regression fromXContent(XContentParser parser, boolean ignoreUnkno return ignoreUnknownFields ? LENIENT_PARSER.apply(parser, null) : STRICT_PARSER.apply(parser, null); } + private static final List PROGRESS_PHASES = Collections.unmodifiableList( + Arrays.asList( + "feature_selection", + "coarse_parameter_search", + "fine_tuning_parameters", + "final_training" + ) + ); + private final String dependentVariable; private final BoostedTreeParams boostedTreeParams; private final String predictionFieldName; @@ -215,7 +225,7 @@ public String getStateDocId(String jobId) { @Override public List getProgressPhases() { - return Collections.singletonList("analyzing"); + return PROGRESS_PHASES; } public static String extractJobIdFromStateDoc(String stateDocId) { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_cat_apis.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_cat_apis.yml index 4f45d2ec9ac3..ebe78698c88e 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_cat_apis.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/data_frame_analytics_cat_apis.yml @@ -81,6 +81,6 @@ setup: - match: $body: | /^ id \s+ t \s+ s \s+ p \s+ source_index \s+ dest_index \n - (dfa\-classification\-job \s+ classification \s+ stopped \s+ reindexing:0,loading_data:0,analyzing:0,writing_results:0 \s+ index-source \s+ index-dest-c \n)+ - (dfa\-outlier\-detection\-job \s+ outlier_detection \s+ stopped \s+ reindexing:0,loading_data:0,analyzing:0,writing_results:0 \s+ index-source \s+ index-dest-od \n)+ - (dfa\-regression\-job \s+ regression \s+ stopped \s+ reindexing:0,loading_data:0,analyzing:0,writing_results:0 \s+ index-source \s+ index-dest-r \n)+ $/ + (dfa\-classification\-job \s+ classification \s+ stopped \s+ reindexing:0,loading_data:0,feature_selection:0,coarse_parameter_search:0,fine_tuning_parameters:0,final_training:0,writing_results:0 \s+ index-source \s+ index-dest-c \n)+ + (dfa\-outlier\-detection\-job \s+ outlier_detection \s+ stopped \s+ reindexing:0,loading_data:0,computing_outliers:0,writing_results:0 \s+ index-source \s+ index-dest-od \n)+ + (dfa\-regression\-job \s+ regression \s+ stopped \s+ reindexing:0,loading_data:0,feature_selection:0,coarse_parameter_search:0,fine_tuning_parameters:0,final_training:0,writing_results:0 \s+ index-source \s+ index-dest-r \n)+ $/