Skip to content

Commit

Permalink
[ML] Adjust DF Analytics process phases (#56107)
Browse files Browse the repository at this point in the history
As of elastic/ml-cpp#1179, the analytics process reports phases
depending on the analysis type. This commit adjusts the phases
of current analyses from `analyzing` to the following:

 - outlier_detection: [`computing_outlier`]
 - regression/classification: [`feature_selection`, `coarse_parameter_search`, `fine_tuning_parameters`, `final_training`]
  • Loading branch information
dimitris-athanasiou authored May 5, 2020
1 parent 0c00a98 commit 4b6d72b
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 7 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1532,7 +1532,7 @@ public void testGetDataFrameAnalyticsStats() throws Exception {
assertThat(progress.size(), equalTo(4));
assertThat(progress.get(0), equalTo(new PhaseProgress("reindexing", 0)));
assertThat(progress.get(1), equalTo(new PhaseProgress("loading_data", 0)));
assertThat(progress.get(2), equalTo(new PhaseProgress("analyzing", 0)));
assertThat(progress.get(2), equalTo(new PhaseProgress("computing_outliers", 0)));
assertThat(progress.get(3), equalTo(new PhaseProgress("writing_results", 0)));
assertThat(stats.getMemoryUsage().getPeakUsageBytes(), equalTo(0L));
assertThat(stats.getDataCounts(), equalTo(new DataCounts(0, 0, 0)));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -104,6 +105,15 @@ public static Classification fromXContent(XContentParser parser, boolean ignoreU
*/
private static final int DEFAULT_NUM_TOP_CLASSES = 2;

private static final List<String> PROGRESS_PHASES = Collections.unmodifiableList(
Arrays.asList(
"feature_selection",
"coarse_parameter_search",
"fine_tuning_parameters",
"final_training"
)
);

private final String dependentVariable;
private final BoostedTreeParams boostedTreeParams;
private final String predictionFieldName;
Expand Down Expand Up @@ -348,7 +358,7 @@ public String getStateDocId(String jobId) {

@Override
public List<String> getProgressPhases() {
return Collections.singletonList("analyzing");
return PROGRESS_PHASES;
}

public static String extractJobIdFromStateDoc(String stateDocId) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ public static OutlierDetection fromXContent(XContentParser parser, boolean ignor
return ignoreUnknownFields ? LENIENT_PARSER.apply(parser, null).build() : STRICT_PARSER.apply(parser, null).build();
}

private static final List<String> PROGRESS_PHASES = Collections.singletonList("computing_outliers");

/**
* The number of neighbors. Leave unspecified for dynamic detection.
*/
Expand Down Expand Up @@ -251,7 +253,7 @@ public String getStateDocId(String jobId) {

@Override
public List<String> getProgressPhases() {
return Collections.singletonList("analyzing");
return PROGRESS_PHASES;
}

public enum Method {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;

import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
Expand Down Expand Up @@ -64,6 +65,15 @@ public static Regression fromXContent(XContentParser parser, boolean ignoreUnkno
return ignoreUnknownFields ? LENIENT_PARSER.apply(parser, null) : STRICT_PARSER.apply(parser, null);
}

private static final List<String> PROGRESS_PHASES = Collections.unmodifiableList(
Arrays.asList(
"feature_selection",
"coarse_parameter_search",
"fine_tuning_parameters",
"final_training"
)
);

private final String dependentVariable;
private final BoostedTreeParams boostedTreeParams;
private final String predictionFieldName;
Expand Down Expand Up @@ -215,7 +225,7 @@ public String getStateDocId(String jobId) {

@Override
public List<String> getProgressPhases() {
return Collections.singletonList("analyzing");
return PROGRESS_PHASES;
}

public static String extractJobIdFromStateDoc(String stateDocId) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,6 @@ setup:
- match:
$body: |
/^ id \s+ t \s+ s \s+ p \s+ source_index \s+ dest_index \n
(dfa\-classification\-job \s+ classification \s+ stopped \s+ reindexing:0,loading_data:0,analyzing:0,writing_results:0 \s+ index-source \s+ index-dest-c \n)+
(dfa\-outlier\-detection\-job \s+ outlier_detection \s+ stopped \s+ reindexing:0,loading_data:0,analyzing:0,writing_results:0 \s+ index-source \s+ index-dest-od \n)+
(dfa\-regression\-job \s+ regression \s+ stopped \s+ reindexing:0,loading_data:0,analyzing:0,writing_results:0 \s+ index-source \s+ index-dest-r \n)+ $/
(dfa\-classification\-job \s+ classification \s+ stopped \s+ reindexing:0,loading_data:0,feature_selection:0,coarse_parameter_search:0,fine_tuning_parameters:0,final_training:0,writing_results:0 \s+ index-source \s+ index-dest-c \n)+
(dfa\-outlier\-detection\-job \s+ outlier_detection \s+ stopped \s+ reindexing:0,loading_data:0,computing_outliers:0,writing_results:0 \s+ index-source \s+ index-dest-od \n)+
(dfa\-regression\-job \s+ regression \s+ stopped \s+ reindexing:0,loading_data:0,feature_selection:0,coarse_parameter_search:0,fine_tuning_parameters:0,final_training:0,writing_results:0 \s+ index-source \s+ index-dest-r \n)+ $/

0 comments on commit 4b6d72b

Please sign in to comment.