From eb9050954b71932ffd3d2bf7a9435ae7e3015e67 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Mon, 16 Jul 2018 13:29:31 +0100 Subject: [PATCH] Move ML integ tests to 3 node cluster --- .../integration/AutodetectMemoryLimitIT.java | 0 .../integration/BasicRenormalizationIT.java | 0 .../ml/integration/CategorizationIT.java | 0 .../xpack/ml/integration/DatafeedJobsIT.java | 0 .../ml/integration/DatafeedJobsRestIT.java | 0 .../ml/integration/DetectionRulesIT.java | 0 .../xpack/ml/integration/ForecastIT.java | 0 ...erimResultsDeletedAfterReopeningJobIT.java | 0 .../xpack/ml/integration/MlJobIT.java | 0 .../xpack/ml/integration/ModelPlotsIT.java | 0 .../ml/integration/OverallBucketsIT.java | 0 .../xpack/ml/integration/PersistJobIT.java | 0 .../integration/RestoreModelSnapshotIT.java | 0 .../ml/integration/RevertModelSnapshotIT.java | 0 .../ml/integration/ScheduledEventsIT.java | 0 .../integration/UpdateInterimResultsIT.java | 0 .../MlNativeAutodetectIntegTestCase.java | 510 ------------------ 17 files changed, 510 deletions(-) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/InterimResultsDeletedAfterReopeningJobIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/ModelPlotsIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/OverallBucketsIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java (100%) rename x-pack/qa/{ml-native-tests => ml-native-multi-node-tests}/src/test/java/org/elasticsearch/xpack/ml/integration/UpdateInterimResultsIT.java (100%) delete mode 100644 x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/BasicRenormalizationIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/CategorizationIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ForecastIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InterimResultsDeletedAfterReopeningJobIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InterimResultsDeletedAfterReopeningJobIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InterimResultsDeletedAfterReopeningJobIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/InterimResultsDeletedAfterReopeningJobIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlJobIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ModelPlotsIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ModelPlotsIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ModelPlotsIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ModelPlotsIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OverallBucketsIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OverallBucketsIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OverallBucketsIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/OverallBucketsIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/PersistJobIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RestoreModelSnapshotIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/RevertModelSnapshotIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/UpdateInterimResultsIT.java b/x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/UpdateInterimResultsIT.java similarity index 100% rename from x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/UpdateInterimResultsIT.java rename to x-pack/qa/ml-native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/UpdateInterimResultsIT.java diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java b/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java deleted file mode 100644 index f70efc72506d3..0000000000000 --- a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java +++ /dev/null @@ -1,510 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.integration; - -import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.cluster.ClusterModule; -import org.elasticsearch.cluster.ClusterState; -import org.elasticsearch.cluster.metadata.MetaData; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.bytes.BytesArray; -import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.common.io.stream.NamedWriteableRegistry; -import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.DeprecationHandler; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.common.xcontent.XContentFactory; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.persistent.PersistentTaskParams; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.SearchModule; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.tasks.Task; -import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.SecuritySettingsSourceField; -import org.elasticsearch.transport.Netty4Plugin; -import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; -import org.elasticsearch.xpack.core.XPackClientPlugin; -import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.ml.MlMetadata; -import org.elasticsearch.xpack.core.ml.action.CloseJobAction; -import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.DeleteJobAction; -import org.elasticsearch.xpack.core.ml.action.FlushJobAction; -import org.elasticsearch.xpack.core.ml.action.ForecastJobAction; -import org.elasticsearch.xpack.core.ml.action.GetBucketsAction; -import org.elasticsearch.xpack.core.ml.action.GetCategoriesAction; -import org.elasticsearch.xpack.core.ml.action.GetJobsAction; -import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; -import org.elasticsearch.xpack.core.ml.action.GetModelSnapshotsAction; -import org.elasticsearch.xpack.core.ml.action.GetRecordsAction; -import org.elasticsearch.xpack.core.ml.action.OpenJobAction; -import org.elasticsearch.xpack.core.ml.action.PersistJobAction; -import org.elasticsearch.xpack.core.ml.action.PostCalendarEventsAction; -import org.elasticsearch.xpack.core.ml.action.PostDataAction; -import org.elasticsearch.xpack.core.ml.action.PutCalendarAction; -import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.PutFilterAction; -import org.elasticsearch.xpack.core.ml.action.PutJobAction; -import org.elasticsearch.xpack.core.ml.action.RevertModelSnapshotAction; -import org.elasticsearch.xpack.core.ml.action.StartDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; -import org.elasticsearch.xpack.core.ml.action.util.PageParams; -import org.elasticsearch.xpack.core.ml.calendars.Calendar; -import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; -import org.elasticsearch.xpack.core.ml.job.config.Job; -import org.elasticsearch.xpack.core.ml.job.config.JobState; -import org.elasticsearch.xpack.core.ml.job.config.JobTaskStatus; -import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; -import org.elasticsearch.xpack.core.ml.job.config.MlFilter; -import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; -import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; -import org.elasticsearch.xpack.core.ml.job.results.Bucket; -import org.elasticsearch.xpack.core.ml.job.results.CategoryDefinition; -import org.elasticsearch.xpack.core.ml.job.results.Forecast; -import org.elasticsearch.xpack.core.ml.job.results.ForecastRequestStats; -import org.elasticsearch.xpack.core.ml.job.results.Result; -import org.elasticsearch.xpack.core.security.SecurityField; -import org.elasticsearch.xpack.core.security.authc.TokenMetaData; - -import java.io.IOException; -import java.net.URISyntaxException; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.function.Function; - -import static org.elasticsearch.test.XContentTestUtils.convertToMap; -import static org.elasticsearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; - -/** - * Base class of ML integration tests that use a native autodetect process - */ -abstract class MlNativeAutodetectIntegTestCase extends ESIntegTestCase { - - private List jobs = new ArrayList<>(); - private List datafeeds = new ArrayList<>(); - @Override - protected Collection> nodePlugins() { - return Arrays.asList(LocalStateCompositeXPackPlugin.class, Netty4Plugin.class); - } - - @Override - protected Collection> transportClientPlugins() { - return Arrays.asList(XPackClientPlugin.class, Netty4Plugin.class); - } - - @Override - protected Settings externalClusterClientSettings() { - Path keyStore; - try { - keyStore = PathUtils.get(getClass().getResource("/test-node.jks").toURI()); - } catch (URISyntaxException e) { - throw new IllegalStateException("error trying to get keystore path", e); - } - Settings.Builder builder = Settings.builder(); - builder.put(NetworkModule.TRANSPORT_TYPE_KEY, SecurityField.NAME4); - builder.put(SecurityField.USER_SETTING.getKey(), "x_pack_rest_user:" + SecuritySettingsSourceField.TEST_PASSWORD_SECURE_STRING); - builder.put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), true); - builder.put("xpack.security.transport.ssl.enabled", true); - builder.put("xpack.security.transport.ssl.keystore.path", keyStore.toAbsolutePath().toString()); - builder.put("xpack.security.transport.ssl.keystore.password", "keypass"); - builder.put("xpack.security.transport.ssl.verification_mode", "certificate"); - return builder.build(); - } - - protected void cleanUp() { - cleanUpDatafeeds(); - cleanUpJobs(); - waitForPendingTasks(); - } - - private void cleanUpDatafeeds() { - for (DatafeedConfig datafeed : datafeeds) { - try { - stopDatafeed(datafeed.getId()); - } catch (Exception e) { - // ignore - } - try { - deleteDatafeed(datafeed.getId()); - } catch (Exception e) { - // ignore - } - } - } - - private void cleanUpJobs() { - for (Job.Builder job : jobs) { - try { - closeJob(job.getId()); - } catch (Exception e) { - // ignore - } - try { - deleteJob(job.getId()); - } catch (Exception e) { - // ignore - } - } - } - - private void waitForPendingTasks() { - ListTasksRequest listTasksRequest = new ListTasksRequest(); - listTasksRequest.setWaitForCompletion(true); - listTasksRequest.setDetailed(true); - listTasksRequest.setTimeout(TimeValue.timeValueSeconds(10)); - try { - admin().cluster().listTasks(listTasksRequest).get(); - } catch (Exception e) { - throw new AssertionError("Failed to wait for pending tasks to complete", e); - } - } - - protected void registerJob(Job.Builder job) { - if (jobs.add(job) == false) { - throw new IllegalArgumentException("job [" + job.getId() + "] is already registered"); - } - } - - protected void registerDatafeed(DatafeedConfig datafeed) { - if (datafeeds.add(datafeed) == false) { - throw new IllegalArgumentException("datafeed [" + datafeed.getId() + "] is already registered"); - } - } - - protected List getJobs() { - return jobs; - } - - protected PutJobAction.Response putJob(Job.Builder job) { - PutJobAction.Request request = new PutJobAction.Request(job); - return client().execute(PutJobAction.INSTANCE, request).actionGet(); - } - - protected OpenJobAction.Response openJob(String jobId) { - OpenJobAction.Request request = new OpenJobAction.Request(jobId); - return client().execute(OpenJobAction.INSTANCE, request).actionGet(); - } - - protected CloseJobAction.Response closeJob(String jobId) { - CloseJobAction.Request request = new CloseJobAction.Request(jobId); - return client().execute(CloseJobAction.INSTANCE, request).actionGet(); - } - - protected FlushJobAction.Response flushJob(String jobId, boolean calcInterim) { - FlushJobAction.Request request = new FlushJobAction.Request(jobId); - request.setCalcInterim(calcInterim); - return client().execute(FlushJobAction.INSTANCE, request).actionGet(); - } - - protected PutJobAction.Response updateJob(String jobId, JobUpdate update) { - UpdateJobAction.Request request = new UpdateJobAction.Request(jobId, update); - return client().execute(UpdateJobAction.INSTANCE, request).actionGet(); - } - - protected DeleteJobAction.Response deleteJob(String jobId) { - DeleteJobAction.Request request = new DeleteJobAction.Request(jobId); - return client().execute(DeleteJobAction.INSTANCE, request).actionGet(); - } - - protected PutDatafeedAction.Response putDatafeed(DatafeedConfig datafeed) { - PutDatafeedAction.Request request = new PutDatafeedAction.Request(datafeed); - return client().execute(PutDatafeedAction.INSTANCE, request).actionGet(); - } - - protected StopDatafeedAction.Response stopDatafeed(String datafeedId) { - StopDatafeedAction.Request request = new StopDatafeedAction.Request(datafeedId); - return client().execute(StopDatafeedAction.INSTANCE, request).actionGet(); - } - - protected DeleteDatafeedAction.Response deleteDatafeed(String datafeedId) { - DeleteDatafeedAction.Request request = new DeleteDatafeedAction.Request(datafeedId); - return client().execute(DeleteDatafeedAction.INSTANCE, request).actionGet(); - } - - protected StartDatafeedAction.Response startDatafeed(String datafeedId, long start, Long end) { - StartDatafeedAction.Request request = new StartDatafeedAction.Request(datafeedId, start); - request.getParams().setEndTime(end); - return client().execute(StartDatafeedAction.INSTANCE, request).actionGet(); - } - - protected void waitUntilJobIsClosed(String jobId) throws Exception { - waitUntilJobIsClosed(jobId, TimeValue.timeValueSeconds(30)); - } - - protected void waitUntilJobIsClosed(String jobId, TimeValue waitTime) throws Exception { - assertBusy(() -> assertThat(getJobStats(jobId).get(0).getState(), equalTo(JobState.CLOSED)), - waitTime.getMillis(), TimeUnit.MILLISECONDS); - } - - protected List getJob(String jobId) { - GetJobsAction.Request request = new GetJobsAction.Request(jobId); - return client().execute(GetJobsAction.INSTANCE, request).actionGet().getResponse().results(); - } - - protected List getJobStats(String jobId) { - GetJobsStatsAction.Request request = new GetJobsStatsAction.Request(jobId); - GetJobsStatsAction.Response response = client().execute(GetJobsStatsAction.INSTANCE, request).actionGet(); - return response.getResponse().results(); - } - - protected List getBuckets(String jobId) { - GetBucketsAction.Request request = new GetBucketsAction.Request(jobId); - return getBuckets(request); - } - - protected List getBuckets(GetBucketsAction.Request request) { - GetBucketsAction.Response response = client().execute(GetBucketsAction.INSTANCE, request).actionGet(); - return response.getBuckets().results(); - } - - protected List getRecords(String jobId) { - GetRecordsAction.Request request = new GetRecordsAction.Request(jobId); - return getRecords(request); - } - - protected List getRecords(GetRecordsAction.Request request) { - GetRecordsAction.Response response = client().execute(GetRecordsAction.INSTANCE, request).actionGet(); - return response.getRecords().results(); - } - - protected List getModelSnapshots(String jobId) { - GetModelSnapshotsAction.Request request = new GetModelSnapshotsAction.Request(jobId, null); - GetModelSnapshotsAction.Response response = client().execute(GetModelSnapshotsAction.INSTANCE, request).actionGet(); - return response.getPage().results(); - } - - protected RevertModelSnapshotAction.Response revertModelSnapshot(String jobId, String snapshotId) { - RevertModelSnapshotAction.Request request = new RevertModelSnapshotAction.Request(jobId, snapshotId); - return client().execute(RevertModelSnapshotAction.INSTANCE, request).actionGet(); - } - - protected List getCategories(String jobId) { - GetCategoriesAction.Request getCategoriesRequest = - new GetCategoriesAction.Request(jobId); - getCategoriesRequest.setPageParams(new PageParams()); - GetCategoriesAction.Response categoriesResponse = client().execute(GetCategoriesAction.INSTANCE, getCategoriesRequest).actionGet(); - return categoriesResponse.getResult().results(); - } - - protected DataCounts postData(String jobId, String data) { - logger.debug("Posting data to job [{}]:\n{}", jobId, data); - PostDataAction.Request request = new PostDataAction.Request(jobId); - request.setContent(new BytesArray(data), XContentType.JSON); - return client().execute(PostDataAction.INSTANCE, request).actionGet().getDataCounts(); - } - - protected String forecast(String jobId, TimeValue duration, TimeValue expiresIn) { - ForecastJobAction.Request request = new ForecastJobAction.Request(jobId); - if (duration != null) { - request.setDuration(duration.getStringRep()); - } - if (expiresIn != null) { - request.setExpiresIn(expiresIn.getStringRep()); - } - return client().execute(ForecastJobAction.INSTANCE, request).actionGet().getForecastId(); - } - - protected void waitForecastToFinish(String jobId, String forecastId) throws Exception { - assertBusy(() -> { - ForecastRequestStats forecastRequestStats = getForecastStats(jobId, forecastId); - assertThat(forecastRequestStats, is(notNullValue())); - assertThat(forecastRequestStats.getStatus(), equalTo(ForecastRequestStats.ForecastRequestStatus.FINISHED)); - }, 30, TimeUnit.SECONDS); - } - - protected ForecastRequestStats getForecastStats(String jobId, String forecastId) { - SearchResponse searchResponse = client().prepareSearch(AnomalyDetectorsIndex.jobResultsAliasedName(jobId)) - .setQuery(QueryBuilders.boolQuery() - .filter(QueryBuilders.termQuery(Result.RESULT_TYPE.getPreferredName(), ForecastRequestStats.RESULT_TYPE_VALUE)) - .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)) - .filter(QueryBuilders.termQuery(ForecastRequestStats.FORECAST_ID.getPreferredName(), forecastId))) - .execute().actionGet(); - SearchHits hits = searchResponse.getHits(); - if (hits.getTotalHits() == 0) { - return null; - } - assertThat(hits.getTotalHits(), equalTo(1L)); - try { - XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( - NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - hits.getHits()[0].getSourceRef().streamInput()); - return ForecastRequestStats.STRICT_PARSER.apply(parser, null); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - protected List getForecastStats() { - List forecastStats = new ArrayList<>(); - - SearchResponse searchResponse = client().prepareSearch(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*") - .setSize(1000) - .setQuery(QueryBuilders.boolQuery() - .filter(QueryBuilders.termQuery(Result.RESULT_TYPE.getPreferredName(), ForecastRequestStats.RESULT_TYPE_VALUE))) - .execute().actionGet(); - SearchHits hits = searchResponse.getHits(); - for (SearchHit hit : hits) { - try { - XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( - NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, hit.getSourceRef().streamInput()); - forecastStats.add(ForecastRequestStats.STRICT_PARSER.apply(parser, null)); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - return forecastStats; - } - - protected long countForecastDocs(String jobId, String forecastId) { - SearchResponse searchResponse = client().prepareSearch(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*") - .setQuery(QueryBuilders.boolQuery() - .filter(QueryBuilders.termQuery(Result.RESULT_TYPE.getPreferredName(), Forecast.RESULT_TYPE_VALUE)) - .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)) - .filter(QueryBuilders.termQuery(Forecast.FORECAST_ID.getPreferredName(), forecastId))) - .execute().actionGet(); - return searchResponse.getHits().getTotalHits(); - } - - protected List getForecasts(String jobId, ForecastRequestStats forecastRequestStats) { - List forecasts = new ArrayList<>(); - - SearchResponse searchResponse = client().prepareSearch(AnomalyDetectorsIndex.jobResultsIndexPrefix() + "*") - .setSize((int) forecastRequestStats.getRecordCount()) - .setQuery(QueryBuilders.boolQuery() - .filter(QueryBuilders.termQuery(Result.RESULT_TYPE.getPreferredName(), Forecast.RESULT_TYPE_VALUE)) - .filter(QueryBuilders.termQuery(Job.ID.getPreferredName(), jobId)) - .filter(QueryBuilders.termQuery(Forecast.FORECAST_ID.getPreferredName(), forecastRequestStats.getForecastId()))) - .addSort(SortBuilders.fieldSort(Result.TIMESTAMP.getPreferredName()).order(SortOrder.ASC)) - .execute().actionGet(); - SearchHits hits = searchResponse.getHits(); - for (SearchHit hit : hits) { - try { - XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser( - NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, - hit.getSourceRef().streamInput()); - forecasts.add(Forecast.STRICT_PARSER.apply(parser, null)); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - return forecasts; - } - - protected boolean putMlFilter(MlFilter filter) { - PutFilterAction.Response response = client().execute(PutFilterAction.INSTANCE, new PutFilterAction.Request(filter)).actionGet(); - return response.isAcknowledged(); - } - - protected PutCalendarAction.Response putCalendar(String calendarId, List jobIds, String description) { - PutCalendarAction.Request request = new PutCalendarAction.Request(new Calendar(calendarId, jobIds, description)); - return client().execute(PutCalendarAction.INSTANCE, request).actionGet(); - } - - protected PostCalendarEventsAction.Response postScheduledEvents(String calendarId, List events) { - PostCalendarEventsAction.Request request = new PostCalendarEventsAction.Request(calendarId, events); - return client().execute(PostCalendarEventsAction.INSTANCE, request).actionGet(); - } - - protected PersistJobAction.Response persistJob(String jobId) { - PersistJobAction.Request request = new PersistJobAction.Request(jobId); - return client().execute(PersistJobAction.INSTANCE, request).actionGet(); - } - - @Override - protected void ensureClusterStateConsistency() throws IOException { - if (cluster() != null && cluster().size() > 0) { - List entries = new ArrayList<>(ClusterModule.getNamedWriteables()); - entries.addAll(new SearchModule(Settings.EMPTY, true, Collections.emptyList()).getNamedWriteables()); - entries.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, "ml", MlMetadata::new)); - entries.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, StartDatafeedAction.TASK_NAME, - StartDatafeedAction.DatafeedParams::new)); - entries.add(new NamedWriteableRegistry.Entry(PersistentTaskParams.class, OpenJobAction.TASK_NAME, - OpenJobAction.JobParams::new)); - entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, JobTaskStatus.NAME, JobTaskStatus::new)); - entries.add(new NamedWriteableRegistry.Entry(Task.Status.class, DatafeedState.NAME, DatafeedState::fromStream)); - entries.add(new NamedWriteableRegistry.Entry(ClusterState.Custom.class, TokenMetaData.TYPE, TokenMetaData::new)); - final NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(entries); - ClusterState masterClusterState = client().admin().cluster().prepareState().all().get().getState(); - byte[] masterClusterStateBytes = ClusterState.Builder.toBytes(masterClusterState); - // remove local node reference - masterClusterState = ClusterState.Builder.fromBytes(masterClusterStateBytes, null, namedWriteableRegistry); - Map masterStateMap = convertToMap(masterClusterState); - int masterClusterStateSize = ClusterState.Builder.toBytes(masterClusterState).length; - String masterId = masterClusterState.nodes().getMasterNodeId(); - for (Client client : cluster().getClients()) { - ClusterState localClusterState = client.admin().cluster().prepareState().all().setLocal(true).get().getState(); - byte[] localClusterStateBytes = ClusterState.Builder.toBytes(localClusterState); - // remove local node reference - localClusterState = ClusterState.Builder.fromBytes(localClusterStateBytes, null, namedWriteableRegistry); - final Map localStateMap = convertToMap(localClusterState); - final int localClusterStateSize = ClusterState.Builder.toBytes(localClusterState).length; - // Check that the non-master node has the same version of the cluster state as the master and - // that the master node matches the master (otherwise there is no requirement for the cluster state to match) - if (masterClusterState.version() == localClusterState.version() && - masterId.equals(localClusterState.nodes().getMasterNodeId())) { - try { - assertEquals("clusterstate UUID does not match", masterClusterState.stateUUID(), localClusterState.stateUUID()); - // We cannot compare serialization bytes since serialization order of maps is not guaranteed - // but we can compare serialization sizes - they should be the same - assertEquals("clusterstate size does not match", masterClusterStateSize, localClusterStateSize); - // Compare JSON serialization - assertNull("clusterstate JSON serialization does not match", - differenceBetweenMapsIgnoringArrayOrder(masterStateMap, localStateMap)); - } catch (AssertionError error) { - logger.error("Cluster state from master:\n{}\nLocal cluster state:\n{}", - masterClusterState.toString(), localClusterState.toString()); - throw error; - } - } - } - } - } - - protected List generateData(long timestamp, TimeValue bucketSpan, int bucketCount, - Function timeToCountFunction) throws IOException { - List data = new ArrayList<>(); - long now = timestamp; - for (int bucketIndex = 0; bucketIndex < bucketCount; bucketIndex++) { - for (int count = 0; count < timeToCountFunction.apply(bucketIndex); count++) { - Map record = new HashMap<>(); - record.put("time", now); - data.add(createJsonRecord(record)); - } - now += bucketSpan.getMillis(); - } - return data; - } - - protected static String createJsonRecord(Map keyValueMap) throws IOException { - return Strings.toString(JsonXContent.contentBuilder().map(keyValueMap)) + "\n"; - } -}