diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/UpgradeTransformsResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/UpgradeTransformsResponseTests.java index 11a4b2c77fa31..e810e36709541 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/UpgradeTransformsResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/transform/UpgradeTransformsResponseTests.java @@ -37,15 +37,9 @@ private static UpgradeTransformsResponse createTestInstance() { private static void toXContent(UpgradeTransformsResponse response, XContentBuilder builder) throws IOException { builder.startObject(); - if (response.getUpdated() != 0) { - builder.field("updated", response.getUpdated()); - } - if (response.getNoAction() != 0) { - builder.field("no_action", response.getNoAction()); - } - if (response.getNeedsUpdate() != 0) { - builder.field("needs_update", response.getNeedsUpdate()); - } + builder.field("updated", response.getUpdated()); + builder.field("no_action", response.getNoAction()); + builder.field("needs_update", response.getNeedsUpdate()); builder.endObject(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java index f42700c66f3f2..daae9e28d0b26 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java @@ -8,6 +8,10 @@ package org.elasticsearch.xpack.core.transform; public class TransformDeprecations { + + public static final String UPGRADE_TRANSFORM_URL = "https://ela.st/es-8-upgrade-transforms"; + + // breaking changes base url for the _next_ major release public static final String BREAKING_CHANGES_BASE_URL = "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html"; @@ -15,5 +19,10 @@ public class TransformDeprecations { public static final String AGGS_BREAKING_CHANGES_URL = BREAKING_CHANGES_BASE_URL + "#breaking_80_aggregations_changes"; + public static final String ACTION_UPGRADE_TRANSFORMS_API = "Use the upgrade transforms API to fix your transforms."; + + public static final String ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED = + "[max_page_search_size] is deprecated inside pivot. Use settings instead."; + private TransformDeprecations() {} } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsAction.java index b32799cd78878..d5449519d2627 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsAction.java @@ -125,15 +125,9 @@ public void writeTo(StreamOutput out) throws IOException { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - if (updated != 0L) { - builder.field("updated", updated); - } - if (noAction != 0L) { - builder.field("no_action", noAction); - } - if (needsUpdate != 0L) { - builder.field("needs_update", needsUpdate); - } + builder.field("updated", updated); + builder.field("no_action", noAction); + builder.field("needs_update", needsUpdate); builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java index 60068cc50ea12..0e2c4166b5606 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java @@ -14,6 +14,8 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.TimeValue; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ObjectParser; @@ -21,8 +23,6 @@ import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.TimeValue; import org.elasticsearch.xpack.core.common.time.TimeUtils; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator; import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.SourceDestValidation; @@ -398,8 +398,8 @@ public List checkForDeprecations(NamedXContentRegistry namedXC new DeprecationIssue( Level.CRITICAL, "Transform [" + id + "] is too old", - TransformDeprecations.BREAKING_CHANGES_BASE_URL, - "The configuration uses an old format, you can use [_update] or [_upgrade] to update", + TransformDeprecations.UPGRADE_TRANSFORM_URL, + TransformDeprecations.ACTION_UPGRADE_TRANSFORMS_API, false, null ) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java index 652ee0577e39d..1851b265ec336 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java @@ -13,13 +13,13 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.xcontent.ConstructingObjectParser; import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ToXContentObject; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.core.Nullable; -import org.elasticsearch.search.aggregations.MultiBucketConsumerService; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; import org.elasticsearch.xpack.core.transform.TransformDeprecations; @@ -91,7 +91,7 @@ public PivotConfig(final GroupConfig groups, final AggregationConfig aggregation deprecationLogger.critical( DeprecationCategory.API, TransformField.MAX_PAGE_SEARCH_SIZE.getPreferredName(), - "[max_page_search_size] is deprecated inside pivot please use settings instead" + TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED ); } } @@ -187,7 +187,7 @@ public void checkForDeprecations(String id, NamedXContentRegistry namedXContentR Level.WARNING, "Transform [" + id + "] uses deprecated max_page_search_size", TransformDeprecations.BREAKING_CHANGES_BASE_URL, - "[max_page_search_size] is deprecated inside pivot please use settings instead", + TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED, false, null ) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java index b6b73ba50b666..d46a9b70ad456 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java @@ -28,6 +28,7 @@ import org.elasticsearch.xpack.core.deprecation.DeprecationIssue; import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level; import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.TransformDeprecations; import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig; import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfigTests; import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig; @@ -510,7 +511,7 @@ public void testRewriteForUpdate() throws IOException { assertTrue(transformConfigRewritten.getSettings().getDatesAsEpochMillis()); assertFalse(transformConfigRewritten.getSettings().getAlignCheckpoints()); - assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead"); + assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED); assertEquals(Version.CURRENT, transformConfigRewritten.getVersion()); } @@ -583,7 +584,7 @@ public void testRewriteForUpdateMaxPageSizeSearchConflicting() throws IOExceptio assertNotNull(transformConfigRewritten.getSettings().getMaxPageSearchSize()); assertEquals(555L, transformConfigRewritten.getSettings().getMaxPageSearchSize().longValue()); assertEquals(Version.CURRENT, transformConfigRewritten.getVersion()); - assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead"); + assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED); } public void testRewriteForBWCOfDateNormalization() throws IOException { @@ -739,7 +740,7 @@ public void testCheckForDeprecations() { TransformConfig deprecatedConfig = randomTransformConfigWithDeprecatedFields(id, Version.CURRENT); // check _and_ clear warnings - assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead"); + assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED); // important: checkForDeprecations does _not_ create new deprecation warnings assertThat( @@ -749,8 +750,8 @@ public void testCheckForDeprecations() { new DeprecationIssue( Level.WARNING, "Transform [" + id + "] uses deprecated max_page_search_size", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html", - "[max_page_search_size] is deprecated inside pivot please use settings instead", + TransformDeprecations.BREAKING_CHANGES_BASE_URL, + TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED, false, null ) @@ -761,7 +762,7 @@ public void testCheckForDeprecations() { deprecatedConfig = randomTransformConfigWithDeprecatedFields(id, Version.V_7_10_0); // check _and_ clear warnings - assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead"); + assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED); // important: checkForDeprecations does _not_ create new deprecation warnings assertThat( @@ -771,8 +772,8 @@ public void testCheckForDeprecations() { new DeprecationIssue( Level.WARNING, "Transform [" + id + "] uses deprecated max_page_search_size", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html", - "[max_page_search_size] is deprecated inside pivot please use settings instead", + TransformDeprecations.BREAKING_CHANGES_BASE_URL, + TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED, false, null ) @@ -783,7 +784,7 @@ public void testCheckForDeprecations() { deprecatedConfig = randomTransformConfigWithDeprecatedFields(id, Version.V_7_4_0); // check _and_ clear warnings - assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead"); + assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED); // important: checkForDeprecations does _not_ create new deprecation warnings assertThat( @@ -793,16 +794,16 @@ public void testCheckForDeprecations() { new DeprecationIssue( Level.CRITICAL, "Transform [" + id + "] is too old", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html", - "The configuration uses an old format, you can use [_update] or [_upgrade] to update", + TransformDeprecations.UPGRADE_TRANSFORM_URL, + TransformDeprecations.ACTION_UPGRADE_TRANSFORMS_API, false, null ), new DeprecationIssue( Level.WARNING, "Transform [" + id + "] uses deprecated max_page_search_size", - "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html", - "[max_page_search_size] is deprecated inside pivot please use settings instead", + TransformDeprecations.BREAKING_CHANGES_BASE_URL, + TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED, false, null ) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java index bb82c92726519..282ec0e93ce49 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.xcontent.XContentParser; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase; +import org.elasticsearch.xpack.core.transform.TransformDeprecations; import java.io.IOException; import java.util.Arrays; @@ -437,7 +438,7 @@ public void testAggNameValidationsWithInvalidFieldnames() { public void testDeprecation() { PivotConfig pivotConfig = randomPivotConfigWithDeprecatedFields(); - assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead"); + assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED); } private static String dotJoin(String... fields) { diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointServiceNodeTests.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointServiceNodeTests.java index c99ee97ee9a3b..e7587d2e3e3b5 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointServiceNodeTests.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointServiceNodeTests.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.shard.ShardPath; import org.elasticsearch.index.store.StoreStats; import org.elasticsearch.index.warmer.WarmerStats; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.search.suggest.completion.CompletionStats; import org.elasticsearch.test.client.NoOpClient; import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint; @@ -138,8 +139,13 @@ public void createComponents() { if (mockClientForCheckpointing == null) { mockClientForCheckpointing = new MockClientForCheckpointing("TransformCheckpointServiceNodeTests"); } - - transformsConfigManager = new IndexBasedTransformConfigManager(client(), xContentRegistry()); + ClusterService clusterService = mock(ClusterService.class); + transformsConfigManager = new IndexBasedTransformConfigManager( + clusterService, + TestIndexNameExpressionResolver.newInstance(), + client(), + xContentRegistry() + ); // use a mock for the checkpoint service TransformAuditor mockAuditor = mock(TransformAuditor.class); diff --git a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java index a0f87ffbaa317..807859487fd89 100644 --- a/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java +++ b/x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java @@ -9,13 +9,29 @@ import org.elasticsearch.ResourceAlreadyExistsException; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.WriteRequest; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.routing.IndexRoutingTable; +import org.elasticsearch.cluster.routing.RoutingTable; +import org.elasticsearch.cluster.routing.ShardRoutingState; +import org.elasticsearch.cluster.routing.TestShardRouting; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.collect.ImmutableOpenMap; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; @@ -32,6 +48,7 @@ import org.elasticsearch.xpack.transform.TransformSingleNodeTestCase; import org.junit.Before; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -49,14 +66,23 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; public class TransformConfigManagerTests extends TransformSingleNodeTestCase { private IndexBasedTransformConfigManager transformConfigManager; + private ClusterService clusterService; @Before public void createComponents() { - transformConfigManager = new IndexBasedTransformConfigManager(client(), xContentRegistry()); + clusterService = mock(ClusterService.class); + transformConfigManager = new IndexBasedTransformConfigManager( + clusterService, + TestIndexNameExpressionResolver.newInstance(), + client(), + xContentRegistry() + ); } public void testGetMissingTransform() throws InterruptedException { @@ -690,6 +716,10 @@ public void testDeleteOldIndices() throws Exception { assertAsync(listener -> transformConfigManager.getTransformConfiguration(transformId, listener), transformConfigNew, null, null); // delete old indices + when(clusterService.state()).thenReturn( + createClusterStateWithTransformIndex(oldIndex, TransformInternalIndexConstants.LATEST_INDEX_NAME) + ); + assertAsync(listener -> transformConfigManager.deleteOldIndices(listener), true, null, null); // the config should still be there @@ -708,4 +738,32 @@ public void testDeleteOldIndices() throws Exception { ); } + private static ClusterState createClusterStateWithTransformIndex(String... indexes) throws IOException { + ImmutableOpenMap.Builder indexMapBuilder = ImmutableOpenMap.builder(); + Metadata.Builder metaBuilder = Metadata.builder(); + ClusterState.Builder csBuilder = ClusterState.builder(ClusterName.DEFAULT); + RoutingTable.Builder routingTableBuilder = RoutingTable.builder(); + + for (String index : indexes) { + IndexMetadata.Builder builder = new IndexMetadata.Builder(index).settings( + Settings.builder() + .put(TransformInternalIndex.settings()) + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT) + .build() + ).numberOfReplicas(0).numberOfShards(1).putMapping(Strings.toString(TransformInternalIndex.mappings())); + indexMapBuilder.put(index, builder.build()); + + routingTableBuilder.add( + IndexRoutingTable.builder(new Index(index, UUIDs.randomBase64UUID())) + .addShard(TestShardRouting.newShardRouting(index, 0, "node_a", null, true, ShardRoutingState.STARTED)) + .build() + ); + + } + csBuilder.routingTable(routingTableBuilder.build()); + metaBuilder.indices(indexMapBuilder.build()); + csBuilder.metadata(metaBuilder.build()); + + return csBuilder.build(); + } } diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java index e948bc6918853..13403f6d5d341 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java @@ -221,7 +221,12 @@ public Collection createComponents( IndexNameExpressionResolver expressionResolver, Supplier repositoriesServiceSupplier ) { - TransformConfigManager configManager = new IndexBasedTransformConfigManager(client, xContentRegistry); + TransformConfigManager configManager = new IndexBasedTransformConfigManager( + clusterService, + expressionResolver, + client, + xContentRegistry + ); TransformAuditor auditor = new TransformAuditor(client, clusterService.getNodeName(), clusterService); TransformCheckpointService checkpointService = new TransformCheckpointService( Clock.systemUTC(), diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java index 1fbe2bc1db618..2f514dbe3043b 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/IndexBasedTransformConfigManager.java @@ -28,9 +28,13 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexNotFoundException; @@ -64,6 +68,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashSet; @@ -99,10 +104,19 @@ public class IndexBasedTransformConfigManager implements TransformConfigManager private static final Logger logger = LogManager.getLogger(IndexBasedTransformConfigManager.class); private static final int MAX_RESULTS_WINDOW = 10_000; + private final ClusterService clusterService; + private final IndexNameExpressionResolver indexNameExpressionResolver; private final Client client; private final NamedXContentRegistry xContentRegistry; - public IndexBasedTransformConfigManager(Client client, NamedXContentRegistry xContentRegistry) { + public IndexBasedTransformConfigManager( + ClusterService clusterService, + IndexNameExpressionResolver indexNameExpressionResolver, + Client client, + NamedXContentRegistry xContentRegistry + ) { + this.clusterService = clusterService; + this.indexNameExpressionResolver = indexNameExpressionResolver; this.client = client; this.xContentRegistry = xContentRegistry; } @@ -255,11 +269,42 @@ public void deleteOldCheckpoints(String transformId, long deleteCheckpointsBelow @Override public void deleteOldIndices(ActionListener listener) { - DeleteIndexRequest deleteRequest = new DeleteIndexRequest( - TransformInternalIndexConstants.INDEX_NAME_PATTERN, - TransformInternalIndexConstants.INDEX_NAME_PATTERN_DEPRECATED, - "-" + TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME - ).indicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN); + ClusterState state = clusterService.state(); + Set indicesToDelete = new HashSet<>(); + + // use the transform context as we access system indexes + try (ThreadContext.StoredContext ctx = client.threadPool().getThreadContext().stashWithOrigin(TRANSFORM_ORIGIN)) { + indicesToDelete.addAll( + Arrays.asList( + indexNameExpressionResolver.concreteIndexNames( + state, + IndicesOptions.lenientExpandHidden(), + TransformInternalIndexConstants.INDEX_NAME_PATTERN + ) + ) + ); + + indicesToDelete.addAll( + Arrays.asList( + indexNameExpressionResolver.concreteIndexNames( + state, + IndicesOptions.lenientExpandHidden(), + TransformInternalIndexConstants.INDEX_NAME_PATTERN_DEPRECATED + ) + ) + ); + + indicesToDelete.remove(TransformInternalIndexConstants.LATEST_INDEX_VERSIONED_NAME); + } + + if (indicesToDelete.isEmpty()) { + listener.onResponse(true); + return; + } + + DeleteIndexRequest deleteRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[0])).indicesOptions( + IndicesOptions.LENIENT_EXPAND_OPEN + ); executeAsyncWithOrigin(client, TRANSFORM_ORIGIN, DeleteIndexAction.INSTANCE, deleteRequest, ActionListener.wrap(response -> { if (response.isAcknowledged() == false) { diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java index 1df06e3329440..64c9845c8f848 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/TransformPersistentTasksExecutorTests.java @@ -398,9 +398,15 @@ private ClusterState buildClusterState(DiscoveryNodes.Builder nodes) { } public TransformPersistentTasksExecutor buildTaskExecutor() { + ClusterService clusterService = mock(ClusterService.class); Client client = mock(Client.class); TransformAuditor mockAuditor = mock(TransformAuditor.class); - IndexBasedTransformConfigManager transformsConfigManager = new IndexBasedTransformConfigManager(client, xContentRegistry()); + IndexBasedTransformConfigManager transformsConfigManager = new IndexBasedTransformConfigManager( + clusterService, + TestIndexNameExpressionResolver.newInstance(), + client, + xContentRegistry() + ); TransformCheckpointService transformCheckpointService = new TransformCheckpointService( Clock.systemUTC(), Settings.EMPTY, @@ -416,7 +422,6 @@ public TransformPersistentTasksExecutor buildTaskExecutor() { ); ClusterSettings cSettings = new ClusterSettings(Settings.EMPTY, Collections.singleton(Transform.NUM_FAILURE_RETRIES_SETTING)); - ClusterService clusterService = mock(ClusterService.class); when(clusterService.getClusterSettings()).thenReturn(cSettings); when(clusterService.state()).thenReturn(TransformInternalIndexTests.randomTransformClusterState()); diff --git a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java index e0864cf3aac7f..227aadc64f055 100644 --- a/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java +++ b/x-pack/plugin/transform/src/test/java/org/elasticsearch/xpack/transform/transforms/pivot/PivotTests.java @@ -20,11 +20,6 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.ValidationException; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.xcontent.DeprecationHandler; -import org.elasticsearch.xcontent.NamedXContentRegistry; -import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xcontent.XContentType; -import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.search.SearchHit; @@ -32,6 +27,12 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.xcontent.DeprecationHandler; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xcontent.json.JsonXContent; +import org.elasticsearch.xpack.core.transform.TransformDeprecations; import org.elasticsearch.xpack.core.transform.transforms.SettingsConfig; import org.elasticsearch.xpack.core.transform.transforms.SourceConfig; import org.elasticsearch.xpack.core.transform.transforms.pivot.AggregationConfig; @@ -136,7 +137,7 @@ public void testInitialPageSize() throws Exception { ); assertThat(pivot.getInitialPageSize(), equalTo(Transform.DEFAULT_INITIAL_MAX_PAGE_SEARCH_SIZE)); - assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead"); + assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED); } public void testSearchFailure() throws Exception {