[Transform] Improve transform upgrader (elastic#79601)

This changes fixes some issues with the upgrader. It inlines the index name resolver as wildcard
deletes are not supported if action.destructive_requires_name is set to true(default). It changes
response to always return all counters in order to avoid an empty response. After documentation has
been created, it switches the deprecation URL to the upgrade API docs.

client/rest-high-level/src/test/java/org/elasticsearch/client/transform/UpgradeTransformsResponseTests.java

@@ -37,15 +37,9 @@ private static UpgradeTransformsResponse createTestInstance() {
 
     private static void toXContent(UpgradeTransformsResponse response, XContentBuilder builder) throws IOException {
         builder.startObject();
-        if (response.getUpdated() != 0) {
-            builder.field("updated", response.getUpdated());
-        }
-        if (response.getNoAction() != 0) {
-            builder.field("no_action", response.getNoAction());
-        }
-        if (response.getNeedsUpdate() != 0) {
-            builder.field("needs_update", response.getNeedsUpdate());
-        }
+        builder.field("updated", response.getUpdated());
+        builder.field("no_action", response.getNoAction());
+        builder.field("needs_update", response.getNeedsUpdate());
         builder.endObject();
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java

@@ -8,12 +8,21 @@
 package org.elasticsearch.xpack.core.transform;
 
 public class TransformDeprecations {
+
+    public static final String UPGRADE_TRANSFORM_URL = "https://ela.st/es-8-upgrade-transforms";
+
+    // breaking changes base url for the _next_ major release
     public static final String BREAKING_CHANGES_BASE_URL =
         "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html";
 
     public static final String QUERY_BREAKING_CHANGES_URL = BREAKING_CHANGES_BASE_URL + "#breaking_80_search_changes";
 
     public static final String AGGS_BREAKING_CHANGES_URL = BREAKING_CHANGES_BASE_URL + "#breaking_80_aggregations_changes";
 
+    public static final String ACTION_UPGRADE_TRANSFORMS_API = "Use the upgrade transforms API to fix your transforms.";
+
+    public static final String ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED =
+        "[max_page_search_size] is deprecated inside pivot. Use settings instead.";
+
     private TransformDeprecations() {}
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/action/UpgradeTransformsAction.java

@@ -125,15 +125,9 @@ public void writeTo(StreamOutput out) throws IOException {
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            if (updated != 0L) {
-                builder.field("updated", updated);
-            }
-            if (noAction != 0L) {
-                builder.field("no_action", noAction);
-            }
-            if (needsUpdate != 0L) {
-                builder.field("needs_update", needsUpdate);
-            }
+            builder.field("updated", updated);
+            builder.field("no_action", noAction);
+            builder.field("needs_update", needsUpdate);
             builder.endObject();
             return builder;
         }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java

@@ -14,15 +14,15 @@
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ObjectParser;
 import org.elasticsearch.xcontent.ParseField;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.xpack.core.common.time.TimeUtils;
 import org.elasticsearch.xpack.core.common.validation.SourceDestValidator;
 import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.SourceDestValidation;
@@ -402,8 +402,8 @@ public List<DeprecationIssue> checkForDeprecations(NamedXContentRegistry namedXC
                 new DeprecationIssue(
                     Level.CRITICAL,
                     "Transform [" + id + "] is too old",
-                    TransformDeprecations.BREAKING_CHANGES_BASE_URL,
-                    "The configuration uses an old format, you can use [_update] or [_upgrade] to update",
+                    TransformDeprecations.UPGRADE_TRANSFORM_URL,
+                    TransformDeprecations.ACTION_UPGRADE_TRANSFORMS_API,
                     false,
                     null
                 )

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfig.java

@@ -13,13 +13,13 @@
 import org.elasticsearch.common.io.stream.Writeable;
 import org.elasticsearch.common.logging.DeprecationCategory;
 import org.elasticsearch.common.logging.DeprecationLogger;
+import org.elasticsearch.core.Nullable;
+import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.xcontent.ConstructingObjectParser;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.core.Nullable;
-import org.elasticsearch.search.aggregations.MultiBucketConsumerService;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level;
 import org.elasticsearch.xpack.core.transform.TransformDeprecations;
@@ -91,7 +91,7 @@ public PivotConfig(final GroupConfig groups, final AggregationConfig aggregation
             deprecationLogger.critical(
                 DeprecationCategory.API,
                 TransformField.MAX_PAGE_SEARCH_SIZE.getPreferredName(),
-                "[max_page_search_size] is deprecated inside pivot please use settings instead"
+                TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED
             );
         }
     }
@@ -187,7 +187,7 @@ public void checkForDeprecations(String id, NamedXContentRegistry namedXContentR
                     Level.WARNING,
                     "Transform [" + id + "] uses deprecated max_page_search_size",
                     TransformDeprecations.BREAKING_CHANGES_BASE_URL,
-                    "[max_page_search_size] is deprecated inside pivot please use settings instead",
+                    TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED,
                     false,
                     null
                 )

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfigTests.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level;
 import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase;
+import org.elasticsearch.xpack.core.transform.TransformDeprecations;
 import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfig;
 import org.elasticsearch.xpack.core.transform.transforms.latest.LatestConfigTests;
 import org.elasticsearch.xpack.core.transform.transforms.pivot.PivotConfig;
@@ -510,7 +511,7 @@ public void testRewriteForUpdate() throws IOException {
         assertTrue(transformConfigRewritten.getSettings().getDatesAsEpochMillis());
         assertFalse(transformConfigRewritten.getSettings().getAlignCheckpoints());
 
-        assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead");
+        assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED);
         assertEquals(Version.CURRENT, transformConfigRewritten.getVersion());
     }
 
@@ -583,7 +584,7 @@ public void testRewriteForUpdateMaxPageSizeSearchConflicting() throws IOExceptio
         assertNotNull(transformConfigRewritten.getSettings().getMaxPageSearchSize());
         assertEquals(555L, transformConfigRewritten.getSettings().getMaxPageSearchSize().longValue());
         assertEquals(Version.CURRENT, transformConfigRewritten.getVersion());
-        assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead");
+        assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED);
     }
 
     public void testRewriteForBWCOfDateNormalization() throws IOException {
@@ -739,7 +740,7 @@ public void testCheckForDeprecations() {
         TransformConfig deprecatedConfig = randomTransformConfigWithDeprecatedFields(id, Version.CURRENT);
 
         // check _and_ clear warnings
-        assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead");
+        assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED);
 
         // important: checkForDeprecations does _not_ create new deprecation warnings
         assertThat(
@@ -749,8 +750,8 @@ public void testCheckForDeprecations() {
                     new DeprecationIssue(
                         Level.WARNING,
                         "Transform [" + id + "] uses deprecated max_page_search_size",
-                        "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html",
-                        "[max_page_search_size] is deprecated inside pivot please use settings instead",
+                        TransformDeprecations.BREAKING_CHANGES_BASE_URL,
+                        TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED,
                         false,
                         null
                     )
@@ -761,7 +762,7 @@ public void testCheckForDeprecations() {
         deprecatedConfig = randomTransformConfigWithDeprecatedFields(id, Version.V_7_10_0);
 
         // check _and_ clear warnings
-        assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead");
+        assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED);
 
         // important: checkForDeprecations does _not_ create new deprecation warnings
         assertThat(
@@ -771,8 +772,8 @@ public void testCheckForDeprecations() {
                     new DeprecationIssue(
                         Level.WARNING,
                         "Transform [" + id + "] uses deprecated max_page_search_size",
-                        "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html",
-                        "[max_page_search_size] is deprecated inside pivot please use settings instead",
+                        TransformDeprecations.BREAKING_CHANGES_BASE_URL,
+                        TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED,
                         false,
                         null
                     )
@@ -783,7 +784,7 @@ public void testCheckForDeprecations() {
         deprecatedConfig = randomTransformConfigWithDeprecatedFields(id, Version.V_7_4_0);
 
         // check _and_ clear warnings
-        assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead");
+        assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED);
 
         // important: checkForDeprecations does _not_ create new deprecation warnings
         assertThat(
@@ -793,16 +794,16 @@ public void testCheckForDeprecations() {
                     new DeprecationIssue(
                         Level.CRITICAL,
                         "Transform [" + id + "] is too old",
-                        "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html",
-                        "The configuration uses an old format, you can use [_update] or [_upgrade] to update",
+                        TransformDeprecations.UPGRADE_TRANSFORM_URL,
+                        TransformDeprecations.ACTION_UPGRADE_TRANSFORMS_API,
                         false,
                         null
                     ),
                     new DeprecationIssue(
                         Level.WARNING,
                         "Transform [" + id + "] uses deprecated max_page_search_size",
-                        "https://www.elastic.co/guide/en/elasticsearch/reference/master/migrating-8.0.html",
-                        "[max_page_search_size] is deprecated inside pivot please use settings instead",
+                        TransformDeprecations.BREAKING_CHANGES_BASE_URL,
+                        TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED,
                         false,
                         null
                     )

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/transform/transforms/pivot/PivotConfigTests.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
 import org.elasticsearch.xpack.core.transform.AbstractSerializingTransformTestCase;
+import org.elasticsearch.xpack.core.transform.TransformDeprecations;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -437,7 +438,7 @@ public void testAggNameValidationsWithInvalidFieldnames() {
 
     public void testDeprecation() {
         PivotConfig pivotConfig = randomPivotConfigWithDeprecatedFields();
-        assertWarnings("[max_page_search_size] is deprecated inside pivot please use settings instead");
+        assertWarnings(TransformDeprecations.ACTION_MAX_PAGE_SEARCH_SIZE_IS_DEPRECATED);
     }
 
     private static String dotJoin(String... fields) {

x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/checkpoint/TransformCheckpointServiceNodeTests.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.index.shard.ShardPath;
 import org.elasticsearch.index.store.StoreStats;
 import org.elasticsearch.index.warmer.WarmerStats;
+import org.elasticsearch.indices.TestIndexNameExpressionResolver;
 import org.elasticsearch.search.suggest.completion.CompletionStats;
 import org.elasticsearch.test.client.NoOpClient;
 import org.elasticsearch.xpack.core.transform.transforms.TransformCheckpoint;
@@ -138,8 +139,13 @@ public void createComponents() {
         if (mockClientForCheckpointing == null) {
             mockClientForCheckpointing = new MockClientForCheckpointing("TransformCheckpointServiceNodeTests");
         }
-
-        transformsConfigManager = new IndexBasedTransformConfigManager(client(), xContentRegistry());
+        ClusterService clusterService = mock(ClusterService.class);
+        transformsConfigManager = new IndexBasedTransformConfigManager(
+            clusterService,
+            TestIndexNameExpressionResolver.newInstance(),
+            client(),
+            xContentRegistry()
+        );
 
         // use a mock for the checkpoint service
         TransformAuditor mockAuditor = mock(TransformAuditor.class);

x-pack/plugin/transform/src/internalClusterTest/java/org/elasticsearch/xpack/transform/persistence/TransformConfigManagerTests.java

@@ -9,14 +9,30 @@
 
 import org.elasticsearch.ResourceAlreadyExistsException;
 import org.elasticsearch.ResourceNotFoundException;
+import org.elasticsearch.Version;
 import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
 import org.elasticsearch.action.admin.indices.refresh.RefreshRequest;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.WriteRequest;
+import org.elasticsearch.cluster.ClusterName;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.cluster.routing.IndexRoutingTable;
+import org.elasticsearch.cluster.routing.RoutingTable;
+import org.elasticsearch.cluster.routing.ShardRoutingState;
+import org.elasticsearch.cluster.routing.TestShardRouting;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.collect.ImmutableOpenMap;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.core.Tuple;
+import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
 import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.indices.TestIndexNameExpressionResolver;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
@@ -33,6 +49,7 @@
 import org.elasticsearch.xpack.transform.TransformSingleNodeTestCase;
 import org.junit.Before;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -50,14 +67,23 @@
 import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class TransformConfigManagerTests extends TransformSingleNodeTestCase {
 
     private IndexBasedTransformConfigManager transformConfigManager;
+    private ClusterService clusterService;
 
     @Before
     public void createComponents() {
-        transformConfigManager = new IndexBasedTransformConfigManager(client(), xContentRegistry());
+        clusterService = mock(ClusterService.class);
+        transformConfigManager = new IndexBasedTransformConfigManager(
+            clusterService,
+            TestIndexNameExpressionResolver.newInstance(),
+            client(),
+            xContentRegistry()
+        );
     }
 
     public void testGetMissingTransform() throws InterruptedException {
@@ -706,6 +732,10 @@ public void testDeleteOldIndices() throws Exception {
         assertAsync(listener -> transformConfigManager.getTransformConfiguration(transformId, listener), transformConfigNew, null, null);
 
         // delete old indices
+        when(clusterService.state()).thenReturn(
+            createClusterStateWithTransformIndex(oldIndex, TransformInternalIndexConstants.LATEST_INDEX_NAME)
+        );
+
         assertAsync(listener -> transformConfigManager.deleteOldIndices(listener), true, null, null);
 
         // the config should still be there
@@ -724,4 +754,32 @@ public void testDeleteOldIndices() throws Exception {
         );
     }
 
+    private static ClusterState createClusterStateWithTransformIndex(String... indexes) throws IOException {
+        ImmutableOpenMap.Builder<String, IndexMetadata> indexMapBuilder = ImmutableOpenMap.builder();
+        Metadata.Builder metaBuilder = Metadata.builder();
+        ClusterState.Builder csBuilder = ClusterState.builder(ClusterName.DEFAULT);
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+
+        for (String index : indexes) {
+            IndexMetadata.Builder builder = new IndexMetadata.Builder(index).settings(
+                Settings.builder()
+                    .put(TransformInternalIndex.settings())
+                    .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT)
+                    .build()
+            ).numberOfReplicas(0).numberOfShards(1).putMapping(Strings.toString(TransformInternalIndex.mappings()));
+            indexMapBuilder.put(index, builder.build());
+
+            routingTableBuilder.add(
+                IndexRoutingTable.builder(new Index(index, UUIDs.randomBase64UUID()))
+                    .addShard(TestShardRouting.newShardRouting(index, 0, "node_a", null, true, ShardRoutingState.STARTED))
+                    .build()
+            );
+
+        }
+        csBuilder.routingTable(routingTableBuilder.build());
+        metaBuilder.indices(indexMapBuilder.build());
+        csBuilder.metadata(metaBuilder.build());
+
+        return csBuilder.build();
+    }
 }

x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/Transform.java

@@ -318,7 +318,12 @@ public Collection<Object> createComponents(
             return emptyList();
         }
 
-        TransformConfigManager configManager = new IndexBasedTransformConfigManager(client, xContentRegistry);
+        TransformConfigManager configManager = new IndexBasedTransformConfigManager(
+            clusterService,
+            expressionResolver,
+            client,
+            xContentRegistry
+        );
         TransformAuditor auditor = new TransformAuditor(client, clusterService.getNodeName(), clusterService);
         TransformCheckpointService checkpointService = new TransformCheckpointService(
             Clock.systemUTC(),