Make stack.templates.enabled a dynamic setting

This change allows the setting for disabling the automatically installed stack templates (the
`logs-*-*`, `metrics-*-*`, and `synthetics-*-*` templates) to be changed dynamically.

As a byproduct, it also moves thes `IndexTemplateRegistry` to use an `initialize()` method so that
constructors are not tempted to use a `this` reference in the constructor (see elastic#37861 for more
information about why to avoid that).

Resolves elastic#62835

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java

@@ -60,6 +60,7 @@ public abstract class IndexTemplateRegistry implements ClusterStateListener {
     protected final Client client;
     protected final ThreadPool threadPool;
     protected final NamedXContentRegistry xContentRegistry;
+    protected final ClusterService clusterService;
     protected final ConcurrentMap<String, AtomicBoolean> templateCreationsInProgress = new ConcurrentHashMap<>();
     protected final ConcurrentMap<String, AtomicBoolean> policyCreationsInProgress = new ConcurrentHashMap<>();
 
@@ -69,6 +70,13 @@ public IndexTemplateRegistry(Settings nodeSettings, ClusterService clusterServic
         this.client = client;
         this.threadPool = threadPool;
         this.xContentRegistry = xContentRegistry;
+        this.clusterService = clusterService;
+    }
+
+    /**
+     * Initialize the template registry, adding it as a listener so templates will be installed as necessary
+     */
+    public void initialize() {
         clusterService.addListener(this);
     }
 

x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycle.java

@@ -180,6 +180,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         @SuppressWarnings("unused")
         ILMHistoryTemplateRegistry ilmTemplateRegistry =
             new ILMHistoryTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry);
+        ilmTemplateRegistry.initialize();
         ilmHistoryStore.set(new ILMHistoryStore(settings, new OriginSettingClient(client, INDEX_LIFECYCLE_ORIGIN),
             clusterService, threadPool));
         indexLifecycleInitialisationService.set(new IndexLifecycleService(settings, client, clusterService, threadPool,
@@ -190,6 +191,7 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
         @SuppressWarnings("unused")
         SnapshotLifecycleTemplateRegistry templateRegistry = new SnapshotLifecycleTemplateRegistry(settings, clusterService, threadPool,
             client, xContentRegistry);
+        templateRegistry.initialize();
         snapshotHistoryStore.set(new SnapshotHistoryStore(settings, new OriginSettingClient(client, INDEX_LIFECYCLE_ORIGIN),
             clusterService));
         snapshotLifecycleService.set(new SnapshotLifecycleService(settings,

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/MachineLearning.java

@@ -583,7 +583,8 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
 
         this.mlUpgradeModeActionFilter.set(new MlUpgradeModeActionFilter(clusterService));
 
-        new MlIndexTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry);
+        MlIndexTemplateRegistry registry = new MlIndexTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry);
+        registry.initialize();
 
         AnomalyDetectionAuditor anomalyDetectionAuditor = new AnomalyDetectionAuditor(client, clusterService);
         DataFrameAnalyticsAuditor dataFrameAnalyticsAuditor = new DataFrameAnalyticsAuditor(client, clusterService);

x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackPlugin.java

@@ -21,7 +21,6 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
 
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -30,19 +29,13 @@
 public class StackPlugin extends Plugin implements ActionPlugin {
     private final Settings settings;
 
-    public static final Setting<Boolean> STACK_TEMPLATES_ENABLED = Setting.boolSetting(
-        "stack.templates.enabled",
-        true,
-        Setting.Property.NodeScope
-    );
-
     public StackPlugin(Settings settings) {
         this.settings = settings;
     }
 
     @Override
     public List<Setting<?>> getSettings() {
-        return Collections.singletonList(STACK_TEMPLATES_ENABLED);
+        return Collections.singletonList(StackTemplateRegistry.STACK_TEMPLATES_ENABLED);
     }
 
     @Override
@@ -59,8 +52,8 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier
     ) {
-        final List<Object> components = new ArrayList<>();
         StackTemplateRegistry templateRegistry = new StackTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry);
-        return components;
+        templateRegistry.initialize();
+        return Collections.singleton(templateRegistry);
     }
 }

x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java

@@ -6,8 +6,11 @@
 
 package org.elasticsearch.xpack.stack;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.client.Client;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -19,17 +22,27 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.stream.Collectors;
 
 public class StackTemplateRegistry extends IndexTemplateRegistry {
+    private static final Logger logger = LogManager.getLogger(StackTemplateRegistry.class);
+
     // The stack template registry should remain at version 0. This is because templates and
     // policies will be changed by the ingest manager once they exist, and ES should only ever put
     // the template in place if it does not exist. If this were incremented we could accidentally
     // overwrite a template or policy changed by the ingest manager.
     public static final int REGISTRY_VERSION = 0;
 
     public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version";
+    public static final Setting<Boolean> STACK_TEMPLATES_ENABLED = Setting.boolSetting(
+        "stack.templates.enabled",
+        true,
+        Setting.Property.NodeScope,
+        Setting.Property.Dynamic
+    );
 
-    private final boolean stackTemplateEnabled;
+    private final ClusterService clusterService;
+    private volatile boolean stackTemplateEnabled;
 
     //////////////////////////////////////////////////////////
     // Logs components (for matching logs-*-* indices)
@@ -129,7 +142,27 @@ public StackTemplateRegistry(
         NamedXContentRegistry xContentRegistry
     ) {
         super(nodeSettings, clusterService, threadPool, client, xContentRegistry);
-        this.stackTemplateEnabled = StackPlugin.STACK_TEMPLATES_ENABLED.get(nodeSettings);
+        this.clusterService = clusterService;
+        this.stackTemplateEnabled = STACK_TEMPLATES_ENABLED.get(nodeSettings);
+    }
+
+    @Override
+    public void initialize() {
+        super.initialize();
+        clusterService.getClusterSettings().addSettingsUpdateConsumer(STACK_TEMPLATES_ENABLED, this::updateEnabledSetting);
+    }
+
+    private void updateEnabledSetting(boolean newValue) {
+        if (newValue) {
+            this.stackTemplateEnabled = true;
+        } else {
+            logger.info(
+                "stack composable templates [{}] and component templates [{}] will not be installed or reinstalled",
+                getComposableTemplateConfigs().stream().map(IndexTemplateConfig::getTemplateName).collect(Collectors.joining(",")),
+                getComponentTemplateConfigs().stream().map(IndexTemplateConfig::getTemplateName).collect(Collectors.joining(","))
+            );
+            this.stackTemplateEnabled = false;
+        }
     }
 
     @Override

x-pack/plugin/stack/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryTests.java

@@ -105,7 +105,7 @@ public void tearDown() throws Exception {
     }
 
     public void testDisabledDoesNotAddTemplates() {
-        Settings settings = Settings.builder().put(StackPlugin.STACK_TEMPLATES_ENABLED.getKey(), false).build();
+        Settings settings = Settings.builder().put(StackTemplateRegistry.STACK_TEMPLATES_ENABLED.getKey(), false).build();
         StackTemplateRegistry disabledRegistry = new StackTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry);
         assertThat(disabledRegistry.getComponentTemplateConfigs(), hasSize(0));
         assertThat(disabledRegistry.getComposableTemplateConfigs(), hasSize(0));

x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java

@@ -272,7 +272,9 @@ public Collection<Object> createComponents(Client client, ClusterService cluster
             throw new UncheckedIOException(e);
         }
 
-        new WatcherIndexTemplateRegistry(environment.settings(), clusterService, threadPool, client, xContentRegistry);
+        WatcherIndexTemplateRegistry templateRegistry = new WatcherIndexTemplateRegistry(environment.settings(),
+            clusterService, threadPool, client, xContentRegistry);
+        templateRegistry.initialize();
 
         final SSLService sslService = getSslService();
         // http client

x-pack/qa/core-rest-tests-with-security/src/test/resources/rest-api-spec/test/stack/10_stack.yml

@@ -0,0 +1,19 @@
+"Stack templates can be disabled":
+  - do:
+      cluster.put_settings:
+        body:
+          transient:
+            stack.templates.enabled: false
+
+  - do:
+      indices.get_index_template:
+        name: logs
+
+  - do:
+      indices.delete_index_template:
+        name: logs
+
+  - do:
+      catch: missing
+      indices.get_index_template:
+        name: logs