Backport: SPI for loading ABC templates (#14659) (#14784)

* SPI for loading ABC templates (#14659)

* SPI for loading ABC templates

Signed-off-by: mgodwan <[email protected]>
Signed-off-by: Mohit Godwani <[email protected]>

CHANGELOG.md

@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Create SystemIndexRegistry with helper method matchesSystemIndex ([#14415](https://github.com/opensearch-project/OpenSearch/pull/14415))
 - Print reason why parent task was cancelled ([#14604](https://github.com/opensearch-project/OpenSearch/issues/14604))
 - Add matchesPluginSystemIndexPattern to SystemIndexRegistry ([#14750](https://github.com/opensearch-project/OpenSearch/pull/14750))
+- Add Plugin interface for loading application based configuration templates (([#14659](https://github.com/opensearch-project/OpenSearch/issues/14659)))
 
 ### Dependencies
 - Update to Apache Lucene 9.11.1 ([#14042](https://github.com/opensearch-project/OpenSearch/pull/14042), [#14576](https://github.com/opensearch-project/OpenSearch/pull/14576))

server/src/main/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoader.java

@@ -0,0 +1,108 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.OpenSearchCorruptionException;
+import org.opensearch.action.admin.indices.template.put.PutComponentTemplateAction;
+import org.opensearch.client.Client;
+import org.opensearch.client.OriginSettingClient;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.ComponentTemplate;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.core.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+/**
+ * Class responsible for loading the component templates provided by a repository into the cluster state.
+ */
+@ExperimentalApi
+public class ClusterStateSystemTemplateLoader implements SystemTemplateLoader {
+
+    private final Client client;
+
+    private final Supplier<ClusterState> clusterStateSupplier;
+
+    private static final Logger logger = LogManager.getLogger(SystemTemplateLoader.class);
+
+    public static final String TEMPLATE_LOADER_IDENTIFIER = "system_template_loader";
+    public static final String TEMPLATE_TYPE_KEY = "_type";
+
+    public ClusterStateSystemTemplateLoader(Client client, Supplier<ClusterState> clusterStateSupplier) {
+        this.client = new OriginSettingClient(client, TEMPLATE_LOADER_IDENTIFIER);
+        this.clusterStateSupplier = clusterStateSupplier;
+    }
+
+    @Override
+    public boolean loadTemplate(SystemTemplate template) throws IOException {
+        final ComponentTemplate existingTemplate = clusterStateSupplier.get()
+            .metadata()
+            .componentTemplates()
+            .get(template.templateMetadata().fullyQualifiedName());
+
+        if (existingTemplate != null
+            && !SystemTemplateMetadata.COMPONENT_TEMPLATE_TYPE.equals(
+                Objects.toString(existingTemplate.metadata().get(TEMPLATE_TYPE_KEY))
+            )) {
+            throw new OpenSearchCorruptionException(
+                "Attempting to create " + template.templateMetadata().name() + " which has already been created through some other source."
+            );
+        }
+
+        if (existingTemplate != null && existingTemplate.version() >= template.templateMetadata().version()) {
+            logger.debug(
+                "Skipping putting template {} as its existing version [{}] is >= fetched version [{}]",
+                template.templateMetadata().fullyQualifiedName(),
+                existingTemplate.version(),
+                template.templateMetadata().version()
+            );
+            return false;
+        }
+
+        ComponentTemplate newTemplate = null;
+        try (
+            XContentParser contentParser = JsonXContent.jsonXContent.createParser(
+                NamedXContentRegistry.EMPTY,
+                DeprecationHandler.IGNORE_DEPRECATIONS,
+                template.templateContent().utf8ToString()
+            )
+        ) {
+            newTemplate = ComponentTemplate.parse(contentParser);
+        }
+
+        if (!Objects.equals(newTemplate.version(), template.templateMetadata().version())) {
+            throw new OpenSearchCorruptionException(
+                "Template version mismatch for "
+                    + template.templateMetadata().name()
+                    + ". Version in metadata: "
+                    + template.templateMetadata().version()
+                    + " , Version in content: "
+                    + newTemplate.version()
+            );
+        }
+
+        final PutComponentTemplateAction.Request request = new PutComponentTemplateAction.Request(
+            template.templateMetadata().fullyQualifiedName()
+        ).componentTemplate(newTemplate);
+
+        return client.admin()
+            .indices()
+            .execute(PutComponentTemplateAction.INSTANCE, request)
+            .actionGet(TimeValue.timeValueMillis(30000))
+            .isAcknowledged();
+    }
+}

server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplate.java

@@ -0,0 +1,43 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.common.bytes.BytesReference;
+
+/**
+ * Encapsulates the information and content about a system template available within a repository.
+ */
+@ExperimentalApi
+public class SystemTemplate {
+
+    private final BytesReference templateContent;
+
+    private final SystemTemplateMetadata templateMetadata;
+
+    private final TemplateRepositoryMetadata repositoryMetadata;
+
+    public SystemTemplate(BytesReference templateContent, SystemTemplateMetadata templateInfo, TemplateRepositoryMetadata repositoryInfo) {
+        this.templateContent = templateContent;
+        this.templateMetadata = templateInfo;
+        this.repositoryMetadata = repositoryInfo;
+    }
+
+    public BytesReference templateContent() {
+        return templateContent;
+    }
+
+    public SystemTemplateMetadata templateMetadata() {
+        return templateMetadata;
+    }
+
+    public TemplateRepositoryMetadata repositoryMetadata() {
+        return repositoryMetadata;
+    }
+}

server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateLoader.java

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Interface to load template into the OpenSearch runtime.
+ */
+@ExperimentalApi
+public interface SystemTemplateLoader {
+
+    /**
+     * @param template Templated to be loaded
+     * @throws IOException If an exceptional situation is encountered while parsing/loading the template
+     */
+    boolean loadTemplate(SystemTemplate template) throws IOException;
+}

server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateMetadata.java

@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * Metadata information about a template available in a template repository.
+ */
+@ExperimentalApi
+public class SystemTemplateMetadata {
+
+    private final long version;
+    private final String type;
+    private final String name;
+
+    private static final String DELIMITER = "@";
+
+    public static final String COMPONENT_TEMPLATE_TYPE = "@abc_template";
+
+    public SystemTemplateMetadata(long version, String type, String name) {
+        this.version = version;
+        this.type = type;
+        this.name = name;
+    }
+
+    public String type() {
+        return type;
+    }
+
+    public String name() {
+        return name;
+    }
+
+    public long version() {
+        return version;
+    }
+
+    /**
+     * Gets the metadata using fully qualified name for the template
+     * @param fullyQualifiedName (e.g. @abc_template@logs@1)
+     * @return Metadata object based on name
+     */
+    public static SystemTemplateMetadata fromComponentTemplate(String fullyQualifiedName) {
+        assert fullyQualifiedName.length() > 1 : "System template name must have at least one component";
+        assert fullyQualifiedName.substring(1, fullyQualifiedName.indexOf(DELIMITER, 1)).equals(COMPONENT_TEMPLATE_TYPE);
+
+        return new SystemTemplateMetadata(
+            Long.parseLong(fullyQualifiedName.substring(fullyQualifiedName.lastIndexOf(DELIMITER))),
+            COMPONENT_TEMPLATE_TYPE,
+            fullyQualifiedName.substring(0, fullyQualifiedName.lastIndexOf(DELIMITER))
+        );
+    }
+
+    public static SystemTemplateMetadata fromComponentTemplateInfo(String name, long version) {
+        return new SystemTemplateMetadata(version, COMPONENT_TEMPLATE_TYPE, name);
+    }
+
+    public final String fullyQualifiedName() {
+        return type + DELIMITER + name + DELIMITER + version;
+    }
+}

server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateRepository.java

@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Repository interface around the templates provided by a store (e.g. code repo, remote file store, etc)
+ */
+@ExperimentalApi
+public interface SystemTemplateRepository extends AutoCloseable {
+
+    /**
+     * @return Metadata about the repository
+     */
+    TemplateRepositoryMetadata metadata();
+
+    /**
+     * @return Metadata for all available templates
+     */
+    Iterable<SystemTemplateMetadata> listTemplates() throws IOException;
+
+    /**
+     *
+     * @param template metadata about template to be fetched
+     * @return The actual template content
+     */
+    SystemTemplate getTemplate(SystemTemplateMetadata template) throws IOException;
+}

server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesPlugin.java

@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Plugin interface to expose the template maintaining logic.
+ */
+@ExperimentalApi
+public interface SystemTemplatesPlugin {
+
+    /**
+     * @return repository implementation from which templates are to be fetched.
+     */
+    SystemTemplateRepository loadRepository() throws IOException;
+
+    /**
+     * @param templateInfo Metadata about the template to load
+     * @return Implementation of TemplateLoader which determines how to make the template available at runtime.
+     */
+    SystemTemplateLoader loaderFor(SystemTemplateMetadata templateInfo);
+}