Backport: SPI for loading ABC templates (#14659)

CHANGELOG.md

@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Create SystemIndexRegistry with helper method matchesSystemIndex ([#14415](https:/opensearch-project/OpenSearch/pull/14415))
 - Print reason why parent task was cancelled ([#14604](https:/opensearch-project/OpenSearch/issues/14604))
 - Add matchesPluginSystemIndexPattern to SystemIndexRegistry ([#14750](https:/opensearch-project/OpenSearch/pull/14750))
+- Add Plugin interface for loading application based configuration templates (([#14659](https:/opensearch-project/OpenSearch/issues/14659)))
 
 ### Dependencies
 - Update to Apache Lucene 9.11.1 ([#14042](https:/opensearch-project/OpenSearch/pull/14042), [#14576](https:/opensearch-project/OpenSearch/pull/14576))

server/src/main/java/org/opensearch/cluster/applicationtemplates/ClusterStateSystemTemplateLoader.java

@@ -0,0 +1,108 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.OpenSearchCorruptionException;
+import org.opensearch.action.admin.indices.template.put.PutComponentTemplateAction;
+import org.opensearch.client.Client;
+import org.opensearch.client.OriginSettingClient;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.ComponentTemplate;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.xcontent.DeprecationHandler;
+import org.opensearch.core.xcontent.NamedXContentRegistry;
+import org.opensearch.core.xcontent.XContentParser;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.function.Supplier;
+
+/**
+ * Class responsible for loading the component templates provided by a repository into the cluster state.
+ */
+@ExperimentalApi
+public class ClusterStateSystemTemplateLoader implements SystemTemplateLoader {
+
+ private final Client client;
+
+ private final Supplier<ClusterState> clusterStateSupplier;
+
+ private static final Logger logger = LogManager.getLogger(SystemTemplateLoader.class);
+
+ public static final String TEMPLATE_LOADER_IDENTIFIER = "system_template_loader";
+ public static final String TEMPLATE_TYPE_KEY = "_type";
+
+ public ClusterStateSystemTemplateLoader(Client client, Supplier<ClusterState> clusterStateSupplier) {
+ this.client = new OriginSettingClient(client, TEMPLATE_LOADER_IDENTIFIER);
+ this.clusterStateSupplier = clusterStateSupplier;
+ }
+
+ @Override
+ public boolean loadTemplate(SystemTemplate template) throws IOException {
+ final ComponentTemplate existingTemplate = clusterStateSupplier.get()
+ .metadata()
+ .componentTemplates()
+ .get(template.templateMetadata().fullyQualifiedName());
+
+ if (existingTemplate != null
+ && !SystemTemplateMetadata.COMPONENT_TEMPLATE_TYPE.equals(
+ Objects.toString(existingTemplate.metadata().get(TEMPLATE_TYPE_KEY))
+ )) {
+ throw new OpenSearchCorruptionException(
+ "Attempting to create " + template.templateMetadata().name() + " which has already been created through some other source."
+ );
+ }
+
+ if (existingTemplate != null && existingTemplate.version() >= template.templateMetadata().version()) {
+ logger.debug(
+ "Skipping putting template {} as its existing version [{}] is >= fetched version [{}]",
+ template.templateMetadata().fullyQualifiedName(),
+ existingTemplate.version(),
+ template.templateMetadata().version()
+ );
+ return false;
+ }
+
+ ComponentTemplate newTemplate = null;
+ try (
+ XContentParser contentParser = JsonXContent.jsonXContent.createParser(
+ NamedXContentRegistry.EMPTY,
+ DeprecationHandler.IGNORE_DEPRECATIONS,
+ template.templateContent().utf8ToString()
+ )
+ ) {
+ newTemplate = ComponentTemplate.parse(contentParser);
+ }
+
+ if (!Objects.equals(newTemplate.version(), template.templateMetadata().version())) {
+ throw new OpenSearchCorruptionException(
+ "Template version mismatch for "
+ + template.templateMetadata().name()
+ + ". Version in metadata: "
+ + template.templateMetadata().version()
+ + " , Version in content: "
+ + newTemplate.version()
+ );
+ }
+
+ final PutComponentTemplateAction.Request request = new PutComponentTemplateAction.Request(
+ template.templateMetadata().fullyQualifiedName()
+ ).componentTemplate(newTemplate);
+
+ return client.admin()
+ .indices()
+ .execute(PutComponentTemplateAction.INSTANCE, request)
+ .actionGet(TimeValue.timeValueMillis(30000))
+ .isAcknowledged();
+ }
+}

server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplate.java

@@ -0,0 +1,43 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.common.bytes.BytesReference;
+
+/**
+ * Encapsulates the information and content about a system template available within a repository.
+ */
+@ExperimentalApi
+public class SystemTemplate {
+
+ private final BytesReference templateContent;
+
+ private final SystemTemplateMetadata templateMetadata;
+
+ private final TemplateRepositoryMetadata repositoryMetadata;
+
+ public SystemTemplate(BytesReference templateContent, SystemTemplateMetadata templateInfo, TemplateRepositoryMetadata repositoryInfo) {
+ this.templateContent = templateContent;
+ this.templateMetadata = templateInfo;
+ this.repositoryMetadata = repositoryInfo;
+ }
+
+ public BytesReference templateContent() {
+ return templateContent;
+ }
+
+ public SystemTemplateMetadata templateMetadata() {
+ return templateMetadata;
+ }
+
+ public TemplateRepositoryMetadata repositoryMetadata() {
+ return repositoryMetadata;
+ }
+}

server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateLoader.java

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Interface to load template into the OpenSearch runtime.
+ */
+@ExperimentalApi
+public interface SystemTemplateLoader {
+
+ /**
+ * @param template Templated to be loaded
+ * @throws IOException If an exceptional situation is encountered while parsing/loading the template
+ */
+ boolean loadTemplate(SystemTemplate template) throws IOException;
+}

server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateMetadata.java

@@ -0,0 +1,68 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * Metadata information about a template available in a template repository.
+ */
+@ExperimentalApi
+public class SystemTemplateMetadata {
+
+ private final long version;
+ private final String type;
+ private final String name;
+
+ private static final String DELIMITER = "@";
+
+ public static final String COMPONENT_TEMPLATE_TYPE = "@abc_template";
+
+ public SystemTemplateMetadata(long version, String type, String name) {
+ this.version = version;
+ this.type = type;
+ this.name = name;
+ }
+
+ public String type() {
+ return type;
+ }
+
+ public String name() {
+ return name;
+ }
+
+ public long version() {
+ return version;
+ }
+
+ /**
+ * Gets the metadata using fully qualified name for the template
+ * @param fullyQualifiedName (e.g. @abc_template@logs@1)
+ * @return Metadata object based on name
+ */
+ public static SystemTemplateMetadata fromComponentTemplate(String fullyQualifiedName) {
+ assert fullyQualifiedName.length() > 1 : "System template name must have at least one component";
+ assert fullyQualifiedName.substring(1, fullyQualifiedName.indexOf(DELIMITER, 1)).equals(COMPONENT_TEMPLATE_TYPE);
+
+ return new SystemTemplateMetadata(
+ Long.parseLong(fullyQualifiedName.substring(fullyQualifiedName.lastIndexOf(DELIMITER))),
+ COMPONENT_TEMPLATE_TYPE,
+ fullyQualifiedName.substring(0, fullyQualifiedName.lastIndexOf(DELIMITER))
+ );
+ }
+
+ public static SystemTemplateMetadata fromComponentTemplateInfo(String name, long version) {
+ return new SystemTemplateMetadata(version, COMPONENT_TEMPLATE_TYPE, name);
+ }
+
+ public final String fullyQualifiedName() {
+ return type + DELIMITER + name + DELIMITER + version;
+ }
+}

server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplateRepository.java

@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Repository interface around the templates provided by a store (e.g. code repo, remote file store, etc)
+ */
+@ExperimentalApi
+public interface SystemTemplateRepository extends AutoCloseable {
+
+ /**
+ * @return Metadata about the repository
+ */
+ TemplateRepositoryMetadata metadata();
+
+ /**
+ * @return Metadata for all available templates
+ */
+ Iterable<SystemTemplateMetadata> listTemplates() throws IOException;
+
+ /**
+ *
+ * @param template metadata about template to be fetched
+ * @return The actual template content
+ */
+ SystemTemplate getTemplate(SystemTemplateMetadata template) throws IOException;
+}

server/src/main/java/org/opensearch/cluster/applicationtemplates/SystemTemplatesPlugin.java

@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.applicationtemplates;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.IOException;
+
+/**
+ * Plugin interface to expose the template maintaining logic.
+ */
+@ExperimentalApi
+public interface SystemTemplatesPlugin {
+
+ /**
+ * @return repository implementation from which templates are to be fetched.
+ */
+ SystemTemplateRepository loadRepository() throws IOException;
+
+ /**
+ * @param templateInfo Metadata about the template to load
+ * @return Implementation of TemplateLoader which determines how to make the template available at runtime.
+ */
+ SystemTemplateLoader loaderFor(SystemTemplateMetadata templateInfo);
+}