diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 221890e4738d6..4883c14f27cd1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -1579,7 +1579,7 @@ public void testDataStreams() throws Exception { CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"@timestamp\":{\"type\":\"date\"}}}"); Template template = new Template(null, mappings, null); ComposableIndexTemplate indexTemplate = new ComposableIndexTemplate(Collections.singletonList(dataStreamName), template, - Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate()); + Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate(), null); PutComposableIndexTemplateRequest putComposableIndexTemplateRequest = new PutComposableIndexTemplateRequest().name("ds-template").create(true).indexTemplate(indexTemplate); AcknowledgedResponse response = execute(putComposableIndexTemplateRequest, @@ -1658,7 +1658,7 @@ public void testIndexTemplates() throws Exception { Template template = new Template(settings, mappings, Map.of("alias", alias)); List pattern = List.of("pattern"); ComposableIndexTemplate indexTemplate = - new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null); + new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null, null); PutComposableIndexTemplateRequest putComposableIndexTemplateRequest = new PutComposableIndexTemplateRequest().name(templateName).create(true).indexTemplate(indexTemplate); @@ -1705,7 +1705,7 @@ public void testSimulateIndexTemplate() throws Exception { Template template = new Template(settings, mappings, Map.of("alias", alias)); List pattern = List.of("pattern"); ComposableIndexTemplate indexTemplate = - new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null); + new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null, null); PutComposableIndexTemplateRequest putComposableIndexTemplateRequest = new PutComposableIndexTemplateRequest().name(templateName).create(true).indexTemplate(indexTemplate); @@ -1716,7 +1716,7 @@ public void testSimulateIndexTemplate() throws Exception { SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest("pattern"); AliasMetadata simulationAlias = AliasMetadata.builder("simulation-alias").writeIndex(true).build(); ComposableIndexTemplate simulationTemplate = new ComposableIndexTemplate(pattern, new Template(null, null, - Map.of("simulation-alias", simulationAlias)), Collections.emptyList(), 2L, 1L, new HashMap<>(), null); + Map.of("simulation-alias", simulationAlias)), Collections.emptyList(), 2L, 1L, new HashMap<>(), null, null); PutComposableIndexTemplateRequest newIndexTemplateReq = new PutComposableIndexTemplateRequest().name("used-for-simulation").create(true).indexTemplate(indexTemplate); newIndexTemplateReq.indexTemplate(simulationTemplate); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java index 23759e7b09a58..f244f16b6a598 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indices/GetComposableIndexTemplatesResponseTests.java @@ -88,6 +88,6 @@ private static ComposableIndexTemplate randomIndexTemplate() { if (randomBoolean()) { dataStreamTemplate = new ComposableIndexTemplate.DataStreamTemplate(); } - return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate); + return new ComposableIndexTemplate(patterns, randomTemplate(), composedOf, priority, version, meta, dataStreamTemplate, null); } } diff --git a/docs/reference/indices/put-component-template.asciidoc b/docs/reference/indices/put-component-template.asciidoc index c5c48cb521fac..e58d370f60f29 100644 --- a/docs/reference/indices/put-component-template.asciidoc +++ b/docs/reference/indices/put-component-template.asciidoc @@ -4,10 +4,10 @@ Put component template ++++ -Creates or updates a component template. -Component templates are building blocks for constructing <>. -that specify index <>, <>, -and <>. +Creates or updates a component template. +Component templates are building blocks for constructing <>. +that specify index <>, <>, +and <>. [source,console] -------------------------------------------------- @@ -55,10 +55,10 @@ DELETE _component_template/template_* [[put-component-template-api-desc]] ==== {api-description-title} -An index template can be composed of multiple component templates. +An index template can be composed of multiple component templates. To use a component template, specify it in an index template's `composed_of` list. -Component templates are only applied to new data streams and indices -as part of a matching index template. +Component templates are only applied to new data streams and indices +as part of a matching index template. Settings and mappings specified directly in the index template or the <> request override any settings or mappings specified in a component template. @@ -112,6 +112,16 @@ include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=settings] Version number used to manage component templates externally. This number is not automatically generated or incremented by {es}. +`allow_auto_create`:: +(Optional, boolean) +This setting overrides the value of the +<> cluster setting. If set to +`true` in a template, then indices can be automatically created using that +template even if auto-creation of indices is disabled via +`actions.auto_create_index`. If set to `false`, then indices or data streams matching the +template must always be explicitly created, and may never be automatically +created. + `_meta`:: (Optional, object) Optional user metadata about the component template. May have any contents. @@ -157,7 +167,7 @@ To be applied, a component template must be included in an index template's `com [[component-templates-version]] ===== Component template versioning -You can use the `version` parameter to add a version number to a component template. +You can use the `version` parameter to add a version number to a component template. External systems can use these version numbers to simplify template management. The `version` parameter is optional and not automatically generated or used by {es}. @@ -182,7 +192,7 @@ To check the `version`, you can use the <false + * disable the automatic creation on indices. + */ + public void testCannotAutoCreateIndexWhenDisabled() throws IOException { + configureAutoCreateIndex(false); + + // Attempt to add a document to a non-existing index. Auto-creating the index should fail owing to the setting above. + final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc/123456"); + indexDocumentRequest.setJsonEntity("{ \"name\": \"Kimchi\" }"); + final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); + + assertThat( + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)), + containsString("no such index [recipe_kr] and [action.auto_create_index] is [false]") + ); + } + + /** + * Check that automatically creating an index is allowed, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} + * is false, when the index name matches a template and that template has allow_auto_create + * set to true. + */ + public void testCanAutoCreateIndexWhenAllowedByTemplate() throws IOException { + configureAutoCreateIndex(false); + + createTemplateWithAllowAutoCreate(true); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + assertOK(this.indexDocument()); + } + + /** + * Check that automatically creating an index is disallowed when the index name matches a template and that template has + * allow_auto_create explicitly to false, even when {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} + * is set to true. + */ + public void testCannotAutoCreateIndexWhenDisallowedByTemplate() throws IOException { + configureAutoCreateIndex(true); + + createTemplateWithAllowAutoCreate(false); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); + + assertThat( + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)), + containsString("no such index [composable template [recipe*] forbids index auto creation]") + ); + } + + + private void configureAutoCreateIndex(boolean value) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("transient") + .field(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), value) + .endObject() + .endObject(); + + final Request settingsRequest = new Request("PUT", "_cluster/settings"); + settingsRequest.setJsonEntity(Strings.toString(builder)); + final Response settingsResponse = client().performRequest(settingsRequest); + assertOK(settingsResponse); + } + + private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .array("index_patterns", "recipe*") + .field("allow_auto_create", allowAutoCreate) + .endObject(); + + final Request createTemplateRequest = new Request("PUT", "_index_template/recipe_template"); + createTemplateRequest.setJsonEntity(Strings.toString(builder)); + final Response createTemplateResponse = client().performRequest(createTemplateRequest); + assertOK(createTemplateResponse); + } + + private Response indexDocument() throws IOException { + final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc/123456"); + indexDocumentRequest.setJsonEntity("{ \"name\": \"Kimchi\" }"); + return client().performRequest(indexDocumentRequest); + } +} diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java index c9494aeb89f94..741817dfd50a3 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/bulk/BulkProcessorClusterSettingsIT.java @@ -49,7 +49,8 @@ public void testBulkProcessorAutoCreateRestrictions() { assertTrue("Missing index should have been flagged", responses[1].isFailed()); assertThat( responses[1].getFailureMessage(), - equalTo("[wontwork] org.elasticsearch.index.IndexNotFoundException: no such index [wontwork]")); + equalTo("[wontwork] org.elasticsearch.index.IndexNotFoundException: no such index [wontwork]" + + " and [action.auto_create_index] is [false]")); assertFalse("Operation on existing index should succeed", responses[2].isFailed()); } } diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java index a84accde6d169..1fbd740f27c60 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/template/ComposableTemplateIT.java @@ -102,7 +102,7 @@ public void testUsageOfDataStreamFails() throws IOException { Exception expectedException = expectThrows(Exception.class, () -> ComposableIndexTemplate.parse(parser)); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of("logs-*-*"), null, null, null, null, - null, new ComposableIndexTemplate.DataStreamTemplate()); + null, new ComposableIndexTemplate.DataStreamTemplate(), null); Exception e = expectThrows(IllegalArgumentException.class, () -> client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("my-it").indexTemplate(template)).actionGet()); Exception actualException = (Exception) e.getCause(); diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index be0082da01849..46334a1ff8155 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -23,15 +23,15 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.ActiveShardsObserver; +import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.master.TransportMasterNodeAction; import org.elasticsearch.cluster.AckedClusterStateUpdateTask; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ack.ClusterStateUpdateResponse; import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlockLevel; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate.DataStreamTemplate; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService; import org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.CreateDataStreamClusterStateUpdateRequest; @@ -40,6 +40,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; @@ -63,17 +64,20 @@ public static final class TransportAction extends TransportMasterNodeAction indices = bulkRequest.requests.stream() - // delete requests should not attempt to create the index (if the index does not - // exists), unless an external versioning is used - .filter(request -> request.opType() != DocWriteRequest.OpType.DELETE - || request.versionType() == VersionType.EXTERNAL - || request.versionType() == VersionType.EXTERNAL_GTE) - .collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2)); - /* Step 2: filter that to indices that don't exist and we can create. At the same time build a map of indices we can't create - * that we'll use when we try to run the requests. */ - final Map indicesThatCannotBeCreated = new HashMap<>(); - Set autoCreateIndices = new HashSet<>(); - ClusterState state = clusterService.state(); - for (Map.Entry indexAndFlag : indices.entrySet()) { - boolean shouldAutoCreate; - final String index = indexAndFlag.getKey(); - try { - shouldAutoCreate = shouldAutoCreate(index, state); - } catch (IndexNotFoundException e) { - shouldAutoCreate = false; - indicesThatCannotBeCreated.put(index, e); - } - // We should only auto create if we are not requiring it to be an alias - if (shouldAutoCreate && (indexAndFlag.getValue() == false)) { - autoCreateIndices.add(index); - } + // Attempt to create all the indices that we're going to need during the bulk before we start. + // Step 1: collect all the indices in the request + final Map indices = bulkRequest.requests.stream() + // delete requests should not attempt to create the index (if the index does not + // exists), unless an external versioning is used + .filter(request -> request.opType() != DocWriteRequest.OpType.DELETE + || request.versionType() == VersionType.EXTERNAL + || request.versionType() == VersionType.EXTERNAL_GTE) + .collect(Collectors.toMap(DocWriteRequest::index, DocWriteRequest::isRequireAlias, (v1, v2) -> v1 || v2)); + + // Step 2: filter the list of indices to find those that don't currently exist. + final Map indicesThatCannotBeCreated = new HashMap<>(); + Set autoCreateIndices = new HashSet<>(); + ClusterState state = clusterService.state(); + for (Map.Entry indexAndFlag : indices.entrySet()) { + final String index = indexAndFlag.getKey(); + boolean shouldAutoCreate = indexNameExpressionResolver.hasIndexAbstraction(index, state) == false; + // We should only auto create if we are not requiring it to be an alias + if (shouldAutoCreate && (indexAndFlag.getValue() == false)) { + autoCreateIndices.add(index); } - // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back. - if (autoCreateIndices.isEmpty()) { - executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated); - } else { - final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size()); - for (String index : autoCreateIndices) { - createIndex(index, bulkRequest.timeout(), minNodeVersion, new ActionListener<>() { - @Override - public void onResponse(CreateIndexResponse result) { - if (counter.decrementAndGet() == 0) { - threadPool.executor(executorName).execute( - () -> executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated)); - } + } + + // Step 3: create all the indices that are missing, if there are any missing. start the bulk after all the creates come back. + if (autoCreateIndices.isEmpty()) { + executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated); + } else { + final AtomicInteger counter = new AtomicInteger(autoCreateIndices.size()); + for (String index : autoCreateIndices) { + createIndex(index, bulkRequest.timeout(), minNodeVersion, new ActionListener<>() { + @Override + public void onResponse(CreateIndexResponse result) { + if (counter.decrementAndGet() == 0) { + threadPool.executor(executorName).execute( + () -> executeBulk(task, bulkRequest, startTime, listener, responses, indicesThatCannotBeCreated)); } + } - @Override - public void onFailure(Exception e) { - if (!(ExceptionsHelper.unwrapCause(e) instanceof ResourceAlreadyExistsException)) { - // fail all requests involving this index, if create didn't work - for (int i = 0; i < bulkRequest.requests.size(); i++) { - DocWriteRequest request = bulkRequest.requests.get(i); - if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) { - bulkRequest.requests.set(i, null); - } + @Override + public void onFailure(Exception e) { + final Throwable cause = ExceptionsHelper.unwrapCause(e); + if (cause instanceof IndexNotFoundException) { + indicesThatCannotBeCreated.put(index, (IndexNotFoundException) e); + } + else if ((cause instanceof ResourceAlreadyExistsException) == false) { + // fail all requests involving this index, if create didn't work + for (int i = 0; i < bulkRequest.requests.size(); i++) { + DocWriteRequest request = bulkRequest.requests.get(i); + if (request != null && setResponseFailureIfIndexMatches(responses, i, request, index, e)) { + bulkRequest.requests.set(i, null); } } - if (counter.decrementAndGet() == 0) { - threadPool.executor(executorName).execute(() -> executeBulk(task, bulkRequest, startTime, - ActionListener.wrap(listener::onResponse, inner -> { + } + if (counter.decrementAndGet() == 0) { + threadPool.executor(executorName).execute(() -> executeBulk(task, bulkRequest, startTime, + ActionListener.wrap(listener::onResponse, inner -> { inner.addSuppressed(e); listener.onFailure(inner); }), responses, indicesThatCannotBeCreated)); - } } - }); - } + } + }); } - } else { - executeBulk(task, bulkRequest, startTime, listener, responses, emptyMap()); } } @@ -351,10 +340,6 @@ boolean isOnlySystem(BulkRequest request, SortedMap in return request.getIndices().stream().allMatch(indexName -> isSystemIndex(indicesLookup, systemIndices, indexName)); } - boolean includesSystem(BulkRequest request, SortedMap indicesLookup, SystemIndices systemIndices) { - return request.getIndices().stream().anyMatch(indexName -> isSystemIndex(indicesLookup, systemIndices, indexName)); - } - private boolean isSystemIndex(SortedMap indicesLookup, SystemIndices systemIndices, String indexName) { final IndexAbstraction abstraction = indicesLookup.get(indexName); if (abstraction != null) { @@ -364,14 +349,6 @@ private boolean isSystemIndex(SortedMap indicesLookup, } } - boolean needToCheck() { - return autoCreateIndex.needToCheck(); - } - - boolean shouldAutoCreate(String index, ClusterState state) { - return autoCreateIndex.shouldAutoCreate(index, state); - } - void createIndex(String index, TimeValue timeout, Version minNodeVersion, diff --git a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java index e9ceca23c41d9..6da8e7bd78f5d 100644 --- a/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java +++ b/server/src/main/java/org/elasticsearch/action/support/AutoCreateIndex.java @@ -20,7 +20,10 @@ package org.elasticsearch.action.support; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.Tuple; @@ -40,7 +43,6 @@ * a write operation is about to happen in a non existing index. */ public final class AutoCreateIndex { - public static final Setting AUTO_CREATE_INDEX_SETTING = new Setting<>("action.auto_create_index", "true", AutoCreate::new, Property.NodeScope, Setting.Property.Dynamic); @@ -58,13 +60,6 @@ public AutoCreateIndex(Settings settings, clusterSettings.addSettingsUpdateConsumer(AUTO_CREATE_INDEX_SETTING, this::setAutoCreate); } - /** - * Do we really need to check if an index should be auto created? - */ - public boolean needToCheck() { - return this.autoCreate.autoCreateIndex; - } - /** * Should the index be auto created? * @throws IndexNotFoundException if the index doesn't exist and shouldn't be auto created @@ -79,6 +74,17 @@ public boolean shouldAutoCreate(String index, ClusterState state) { return true; } + // Templates can override the AUTO_CREATE_INDEX_SETTING setting + final ComposableIndexTemplate template = findTemplate(index, state.metadata()); + if (template != null && template.getAllowAutoCreate() != null) { + if (template.getAllowAutoCreate()) { + return true; + } else { + // An explicit false value overrides AUTO_CREATE_INDEX_SETTING + throw new IndexNotFoundException("composable template " + template.indexPatterns() + " forbids index auto creation"); + } + } + // One volatile read, so that all checks are done against the same instance: final AutoCreate autoCreate = this.autoCreate; if (autoCreate.autoCreateIndex == false) { @@ -112,6 +118,11 @@ void setAutoCreate(AutoCreate autoCreate) { this.autoCreate = autoCreate; } + private ComposableIndexTemplate findTemplate(String indexName, Metadata metadata) { + final String templateName = MetadataIndexTemplateService.findV2Template(metadata, indexName, false); + return metadata.templatesV2().get(templateName); + } + static class AutoCreate { private final boolean autoCreateIndex; private final List> expressions; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index a8af8d10faa7a..28efd3e0593d4 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -55,6 +55,9 @@ public class ComposableIndexTemplate extends AbstractDiffable PARSER = new ConstructingObjectParser<>("index_template", @@ -65,7 +68,8 @@ public class ComposableIndexTemplate extends AbstractDiffable) a[5], - (DataStreamTemplate) a[6])); + (DataStreamTemplate) a[6], + (Boolean) a[7])); static { PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), INDEX_PATTERNS); @@ -75,6 +79,7 @@ public class ComposableIndexTemplate extends AbstractDiffable p.map(), METADATA); PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), DataStreamTemplate.PARSER, DATA_STREAM); + PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), ALLOW_AUTO_CREATE); } private final List indexPatterns; @@ -90,6 +95,8 @@ public class ComposableIndexTemplate extends AbstractDiffable metadata; @Nullable private final DataStreamTemplate dataStreamTemplate; + @Nullable + private final Boolean allowAutoCreate; static Diff readITV2DiffFrom(StreamInput in) throws IOException { return AbstractDiffable.readDiffFrom(ComposableIndexTemplate::new, in); @@ -101,12 +108,12 @@ public static ComposableIndexTemplate parse(XContentParser parser) throws IOExce public ComposableIndexTemplate(List indexPatterns, @Nullable Template template, @Nullable List componentTemplates, @Nullable Long priority, @Nullable Long version, @Nullable Map metadata) { - this(indexPatterns, template, componentTemplates, priority, version, metadata, null); + this(indexPatterns, template, componentTemplates, priority, version, metadata, null, null); } public ComposableIndexTemplate(List indexPatterns, @Nullable Template template, @Nullable List componentTemplates, @Nullable Long priority, @Nullable Long version, @Nullable Map metadata, - @Nullable DataStreamTemplate dataStreamTemplate) { + @Nullable DataStreamTemplate dataStreamTemplate, @Nullable Boolean allowAutoCreate) { this.indexPatterns = indexPatterns; this.template = template; this.componentTemplates = componentTemplates; @@ -114,6 +121,7 @@ public ComposableIndexTemplate(List indexPatterns, @Nullable Template te this.version = version; this.metadata = metadata; this.dataStreamTemplate = dataStreamTemplate; + this.allowAutoCreate = allowAutoCreate; } public ComposableIndexTemplate(StreamInput in) throws IOException { @@ -132,6 +140,11 @@ public ComposableIndexTemplate(StreamInput in) throws IOException { } else { this.dataStreamTemplate = null; } + if (in.getVersion().onOrAfter(ALLOW_AUTO_CREATE_VERSION)) { + this.allowAutoCreate = in.readOptionalBoolean(); + } else { + this.allowAutoCreate = null; + } } public List indexPatterns() { @@ -173,6 +186,11 @@ public DataStreamTemplate getDataStreamTemplate() { return dataStreamTemplate; } + @Nullable + public Boolean getAllowAutoCreate() { + return this.allowAutoCreate; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeStringCollection(this.indexPatterns); @@ -189,6 +207,9 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_9_0)) { out.writeOptionalWriteable(dataStreamTemplate); } + if (out.getVersion().onOrAfter(ALLOW_AUTO_CREATE_VERSION)) { + out.writeOptionalBoolean(allowAutoCreate); + } } @Override @@ -213,6 +234,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (this.dataStreamTemplate != null) { builder.field(DATA_STREAM.getPreferredName(), dataStreamTemplate); } + if (this.allowAutoCreate != null) { + builder.field(ALLOW_AUTO_CREATE.getPreferredName(), allowAutoCreate); + } builder.endObject(); return builder; } @@ -220,7 +244,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws @Override public int hashCode() { return Objects.hash(this.indexPatterns, this.template, this.componentTemplates, this.priority, this.version, - this.metadata, this.dataStreamTemplate); + this.metadata, this.dataStreamTemplate, this.allowAutoCreate); } @Override @@ -238,7 +262,8 @@ public boolean equals(Object obj) { Objects.equals(this.priority, other.priority) && Objects.equals(this.version, other.version) && Objects.equals(this.metadata, other.metadata) && - Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate); + Objects.equals(this.dataStreamTemplate, other.dataStreamTemplate) && + Objects.equals(this.allowAutoCreate, other.allowAutoCreate); } @Override diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java index fa9746129c597..0504cf53a4255 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateService.java @@ -501,7 +501,8 @@ public ClusterState addIndexTemplateV2(final ClusterState currentState, final bo final Template finalTemplate = new Template(finalSettings, stringMappings == null ? null : new CompressedXContent(stringMappings), innerTemplate.aliases()); finalIndexTemplate = new ComposableIndexTemplate(template.indexPatterns(), finalTemplate, template.composedOf(), - template.priority(), template.version(), template.metadata(), template.getDataStreamTemplate()); + template.priority(), template.version(), template.metadata(), template.getDataStreamTemplate(), + template.getAllowAutoCreate()); } if (finalIndexTemplate.equals(existing)) { diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java index 07d48b8b077bd..24cb4c37fb79c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/AutoCreateActionTests.java @@ -32,35 +32,38 @@ public class AutoCreateActionTests extends ESTestCase { - public void testResolveAutoCreateDataStreams() { + public void testResolveTemplates() { Metadata metadata; { Metadata.Builder mdBuilder = new Metadata.Builder(); DataStreamTemplate dataStreamTemplate = new DataStreamTemplate(); - mdBuilder.put("1", new ComposableIndexTemplate(List.of("legacy-logs-*"), null, null, 10L, null, null, null)); - mdBuilder.put("2", new ComposableIndexTemplate(List.of("logs-*"), null, null, 20L, null, null, dataStreamTemplate)); - mdBuilder.put("3", new ComposableIndexTemplate(List.of("logs-foobar"), null, null, 30L, null, null, dataStreamTemplate)); + mdBuilder.put("1", new ComposableIndexTemplate(List.of("legacy-logs-*"), null, null, 10L, null, null, null, null)); + mdBuilder.put("2", new ComposableIndexTemplate(List.of("logs-*"), null, null, 20L, null, null, dataStreamTemplate, null)); + mdBuilder.put("3", new ComposableIndexTemplate(List.of("logs-foobar"), null, null, 30L, null, null, dataStreamTemplate, null)); metadata = mdBuilder.build(); } CreateIndexRequest request = new CreateIndexRequest("logs-foobar"); - DataStreamTemplate result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + ComposableIndexTemplate result = AutoCreateAction.resolveTemplate(request, metadata); assertThat(result, notNullValue()); - assertThat(result.getTimestampField(), equalTo("@timestamp")); + assertThat(result.getDataStreamTemplate(), notNullValue()); + assertThat(result.getDataStreamTemplate().getTimestampField(), equalTo("@timestamp")); request = new CreateIndexRequest("logs-barbaz"); - result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + result = AutoCreateAction.resolveTemplate(request, metadata); assertThat(result, notNullValue()); - assertThat(result.getTimestampField(), equalTo("@timestamp")); + assertThat(result.getDataStreamTemplate(), notNullValue()); + assertThat(result.getDataStreamTemplate().getTimestampField(), equalTo("@timestamp")); // An index that matches with a template without a data steam definition request = new CreateIndexRequest("legacy-logs-foobaz"); - result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); - assertThat(result, nullValue()); + result = AutoCreateAction.resolveTemplate(request, metadata); + assertThat(result, notNullValue()); + assertThat(result.getDataStreamTemplate(), nullValue()); // An index that doesn't match with an index template request = new CreateIndexRequest("my-index"); - result = AutoCreateAction.resolveAutoCreateDataStream(request, metadata); + result = AutoCreateAction.resolveTemplate(request, metadata); assertThat(result, nullValue()); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index d9a853a4eb45e..d0b14621ae98b 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -375,7 +375,7 @@ public void testRejectDuplicateAliasV2() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("foo-*", "bar-*"), new Template(null, null, aliases), - null, null, null, null, null); + null, null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("test-template", template).build(); @@ -392,7 +392,7 @@ public void testRejectDuplicateAliasV2UsingComponentTemplates() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null); final ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("foo-*", "bar-*"), null, - Collections.singletonList("ct"), null, null, null, null); + Collections.singletonList("ct"), null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("ct", ct) @@ -428,7 +428,7 @@ public void testHiddenAffectsResolvedV2Templates() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("*"), new Template(null, null, aliases), - null, null, null, null, null); + null, null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("test-template", template).build(); @@ -449,7 +449,7 @@ public void testHiddenAffectsResolvedV2ComponentTemplates() { aliases.put("bar-write", AliasMetadata.builder("bar-write").writeIndex(randomBoolean()).build()); final ComponentTemplate ct = new ComponentTemplate(new Template(null, null, aliases), null, null); final ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("*"), null, - Collections.singletonList("ct"), null, null, null, null); + Collections.singletonList("ct"), null, null, null, null, null); final Metadata metadata = Metadata.builder().put(createMetadata(randomAlphaOfLengthBetween(5, 7)), false) .put("ct", ct) @@ -541,7 +541,7 @@ public void testRolloverClusterState() throws Exception { public void testRolloverClusterStateForDataStream() throws Exception { final DataStream dataStream = DataStreamTestHelper.randomInstance(); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStream.getName() + "*"), null, null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); Metadata.Builder builder = Metadata.builder(); builder.put("template", template); for (Index index : dataStream.getIndices()) { @@ -640,7 +640,7 @@ public void testValidation() throws Exception { sourceIndexName = dataStream.getIndices().get(dataStream.getIndices().size() - 1).getName(); defaultRolloverIndexName = DataStream.getDefaultBackingIndexName(dataStream.getName(), dataStream.getGeneration() + 1); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStream.getName() + "*"), null, null, null, null, - null, new ComposableIndexTemplate.DataStreamTemplate()); + null, new ComposableIndexTemplate.DataStreamTemplate(), null); builder.put("template", template); for (Index index : dataStream.getIndices()) { builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index)); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java index 09c0e6dcac1f1..11db2728b3388 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateIndexTemplateRequestTests.java @@ -63,7 +63,7 @@ public void testIndexNameCannotBeNullOrEmpty() { public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); - ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null); + ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null, null); PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java index ab8b49e03a4e4..b7530c29d2163 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/post/SimulateTemplateRequestTests.java @@ -64,7 +64,7 @@ public void testIndexNameCannotBeNullOrEmpty() { public void testAddingGlobalTemplateWithHiddenIndexSettingIsIllegal() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); - ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null); + ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null, null); PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java index 47e25a9cd53f8..e48f6694bc959 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/template/put/PutComposableIndexTemplateRequestTests.java @@ -58,7 +58,7 @@ protected PutComposableIndexTemplateAction.Request mutateInstance(PutComposableI public void testPutGlobalTemplatesCannotHaveHiddenIndexSetting() { Template template = new Template(Settings.builder().put(IndexMetadata.SETTING_INDEX_HIDDEN, true).build(), null, null); - ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null); + ComposableIndexTemplate globalTemplate = new ComposableIndexTemplate(List.of("*"), template, null, null, null, null, null, null); PutComposableIndexTemplateAction.Request request = new PutComposableIndexTemplateAction.Request("test"); request.indexTemplate(globalTemplate); @@ -84,7 +84,7 @@ public void testPutIndexTemplateV2RequestMustContainTemplate() { public void testValidationOfPriority() { PutComposableIndexTemplateAction.Request req = new PutComposableIndexTemplateAction.Request("test"); - req.indexTemplate(new ComposableIndexTemplate(Arrays.asList("foo", "bar"), null, null, -5L, null, null, null)); + req.indexTemplate(new ComposableIndexTemplate(Arrays.asList("foo", "bar"), null, null, -5L, null, null, null, null)); ActionRequestValidationException validationException = req.validate(); assertThat(validationException, is(notNullValue())); List validationErrors = validationException.validationErrors(); diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java index d790061c82a5b..bcaf58da072fb 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIndicesThatCannotBeCreatedTests.java @@ -27,6 +27,7 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodes; @@ -35,6 +36,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.IndexingPressure; import org.elasticsearch.index.VersionType; @@ -45,34 +47,32 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; -import java.util.Arrays; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ExecutorService; -import java.util.function.Predicate; +import java.util.function.Consumer; +import java.util.function.Function; import static java.util.Collections.emptySet; -import static java.util.Collections.singleton; import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; public class TransportBulkActionIndicesThatCannotBeCreatedTests extends ESTestCase { + private static final Consumer noop = index -> {}; + public void testNonExceptional() { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); bulkRequest.add(new IndexRequest(randomAlphaOfLength(5))); bulkRequest.add(new DeleteRequest(randomAlphaOfLength(5))); bulkRequest.add(new UpdateRequest(randomAlphaOfLength(5), randomAlphaOfLength(5))); - // Test emulating auto_create_index=false - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, null); - // Test emulating auto_create_index=true - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> true); - // Test emulating all indices already created - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> false); + // Test emulating that index can be auto-created + indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> true, noop); + // Test emulating that index cannot be auto-created + indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> false, noop); // Test emulating auto_create_index=true with some indices already created. - indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> randomBoolean()); + indicesThatCannotBeCreatedTestCase(emptySet(), bulkRequest, index -> randomBoolean(), noop); } public void testAllFail() { @@ -81,7 +81,7 @@ public void testAllFail() { bulkRequest.add(new IndexRequest("can't")); bulkRequest.add(new DeleteRequest("do").version(0).versionType(VersionType.EXTERNAL)); bulkRequest.add(new UpdateRequest("nothin", randomAlphaOfLength(5))); - indicesThatCannotBeCreatedTestCase(new HashSet<>(Arrays.asList("no", "can't", "do", "nothin")), bulkRequest, index -> { + indicesThatCannotBeCreatedTestCase(Set.of("no", "can't", "do", "nothin"), bulkRequest, index -> true, index -> { throw new IndexNotFoundException("Can't make it because I say so"); }); } @@ -91,40 +91,43 @@ public void testSomeFail() { bulkRequest.add(new IndexRequest("ok")); bulkRequest.add(new IndexRequest("bad")); // Emulate auto_create_index=-bad,+* - indicesThatCannotBeCreatedTestCase(singleton("bad"), bulkRequest, index -> { - if (index.equals("bad")) { - throw new IndexNotFoundException("Can't make it because I say so"); - } - return true; - }); - // Emulate auto_create_index=false but the "ok" index already exists - indicesThatCannotBeCreatedTestCase(singleton("bad"), bulkRequest, index -> { + indicesThatCannotBeCreatedTestCase(Set.of("bad"), bulkRequest, index -> true, index -> { if (index.equals("bad")) { throw new IndexNotFoundException("Can't make it because I say so"); } - return false; }); } private void indicesThatCannotBeCreatedTestCase(Set expected, - BulkRequest bulkRequest, Predicate shouldAutoCreate) { + BulkRequest bulkRequest, Function shouldAutoCreate, Consumer simulateAutoCreate) { ClusterService clusterService = mock(ClusterService.class); ClusterState state = mock(ClusterState.class); when(state.getMetadata()).thenReturn(Metadata.EMPTY_METADATA); when(state.metadata()).thenReturn(Metadata.EMPTY_METADATA); when(clusterService.state()).thenReturn(state); + DiscoveryNodes discoveryNodes = mock(DiscoveryNodes.class); when(state.getNodes()).thenReturn(discoveryNodes); when(discoveryNodes.getMinNodeVersion()).thenReturn(VersionUtils.randomCompatibleVersion(random(), Version.CURRENT)); + DiscoveryNode localNode = mock(DiscoveryNode.class); when(clusterService.localNode()).thenReturn(localNode); when(localNode.isIngestNode()).thenReturn(randomBoolean()); + final ThreadPool threadPool = mock(ThreadPool.class); final ExecutorService direct = EsExecutors.newDirectExecutorService(); when(threadPool.executor(anyString())).thenReturn(direct); + + final IndexNameExpressionResolver indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)) { + @Override + public boolean hasIndexAbstraction(String indexAbstraction, ClusterState state) { + return shouldAutoCreate.apply(indexAbstraction) == false; + } + }; + TransportBulkAction action = new TransportBulkAction(threadPool, mock(TransportService.class), clusterService, - null, null, mock(ActionFilters.class), null, null, + null, null, mock(ActionFilters.class), indexNameExpressionResolver, new IndexingPressure(Settings.EMPTY), new SystemIndices(Map.of())) { @Override void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, ActionListener listener, @@ -133,21 +136,25 @@ void executeBulk(Task task, BulkRequest bulkRequest, long startTimeNanos, Action } @Override - boolean needToCheck() { - return null != shouldAutoCreate; // Use "null" to mean "no indices can be created so don't bother checking" + void createIndex(String index, TimeValue timeout, Version minNodeVersion, ActionListener listener) { + try { + simulateAutoCreate.accept(index); + // If we try to create an index just immediately assume it worked + listener.onResponse(new CreateIndexResponse(true, true, index) { + }); + } catch (Exception e) { + listener.onFailure(e); + } } - + }; + action.doExecute(null, bulkRequest, new ActionListener<>() { @Override - boolean shouldAutoCreate(String index, ClusterState state) { - return shouldAutoCreate.test(index); - } + public void onResponse(BulkResponse bulkItemResponses) {} @Override - void createIndex(String index, TimeValue timeout, Version minNodeVersion, ActionListener listener) { - // If we try to create an index just immediately assume it worked - listener.onResponse(new CreateIndexResponse(true, true, index) {}); + public void onFailure(Exception e) { + throw new AssertionError(e); } - }; - action.doExecute(null, bulkRequest, null); + }); } } diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java index 78f7dbabbe252..691c9bfe17eff 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionIngestTests.java @@ -34,10 +34,10 @@ import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateApplier; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; -import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -45,7 +45,6 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.collect.ImmutableOpenMap; -import org.elasticsearch.common.settings.ClusterSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.AtomicArray; @@ -143,18 +142,11 @@ class TestTransportBulkAction extends TransportBulkAction { TestTransportBulkAction() { super(threadPool, transportService, clusterService, ingestService, - null, new ActionFilters(Collections.emptySet()), null, - new AutoCreateIndex( - SETTINGS, new ClusterSettings(SETTINGS, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS), - new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), - new SystemIndices(Map.of()) - ), new IndexingPressure(SETTINGS), new SystemIndices(Map.of()) + null, new ActionFilters(Collections.emptySet()), new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)), + new IndexingPressure(SETTINGS), new SystemIndices(Map.of()) ); } - @Override - protected boolean needToCheck() { - return needToCheck; - } + @Override void executeBulk(Task task, final BulkRequest bulkRequest, final long startTimeNanos, final ActionListener listener, final AtomicArray responses, Map indicesThatCannotBeCreated) { @@ -633,7 +625,7 @@ public void testFindDefaultPipelineFromV2TemplateMatch() { ComposableIndexTemplate t1 = new ComposableIndexTemplate(Collections.singletonList("missing_*"), new Template(Settings.builder().put(IndexSettings.DEFAULT_PIPELINE.getKey(), "pipeline2").build(), null, null), - null, null, null, null, null); + null, null, null, null, null, null); ClusterState state = clusterService.state(); Metadata metadata = Metadata.builder() diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java index df3c654191bfd..75bfcad1a9caf 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; -import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.cluster.ClusterState; @@ -85,15 +84,9 @@ class TestTransportBulkAction extends TransportBulkAction { TestTransportBulkAction() { super(TransportBulkActionTests.this.threadPool, transportService, clusterService, null, null, new ActionFilters(Collections.emptySet()), new Resolver(), - new AutoCreateIndex(Settings.EMPTY, clusterService.getClusterSettings(), new Resolver(), new SystemIndices(Map.of())), new IndexingPressure(Settings.EMPTY), new SystemIndices(Map.of())); } - @Override - protected boolean needToCheck() { - return true; - } - @Override void createIndex(String index, TimeValue timeout, Version minNodeVersion, ActionListener listener) { indexCreated = true; @@ -268,27 +261,6 @@ public void testOnlySystem() { assertFalse(bulkAction.isOnlySystem(buildBulkRequest(mixed), indicesLookup, systemIndices)); } - public void testIncludesSystem() { - SortedMap indicesLookup = new TreeMap<>(); - Settings settings = Settings.builder().put("index.version.created", Version.CURRENT).build(); - indicesLookup.put(".foo", - new Index(IndexMetadata.builder(".foo").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); - indicesLookup.put(".bar", - new Index(IndexMetadata.builder(".bar").settings(settings).system(true).numberOfShards(1).numberOfReplicas(0).build())); - SystemIndices systemIndices = new SystemIndices(Map.of("plugin", List.of(new SystemIndexDescriptor(".test", "")))); - List onlySystem = List.of(".foo", ".bar"); - assertTrue(bulkAction.includesSystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices)); - - onlySystem = List.of(".foo", ".bar", ".test"); - assertTrue(bulkAction.includesSystem(buildBulkRequest(onlySystem), indicesLookup, systemIndices)); - - List nonSystem = List.of("foo", "bar"); - assertFalse(bulkAction.includesSystem(buildBulkRequest(nonSystem), indicesLookup, systemIndices)); - - List mixed = List.of(".foo", ".test", "other"); - assertTrue(bulkAction.includesSystem(buildBulkRequest(mixed), indicesLookup, systemIndices)); - } - private BulkRequest buildBulkRequest(List indices) { BulkRequest request = new BulkRequest(); for (String index : indices) { diff --git a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java index 792b1aed322ef..df1abbcd14e3b 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/TransportBulkActionTookTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.support.ActionFilters; -import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -126,7 +125,6 @@ void doExecute(ActionType action, Request request, ActionListener + autoCreateIndex.shouldAutoCreate(randomIndex, clusterState)); + assertEquals("no such index [" + randomIndex + "] and [action.auto_create_index] is [false]", e.getMessage()); + } + + /** + * Check that if a template matches the index to be created, but that template has the allow_auto_create + * setting turned off, then it overrides the global setting. + */ + public void testDisabledAutoCreateTemplateSettingDoesNotOverride() { + String randomIndex = randomAlphaOfLengthBetween(2, 10); + final ComposableIndexTemplate template = new ComposableIndexTemplate( + List.of(randomIndex.charAt(0) + "*"), + null, + List.of(), + null, + null, + Map.of(), + null, + false + ); + + final Metadata metadata = Metadata.builder().indexTemplates(Map.of("test_template", template)).build(); + final ClusterState clusterState = ClusterState.builder(buildClusterState()).metadata(metadata).build(); + + Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false).build(); + AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings); + IndexNotFoundException e = expectThrows(IndexNotFoundException.class, () -> + autoCreateIndex.shouldAutoCreate(randomIndex, clusterState)); + assertEquals("no such index [composable template [" + randomIndex.charAt(0) + "*] forbids index auto creation]", e.getMessage()); + } + + /** + * Check that if a template matches the index to be created, and that template has the allow_auto_create + * setting enabled, then it overrides the global setting. + */ + public void testEnabledAutoCreateTemplateSettingDoesOverride() { + String randomIndex = randomAlphaOfLengthBetween(2, 10); + final ComposableIndexTemplate template = new ComposableIndexTemplate( + List.of(randomIndex.charAt(0) + "*"), + null, + List.of(), + null, + null, + Map.of(), + null, + true + ); + + final Metadata metadata = Metadata.builder().indexTemplates(Map.of("test_template", template)).build(); + final ClusterState clusterState = ClusterState.builder(buildClusterState()).metadata(metadata).build(); + + Settings settings = Settings.builder().put(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), false).build(); + AutoCreateIndex autoCreateIndex = newAutoCreateIndex(settings); + assertThat(autoCreateIndex.shouldAutoCreate(randomIndex, clusterState), equalTo(true)); + } + private static ClusterState buildClusterState(String... indices) { Metadata.Builder metadata = Metadata.builder(); for (String index : indices) { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java index 24efa12b3c966..10d805f780502 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplateTests.java @@ -97,7 +97,9 @@ public static ComposableIndexTemplate randomInstance() { randomBoolean() ? null : randomNonNegativeLong(), randomBoolean() ? null : randomNonNegativeLong(), meta, - dataStreamTemplate); + dataStreamTemplate, + randomBoolean() ? null : randomBoolean() + ); } private static Map randomAliases() { @@ -163,7 +165,7 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori List newIndexPatterns = randomValueOtherThan(orig.indexPatterns(), () -> randomList(1, 4, () -> randomAlphaOfLength(4))); return new ComposableIndexTemplate(newIndexPatterns, orig.template(), orig.composedOf(), - orig.priority(), orig.version(), orig.metadata(), orig.getDataStreamTemplate()); + orig.priority(), orig.version(), orig.metadata(), orig.getDataStreamTemplate(), null); case 1: return new ComposableIndexTemplate(orig.indexPatterns(), randomValueOtherThan(orig.template(), () -> new Template(randomSettings(), @@ -172,7 +174,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 2: List newComposedOf = randomValueOtherThan(orig.composedOf(), () -> randomList(0, 10, () -> randomAlphaOfLength(5))); @@ -182,7 +185,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 3: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -190,7 +194,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori randomValueOtherThan(orig.priority(), ESTestCase::randomNonNegativeLong), orig.version(), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 4: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -198,7 +203,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), randomValueOtherThan(orig.version(), ESTestCase::randomNonNegativeLong), orig.metadata(), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 5: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -206,7 +212,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), randomValueOtherThan(orig.metadata(), ComposableIndexTemplateTests::randomMeta), - orig.getDataStreamTemplate()); + orig.getDataStreamTemplate(), + orig.getAllowAutoCreate()); case 6: return new ComposableIndexTemplate(orig.indexPatterns(), orig.template(), @@ -214,7 +221,8 @@ public static ComposableIndexTemplate mutateTemplate(ComposableIndexTemplate ori orig.priority(), orig.version(), orig.metadata(), - randomValueOtherThan(orig.getDataStreamTemplate(), ComposableIndexTemplateTests::randomDataStreamTemplate)); + randomValueOtherThan(orig.getDataStreamTemplate(), ComposableIndexTemplateTests::randomDataStreamTemplate), + orig.getAllowAutoCreate()); default: throw new IllegalStateException("illegal randomization branch"); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index d52f36afb1fb1..3e338058732c1 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -48,7 +48,7 @@ public void testCreateDataStream() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); @@ -126,7 +126,8 @@ public void testCreateDataStreamNoTemplate() throws Exception { public void testCreateDataStreamNoValidTemplate() throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); final String dataStreamName = "my-data-stream"; - ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, null); + ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, null, + null); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); @@ -141,7 +142,7 @@ public void testCreateDataStreamNoValidTemplate() throws Exception { public static ClusterState createDataStream(final String dataStreamName) throws Exception { final MetadataCreateIndexService metadataCreateIndexService = getMetadataCreateIndexService(); ComposableIndexTemplate template = new ComposableIndexTemplate(List.of(dataStreamName + "*"), null, null, null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); ClusterState cs = ClusterState.builder(new ClusterName("_name")) .metadata(Metadata.builder().put("template", template).build()) .build(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index a5ecb6efc2cec..999ace6007901 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -368,15 +368,15 @@ public void testUpdateComponentTemplateWithIndexHiddenSetting() throws Exception assertNotNull(state.metadata().componentTemplates().get("foo")); ComposableIndexTemplate firstGlobalIndexTemplate = - new ComposableIndexTemplate(List.of("*"), template, List.of("foo"), 1L, null, null, null); + new ComposableIndexTemplate(List.of("*"), template, List.of("foo"), 1L, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalindextemplate1", firstGlobalIndexTemplate); ComposableIndexTemplate secondGlobalIndexTemplate = - new ComposableIndexTemplate(List.of("*"), template, List.of("foo"), 2L, null, null, null); + new ComposableIndexTemplate(List.of("*"), template, List.of("foo"), 2L, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "globalindextemplate2", secondGlobalIndexTemplate); ComposableIndexTemplate fooPatternIndexTemplate = - new ComposableIndexTemplate(List.of("foo-*"), template, List.of("foo"), 3L, null, null, null); + new ComposableIndexTemplate(List.of("foo-*"), template, List.of("foo"), 3L, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, true, "foopatternindextemplate", fooPatternIndexTemplate); // update the component template to set the index.hidden setting @@ -428,7 +428,7 @@ public void testUpdateIndexTemplateV2() throws Exception { List patterns = new ArrayList<>(template.indexPatterns()); patterns.add("new-pattern"); template = new ComposableIndexTemplate(patterns, template.template(), template.composedOf(), template.priority(), - template.version(), template.metadata(), null); + template.version(), template.metadata(), null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo", template); assertNotNull(state.metadata().templatesV2().get("foo")); @@ -466,7 +466,7 @@ public void testPuttingV2TemplateGeneratesWarning() throws Exception { .build(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -503,7 +503,7 @@ public void onFailure(Exception e) { waitToCreateComponentTemplate.await(10, TimeUnit.SECONDS); ComposableIndexTemplate globalIndexTemplate = new ComposableIndexTemplate(List.of("*"), - null, List.of("ct-with-index-hidden-setting"), null, null, null, null); + null, List.of("ct-with-index-hidden-setting"), null, null, null, null, null); expectThrows(InvalidIndexTemplateException.class, () -> metadataIndexTemplateService.putIndexTemplateV2("testing", true, "template-referencing-ct-with-hidden-index-setting", @@ -528,7 +528,7 @@ public void onFailure(Exception e) { public void testPuttingV1StarTemplateGeneratesWarning() throws Exception { final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); @@ -549,7 +549,7 @@ public void testPuttingV1StarTemplateGeneratesWarning() throws Exception { public void testPuttingV1NonStarTemplateGeneratesError() throws Exception { final MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "v2-template", v2Template); MetadataIndexTemplateService.PutRequest req = new MetadataIndexTemplateService.PutRequest("cause", "v1-template"); @@ -583,7 +583,7 @@ public void testUpdatingV1NonStarTemplateWithUnchangedPatternsGeneratesWarning() .build(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -624,7 +624,7 @@ public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() thr .build(); ComposableIndexTemplate v2Template = new ComposableIndexTemplate(Arrays.asList("foo-bar-*", "eggplant"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); state = metadataIndexTemplateService.addIndexTemplateV2(state, false, "v2-template", v2Template); assertWarnings("index template [v2-template] has index patterns [foo-bar-*, eggplant] matching patterns " + @@ -650,11 +650,11 @@ public void testUpdatingV1NonStarWithChangedPatternsTemplateGeneratesError() thr public void testPuttingOverlappingV2Template() throws Exception { { ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("egg*", "baz"), - null, null, 1L, null, null, null); + null, null, 1L, null, null, null, null); MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Arrays.asList("abc", "baz*"), - null, null, 1L, null, null, null); + null, null, 1L, null, null, null, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo2", newTemplate)); assertThat(e.getMessage(), equalTo("index template [foo2] has index patterns [abc, baz*] matching patterns from existing " + @@ -664,11 +664,11 @@ public void testPuttingOverlappingV2Template() throws Exception { { ComposableIndexTemplate template = new ComposableIndexTemplate(Arrays.asList("egg*", "baz"), - null, null, null, null, null, null); + null, null, null, null, null, null, null); MetadataIndexTemplateService metadataIndexTemplateService = getMetadataIndexTemplateService(); ClusterState state = metadataIndexTemplateService.addIndexTemplateV2(ClusterState.EMPTY_STATE, false, "foo", template); ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Arrays.asList("abc", "baz*"), - null, null, 0L, null, null, null); + null, null, 0L, null, null, null, null); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> metadataIndexTemplateService.addIndexTemplateV2(state, false, "foo2", newTemplate)); assertThat(e.getMessage(), equalTo("index template [foo2] has index patterns [abc, baz*] matching patterns from existing " + @@ -684,9 +684,9 @@ public void testFindV2Templates() throws Exception { ComponentTemplate ct = ComponentTemplateTests.randomInstance(); state = service.addComponentTemplate(state, true, "ct", ct); - ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), null, List.of("ct"), null, 1L, null, null); + ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), null, List.of("ct"), null, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); - ComposableIndexTemplate it2 = new ComposableIndexTemplate(List.of("in*"), null, List.of("ct"), 10L, 2L, null, null); + ComposableIndexTemplate it2 = new ComposableIndexTemplate(List.of("in*"), null, List.of("ct"), 10L, 2L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template2", it2); String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", randomBoolean()); @@ -701,9 +701,9 @@ public void testFindV2TemplatesForHiddenIndex() throws Exception { ComponentTemplate ct = ComponentTemplateTests.randomInstance(); state = service.addComponentTemplate(state, true, "ct", ct); - ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), null, List.of("ct"), 0L, 1L, null, null); + ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), null, List.of("ct"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); - ComposableIndexTemplate it2 = new ComposableIndexTemplate(List.of("*"), null, List.of("ct"), 10L, 2L, null, null); + ComposableIndexTemplate it2 = new ComposableIndexTemplate(List.of("*"), null, List.of("ct"), 10L, 2L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template2", it2); String result = MetadataIndexTemplateService.findV2Template(state.metadata(), "index", true); @@ -716,7 +716,7 @@ public void testFindV2InvalidGlobalTemplate() { try { // add an invalid global template that specifies the `index.hidden` setting ComposableIndexTemplate invalidGlobalTemplate = new ComposableIndexTemplate(List.of("*"), - templateWithHiddenSetting, List.of("ct"), 5L, 1L, null, null); + templateWithHiddenSetting, List.of("ct"), 5L, 1L, null, null, null); Metadata invalidGlobalTemplateMetadata = Metadata.builder().putCustom(ComposableIndexTemplateMetadata.TYPE, new ComposableIndexTemplateMetadata(Map.of("invalid_global_template", invalidGlobalTemplate))).build(); @@ -760,7 +760,7 @@ public void testResolveConflictingMappings() throws Exception { " }\n" + " }\n" + " }"), null), - List.of("ct_low", "ct_high"), 0L, 1L, null, null); + List.of("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index", @@ -824,7 +824,7 @@ public void testResolveMappings() throws Exception { " }\n" + " }\n" + " }"), null), - List.of("ct_low", "ct_high"), 0L, 1L, null, null); + List.of("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "my-template", "my-index", @@ -876,7 +876,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex " }\n" + " }\n" + " }"), null), - List.of("ct1"), 0L, 1L, null, new ComposableIndexTemplate.DataStreamTemplate()); + List.of("ct1"), 0L, 1L, null, new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, true, "logs-data-stream-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "logs-data-stream-template", @@ -915,7 +915,7 @@ public void testDefinedTimestampMappingIsAddedForDataStreamTemplates() throws Ex " }\n" + " }\n" + " }"), null), - List.of("ct1"), 0L, 1L, null, null); + List.of("ct1"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", "timeseries", @@ -983,7 +983,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception state = service.addComponentTemplate(state, true, "ct1", ct1); ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("logs*"), null, List.of("ct1"), 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, true, "logs-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "logs-template", @@ -1018,7 +1018,7 @@ public void testUserDefinedMappingTakesPrecedenceOverDefault() throws Exception " }\n" + " }"), null); ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("timeseries*"), template, null, 0L, 1L, null, - new ComposableIndexTemplate.DataStreamTemplate()); + new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, true, "timeseries-template", it); List mappings = MetadataIndexTemplateService.collectMappings(state, "timeseries-template", @@ -1065,7 +1065,7 @@ public void testResolveSettings() throws Exception { .put("index.blocks.write", false) .put("index.number_of_shards", 3) .build(), null, null), - List.of("ct_low", "ct_high"), 0L, 1L, null, null); + List.of("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), "my-template"); @@ -1093,7 +1093,7 @@ public void testResolveAliases() throws Exception { state = service.addComponentTemplate(state, true, "ct_low", ct2); ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), new Template(null, null, a3), - List.of("ct_low", "ct_high"), 0L, 1L, null, null); + List.of("ct_low", "ct_high"), 0L, 1L, null, null, null); state = service.addIndexTemplateV2(state, true, "my-template", it); List> resolvedAliases = MetadataIndexTemplateService.resolveAliases(state.metadata(), "my-template"); @@ -1209,7 +1209,7 @@ public void testIndexTemplateFailsToOverrideComponentTemplateMappingField() thr " }\n" + " }\n" + " }"), null), - randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null); + randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null, null); final ClusterState finalState = state; IllegalArgumentException e = expectThrows(IllegalArgumentException.class, @@ -1260,7 +1260,7 @@ public void testUpdateComponentTemplateFailsIfResolvedIndexTemplatesWouldBeInval state = service.addComponentTemplate(state, true, "c2", ct2); ComposableIndexTemplate it = new ComposableIndexTemplate(List.of("i*"), new Template(null, null, null), - randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null); + randomBoolean() ? Arrays.asList("c1", "c2") : Arrays.asList("c2", "c1"), 0L, 1L, null, null, null); // Great, the templates aren't invalid state = service.addIndexTemplateV2(state, randomBoolean(), "my-template", it); @@ -1332,7 +1332,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { .build(); ComposableIndexTemplate template = new ComposableIndexTemplate(Collections.singletonList("logs-*-*"), - null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); state = service.addIndexTemplateV2(state, false, "logs", template); @@ -1354,7 +1354,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Test replacing it with a version without the data stream config IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> { ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-*-*"), null, null, - 100L, null, null, null); + 100L, null, null, null, null); service.addIndexTemplateV2(stateWithDS, false, "logs", nonDSTemplate); }); @@ -1365,7 +1365,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Test adding a higher priority version that would cause problems e = expectThrows(IllegalArgumentException.class, () -> { ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-my*-*"), null, null, - 105L, null, null, null); + 105L, null, null, null, null); service.addIndexTemplateV2(stateWithDS, false, "logs2", nonDSTemplate); }); @@ -1376,7 +1376,7 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Change the pattern to one that doesn't match the data stream e = expectThrows(IllegalArgumentException.class, () -> { ComposableIndexTemplate newTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-postgres-*"), null, - null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, 100L, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); service.addIndexTemplateV2(stateWithDS, false, "logs", newTemplate); }); @@ -1386,12 +1386,12 @@ public void testUnreferencedDataStreamsWhenAddingTemplate() throws Exception { // Add an additional template that matches our data stream at a lower priority ComposableIndexTemplate mysqlTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-mysql-*"), null, - null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate()); + null, 50L, null, null, new ComposableIndexTemplate.DataStreamTemplate(), null); ClusterState stateWithDSAndTemplate = service.addIndexTemplateV2(stateWithDS, false, "logs-mysql", mysqlTemplate); // We should be able to replace the "logs" template, because we have the "logs-mysql" template that can handle the data stream ComposableIndexTemplate nonDSTemplate = new ComposableIndexTemplate(Collections.singletonList("logs-postgres-*"), null, null, - 100L, null, null, null); + 100L, null, null, null, null); service.addIndexTemplateV2(stateWithDSAndTemplate, false, "logs", nonDSTemplate); } @@ -1475,9 +1475,9 @@ clusterService, createIndexService, new AliasValidator(), indicesService, @SuppressWarnings("unchecked") public static void assertTemplatesEqual(ComposableIndexTemplate actual, ComposableIndexTemplate expected) { ComposableIndexTemplate actualNoTemplate = new ComposableIndexTemplate(actual.indexPatterns(), null, - actual.composedOf(), actual.priority(), actual.version(), actual.metadata(), actual.getDataStreamTemplate()); + actual.composedOf(), actual.priority(), actual.version(), actual.metadata(), actual.getDataStreamTemplate(), null); ComposableIndexTemplate expectedNoTemplate = new ComposableIndexTemplate(expected.indexPatterns(), null, - expected.composedOf(), expected.priority(), expected.version(), expected.metadata(), expected.getDataStreamTemplate()); + expected.composedOf(), expected.priority(), expected.version(), expected.metadata(), expected.getDataStreamTemplate(), null); assertThat(actualNoTemplate, equalTo(expectedNoTemplate)); Template actualTemplate = actual.template(); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index ab42a213d7b2d..e20b342d147d0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -79,7 +79,8 @@ public void testSimpleJsonFromAndTo() throws IOException { 5L, 4L, Collections.singletonMap("my_meta", Collections.singletonMap("potato", "chicken")), - randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate())) + randomBoolean() ? null : new ComposableIndexTemplate.DataStreamTemplate(), + null)) .put(IndexMetadata.builder("test12") .settings(settings(Version.CURRENT) .put("setting1", "value1") diff --git a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java index ae0ad31c0d9fc..42beca2873ee9 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/SnapshotResiliencyTests.java @@ -81,7 +81,6 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.ActiveShardCount; -import org.elasticsearch.action.support.AutoCreateIndex; import org.elasticsearch.action.support.DestructiveOperations; import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.PlainActionFuture; @@ -1587,7 +1586,6 @@ allocationService, new AliasValidator(), shardLimitValidator, environment, index new AnalysisModule(environment, Collections.emptyList()).getAnalysisRegistry(), Collections.emptyList(), client), client, actionFilters, indexNameExpressionResolver, - new AutoCreateIndex(settings, clusterSettings, indexNameExpressionResolver, new SystemIndices(Map.of())), new IndexingPressure(settings), new SystemIndices(Map.of()) )); diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java index 4e336903aa84c..0662b80d1e7b2 100644 --- a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/cluster/routing/allocation/DataTierIT.java @@ -168,7 +168,7 @@ public void testTemplateOverridesDefaults() { Template t = new Template(Settings.builder() .put(DataTierAllocationDecider.INDEX_ROUTING_REQUIRE, DataTier.DATA_WARM) .build(), null, null); - ComposableIndexTemplate ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null); + ComposableIndexTemplate ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null, null); client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("template").indexTemplate(ct)).actionGet(); @@ -185,7 +185,7 @@ public void testTemplateOverridesDefaults() { t = new Template(Settings.builder() .putNull(DataTierAllocationDecider.INDEX_ROUTING_PREFER) .build(), null, null); - ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null); + ct = new ComposableIndexTemplate(Collections.singletonList(index), t, null, null, null, null, null, null); client().execute(PutComposableIndexTemplateAction.INSTANCE, new PutComposableIndexTemplateAction.Request("template").indexTemplate(ct)).actionGet(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java index e3f80404e23bb..994e996510b11 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/slm/history/SnapshotLifecycleTemplateRegistry.java @@ -38,7 +38,8 @@ public class SnapshotLifecycleTemplateRegistry extends IndexTemplateRegistry { // version 2: converted to hidden index // version 3: templates moved to composable templates // version 4:converted data stream - public static final int INDEX_TEMPLATE_VERSION = 4; + // version 5: add `allow_auto_create` setting + public static final int INDEX_TEMPLATE_VERSION = 5; public static final String SLM_TEMPLATE_VERSION_VARIABLE = "xpack.slm.template.version"; public static final String SLM_TEMPLATE_NAME = ".slm-history"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java index 82994613a8ade..6a072e25df245 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/watcher/support/WatcherIndexTemplateRegistryField.java @@ -17,8 +17,9 @@ public final class WatcherIndexTemplateRegistryField { // version 10: add support for foreach path in actions // version 11: watch history indices are hidden // version 12: templates changed to composable templates + // version 13: add `allow_auto_create` setting // Note: if you change this, also inform the kibana team around the watcher-ui - public static final int INDEX_TEMPLATE_VERSION = 12; + public static final int INDEX_TEMPLATE_VERSION = 13; public static final String HISTORY_TEMPLATE_NAME = ".watch-history-" + INDEX_TEMPLATE_VERSION; public static final String HISTORY_TEMPLATE_NAME_NO_ILM = ".watch-history-no-ilm-" + INDEX_TEMPLATE_VERSION; public static final String TRIGGERED_TEMPLATE_NAME = ".triggered_watches"; diff --git a/x-pack/plugin/core/src/main/resources/ilm-history.json b/x-pack/plugin/core/src/main/resources/ilm-history.json index 8fa18c742c277..44ad66d9dabdc 100644 --- a/x-pack/plugin/core/src/main/resources/ilm-history.json +++ b/x-pack/plugin/core/src/main/resources/ilm-history.json @@ -80,6 +80,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for ILM history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/logs-template.json b/x-pack/plugin/core/src/main/resources/logs-template.json index 0e23872ec0170..9c21a2b6ae504 100644 --- a/x-pack/plugin/core/src/main/resources/logs-template.json +++ b/x-pack/plugin/core/src/main/resources/logs-template.json @@ -6,6 +6,7 @@ "logs-mappings", "logs-settings" ], + "allow_auto_create": true, "_meta": { "description": "default logs template installed by x-pack", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/logstash-management.json b/x-pack/plugin/core/src/main/resources/logstash-management.json index d9528238dc0fb..00b0826a02574 100644 --- a/x-pack/plugin/core/src/main/resources/logstash-management.json +++ b/x-pack/plugin/core/src/main/resources/logstash-management.json @@ -7,6 +7,7 @@ "codec": "best_compression" } }, + "allow_auto_create": true, "mappings" : { "_doc" : { "_meta": { diff --git a/x-pack/plugin/core/src/main/resources/metrics-template.json b/x-pack/plugin/core/src/main/resources/metrics-template.json index cccd852196d41..0da4169de79b7 100644 --- a/x-pack/plugin/core/src/main/resources/metrics-template.json +++ b/x-pack/plugin/core/src/main/resources/metrics-template.json @@ -6,6 +6,7 @@ "metrics-mappings", "metrics-settings" ], + "allow_auto_create": true, "_meta": { "description": "default metrics template installed by x-pack", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/slm-history.json b/x-pack/plugin/core/src/main/resources/slm-history.json index 0ba7e4199daea..794a0ecf9423d 100644 --- a/x-pack/plugin/core/src/main/resources/slm-history.json +++ b/x-pack/plugin/core/src/main/resources/slm-history.json @@ -55,6 +55,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for SLM history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/synthetics-template.json b/x-pack/plugin/core/src/main/resources/synthetics-template.json index 9bbbe0ef9e3df..1036a29cf5a9d 100644 --- a/x-pack/plugin/core/src/main/resources/synthetics-template.json +++ b/x-pack/plugin/core/src/main/resources/synthetics-template.json @@ -6,6 +6,7 @@ "synthetics-mappings", "synthetics-settings" ], + "allow_auto_create": true, "_meta": { "description": "default synthetics template installed by x-pack", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/triggered-watches.json b/x-pack/plugin/core/src/main/resources/triggered-watches.json index 1cb43c51c6ef5..e11f7d9853186 100644 --- a/x-pack/plugin/core/src/main/resources/triggered-watches.json +++ b/x-pack/plugin/core/src/main/resources/triggered-watches.json @@ -37,6 +37,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for triggered watches indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json index 2f4433536d4ae..d9de434e08275 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json +++ b/x-pack/plugin/core/src/main/resources/watch-history-no-ilm.json @@ -613,6 +613,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for watcher history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/watch-history.json b/x-pack/plugin/core/src/main/resources/watch-history.json index c5b380165fd78..7ee688da832d0 100644 --- a/x-pack/plugin/core/src/main/resources/watch-history.json +++ b/x-pack/plugin/core/src/main/resources/watch-history.json @@ -564,6 +564,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for watcher history indices", "managed": true diff --git a/x-pack/plugin/core/src/main/resources/watches.json b/x-pack/plugin/core/src/main/resources/watches.json index ab9ba4b1d9ec6..4c1108c9f38ad 100644 --- a/x-pack/plugin/core/src/main/resources/watches.json +++ b/x-pack/plugin/core/src/main/resources/watches.json @@ -59,6 +59,7 @@ } } }, + "allow_auto_create": true, "_meta": { "description": "index template for watches indices", "managed": true diff --git a/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java new file mode 100644 index 0000000000000..83dbf56a0a26b --- /dev/null +++ b/x-pack/plugin/data-streams/qa/rest/src/yamlRestTest/java/org/elasticsearch/xpack/datastreams/AutoCreateDataStreamIT.java @@ -0,0 +1,118 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.xpack.datastreams; + +import org.elasticsearch.action.support.AutoCreateIndex; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.test.rest.ESRestTestCase; + +import java.io.IOException; +import java.io.InputStreamReader; +import java.time.Instant; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.hamcrest.Matchers.containsString; + +public class AutoCreateDataStreamIT extends ESRestTestCase { + + /** + * Check that setting {@link AutoCreateIndex#AUTO_CREATE_INDEX_SETTING} to false + * does not affect the ability to auto-create data streams, which are not subject to the setting. + */ + public void testCanAutoCreateDataStreamWhenAutoCreateIndexDisabled() throws IOException { + configureAutoCreateIndex(false); + createTemplateWithAllowAutoCreate(null); + assertOK(this.indexDocument()); + } + + /** + * Check that automatically creating a data stream is allowed when the index name matches a template + * and that template has allow_auto_create set to true. + */ + public void testCanAutoCreateDataStreamWhenExplicitlyAllowedByTemplate() throws IOException { + createTemplateWithAllowAutoCreate(true); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + assertOK(this.indexDocument()); + } + + /** + * Check that automatically creating a data stream is disallowed when the data stream name matches a template and that template has + * allow_auto_create explicitly to false. + */ + public void testCannotAutoCreateDataStreamWhenDisallowedByTemplate() throws IOException { + createTemplateWithAllowAutoCreate(false); + + // Attempt to add a document to a non-existing index. Auto-creating the index should succeed because the index name + // matches the template pattern + final ResponseException responseException = expectThrows(ResponseException.class, this::indexDocument); + + assertThat( + Streams.copyToString(new InputStreamReader(responseException.getResponse().getEntity().getContent(), UTF_8)), + containsString("no such index [composable template [recipe*] forbids index auto creation]") + ); + } + + private void configureAutoCreateIndex(boolean value) throws IOException { + XContentBuilder builder = JsonXContent.contentBuilder() + .startObject() + .startObject("transient") + .field(AutoCreateIndex.AUTO_CREATE_INDEX_SETTING.getKey(), value) + .endObject() + .endObject(); + + final Request settingsRequest = new Request("PUT", "_cluster/settings"); + settingsRequest.setJsonEntity(Strings.toString(builder)); + final Response settingsResponse = client().performRequest(settingsRequest); + assertOK(settingsResponse); + } + + private void createTemplateWithAllowAutoCreate(Boolean allowAutoCreate) throws IOException { + XContentBuilder b = JsonXContent.contentBuilder(); + b.startObject(); + { + b.array("index_patterns", "recipe*"); + if (allowAutoCreate != null) { + b.field("allow_auto_create", allowAutoCreate); + } + b.startObject("data_stream"); + b.endObject(); + } + b.endObject(); + + final Request createTemplateRequest = new Request("PUT", "_index_template/recipe_template"); + createTemplateRequest.setJsonEntity(Strings.toString(b)); + final Response createTemplateResponse = client().performRequest(createTemplateRequest); + assertOK(createTemplateResponse); + } + + private Response indexDocument() throws IOException { + final Request indexDocumentRequest = new Request("POST", "recipe_kr/_doc"); + indexDocumentRequest.setJsonEntity("{ \"@timestamp\": \"" + Instant.now() + "\", \"name\": \"Kimchi\" }"); + return client().performRequest(indexDocumentRequest); + } +} diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 25cf50926de65..94daa9b951db0 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -426,7 +426,8 @@ public void testComposableTemplateOnlyMatchingWithDataStreamName() throws Except null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); @@ -509,7 +510,8 @@ public void testTimeStampValidationInvalidFieldMapping() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); @@ -968,7 +970,8 @@ public void testMixedAutoCreate() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, createTemplateRequest).actionGet(); @@ -1197,7 +1200,8 @@ static void putComposableIndexTemplate(String id, @Nullable String mappings, Lis null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ) ); client().execute(PutComposableIndexTemplateAction.INSTANCE, request).actionGet(); diff --git a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java index 36f0fdd8a1712..78b0706a7e395 100644 --- a/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java +++ b/x-pack/plugin/data-streams/src/internalClusterTest/java/org/elasticsearch/xpack/datastreams/DataTierDataStreamIT.java @@ -43,7 +43,8 @@ public void testDefaultDataStreamAllocateToHot() { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ); client().execute( PutComposableIndexTemplateAction.INSTANCE, diff --git a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java index 2d732b534ef44..5b9edc96bf6e4 100644 --- a/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java +++ b/x-pack/plugin/data-streams/src/test/java/org/elasticsearch/xpack/datastreams/DataStreamsStatsTests.java @@ -218,7 +218,8 @@ private String createDataStream() throws Exception { null, null, null, - new ComposableIndexTemplate.DataStreamTemplate() + new ComposableIndexTemplate.DataStreamTemplate(), + null ); assertTrue( client().execute( diff --git a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java index 08494b7fb70e8..c8835acbd0b39 100644 --- a/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java +++ b/x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/history/ILMHistoryTemplateRegistry.java @@ -29,7 +29,8 @@ public class ILMHistoryTemplateRegistry extends IndexTemplateRegistry { // version 1: initial // version 2: convert to hidden index // version 3: templates moved to composable templates - public static final int INDEX_TEMPLATE_VERSION = 3; + // version 4: add `allow_auto_create` setting + public static final int INDEX_TEMPLATE_VERSION = 4; public static final String ILM_TEMPLATE_VERSION_VARIABLE = "xpack.ilm_history.template.version"; public static final String ILM_TEMPLATE_NAME = "ilm-history"; diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java index 2fc7e7d68c261..c0d2e93f45d93 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/MlNativeIntegTestCase.java @@ -313,7 +313,8 @@ protected static void createDataStreamAndTemplate(String dataStreamName, String null, null, null, - new ComposableIndexTemplate.DataStreamTemplate()))) + new ComposableIndexTemplate.DataStreamTemplate(), + null))) .actionGet(); client().execute(CreateDataStreamAction.INSTANCE, new CreateDataStreamAction.Request(dataStreamName)).actionGet(); } diff --git a/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/put_watch/90_auto_create_index.yml b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/put_watch/90_auto_create_index.yml new file mode 100644 index 0000000000000..b9b9a39dab776 --- /dev/null +++ b/x-pack/plugin/watcher/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/watcher/put_watch/90_auto_create_index.yml @@ -0,0 +1,52 @@ +--- +setup: + - do: + cluster.health: + wait_for_status: yellow + - do: + cluster.put_settings: + body: + transient: + action.auto_create_index: false + +--- +teardown: + - do: + watcher.delete_watch: + id: "test_watch" + ignore: 404 + - do: + cluster.put_settings: + body: + transient: + action.auto_create_index: null + +--- +"Ensure watch is created even when auto_create_index is disabled": + - do: + watcher.put_watch: + id: "test_watch" + body: > + { + "trigger": { + "schedule" : { "cron" : "0 0 0 1 * ? 2099" } + }, + "input": { + "simple": { + "foo": "bar" + } + }, + "condition": { + "always": {} + }, + "actions": { + "indexme" : { + "index" : { + "index" : "my_test_index", + "doc_id": "my-id" + } + } + } + } + - match: { _id: "test_watch" } + - match: { created: true }