From f6db6dc17142da59a0124b8bcf2af9906417c04b Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 24 Feb 2020 15:58:23 -0500 Subject: [PATCH 1/5] [ML] Add indices_options to datafeed config --- .../client/ml/datafeed/DatafeedConfig.java | 38 +++++++++- .../client/ml/datafeed/DatafeedUpdate.java | 31 +++++++- .../ml/datafeed/DatafeedConfigTests.java | 8 ++ .../ml/datafeed/DatafeedUpdateTests.java | 8 ++ .../apis/put-datafeed.asciidoc | 5 ++ .../apis/update-datafeed.asciidoc | 4 + docs/reference/ml/ml-shared.asciidoc | 13 ++++ .../core/ml/action/PutDatafeedAction.java | 7 +- .../core/ml/action/StartDatafeedAction.java | 35 ++++++++- .../core/ml/action/UpdateDatafeedAction.java | 7 +- .../core/ml/datafeed/DatafeedConfig.java | 45 ++++++++++- .../core/ml/datafeed/DatafeedUpdate.java | 51 +++++++++++-- .../ml/job/results/ReservedFieldNames.java | 1 + .../xpack/core/ml/config_index_mappings.json | 4 + .../action/PutDatafeedActionRequestTests.java | 2 +- .../UpdateDatafeedActionRequestTests.java | 2 +- .../core/ml/datafeed/DatafeedConfigTests.java | 21 ++++- .../core/ml/datafeed/DatafeedUpdateTests.java | 31 +++++++- .../ml/integration/DatafeedJobsRestIT.java | 76 +++++++++++++++++++ .../action/TransportStartDatafeedAction.java | 10 ++- .../ml/datafeed/DatafeedNodeSelector.java | 6 +- .../DatafeedDelayedDataDetector.java | 9 ++- .../DelayedDataDetectorFactory.java | 1 + .../extractor/DataExtractorFactory.java | 4 +- .../aggregation/AggregationDataExtractor.java | 1 + .../AggregationDataExtractorContext.java | 7 +- .../AggregationDataExtractorFactory.java | 3 +- .../aggregation/RollupDataExtractor.java | 4 +- .../RollupDataExtractorFactory.java | 3 +- .../chunked/ChunkedDataExtractor.java | 5 +- .../chunked/ChunkedDataExtractorContext.java | 6 +- .../chunked/ChunkedDataExtractorFactory.java | 3 +- .../extractor/scroll/ScrollDataExtractor.java | 1 + .../scroll/ScrollDataExtractorContext.java | 7 +- .../scroll/ScrollDataExtractorFactory.java | 16 ++-- .../rest/datafeeds/RestPutDatafeedAction.java | 11 ++- .../datafeeds/RestUpdateDatafeedAction.java | 11 ++- .../datafeed/DatafeedNodeSelectorTests.java | 44 +++++------ .../AggregationDataExtractorTests.java | 2 +- .../chunked/ChunkedDataExtractorTests.java | 2 +- .../scroll/ScrollDataExtractorTests.java | 4 +- .../rest-api-spec/api/ml.put_datafeed.json | 25 ++++++ .../rest-api-spec/api/ml.update_datafeed.json | 25 ++++++ .../rest-api-spec/test/ml/datafeeds_crud.yml | 75 ++++++++++++++++++ 44 files changed, 602 insertions(+), 72 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedConfig.java index f192b420eba0e..c1b5b589b2f15 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedConfig.java @@ -18,8 +18,10 @@ */ package org.elasticsearch.client.ml.datafeed; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.ml.job.config.Job; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; @@ -63,6 +65,7 @@ public class DatafeedConfig implements ToXContentObject { public static final ParseField CHUNKING_CONFIG = new ParseField("chunking_config"); public static final ParseField DELAYED_DATA_CHECK_CONFIG = new ParseField("delayed_data_check_config"); public static final ParseField MAX_EMPTY_SEARCHES = new ParseField("max_empty_searches"); + public static final ParseField INDICES_OPTIONS = new ParseField("indices_options"); public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "datafeed_config", true, a -> new Builder((String)a[0], (String)a[1])); @@ -90,6 +93,9 @@ public class DatafeedConfig implements ToXContentObject { PARSER.declareObject(Builder::setChunkingConfig, ChunkingConfig.PARSER, CHUNKING_CONFIG); PARSER.declareObject(Builder::setDelayedDataCheckConfig, DelayedDataCheckConfig.PARSER, DELAYED_DATA_CHECK_CONFIG); PARSER.declareInt(Builder::setMaxEmptySearches, MAX_EMPTY_SEARCHES); + PARSER.declareObject(Builder::setIndicesOptions, + (p, c) -> IndicesOptions.fromMap(p.map(), new IndicesOptions(IndicesOptions.Option.NONE, IndicesOptions.WildcardStates.NONE)), + INDICES_OPTIONS); } private static BytesReference parseBytes(XContentParser parser) throws IOException { @@ -110,11 +116,12 @@ private static BytesReference parseBytes(XContentParser parser) throws IOExcepti private final ChunkingConfig chunkingConfig; private final DelayedDataCheckConfig delayedDataCheckConfig; private final Integer maxEmptySearches; + private final IndicesOptions indicesOptions; private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List indices, BytesReference query, BytesReference aggregations, List scriptFields, Integer scrollSize, ChunkingConfig chunkingConfig, DelayedDataCheckConfig delayedDataCheckConfig, - Integer maxEmptySearches) { + Integer maxEmptySearches, IndicesOptions indicesOptions) { this.id = id; this.jobId = jobId; this.queryDelay = queryDelay; @@ -127,6 +134,7 @@ private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue this.chunkingConfig = chunkingConfig; this.delayedDataCheckConfig = delayedDataCheckConfig; this.maxEmptySearches = maxEmptySearches; + this.indicesOptions = indicesOptions; } public String getId() { @@ -177,6 +185,10 @@ public Integer getMaxEmptySearches() { return maxEmptySearches; } + public IndicesOptions getIndicesOptions() { + return indicesOptions; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -216,6 +228,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (maxEmptySearches != null) { builder.field(MAX_EMPTY_SEARCHES.getPreferredName(), maxEmptySearches); } + if (indicesOptions != null) { + builder.startObject(INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } builder.endObject(); return builder; @@ -225,6 +242,11 @@ private static Map asMap(BytesReference bytesReference) { return bytesReference == null ? null : XContentHelper.convertToMap(bytesReference, true, XContentType.JSON).v2(); } + @Override + public String toString() { + return Strings.toString(this, true, true); + } + /** * The lists of indices and types are compared for equality but they are not * sorted first so this test could fail simply because the indices and types @@ -257,7 +279,8 @@ public boolean equals(Object other) { && Objects.equals(this.scriptFields, that.scriptFields) && Objects.equals(this.chunkingConfig, that.chunkingConfig) && Objects.equals(this.delayedDataCheckConfig, that.delayedDataCheckConfig) - && Objects.equals(this.maxEmptySearches, that.maxEmptySearches); + && Objects.equals(this.maxEmptySearches, that.maxEmptySearches) + && Objects.equals(this.indicesOptions, that.indicesOptions); } /** @@ -268,7 +291,7 @@ public boolean equals(Object other) { @Override public int hashCode() { return Objects.hash(id, jobId, frequency, queryDelay, indices, asMap(query), scrollSize, asMap(aggregations), scriptFields, - chunkingConfig, delayedDataCheckConfig, maxEmptySearches); + chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions); } public static Builder builder(String id, String jobId) { @@ -289,6 +312,7 @@ public static class Builder { private ChunkingConfig chunkingConfig; private DelayedDataCheckConfig delayedDataCheckConfig; private Integer maxEmptySearches; + private IndicesOptions indicesOptions; public Builder(String id, String jobId) { this.id = Objects.requireNonNull(id, ID.getPreferredName()); @@ -308,6 +332,7 @@ public Builder(DatafeedConfig config) { this.chunkingConfig = config.chunkingConfig; this.delayedDataCheckConfig = config.getDelayedDataCheckConfig(); this.maxEmptySearches = config.getMaxEmptySearches(); + this.indicesOptions = config.indicesOptions; } public Builder setIndices(List indices) { @@ -395,9 +420,14 @@ public Builder setMaxEmptySearches(int maxEmptySearches) { return this; } + public Builder setIndicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + public DatafeedConfig build() { return new DatafeedConfig(id, jobId, queryDelay, frequency, indices, query, aggregations, scriptFields, scrollSize, - chunkingConfig, delayedDataCheckConfig, maxEmptySearches); + chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions); } private static BytesReference xContentToBytes(ToXContentObject object) throws IOException { diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdate.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdate.java index a7d76e9a69c96..0c2f88f7e21a6 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdate.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdate.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.client.ml.datafeed; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -78,6 +79,9 @@ public class DatafeedUpdate implements ToXContentObject { DelayedDataCheckConfig.PARSER, DatafeedConfig.DELAYED_DATA_CHECK_CONFIG); PARSER.declareInt(Builder::setMaxEmptySearches, DatafeedConfig.MAX_EMPTY_SEARCHES); + PARSER.declareObject(Builder::setIndicesOptions, + (p, c) -> IndicesOptions.fromMap(p.map(), new IndicesOptions(IndicesOptions.Option.NONE, IndicesOptions.WildcardStates.NONE)), + DatafeedConfig.INDICES_OPTIONS); } private static BytesReference parseBytes(XContentParser parser) throws IOException { @@ -97,11 +101,12 @@ private static BytesReference parseBytes(XContentParser parser) throws IOExcepti private final ChunkingConfig chunkingConfig; private final DelayedDataCheckConfig delayedDataCheckConfig; private final Integer maxEmptySearches; + private final IndicesOptions indicesOptions; private DatafeedUpdate(String id, TimeValue queryDelay, TimeValue frequency, List indices, BytesReference query, BytesReference aggregations, List scriptFields, Integer scrollSize, ChunkingConfig chunkingConfig, DelayedDataCheckConfig delayedDataCheckConfig, - Integer maxEmptySearches) { + Integer maxEmptySearches, IndicesOptions indicesOptions) { this.id = id; this.queryDelay = queryDelay; this.frequency = frequency; @@ -113,6 +118,7 @@ private DatafeedUpdate(String id, TimeValue queryDelay, TimeValue frequency, Lis this.chunkingConfig = chunkingConfig; this.delayedDataCheckConfig = delayedDataCheckConfig; this.maxEmptySearches = maxEmptySearches; + this.indicesOptions = indicesOptions; } /** @@ -152,6 +158,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws addOptionalField(builder, DatafeedConfig.SCROLL_SIZE, scrollSize); addOptionalField(builder, DatafeedConfig.CHUNKING_CONFIG, chunkingConfig); addOptionalField(builder, DatafeedConfig.MAX_EMPTY_SEARCHES, maxEmptySearches); + if (indicesOptions != null) { + builder.startObject(DatafeedConfig.INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } builder.endObject(); return builder; } @@ -202,6 +213,10 @@ public Integer getMaxEmptySearches() { return maxEmptySearches; } + public IndicesOptions getIndicesOptions() { + return indicesOptions; + } + private static Map asMap(BytesReference bytesReference) { return bytesReference == null ? null : XContentHelper.convertToMap(bytesReference, true, XContentType.JSON).v2(); } @@ -237,7 +252,8 @@ public boolean equals(Object other) { && Objects.equals(this.delayedDataCheckConfig, that.delayedDataCheckConfig) && Objects.equals(this.scriptFields, that.scriptFields) && Objects.equals(this.chunkingConfig, that.chunkingConfig) - && Objects.equals(this.maxEmptySearches, that.maxEmptySearches); + && Objects.equals(this.maxEmptySearches, that.maxEmptySearches) + && Objects.equals(this.indicesOptions, that.indicesOptions); } /** @@ -248,7 +264,7 @@ public boolean equals(Object other) { @Override public int hashCode() { return Objects.hash(id, frequency, queryDelay, indices, asMap(query), scrollSize, asMap(aggregations), scriptFields, - chunkingConfig, delayedDataCheckConfig, maxEmptySearches); + chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions); } public static Builder builder(String id) { @@ -268,6 +284,7 @@ public static class Builder { private ChunkingConfig chunkingConfig; private DelayedDataCheckConfig delayedDataCheckConfig; private Integer maxEmptySearches; + private IndicesOptions indicesOptions; public Builder(String id) { this.id = Objects.requireNonNull(id, DatafeedConfig.ID.getPreferredName()); @@ -285,6 +302,7 @@ public Builder(DatafeedUpdate config) { this.chunkingConfig = config.chunkingConfig; this.delayedDataCheckConfig = config.delayedDataCheckConfig; this.maxEmptySearches = config.maxEmptySearches; + this.indicesOptions = config.indicesOptions; } public Builder setIndices(List indices) { @@ -363,9 +381,14 @@ public Builder setMaxEmptySearches(int maxEmptySearches) { return this; } + public Builder setIndicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + public DatafeedUpdate build() { return new DatafeedUpdate(id, queryDelay, frequency, indices, query, aggregations, scriptFields, scrollSize, - chunkingConfig, delayedDataCheckConfig, maxEmptySearches); + chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions); } private static BytesReference xContentToBytes(ToXContentObject object) throws IOException { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedConfigTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedConfigTests.java index 7f7a03ab2e182..16bd25bb49298 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedConfigTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedConfigTests.java @@ -19,6 +19,7 @@ package org.elasticsearch.client.ml.datafeed; import com.carrotsearch.randomizedtesting.generators.CodepointSetGenerator; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -109,6 +110,13 @@ public static DatafeedConfig.Builder createRandomBuilder() { if (randomBoolean()) { builder.setMaxEmptySearches(randomIntBetween(10, 100)); } + if (randomBoolean()) { + builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean())); + } return builder; } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdateTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdateTests.java index 72ab156406fe5..61301324f7660 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdateTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ml/datafeed/DatafeedUpdateTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.client.ml.datafeed; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryBuilders; @@ -83,6 +84,13 @@ public static DatafeedUpdate createRandom() { if (randomBoolean()) { builder.setMaxEmptySearches(randomIntBetween(10, 100)); } + if (randomBoolean()) { + builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean())); + } return builder.build(); } diff --git a/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc index 0835d6fcd7c58..86bc85c34148f 100644 --- a/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/put-datafeed.asciidoc @@ -98,6 +98,11 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=script-fields] (Optional, unsigned integer) include::{docdir}/ml/ml-shared.asciidoc[tag=scroll-size] +`indices_options`:: +(Optional, object) +include::{docdir}/ml/ml-shared.asciidoc[tag=indices-options] + + [[ml-put-datafeed-example]] ==== {api-examples-title} diff --git a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc index 044c65f11abcf..3a6d42ee00484 100644 --- a/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc +++ b/docs/reference/ml/anomaly-detection/apis/update-datafeed.asciidoc @@ -101,6 +101,10 @@ include::{docdir}/ml/ml-shared.asciidoc[tag=script-fields] (Optional, unsigned integer) include::{docdir}/ml/ml-shared.asciidoc[tag=scroll-size] +`indices_options`:: +(Optional, object) +include::{docdir}/ml/ml-shared.asciidoc[tag=indices-options] + [[ml-update-datafeed-example]] ==== {api-examples-title} diff --git a/docs/reference/ml/ml-shared.asciidoc b/docs/reference/ml/ml-shared.asciidoc index 57740a8884a60..bbd945b690e26 100644 --- a/docs/reference/ml/ml-shared.asciidoc +++ b/docs/reference/ml/ml-shared.asciidoc @@ -655,6 +655,19 @@ not be set to `false` on any {ml} nodes. -- end::indices[] +tag::indices-options[] +Object specifying index expansion options used during search. +For example: +``` +{ + "expand_wildcards": ["all"], + "ignore_unavailable": true, + "allow_no_indices": "false", + "ignore_throttled": true +} +``` +end::indices-options[] + tag::influencers[] A comma separated list of influencer field names. Typically these can be the by, over, or partition fields that are used in the detector configuration. You might diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java index 481d6156204c4..5b2ada2a48837 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java @@ -8,9 +8,11 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -32,8 +34,11 @@ private PutDatafeedAction() { public static class Request extends AcknowledgedRequest implements ToXContentObject { - public static Request parseRequest(String datafeedId, XContentParser parser) { + public static Request parseRequest(String datafeedId, @Nullable IndicesOptions indicesOptions, XContentParser parser) { DatafeedConfig.Builder datafeed = DatafeedConfig.STRICT_PARSER.apply(parser, null); + if (indicesOptions != null) { + datafeed.setIndicesOptions(indicesOptions); + } datafeed.setId(datafeedId); return new Request(datafeed.build()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java index afd58e0b70554..904e78eb12ec4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.ValidateActions; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeRequest; import org.elasticsearch.client.ElasticsearchClient; @@ -150,6 +151,9 @@ public static class DatafeedParams implements PersistentTaskParams { params.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT); PARSER.declareString(DatafeedParams::setJobId, Job.ID); PARSER.declareStringArray(DatafeedParams::setDatafeedIndices, INDICES); + PARSER.declareObject(DatafeedParams::setIndicesOptions, + (p, c) -> IndicesOptions.fromMap(p.map(), IndicesOptions.lenientExpandOpen()), + DatafeedConfig.INDICES_OPTIONS); } static long parseDateOrThrow(String date, ParseField paramName, LongSupplier now) { @@ -191,6 +195,11 @@ public DatafeedParams(StreamInput in) throws IOException { timeout = TimeValue.timeValueMillis(in.readVLong()); jobId = in.readOptionalString(); datafeedIndices = in.readStringList(); + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + indicesOptions = in.readBoolean() ? IndicesOptions.readIndicesOptions(in) : null; + } else { + indicesOptions = null; + } } DatafeedParams() { @@ -202,6 +211,7 @@ public DatafeedParams(StreamInput in) throws IOException { private TimeValue timeout = TimeValue.timeValueSeconds(20); private List datafeedIndices = Collections.emptyList(); private String jobId; + private IndicesOptions indicesOptions; public String getDatafeedId() { @@ -248,6 +258,15 @@ public void setDatafeedIndices(List datafeedIndices) { this.datafeedIndices = datafeedIndices; } + public IndicesOptions getIndicesOptions() { + return indicesOptions; + } + + public DatafeedParams setIndicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + @Override public String getWriteableName() { return MlTasks.DATAFEED_TASK_NAME; @@ -266,6 +285,14 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVLong(timeout.millis()); out.writeOptionalString(jobId); out.writeStringCollection(datafeedIndices); + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (indicesOptions != null) { + out.writeBoolean(true); + indicesOptions.writeIndicesOptions(out); + } else { + out.writeBoolean(false); + } + } } @Override @@ -283,13 +310,18 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par if (datafeedIndices.isEmpty() == false) { builder.field(INDICES.getPreferredName(), datafeedIndices); } + if (indicesOptions != null) { + builder.startObject(DatafeedConfig.INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } builder.endObject(); return builder; } @Override public int hashCode() { - return Objects.hash(datafeedId, startTime, endTime, timeout, jobId, datafeedIndices); + return Objects.hash(datafeedId, startTime, endTime, timeout, jobId, datafeedIndices, indicesOptions); } @Override @@ -306,6 +338,7 @@ public boolean equals(Object obj) { Objects.equals(endTime, other.endTime) && Objects.equals(timeout, other.timeout) && Objects.equals(jobId, other.jobId) && + Objects.equals(indicesOptions, other.indicesOptions) && Objects.equals(datafeedIndices, other.datafeedIndices); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedAction.java index 5d82a21662648..a5aebc61df227 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedAction.java @@ -7,9 +7,11 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -31,8 +33,11 @@ private UpdateDatafeedAction() { public static class Request extends AcknowledgedRequest implements ToXContentObject { - public static Request parseRequest(String datafeedId, XContentParser parser) { + public static Request parseRequest(String datafeedId, @Nullable IndicesOptions indicesOptions, XContentParser parser) { DatafeedUpdate.Builder update = DatafeedUpdate.PARSER.apply(parser, null); + if (indicesOptions != null) { + update.setIndicesOptions(indicesOptions); + } update.setId(datafeedId); return new Request(update.build()); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 951f259a9d219..44a9034c3b18a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; @@ -92,6 +93,7 @@ public class DatafeedConfig extends AbstractDiffable implements public static final ParseField HEADERS = new ParseField("headers"); public static final ParseField DELAYED_DATA_CHECK_CONFIG = new ParseField("delayed_data_check_config"); public static final ParseField MAX_EMPTY_SEARCHES = new ParseField("max_empty_searches"); + public static final ParseField INDICES_OPTIONS = new ParseField("indices_options"); // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly public static final ObjectParser LENIENT_PARSER = createParser(true); @@ -154,6 +156,9 @@ private static ObjectParser createParser(boolean ignoreUnknownFie ignoreUnknownFields ? DelayedDataCheckConfig.LENIENT_PARSER : DelayedDataCheckConfig.STRICT_PARSER, DELAYED_DATA_CHECK_CONFIG); parser.declareInt(Builder::setMaxEmptySearches, MAX_EMPTY_SEARCHES); + parser.declareObject(Builder::setIndicesOptions, + (p, c) -> IndicesOptions.fromMap(p.map(), new IndicesOptions(IndicesOptions.Option.NONE, IndicesOptions.WildcardStates.NONE)), + INDICES_OPTIONS); return parser; } @@ -179,11 +184,12 @@ private static ObjectParser createParser(boolean ignoreUnknownFie private final Map headers; private final DelayedDataCheckConfig delayedDataCheckConfig; private final Integer maxEmptySearches; + private final IndicesOptions indicesOptions; private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List indices, QueryProvider queryProvider, AggProvider aggProvider, List scriptFields, Integer scrollSize, ChunkingConfig chunkingConfig, Map headers, - DelayedDataCheckConfig delayedDataCheckConfig, Integer maxEmptySearches) { + DelayedDataCheckConfig delayedDataCheckConfig, Integer maxEmptySearches, IndicesOptions indicesOptions) { this.id = id; this.jobId = jobId; this.queryDelay = queryDelay; @@ -197,6 +203,7 @@ private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue this.headers = Collections.unmodifiableMap(headers); this.delayedDataCheckConfig = delayedDataCheckConfig; this.maxEmptySearches = maxEmptySearches; + this.indicesOptions = indicesOptions; } public DatafeedConfig(StreamInput in) throws IOException { @@ -228,6 +235,11 @@ public DatafeedConfig(StreamInput in) throws IOException { } else { maxEmptySearches = null; } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + indicesOptions = in.readBoolean() ? IndicesOptions.readIndicesOptions(in) : null; + } else { + indicesOptions = null; + } } /** @@ -400,6 +412,10 @@ public Integer getMaxEmptySearches() { return maxEmptySearches; } + public IndicesOptions getIndicesOptions() { + return indicesOptions; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(id); @@ -431,6 +447,14 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_5_0)) { out.writeOptionalVInt(maxEmptySearches); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (indicesOptions != null) { + out.writeBoolean(true); + indicesOptions.writeIndicesOptions(out); + } else { + out.writeBoolean(false); + } + } } @Override @@ -470,6 +494,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (maxEmptySearches != null) { builder.field(MAX_EMPTY_SEARCHES.getPreferredName(), maxEmptySearches); } + if (indicesOptions != null) { + builder.startObject(INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } builder.endObject(); return builder; } @@ -503,13 +532,14 @@ public boolean equals(Object other) { && Objects.equals(this.chunkingConfig, that.chunkingConfig) && Objects.equals(this.headers, that.headers) && Objects.equals(this.delayedDataCheckConfig, that.delayedDataCheckConfig) - && Objects.equals(this.maxEmptySearches, that.maxEmptySearches); + && Objects.equals(this.maxEmptySearches, that.maxEmptySearches) + && Objects.equals(this.indicesOptions, that.indicesOptions); } @Override public int hashCode() { return Objects.hash(id, jobId, frequency, queryDelay, indices, queryProvider, scrollSize, aggProvider, scriptFields, chunkingConfig, - headers, delayedDataCheckConfig, maxEmptySearches); + headers, delayedDataCheckConfig, maxEmptySearches, indicesOptions); } @Override @@ -583,6 +613,7 @@ public static class Builder { private Map headers = Collections.emptyMap(); private DelayedDataCheckConfig delayedDataCheckConfig = DelayedDataCheckConfig.defaultDelayedDataCheckConfig(); private Integer maxEmptySearches; + private IndicesOptions indicesOptions; public Builder() { } @@ -606,6 +637,7 @@ public Builder(DatafeedConfig config) { this.headers = new HashMap<>(config.headers); this.delayedDataCheckConfig = config.getDelayedDataCheckConfig(); this.maxEmptySearches = config.getMaxEmptySearches(); + this.indicesOptions = config.indicesOptions; } public void setId(String datafeedId) { @@ -711,6 +743,11 @@ public void setMaxEmptySearches(int maxEmptySearches) { } } + public Builder setIndicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + public DatafeedConfig build() { ExceptionsHelper.requireNonNull(id, ID.getPreferredName()); ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); @@ -726,7 +763,7 @@ public DatafeedConfig build() { setDefaultQueryDelay(); return new DatafeedConfig(id, jobId, queryDelay, frequency, indices, queryProvider, aggProvider, scriptFields, scrollSize, - chunkingConfig, headers, delayedDataCheckConfig, maxEmptySearches); + chunkingConfig, headers, delayedDataCheckConfig, maxEmptySearches, indicesOptions); } void validateScriptFields() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java index 53a370f46117b..f1c6f98857b8d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.ml.datafeed; import org.elasticsearch.Version; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.stream.StreamInput; @@ -79,6 +80,9 @@ public class DatafeedUpdate implements Writeable, ToXContentObject { DelayedDataCheckConfig.STRICT_PARSER, DatafeedConfig.DELAYED_DATA_CHECK_CONFIG); PARSER.declareInt(Builder::setMaxEmptySearches, DatafeedConfig.MAX_EMPTY_SEARCHES); + PARSER.declareObject(Builder::setIndicesOptions, + (p, c) -> IndicesOptions.fromMap(p.map(), new IndicesOptions(IndicesOptions.Option.NONE, IndicesOptions.WildcardStates.NONE)), + DatafeedConfig.INDICES_OPTIONS); } private final String id; @@ -93,12 +97,13 @@ public class DatafeedUpdate implements Writeable, ToXContentObject { private final ChunkingConfig chunkingConfig; private final DelayedDataCheckConfig delayedDataCheckConfig; private final Integer maxEmptySearches; + private final IndicesOptions indicesOptions; private DatafeedUpdate(String id, String jobId, TimeValue queryDelay, TimeValue frequency, List indices, QueryProvider queryProvider, AggProvider aggProvider, List scriptFields, Integer scrollSize, ChunkingConfig chunkingConfig, DelayedDataCheckConfig delayedDataCheckConfig, - Integer maxEmptySearches) { + Integer maxEmptySearches, IndicesOptions indicesOptions) { this.id = id; this.jobId = jobId; this.queryDelay = queryDelay; @@ -111,6 +116,7 @@ private DatafeedUpdate(String id, String jobId, TimeValue queryDelay, TimeValue this.chunkingConfig = chunkingConfig; this.delayedDataCheckConfig = delayedDataCheckConfig; this.maxEmptySearches = maxEmptySearches; + this.indicesOptions = indicesOptions; } public DatafeedUpdate(StreamInput in) throws IOException { @@ -149,6 +155,11 @@ public DatafeedUpdate(StreamInput in) throws IOException { } else { maxEmptySearches = null; } + if (in.getVersion().onOrAfter(Version.V_8_0_0)) { + indicesOptions = in.readBoolean() ? IndicesOptions.readIndicesOptions(in) : null; + } else { + indicesOptions = null; + } } /** @@ -195,6 +206,14 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_5_0)) { out.writeOptionalInt(maxEmptySearches); } + if (out.getVersion().onOrAfter(Version.V_8_0_0)) { + if (indicesOptions != null) { + out.writeBoolean(true); + indicesOptions.writeIndicesOptions(out); + } else { + out.writeBoolean(false); + } + } } @Override @@ -226,7 +245,11 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws addOptionalField(builder, DatafeedConfig.CHUNKING_CONFIG, chunkingConfig); addOptionalField(builder, DatafeedConfig.DELAYED_DATA_CHECK_CONFIG, delayedDataCheckConfig); addOptionalField(builder, DatafeedConfig.MAX_EMPTY_SEARCHES, maxEmptySearches); - + if (indicesOptions != null) { + builder.startObject(DatafeedConfig.INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + } builder.endObject(); return builder; } @@ -299,6 +322,10 @@ public Integer getMaxEmptySearches() { return maxEmptySearches; } + public IndicesOptions getIndicesOptions() { + return indicesOptions; + } + /** * Applies the update to the given {@link DatafeedConfig} * @return a new {@link DatafeedConfig} that contains the update @@ -346,6 +373,9 @@ public DatafeedConfig apply(DatafeedConfig datafeedConfig, Map h if (maxEmptySearches != null) { builder.setMaxEmptySearches(maxEmptySearches); } + if (indicesOptions != null) { + builder.setIndicesOptions(indicesOptions); + } if (headers.isEmpty() == false) { // Adjust the request, adding security headers from the current thread context @@ -386,13 +416,14 @@ public boolean equals(Object other) { && Objects.equals(this.delayedDataCheckConfig, that.delayedDataCheckConfig) && Objects.equals(this.scriptFields, that.scriptFields) && Objects.equals(this.chunkingConfig, that.chunkingConfig) - && Objects.equals(this.maxEmptySearches, that.maxEmptySearches); + && Objects.equals(this.maxEmptySearches, that.maxEmptySearches) + && Objects.equals(this.indicesOptions, that.indicesOptions); } @Override public int hashCode() { return Objects.hash(id, jobId, frequency, queryDelay, indices, queryProvider, scrollSize, aggProvider, scriptFields, chunkingConfig, - delayedDataCheckConfig, maxEmptySearches); + delayedDataCheckConfig, maxEmptySearches, indicesOptions); } @Override @@ -411,7 +442,8 @@ boolean isNoop(DatafeedConfig datafeed) { && (delayedDataCheckConfig == null || Objects.equals(delayedDataCheckConfig, datafeed.getDelayedDataCheckConfig())) && (chunkingConfig == null || Objects.equals(chunkingConfig, datafeed.getChunkingConfig())) && (maxEmptySearches == null || Objects.equals(maxEmptySearches, datafeed.getMaxEmptySearches()) - || (maxEmptySearches == -1 && datafeed.getMaxEmptySearches() == null)); + || (maxEmptySearches == -1 && datafeed.getMaxEmptySearches() == null)) + && (indicesOptions == null || Objects.equals(indicesOptions, datafeed.getIndicesOptions())); } public static class Builder { @@ -428,6 +460,7 @@ public static class Builder { private ChunkingConfig chunkingConfig; private DelayedDataCheckConfig delayedDataCheckConfig; private Integer maxEmptySearches; + private IndicesOptions indicesOptions; public Builder() { } @@ -449,6 +482,7 @@ public Builder(DatafeedUpdate config) { this.chunkingConfig = config.chunkingConfig; this.delayedDataCheckConfig = config.delayedDataCheckConfig; this.maxEmptySearches = config.maxEmptySearches; + this.indicesOptions = config.indicesOptions; } public Builder setId(String datafeedId) { @@ -526,9 +560,14 @@ public Builder setMaxEmptySearches(int maxEmptySearches) { return this; } + public Builder setIndicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + public DatafeedUpdate build() { return new DatafeedUpdate(id, jobId, queryDelay, frequency, indices, queryProvider, aggProvider, scriptFields, scrollSize, - chunkingConfig, delayedDataCheckConfig, maxEmptySearches); + chunkingConfig, delayedDataCheckConfig, maxEmptySearches, indicesOptions); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java index e1be33e5ff8a2..9ae1aa2957adf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/results/ReservedFieldNames.java @@ -286,6 +286,7 @@ public final class ReservedFieldNames { DatafeedConfig.CHUNKING_CONFIG.getPreferredName(), DatafeedConfig.HEADERS.getPreferredName(), DatafeedConfig.DELAYED_DATA_CHECK_CONFIG.getPreferredName(), + DatafeedConfig.INDICES_OPTIONS.getPreferredName(), DelayedDataCheckConfig.ENABLED.getPreferredName(), DelayedDataCheckConfig.CHECK_WINDOW.getPreferredName(), diff --git a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json index 7c10d342bea15..1abd7e42e8ad1 100644 --- a/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json +++ b/x-pack/plugin/core/src/main/resources/org/elasticsearch/xpack/core/ml/config_index_mappings.json @@ -290,6 +290,10 @@ "indices" : { "type" : "keyword" }, + "indices_options": { + "type" : "object", + "enabled" : false + }, "job_id" : { "type" : "keyword" }, diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java index 2de9f9ec23165..7c9f7c46eab62 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java @@ -47,7 +47,7 @@ protected boolean supportsUnknownFields() { @Override protected Request doParseInstance(XContentParser parser) { - return Request.parseRequest(datafeedId, parser); + return Request.parseRequest(datafeedId, null, parser); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java index 97792d1bbad1d..c3570e16efc5c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/UpdateDatafeedActionRequestTests.java @@ -45,7 +45,7 @@ protected boolean supportsUnknownFields() { @Override protected Request doParseInstance(XContentParser parser) { - return Request.parseRequest(datafeedId, parser); + return Request.parseRequest(datafeedId, null, parser); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java index 672dc65c43c21..b5207bcfe71b4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; @@ -138,6 +139,13 @@ private static DatafeedConfig.Builder createRandomizedDatafeedConfigBuilder(Stri if (randomBoolean()) { builder.setMaxEmptySearches(randomIntBetween(10, 100)); } + if (randomBoolean()) { + builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean())); + } return builder; } @@ -865,7 +873,7 @@ private static DatafeedConfig createDatafeedWithDateHistogram(DateHistogramAggre @Override protected DatafeedConfig mutateInstance(DatafeedConfig instance) throws IOException { DatafeedConfig.Builder builder = new DatafeedConfig.Builder(instance); - switch (between(0, 10)) { + switch (between(0, 11)) { case 0: builder.setId(instance.getId() + randomValidDatafeedId()); break; @@ -937,6 +945,17 @@ protected DatafeedConfig mutateInstance(DatafeedConfig instance) throws IOExcept builder.setMaxEmptySearches(instance.getMaxEmptySearches() + 1); } break; + case 11: + builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean())); + break; default: throw new AssertionError("Illegal randomisation branch"); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java index b988e65716c06..2b77ecc6c164a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java @@ -7,6 +7,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; @@ -108,6 +109,13 @@ public static DatafeedUpdate createRandomized(String datafeedId, @Nullable Dataf if (randomBoolean()) { builder.setMaxEmptySearches(randomBoolean() ? -1 : randomIntBetween(10, 100)); } + if (randomBoolean()) { + builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean())); + } return builder.build(); } @@ -275,6 +283,16 @@ public void testApply_givenAggregations() throws IOException { assertThat(updatedDatafeed.getAggregations(), equalTo(aggProvider.getAggs())); } + public void testApply_givenIndicesOptions() { + DatafeedConfig datafeed = DatafeedConfigTests.createRandomizedDatafeedConfig("foo"); + DatafeedConfig updatedDatafeed = new DatafeedUpdate.Builder(datafeed.getId()) + .setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN) + .build() + .apply(datafeed, Collections.emptyMap()); + assertThat(datafeed.getIndicesOptions(), is(not(equalTo(updatedDatafeed.getIndicesOptions())))); + assertThat(updatedDatafeed.getIndicesOptions(), equalTo(IndicesOptions.LENIENT_EXPAND_OPEN_HIDDEN)); + } + public void testApply_GivenRandomUpdates_AssertImmutability() { for (int i = 0; i < 100; ++i) { DatafeedConfig datafeed = DatafeedConfigTests.createRandomizedDatafeedConfig(JobTests.randomValidJobId()); @@ -346,7 +364,7 @@ public void testSerializationOfComplexAggsBetweenVersions() throws IOException { @Override protected DatafeedUpdate mutateInstance(DatafeedUpdate instance) throws IOException { DatafeedUpdate.Builder builder = new DatafeedUpdate.Builder(instance); - switch (between(1, 10)) { + switch (between(1, 11)) { case 1: builder.setId(instance.getId() + DatafeedConfigTests.randomValidDatafeedId()); break; @@ -425,6 +443,17 @@ protected DatafeedUpdate mutateInstance(DatafeedUpdate instance) throws IOExcept builder.setMaxEmptySearches(instance.getMaxEmptySearches() + 100); } break; + case 11: + builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean())); + break; default: throw new AssertionError("Illegal randomisation branch"); } diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java index f5e229d572bcd..b78cdfa953645 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java @@ -389,6 +389,75 @@ public void testLookbackWithGeo() throws Exception { assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0")); } + public void testLookbackWithIndicesOptions() throws Exception { + String jobId = "test-lookback-only-with-indices-options"; + Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId); + createJobRequest.setJsonEntity("{\n" + + " \"description\": \"custom indices options\",\n" + + " \"analysis_config\": {\n" + + " \"bucket_span\": \"15m\",\n" + + " \"detectors\": [\n" + + " {\n" + + " \"function\": \"count\"\n" + + " }\n" + + " ]\n" + + " }," + + " \"data_description\": {\"time_field\": \"time\"}\n" + + "}"); + client().performRequest(createJobRequest); + String datafeedId = jobId + "-datafeed"; + new DatafeedBuilder(datafeedId, jobId, "*hidden-*") + .setIndicesOptions("{" + + "\"expand_wildcards\": [\"all\"]," + + "\"allow_no_indices\": true"+ + "}") + .build(); + + StringBuilder bulk = new StringBuilder(); + + Request createGeoData = new Request("PUT", "/.hidden-index"); + createGeoData.setJsonEntity("{" + + " \"mappings\": {" + + " \"properties\": {" + + " \"time\": { \"type\":\"date\"}," + + " \"value\": { \"type\":\"long\"}" + + " }" + + " }, \"settings\": {\"index.hidden\": true} " + + "}"); + client().performRequest(createGeoData); + + bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 1}}\n"); + bulk.append("{\"time\":\"2016-06-01T00:00:00Z\",\"value\": 1000}\n"); + bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 2}}\n"); + bulk.append("{\"time\":\"2016-06-01T00:05:00Z\",\"value\":1500}\n"); + bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 3}}\n"); + bulk.append("{\"time\":\"2016-06-01T00:10:00Z\",\"value\":1600}\n"); + bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 4}}\n"); + bulk.append("{\"time\":\"2016-06-01T00:15:00Z\",\"value\":100}\n"); + bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 5}}\n"); + bulk.append("{\"time\":\"2016-06-01T00:20:00Z\",\"value\":1}\n"); + bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 6}}\n"); + bulk.append("{\"time\":\"2016-06-01T00:25:00Z\",\"value\":1500}\n"); + bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 7}}\n"); + bulk.append("{\"time\":\"2016-06-01T00:30:00Z\",\"value\":1500}\n"); + bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 8}}\n"); + bulk.append("{\"time\":\"2016-06-01T00:40:00Z\",\"value\":2100}\n"); + bulk.append("{\"index\": {\"_index\": \".hidden-index\", \"_id\": 9}}\n"); + bulk.append("{\"time\":\"2016-06-01T00:41:00Z\",\"value\":0}\n"); + bulkIndex(bulk.toString()); + + openJob(client(), jobId); + + startDatafeedAndWaitUntilStopped(datafeedId); + waitUntilJobIsClosed(jobId); + Response jobStatsResponse = client().performRequest( + new Request("GET", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId + "/_stats")); + String jobStatsResponseAsString = EntityUtils.toString(jobStatsResponse.getEntity()); + assertThat(jobStatsResponseAsString, containsString("\"input_record_count\":9")); + assertThat(jobStatsResponseAsString, containsString("\"processed_record_count\":9")); + assertThat(jobStatsResponseAsString, containsString("\"missing_field_count\":0")); + } + public void testLookbackOnlyGivenEmptyIndex() throws Exception { new LookbackOnlyTestHelper("test-lookback-only-given-empty-index", "airline-data-empty") .setShouldSucceedInput(false).setShouldSucceedProcessing(false).execute(); @@ -1229,6 +1298,7 @@ private static class DatafeedBuilder { String aggregations; String authHeader = BASIC_AUTH_VALUE_SUPER_USER; String chunkingTimespan; + String indicesOptions; DatafeedBuilder(String datafeedId, String jobId, String index) { this.datafeedId = datafeedId; @@ -1261,6 +1331,11 @@ DatafeedBuilder setChunkingTimespan(String timespan) { return this; } + DatafeedBuilder setIndicesOptions(String indicesOptions) { + this.indicesOptions = indicesOptions; + return this; + } + Response build() throws IOException { Request request = new Request("PUT", MachineLearning.BASE_PATH + "datafeeds/" + datafeedId); request.setJsonEntity("{" @@ -1268,6 +1343,7 @@ Response build() throws IOException { + (source ? ",\"_source\":true" : "") + (scriptedFields == null ? "" : ",\"script_fields\":" + scriptedFields) + (aggregations == null ? "" : ",\"aggs\":" + aggregations) + + (indicesOptions == null ? "" : ",\"indices_options\":" + indicesOptions) + (chunkingTimespan == null ? "" : ",\"chunking_config\":{\"mode\":\"MANUAL\",\"time_span\":\"" + chunkingTimespan + "\"}") + "}"); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java index 35e408bdbdb1b..40c4cea806f6f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java @@ -243,6 +243,7 @@ public void onFailure(Exception e) { DatafeedConfig datafeedConfig = datafeedBuilder.build(); params.setDatafeedIndices(datafeedConfig.getIndices()); params.setJobId(datafeedConfig.getJobId()); + params.setIndicesOptions(datafeedConfig.getIndicesOptions()); datafeedConfigHolder.set(datafeedConfig); jobConfigProvider.getJob(datafeedConfig.getJobId(), jobListener); } catch (Exception e) { @@ -378,12 +379,17 @@ public StartDatafeedPersistentTasksExecutor(DatafeedManager datafeedManager, Ind public PersistentTasksCustomMetaData.Assignment getAssignment(StartDatafeedAction.DatafeedParams params, ClusterState clusterState) { return new DatafeedNodeSelector(clusterState, resolver, params.getDatafeedId(), params.getJobId(), - params.getDatafeedIndices()).selectNode(); + params.getDatafeedIndices(), params.getIndicesOptions()).selectNode(); } @Override public void validate(StartDatafeedAction.DatafeedParams params, ClusterState clusterState) { - new DatafeedNodeSelector(clusterState, resolver, params.getDatafeedId(), params.getJobId(), params.getDatafeedIndices()) + new DatafeedNodeSelector(clusterState, + resolver, + params.getDatafeedId(), + params.getJobId(), + params.getDatafeedIndices(), + params.getIndicesOptions()) .checkDatafeedTaskCanBeCreated(); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java index e29fc8ef5c326..9377bba89772b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java @@ -37,9 +37,10 @@ public class DatafeedNodeSelector { private final PersistentTasksCustomMetaData.PersistentTask jobTask; private final ClusterState clusterState; private final IndexNameExpressionResolver resolver; + private final IndicesOptions indicesOptions; public DatafeedNodeSelector(ClusterState clusterState, IndexNameExpressionResolver resolver, String datafeedId, - String jobId, List datafeedIndices) { + String jobId, List datafeedIndices, IndicesOptions indicesOptions) { PersistentTasksCustomMetaData tasks = clusterState.getMetaData().custom(PersistentTasksCustomMetaData.TYPE); this.datafeedId = datafeedId; this.jobId = jobId; @@ -47,6 +48,7 @@ public DatafeedNodeSelector(ClusterState clusterState, IndexNameExpressionResolv this.jobTask = MlTasks.getJobTask(jobId, tasks); this.clusterState = Objects.requireNonNull(clusterState); this.resolver = Objects.requireNonNull(resolver); + this.indicesOptions = indicesOptions == null ? IndicesOptions.lenientExpandOpen() : indicesOptions; } public void checkDatafeedTaskCanBeCreated() { @@ -121,7 +123,7 @@ private AssignmentFailure verifyIndicesActive() { + index + "] does not exist, is closed, or is still initializing."; try { - concreteIndices = resolver.concreteIndexNames(clusterState, IndicesOptions.lenientExpandOpen(), index); + concreteIndices = resolver.concreteIndexNames(clusterState, indicesOptions, index); if (concreteIndices.length == 0) { return new AssignmentFailure(reason, true); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DatafeedDelayedDataDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DatafeedDelayedDataDetector.java index d2a933ffa466f..96b183d216589 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DatafeedDelayedDataDetector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DatafeedDelayedDataDetector.java @@ -8,7 +8,9 @@ import org.elasticsearch.action.search.SearchAction; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -46,15 +48,17 @@ public class DatafeedDelayedDataDetector implements DelayedDataDetector { private final String jobId; private final QueryBuilder datafeedQuery; private final String[] datafeedIndices; + private final IndicesOptions indicesOptions; DatafeedDelayedDataDetector(long bucketSpan, long window, String jobId, String timeField, QueryBuilder datafeedQuery, - String[] datafeedIndices, Client client) { + String[] datafeedIndices, @Nullable IndicesOptions indicesOptions, Client client) { this.bucketSpan = bucketSpan; this.window = window; this.jobId = jobId; this.timeField = timeField; this.datafeedQuery = datafeedQuery; this.datafeedIndices = datafeedIndices; + this.indicesOptions = indicesOptions; this.client = client; } @@ -116,6 +120,9 @@ private Map checkCurrentBucketEventCount(long start, long end) { .query(ExtractorUtils.wrapInTimeRangeQuery(datafeedQuery, timeField, start, end)); SearchRequest searchRequest = new SearchRequest(datafeedIndices).source(searchSourceBuilder); + if (indicesOptions != null) { + searchRequest.indicesOptions(indicesOptions); + } try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) { SearchResponse response = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); List buckets = ((Histogram)response.getAggregations().get(DATE_BUCKETS)).getBuckets(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DelayedDataDetectorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DelayedDataDetectorFactory.java index 88f8e6caadf09..23a5de6baa0c5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DelayedDataDetectorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DelayedDataDetectorFactory.java @@ -51,6 +51,7 @@ public static DelayedDataDetector buildDetector(Job job, job.getDataDescription().getTimeField(), datafeedConfig.getParsedQuery(xContentRegistry), datafeedConfig.getIndices().toArray(new String[0]), + datafeedConfig.getIndicesOptions(), client); } else { return new NullDelayedDataDetector(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java index d43ede48d0578..91d059ff6336c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java @@ -79,7 +79,9 @@ static void create(Client client, client, ClientHelper.ML_ORIGIN, GetRollupIndexCapsAction.INSTANCE, - new GetRollupIndexCapsAction.Request(datafeed.getIndices().toArray(new String[0])), + datafeed.getIndicesOptions() == null ? + new GetRollupIndexCapsAction.Request(datafeed.getIndices().toArray(new String[0])) : + new GetRollupIndexCapsAction.Request(datafeed.getIndices().toArray(new String[0]), datafeed.getIndicesOptions()), getRollupIndexCapsActionHandler); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java index 031db35ebac15..aea9c9d7a7553 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractor.java @@ -28,6 +28,7 @@ class AggregationDataExtractor extends AbstractAggregationDataExtractor headers; + final IndicesOptions indicesOptions; AggregationDataExtractorContext(String jobId, String timeField, Set fields, List indices, QueryBuilder query, AggregatorFactories.Builder aggs, long start, long end, boolean includeDocCount, - Map headers) { + Map headers, @Nullable IndicesOptions indicesOptions) { this.jobId = Objects.requireNonNull(jobId); this.timeField = Objects.requireNonNull(timeField); this.fields = Objects.requireNonNull(fields); @@ -39,5 +43,6 @@ class AggregationDataExtractorContext { this.end = end; this.includeDocCount = includeDocCount; this.headers = headers; + this.indicesOptions = indicesOptions == null ? SearchRequest.DEFAULT_INDICES_OPTIONS : indicesOptions; } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactory.java index 40d186542df04..93c5a12d82d55 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorFactory.java @@ -50,7 +50,8 @@ public DataExtractor newExtractor(long start, long end) { Intervals.alignToCeil(start, histogramInterval), Intervals.alignToFloor(end, histogramInterval), job.getAnalysisConfig().getSummaryCountFieldName().equals(DatafeedConfig.DOC_COUNT), - datafeedConfig.getHeaders()); + datafeedConfig.getHeaders(), + datafeedConfig.getIndicesOptions()); return new AggregationDataExtractor(client, dataExtractorContext, timingStatsReporter); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractor.java index 9341161057717..75b215e34c54f 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/RollupDataExtractor.java @@ -26,7 +26,9 @@ class RollupDataExtractor extends AbstractAggregationDataExtractor headers; final boolean hasAggregations; final Long histogramInterval; + final IndicesOptions indicesOptions; ChunkedDataExtractorContext(String jobId, String timeField, List indices, QueryBuilder query, int scrollSize, long start, long end, @Nullable TimeValue chunkSpan, TimeAligner timeAligner, Map headers, - boolean hasAggregations, @Nullable Long histogramInterval) { + boolean hasAggregations, @Nullable Long histogramInterval, @Nullable IndicesOptions indicesOptions) { this.jobId = Objects.requireNonNull(jobId); this.timeField = Objects.requireNonNull(timeField); this.indices = indices.toArray(new String[indices.size()]); @@ -48,5 +51,6 @@ interface TimeAligner { this.headers = headers; this.hasAggregations = hasAggregations; this.histogramInterval = histogramInterval; + this.indicesOptions = indicesOptions == null ? SearchRequest.DEFAULT_INDICES_OPTIONS : indicesOptions; } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java index 028409f69398b..e153bafa096e1 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorFactory.java @@ -54,7 +54,8 @@ public DataExtractor newExtractor(long start, long end) { timeAligner, datafeedConfig.getHeaders(), datafeedConfig.hasAggregations(), - datafeedConfig.hasAggregations() ? datafeedConfig.getHistogramIntervalMillis(xContentRegistry) : null + datafeedConfig.hasAggregations() ? datafeedConfig.getHistogramIntervalMillis(xContentRegistry) : null, + datafeedConfig.getIndicesOptions() ); return new ChunkedDataExtractor(client, dataExtractorFactory, dataExtractorContext, timingStatsReporter); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java index e0bf14b1bb1b3..8745ee49a1166 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractor.java @@ -125,6 +125,7 @@ private SearchRequestBuilder buildSearchRequest(long start) { .setScroll(SCROLL_TIMEOUT) .addSort(context.extractedFields.timeField(), SortOrder.ASC) .setIndices(context.indices) + .setIndicesOptions(context.indicesOptions) .setSize(context.scrollSize) .setQuery(ExtractorUtils.wrapInTimeRangeQuery( context.query, context.extractedFields.timeField(), start, context.end)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java index 19dbfa669b111..e39f45fc2fe64 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java @@ -5,6 +5,9 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.Nullable; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -23,10 +26,11 @@ class ScrollDataExtractorContext { final long start; final long end; final Map headers; + final IndicesOptions indicesOptions; ScrollDataExtractorContext(String jobId, TimeBasedExtractedFields extractedFields, List indices, QueryBuilder query, List scriptFields, int scrollSize, long start, long end, - Map headers) { + Map headers, @Nullable IndicesOptions indicesOptions) { this.jobId = Objects.requireNonNull(jobId); this.extractedFields = Objects.requireNonNull(extractedFields); this.indices = indices.toArray(new String[indices.size()]); @@ -36,5 +40,6 @@ class ScrollDataExtractorContext { this.start = start; this.end = end; this.headers = headers; + this.indicesOptions = indicesOptions == null ? SearchRequest.DEFAULT_INDICES_OPTIONS : indicesOptions; } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java index 0c626ea7e22f7..2988dfedfa968 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java @@ -25,7 +25,6 @@ import java.util.Objects; public class ScrollDataExtractorFactory implements DataExtractorFactory { - private final Client client; private final DatafeedConfig datafeedConfig; private final Job job; @@ -54,7 +53,9 @@ public DataExtractor newExtractor(long start, long end) { datafeedConfig.getScrollSize(), start, end, - datafeedConfig.getHeaders()); + datafeedConfig.getHeaders(), + datafeedConfig.getIndicesOptions() + ); return new ScrollDataExtractor(client, dataExtractorContext, timingStatsReporter); } @@ -87,11 +88,16 @@ public static void create(Client client, // Step 1. Get field capabilities necessary to build the information of how to extract fields FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest(); - fieldCapabilitiesRequest.indices(datafeed.getIndices().toArray(new String[datafeed.getIndices().size()])); + fieldCapabilitiesRequest.indices(datafeed.getIndices().toArray(new String[0])); + if (datafeed.getIndicesOptions() != null) { + fieldCapabilitiesRequest.indicesOptions(datafeed.getIndicesOptions()); + } // We need capabilities for all fields matching the requested fields' parents so that we can work around // multi-fields that are not in source. - String[] requestFields = job.allInputFields().stream().map(f -> MlStrings.getParentField(f) + "*") - .toArray(size -> new String[size]); + String[] requestFields = job.allInputFields() + .stream() + .map(f -> MlStrings.getParentField(f) + "*") + .toArray(String[]::new); fieldCapabilitiesRequest.fields(requestFields); ClientHelper. executeWithHeaders(datafeed.getHeaders(), ClientHelper.ML_ORIGIN, client, () -> { client.execute(FieldCapabilitiesAction.INSTANCE, fieldCapabilitiesRequest, fieldCapabilitiesHandler); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java index 8b71e137bea8f..ba6a30e0d35e2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.ml.rest.datafeeds; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; @@ -44,8 +46,15 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName()); + IndicesOptions indicesOptions = null; + if (restRequest.hasParam("expand_wildcards") + || restRequest.hasParam("ignore_unavailable") + || restRequest.hasParam("allow_no_indices") + || restRequest.hasParam("ignore_throttled")) { + indicesOptions = IndicesOptions.fromRequest(restRequest, SearchRequest.DEFAULT_INDICES_OPTIONS); + } XContentParser parser = restRequest.contentParser(); - PutDatafeedAction.Request putDatafeedRequest = PutDatafeedAction.Request.parseRequest(datafeedId, parser); + PutDatafeedAction.Request putDatafeedRequest = PutDatafeedAction.Request.parseRequest(datafeedId, indicesOptions, parser); putDatafeedRequest.timeout(restRequest.paramAsTime("timeout", putDatafeedRequest.timeout())); putDatafeedRequest.masterNodeTimeout(restRequest.paramAsTime("master_timeout", putDatafeedRequest.masterNodeTimeout())); return channel -> client.execute(PutDatafeedAction.INSTANCE, putDatafeedRequest, new RestToXContentListener<>(channel)); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestUpdateDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestUpdateDatafeedAction.java index 0375be2907b35..cf8046921336a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestUpdateDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestUpdateDatafeedAction.java @@ -5,6 +5,8 @@ */ package org.elasticsearch.xpack.ml.rest.datafeeds; +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.rest.BaseRestHandler; @@ -44,8 +46,15 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName()); + IndicesOptions indicesOptions = null; + if (restRequest.hasParam("expand_wildcards") + || restRequest.hasParam("ignore_unavailable") + || restRequest.hasParam("allow_no_indices") + || restRequest.hasParam("ignore_throttled")) { + indicesOptions = IndicesOptions.fromRequest(restRequest, SearchRequest.DEFAULT_INDICES_OPTIONS); + } XContentParser parser = restRequest.contentParser(); - UpdateDatafeedAction.Request updateDatafeedRequest = UpdateDatafeedAction.Request.parseRequest(datafeedId, parser); + UpdateDatafeedAction.Request updateDatafeedRequest = UpdateDatafeedAction.Request.parseRequest(datafeedId, indicesOptions, parser); updateDatafeedRequest.timeout(restRequest.paramAsTime("timeout", updateDatafeedRequest.timeout())); updateDatafeedRequest.masterNodeTimeout(restRequest.paramAsTime("master_timeout", updateDatafeedRequest.masterNodeTimeout())); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java index 39f3a3f4889c1..fad7b539f41e2 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java @@ -76,9 +76,9 @@ public void testSelectNode_GivenJobIsOpened() { givenClusterState("foo", 1, 0); PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); assertEquals("node_id", result.getExecutorNode()); - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).checkDatafeedTaskCanBeCreated(); } public void testSelectNode_GivenJobIsOpening() { @@ -92,9 +92,9 @@ public void testSelectNode_GivenJobIsOpening() { givenClusterState("foo", 1, 0); PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); assertEquals("node_id", result.getExecutorNode()); - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).checkDatafeedTaskCanBeCreated(); } public void testNoJobTask() { @@ -108,13 +108,13 @@ public void testNoJobTask() { givenClusterState("foo", 1, 0); PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); assertNull(result.getExecutorNode()); assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id], because the job's [job_id] state is " + "[closed] while state [opened] is required")); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()) + () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) .checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " + "[cannot start datafeed [datafeed_id], because the job's [job_id] state is [closed] while state [opened] is required]")); @@ -132,13 +132,13 @@ public void testSelectNode_GivenJobFailedOrClosed() { givenClusterState("foo", 1, 0); PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); assertNull(result.getExecutorNode()); assertEquals("cannot start datafeed [datafeed_id], because the job's [job_id] state is [" + jobState + "] while state [opened] is required", result.getExplanation()); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()) + () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) .checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " + "[cannot start datafeed [datafeed_id], because the job's [job_id] state is [" + jobState @@ -161,12 +161,12 @@ public void testShardUnassigned() { givenClusterState("foo", 1, 0, states); PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); assertNull(result.getExecutorNode()); assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id] because index [foo] " + "does not have all primary shards active yet.")); - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).checkDatafeedTaskCanBeCreated(); } public void testShardNotAllActive() { @@ -186,12 +186,12 @@ public void testShardNotAllActive() { givenClusterState("foo", 2, 0, states); PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); assertNull(result.getExecutorNode()); assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id] because index [foo] " + "does not have all primary shards active yet.")); - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).checkDatafeedTaskCanBeCreated(); } public void testIndexDoesntExist() { @@ -205,13 +205,13 @@ public void testIndexDoesntExist() { givenClusterState("foo", 1, 0); PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); assertNull(result.getExecutorNode()); assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id] because index [not_foo] " + "does not exist, is closed, or is still initializing.")); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()) + () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) .checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " + "[cannot start datafeed [datafeed_id] because index [not_foo] does not exist, is closed, or is still initializing.]")); @@ -228,7 +228,7 @@ public void testRemoteIndex() { givenClusterState("foo", 1, 0); PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); assertNotNull(result.getExecutorNode()); } @@ -246,13 +246,13 @@ public void testSelectNode_jobTaskStale() { givenClusterState("foo", 1, 0); PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); assertNull(result.getExecutorNode()); assertEquals("cannot start datafeed [datafeed_id], because the job's [job_id] state is stale", result.getExplanation()); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()) + () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) .checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " + "[cannot start datafeed [datafeed_id], because the job's [job_id] state is stale]")); @@ -261,9 +261,9 @@ public void testSelectNode_jobTaskStale() { addJobTask(job.getId(), "node_id1", JobState.OPENED, tasksBuilder); tasks = tasksBuilder.build(); givenClusterState("foo", 1, 0); - result = new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); + result = new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); assertEquals("node_id1", result.getExecutorNode()); - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).checkDatafeedTaskCanBeCreated(); } public void testSelectNode_GivenJobOpeningAndIndexDoesNotExist() { @@ -280,7 +280,7 @@ public void testSelectNode_GivenJobOpeningAndIndexDoesNotExist() { givenClusterState("foo", 1, 0); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()) + () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) .checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " + "[cannot start datafeed [datafeed_id] because index [not_foo] does not exist, is closed, or is still initializing.]")); @@ -298,7 +298,7 @@ public void testSelectNode_GivenMlUpgradeMode() { givenClusterState("foo", 1, 0); PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()).selectNode(); + new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); assertThat(result, equalTo(MlTasks.AWAITING_UPGRADE)); } @@ -314,7 +314,7 @@ public void testCheckDatafeedTaskCanBeCreated_GivenMlUpgradeMode() { givenClusterState("foo", 1, 0); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices()) + () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) .checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), equalTo("Could not start datafeed [datafeed_id] as indices are being upgraded")); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java index 537a6b44d0bc6..3708f2052d22c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java @@ -272,7 +272,7 @@ public void testExtractionGivenInitSearchResponseEncounteredUnavailableShards() private AggregationDataExtractorContext createContext(long start, long end) { return new AggregationDataExtractorContext(jobId, timeField, fields, indices, query, aggs, start, end, true, - Collections.emptyMap()); + Collections.emptyMap(), null); } @SuppressWarnings("unchecked") diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java index e7ac362543945..68ef6f8f41476 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java @@ -565,7 +565,7 @@ private ChunkedDataExtractorContext createContext(long start, long end) { private ChunkedDataExtractorContext createContext(long start, long end, boolean hasAggregations, Long histogramInterval) { return new ChunkedDataExtractorContext(jobId, timeField, indices, query, scrollSize, start, end, chunkSpan, - ChunkedDataExtractorFactory.newIdentityTimeAligner(), Collections.emptyMap(), hasAggregations, histogramInterval); + ChunkedDataExtractorFactory.newIdentityTimeAligner(), Collections.emptyMap(), hasAggregations, histogramInterval, null); } private static class StubSubExtractor implements DataExtractor { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java index 962bd9ee6d3df..e198b46621aae 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java @@ -444,7 +444,7 @@ public void testDomainSplitScriptField() throws IOException { List sFields = Arrays.asList(withoutSplit, withSplit); ScrollDataExtractorContext context = new ScrollDataExtractorContext(jobId, extractedFields, indices, - query, sFields, scrollSize, 1000, 2000, Collections.emptyMap()); + query, sFields, scrollSize, 1000, 2000, Collections.emptyMap(), null); TestDataExtractor extractor = new TestDataExtractor(context); @@ -490,7 +490,7 @@ public void testDomainSplitScriptField() throws IOException { private ScrollDataExtractorContext createContext(long start, long end) { return new ScrollDataExtractorContext(jobId, extractedFields, indices, query, scriptFields, scrollSize, start, end, - Collections.emptyMap()); + Collections.emptyMap(), null); } private SearchResponse createEmptySearchResponse() { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_datafeed.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_datafeed.json index 3991c682ac2eb..30985f178005e 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_datafeed.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.put_datafeed.json @@ -23,6 +23,31 @@ "body":{ "description":"The datafeed config", "required":true + }, + "params": { + "ignore_unavailable":{ + "type":"boolean", + "description":"Ignore unavailable indexes (default: false)" + }, + "allow_no_indices":{ + "type":"boolean", + "description":"Ignore if the source indices expressions resolves to no concrete indices (default: true)" + }, + "ignore_throttled":{ + "type":"boolean", + "description":"Ignore indices that are marked as throttled (default: true)" + }, + "expand_wildcards":{ + "type":"enum", + "options":[ + "open", + "closed", + "hidden", + "none", + "all" + ], + "description":"Whether source index expressions should get expanded to open or closed indices (default: open)" + } } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_datafeed.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_datafeed.json index d4c4a1f63d78d..f820933c76955 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_datafeed.json +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/ml.update_datafeed.json @@ -23,6 +23,31 @@ "body":{ "description":"The datafeed update settings", "required":true + }, + "params": { + "ignore_unavailable":{ + "type":"boolean", + "description":"Ignore unavailable indexes (default: false)" + }, + "allow_no_indices":{ + "type":"boolean", + "description":"Ignore if the source indices expressions resolves to no concrete indices (default: true)" + }, + "ignore_throttled":{ + "type":"boolean", + "description":"Ignore indices that are marked as throttled (default: true)" + }, + "expand_wildcards":{ + "type":"enum", + "options":[ + "open", + "closed", + "hidden", + "none", + "all" + ], + "description":"Whether source index expressions should get expanded to open or closed indices (default: open)" + } } } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml index fe1e8f5ad2f8d..4415a8eefa427 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/datafeeds_crud.yml @@ -408,3 +408,78 @@ setup: datafeed_id: test-datafeed-1 force: true - match: { acknowledged: true } +--- +"Test put and update datafeed with indices options": + - do: + ml.put_datafeed: + datafeed_id: test-datafeed-indices-options-1 + body: > + { + "job_id":"datafeeds-crud-1", + "indexes":["index-foo"], + "indices_options": { + "expand_wildcards": ["closed", "open"], + "ignore_throttled": false + } + } + - match: { datafeed_id: "test-datafeed-indices-options-1" } + + - do: + ml.get_datafeeds: + datafeed_id: test-datafeed-indices-options-1 + + - match: { datafeeds.0.indices_options.ignore_throttled: false } + - length: { datafeeds.0.indices_options.expand_wildcards: 2 } + - match: { datafeeds.0.indices_options.expand_wildcards.0: open } + - match: { datafeeds.0.indices_options.expand_wildcards.1: closed } + + - do: + ml.update_datafeed: + datafeed_id: test-datafeed-indices-options-1 + body: > + { + "indices_options": { + "ignore_throttled": true + } + } + - match: { datafeed_id: "test-datafeed-indices-options-1" } + - do: + ml.get_datafeeds: + datafeed_id: test-datafeed-indices-options-1 + + - match: { datafeeds.0.indices_options.ignore_throttled: true } +--- +"Test put and update datafeed with indices options in params": + - do: + ml.put_datafeed: + datafeed_id: test-datafeed-indices-options-params-1 + ignore_throttled: false + body: > + { + "job_id":"datafeeds-crud-1", + "indexes":["index-foo"] + } + - match: { datafeed_id: "test-datafeed-indices-options-params-1" } + + - do: + ml.get_datafeeds: + datafeed_id: test-datafeed-indices-options-params-1 + + - match: { datafeeds.0.indices_options.ignore_throttled: false } + + + - do: + ml.update_datafeed: + datafeed_id: test-datafeed-indices-options-params-1 + ignore_throttled: false + allow_no_indices: false + body: > + { + } + - match: { datafeed_id: "test-datafeed-indices-options-params-1" } + - do: + ml.get_datafeeds: + datafeed_id: test-datafeed-indices-options-params-1 + + - match: { datafeeds.0.indices_options.ignore_throttled: false } + - match: { datafeeds.0.indices_options.allow_no_indices: false } From 2070eb87213af8185dc2300e84eb1338d6b81085 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 26 Feb 2020 10:34:27 -0500 Subject: [PATCH 2/5] addressing PR comments --- .../client/ml/datafeed/DatafeedConfig.java | 6 - .../core/ml/action/PutDatafeedAction.java | 5 +- .../core/ml/action/StartDatafeedAction.java | 28 ++- .../core/ml/datafeed/DatafeedConfig.java | 30 +-- .../action/PutDatafeedActionRequestTests.java | 8 +- .../core/ml/datafeed/DatafeedConfigTests.java | 32 ++- .../core/ml/datafeed/DatafeedUpdateTests.java | 4 - .../ml/datafeed/DatafeedNodeSelector.java | 22 +- .../DatafeedDelayedDataDetector.java | 11 +- .../extractor/DataExtractorFactory.java | 4 +- .../AggregationDataExtractorContext.java | 6 +- .../chunked/ChunkedDataExtractorContext.java | 5 +- .../scroll/ScrollDataExtractorContext.java | 6 +- .../scroll/ScrollDataExtractorFactory.java | 5 +- .../rest/datafeeds/RestPutDatafeedAction.java | 8 +- .../datafeed/DatafeedNodeSelectorTests.java | 189 ++++++++++++++---- .../AggregationDataExtractorTests.java | 3 +- .../chunked/ChunkedDataExtractorTests.java | 3 +- .../scroll/ScrollDataExtractorTests.java | 5 +- 19 files changed, 228 insertions(+), 152 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedConfig.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedConfig.java index c1b5b589b2f15..0b53c2e185268 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedConfig.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ml/datafeed/DatafeedConfig.java @@ -21,7 +21,6 @@ import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.ml.job.config.Job; import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; @@ -242,11 +241,6 @@ private static Map asMap(BytesReference bytesReference) { return bytesReference == null ? null : XContentHelper.convertToMap(bytesReference, true, XContentType.JSON).v2(); } - @Override - public String toString() { - return Strings.toString(this, true, true); - } - /** * The lists of indices and types are compared for equality but they are not * sorted first so this test could fail simply because the indices and types diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java index 5b2ada2a48837..afc383619e3bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedAction.java @@ -12,7 +12,6 @@ import org.elasticsearch.action.support.master.AcknowledgedRequest; import org.elasticsearch.action.support.master.MasterNodeOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ToXContentObject; @@ -34,9 +33,9 @@ private PutDatafeedAction() { public static class Request extends AcknowledgedRequest implements ToXContentObject { - public static Request parseRequest(String datafeedId, @Nullable IndicesOptions indicesOptions, XContentParser parser) { + public static Request parseRequest(String datafeedId, IndicesOptions indicesOptions, XContentParser parser) { DatafeedConfig.Builder datafeed = DatafeedConfig.STRICT_PARSER.apply(parser, null); - if (indicesOptions != null) { + if (datafeed.getIndicesOptions() == null) { datafeed.setIndicesOptions(indicesOptions); } datafeed.setId(datafeedId); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java index 904e78eb12ec4..cebfa97a8d05a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/StartDatafeedAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ActionType; import org.elasticsearch.action.ValidateActions; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.support.master.MasterNodeRequest; @@ -152,7 +153,7 @@ public static class DatafeedParams implements PersistentTaskParams { PARSER.declareString(DatafeedParams::setJobId, Job.ID); PARSER.declareStringArray(DatafeedParams::setDatafeedIndices, INDICES); PARSER.declareObject(DatafeedParams::setIndicesOptions, - (p, c) -> IndicesOptions.fromMap(p.map(), IndicesOptions.lenientExpandOpen()), + (p, c) -> IndicesOptions.fromMap(p.map(), SearchRequest.DEFAULT_INDICES_OPTIONS), DatafeedConfig.INDICES_OPTIONS); } @@ -196,9 +197,9 @@ public DatafeedParams(StreamInput in) throws IOException { jobId = in.readOptionalString(); datafeedIndices = in.readStringList(); if (in.getVersion().onOrAfter(Version.V_8_0_0)) { - indicesOptions = in.readBoolean() ? IndicesOptions.readIndicesOptions(in) : null; + indicesOptions = IndicesOptions.readIndicesOptions(in); } else { - indicesOptions = null; + indicesOptions = SearchRequest.DEFAULT_INDICES_OPTIONS; } } @@ -211,7 +212,7 @@ public DatafeedParams(StreamInput in) throws IOException { private TimeValue timeout = TimeValue.timeValueSeconds(20); private List datafeedIndices = Collections.emptyList(); private String jobId; - private IndicesOptions indicesOptions; + private IndicesOptions indicesOptions = SearchRequest.DEFAULT_INDICES_OPTIONS; public String getDatafeedId() { @@ -263,7 +264,7 @@ public IndicesOptions getIndicesOptions() { } public DatafeedParams setIndicesOptions(IndicesOptions indicesOptions) { - this.indicesOptions = indicesOptions; + this.indicesOptions = ExceptionsHelper.requireNonNull(indicesOptions, DatafeedConfig.INDICES_OPTIONS); return this; } @@ -286,12 +287,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalString(jobId); out.writeStringCollection(datafeedIndices); if (out.getVersion().onOrAfter(Version.V_8_0_0)) { - if (indicesOptions != null) { - out.writeBoolean(true); - indicesOptions.writeIndicesOptions(out); - } else { - out.writeBoolean(false); - } + indicesOptions.writeIndicesOptions(out); } } @@ -310,11 +306,11 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par if (datafeedIndices.isEmpty() == false) { builder.field(INDICES.getPreferredName(), datafeedIndices); } - if (indicesOptions != null) { - builder.startObject(DatafeedConfig.INDICES_OPTIONS.getPreferredName()); - indicesOptions.toXContent(builder, params); - builder.endObject(); - } + + builder.startObject(DatafeedConfig.INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + builder.endObject(); return builder; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 44a9034c3b18a..1fb94356ecb05 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.AbstractDiffable; import org.elasticsearch.common.ParseField; @@ -203,7 +204,7 @@ private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue this.headers = Collections.unmodifiableMap(headers); this.delayedDataCheckConfig = delayedDataCheckConfig; this.maxEmptySearches = maxEmptySearches; - this.indicesOptions = indicesOptions; + this.indicesOptions = ExceptionsHelper.requireNonNull(indicesOptions, INDICES_OPTIONS); } public DatafeedConfig(StreamInput in) throws IOException { @@ -236,9 +237,9 @@ public DatafeedConfig(StreamInput in) throws IOException { maxEmptySearches = null; } if (in.getVersion().onOrAfter(Version.V_8_0_0)) { - indicesOptions = in.readBoolean() ? IndicesOptions.readIndicesOptions(in) : null; + indicesOptions = IndicesOptions.readIndicesOptions(in); } else { - indicesOptions = null; + indicesOptions = SearchRequest.DEFAULT_INDICES_OPTIONS; } } @@ -448,12 +449,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeOptionalVInt(maxEmptySearches); } if (out.getVersion().onOrAfter(Version.V_8_0_0)) { - if (indicesOptions != null) { - out.writeBoolean(true); - indicesOptions.writeIndicesOptions(out); - } else { - out.writeBoolean(false); - } + indicesOptions.writeIndicesOptions(out); } } @@ -494,11 +490,10 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (maxEmptySearches != null) { builder.field(MAX_EMPTY_SEARCHES.getPreferredName(), maxEmptySearches); } - if (indicesOptions != null) { - builder.startObject(INDICES_OPTIONS.getPreferredName()); - indicesOptions.toXContent(builder, params); - builder.endObject(); - } + builder.startObject(INDICES_OPTIONS.getPreferredName()); + indicesOptions.toXContent(builder, params); + builder.endObject(); + builder.endObject(); return builder; } @@ -748,6 +743,10 @@ public Builder setIndicesOptions(IndicesOptions indicesOptions) { return this; } + public IndicesOptions getIndicesOptions() { + return this.indicesOptions; + } + public DatafeedConfig build() { ExceptionsHelper.requireNonNull(id, ID.getPreferredName()); ExceptionsHelper.requireNonNull(jobId, Job.ID.getPreferredName()); @@ -762,6 +761,9 @@ public DatafeedConfig build() { setDefaultChunkingConfig(); setDefaultQueryDelay(); + if (indicesOptions == null) { + indicesOptions = SearchRequest.DEFAULT_INDICES_OPTIONS; + } return new DatafeedConfig(id, jobId, queryDelay, frequency, indices, queryProvider, aggProvider, scriptFields, scrollSize, chunkingConfig, headers, delayedDataCheckConfig, maxEmptySearches, indicesOptions); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java index 7c9f7c46eab62..31b7f3394ef06 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/PutDatafeedActionRequestTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.ml.action; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; @@ -13,7 +14,6 @@ import org.elasticsearch.search.SearchModule; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ml.action.PutDatafeedAction.Request; -import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfigTests; import org.junit.Before; @@ -30,9 +30,7 @@ public void setUpDatafeedId() { @Override protected Request createTestInstance() { - DatafeedConfig.Builder datafeedConfig = new DatafeedConfig.Builder(datafeedId, randomAlphaOfLength(10)); - datafeedConfig.setIndices(Collections.singletonList(randomAlphaOfLength(10))); - return new Request(datafeedConfig.build()); + return new Request(DatafeedConfigTests.createRandomizedDatafeedConfig(randomAlphaOfLength(10), datafeedId, 3600)); } @Override @@ -47,7 +45,7 @@ protected boolean supportsUnknownFields() { @Override protected Request doParseInstance(XContentParser parser) { - return Request.parseRequest(datafeedId, null, parser); + return Request.parseRequest(datafeedId, SearchRequest.DEFAULT_INDICES_OPTIONS, parser); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java index b5207bcfe71b4..0cf14ba937e0b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java @@ -84,11 +84,15 @@ public static DatafeedConfig createRandomizedDatafeedConfig(String jobId) { } public static DatafeedConfig createRandomizedDatafeedConfig(String jobId, long bucketSpanMillis) { - return createRandomizedDatafeedConfigBuilder(jobId, bucketSpanMillis).build(); + return createRandomizedDatafeedConfig(jobId, randomValidDatafeedId(), bucketSpanMillis); } - private static DatafeedConfig.Builder createRandomizedDatafeedConfigBuilder(String jobId, long bucketSpanMillis) { - DatafeedConfig.Builder builder = new DatafeedConfig.Builder(randomValidDatafeedId(), jobId); + public static DatafeedConfig createRandomizedDatafeedConfig(String jobId, String datafeedId, long bucketSpanMillis) { + return createRandomizedDatafeedConfigBuilder(jobId, datafeedId, bucketSpanMillis).build(); + } + + private static DatafeedConfig.Builder createRandomizedDatafeedConfigBuilder(String jobId, String datafeedId, long bucketSpanMillis) { + DatafeedConfig.Builder builder = new DatafeedConfig.Builder(datafeedId, jobId); builder.setIndices(randomStringList(1, 10)); if (randomBoolean()) { builder.setQueryProvider(createRandomValidQueryProvider(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10))); @@ -139,13 +143,11 @@ private static DatafeedConfig.Builder createRandomizedDatafeedConfigBuilder(Stri if (randomBoolean()) { builder.setMaxEmptySearches(randomIntBetween(10, 100)); } - if (randomBoolean()) { - builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean())); - } + builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean(), + randomBoolean())); return builder; } @@ -372,7 +374,7 @@ public void testMultipleDefinedAggParse() throws IOException { } public void testToXContentForInternalStorage() throws IOException { - DatafeedConfig.Builder builder = createRandomizedDatafeedConfigBuilder("foo", 300); + DatafeedConfig.Builder builder = createRandomizedDatafeedConfigBuilder("foo", randomValidDatafeedId(), 300); // headers are only persisted to cluster state Map headers = new HashMap<>(); @@ -757,7 +759,7 @@ public void testSerializationOfComplexAggs() throws IOException { xContentType); DatafeedConfig parsedDatafeedConfig = doParseInstance(parser); - assertEquals(datafeedConfig, parsedDatafeedConfig); + assertEquals(datafeedConfig.getAggregations(), parsedDatafeedConfig.getAggregations()); // Assert that the parsed versions of our aggs and queries work as well assertEquals(aggBuilder, parsedDatafeedConfig.getParsedAggregations(xContentRegistry())); @@ -769,7 +771,7 @@ public void testSerializationOfComplexAggs() throws IOException { datafeedConfig.writeTo(output); try(StreamInput streamInput = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { DatafeedConfig streamedDatafeedConfig = new DatafeedConfig(streamInput); - assertEquals(datafeedConfig, streamedDatafeedConfig); + assertEquals(datafeedConfig.getAggregations(), streamedDatafeedConfig.getAggregations()); // Assert that the parsed versions of our aggs and queries work as well assertEquals(aggBuilder, streamedDatafeedConfig.getParsedAggregations(xContentRegistry())); @@ -947,10 +949,6 @@ protected DatafeedConfig mutateInstance(DatafeedConfig instance) throws IOExcept break; case 11: builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java index 2b77ecc6c164a..88e644721242c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java @@ -445,10 +445,6 @@ protected DatafeedUpdate mutateInstance(DatafeedUpdate instance) throws IOExcept break; case 11: builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java index 9377bba89772b..03c8272a979d8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java @@ -7,6 +7,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.cluster.ClusterState; @@ -48,7 +49,7 @@ public DatafeedNodeSelector(ClusterState clusterState, IndexNameExpressionResolv this.jobTask = MlTasks.getJobTask(jobId, tasks); this.clusterState = Objects.requireNonNull(clusterState); this.resolver = Objects.requireNonNull(resolver); - this.indicesOptions = indicesOptions == null ? IndicesOptions.lenientExpandOpen() : indicesOptions; + this.indicesOptions = Objects.requireNonNull(indicesOptions); } public void checkDatafeedTaskCanBeCreated() { @@ -119,25 +120,28 @@ private AssignmentFailure verifyIndicesActive() { } String[] concreteIndices; - String reason = "cannot start datafeed [" + datafeedId + "] because index [" - + index + "] does not exist, is closed, or is still initializing."; try { concreteIndices = resolver.concreteIndexNames(clusterState, indicesOptions, index); if (concreteIndices.length == 0) { - return new AssignmentFailure(reason, true); + return new AssignmentFailure("cannot start datafeed [" + datafeedId + "] because index [" + + index + "] does not exist, is closed, or is still initializing.", true); } } catch (Exception e) { - LOGGER.debug(reason, e); - return new AssignmentFailure(reason, true); + String msg = new ParameterizedMessage("failed resolving indices given [{}] and indices_options [{}]", + index, + indicesOptions).getFormattedMessage(); + LOGGER.debug("[" + datafeedId + "] " + msg, e); + return new AssignmentFailure( + "cannot start datafeed [" + datafeedId + "] because it " + msg + " with exception [" + e.getMessage() + "]", + true); } for (String concreteIndex : concreteIndices) { IndexRoutingTable routingTable = clusterState.getRoutingTable().index(concreteIndex); if (routingTable == null || !routingTable.allPrimaryShardsActive()) { - reason = "cannot start datafeed [" + datafeedId + "] because index [" - + concreteIndex + "] does not have all primary shards active yet."; - return new AssignmentFailure(reason, false); + return new AssignmentFailure("cannot start datafeed [" + datafeedId + "] because index [" + + concreteIndex + "] does not have all primary shards active yet.", false); } } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DatafeedDelayedDataDetector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DatafeedDelayedDataDetector.java index 96b183d216589..0fa4411d839e3 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DatafeedDelayedDataDetector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/delayeddatacheck/DatafeedDelayedDataDetector.java @@ -10,7 +10,6 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.Client; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; @@ -29,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; @@ -51,14 +51,14 @@ public class DatafeedDelayedDataDetector implements DelayedDataDetector { private final IndicesOptions indicesOptions; DatafeedDelayedDataDetector(long bucketSpan, long window, String jobId, String timeField, QueryBuilder datafeedQuery, - String[] datafeedIndices, @Nullable IndicesOptions indicesOptions, Client client) { + String[] datafeedIndices, IndicesOptions indicesOptions, Client client) { this.bucketSpan = bucketSpan; this.window = window; this.jobId = jobId; this.timeField = timeField; this.datafeedQuery = datafeedQuery; this.datafeedIndices = datafeedIndices; - this.indicesOptions = indicesOptions; + this.indicesOptions = Objects.requireNonNull(indicesOptions); this.client = client; } @@ -119,10 +119,7 @@ private Map checkCurrentBucketEventCount(long start, long end) { .fixedInterval(new DateHistogramInterval(bucketSpan + "ms")).field(timeField)) .query(ExtractorUtils.wrapInTimeRangeQuery(datafeedQuery, timeField, start, end)); - SearchRequest searchRequest = new SearchRequest(datafeedIndices).source(searchSourceBuilder); - if (indicesOptions != null) { - searchRequest.indicesOptions(indicesOptions); - } + SearchRequest searchRequest = new SearchRequest(datafeedIndices).source(searchSourceBuilder).indicesOptions(indicesOptions); try (ThreadContext.StoredContext ignore = client.threadPool().getThreadContext().stashWithOrigin(ML_ORIGIN)) { SearchResponse response = client.execute(SearchAction.INSTANCE, searchRequest).actionGet(); List buckets = ((Histogram)response.getAggregations().get(DATE_BUCKETS)).getBuckets(); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java index 91d059ff6336c..5863a4e05fd27 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/DataExtractorFactory.java @@ -79,9 +79,7 @@ static void create(Client client, client, ClientHelper.ML_ORIGIN, GetRollupIndexCapsAction.INSTANCE, - datafeed.getIndicesOptions() == null ? - new GetRollupIndexCapsAction.Request(datafeed.getIndices().toArray(new String[0])) : - new GetRollupIndexCapsAction.Request(datafeed.getIndices().toArray(new String[0]), datafeed.getIndicesOptions()), + new GetRollupIndexCapsAction.Request(datafeed.getIndices().toArray(new String[0]), datafeed.getIndicesOptions()), getRollupIndexCapsActionHandler); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorContext.java index 2dee3b6759b52..077dc6fca6dc5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorContext.java @@ -5,9 +5,7 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation; -import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.common.Nullable; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories; @@ -32,7 +30,7 @@ class AggregationDataExtractorContext { AggregationDataExtractorContext(String jobId, String timeField, Set fields, List indices, QueryBuilder query, AggregatorFactories.Builder aggs, long start, long end, boolean includeDocCount, - Map headers, @Nullable IndicesOptions indicesOptions) { + Map headers, IndicesOptions indicesOptions) { this.jobId = Objects.requireNonNull(jobId); this.timeField = Objects.requireNonNull(timeField); this.fields = Objects.requireNonNull(fields); @@ -43,6 +41,6 @@ class AggregationDataExtractorContext { this.end = end; this.includeDocCount = includeDocCount; this.headers = headers; - this.indicesOptions = indicesOptions == null ? SearchRequest.DEFAULT_INDICES_OPTIONS : indicesOptions; + this.indicesOptions = Objects.requireNonNull(indicesOptions); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java index 6344fbe19f418..cf8f721fee694 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorContext.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor.chunked; -import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.unit.TimeValue; @@ -38,7 +37,7 @@ interface TimeAligner { ChunkedDataExtractorContext(String jobId, String timeField, List indices, QueryBuilder query, int scrollSize, long start, long end, @Nullable TimeValue chunkSpan, TimeAligner timeAligner, Map headers, - boolean hasAggregations, @Nullable Long histogramInterval, @Nullable IndicesOptions indicesOptions) { + boolean hasAggregations, @Nullable Long histogramInterval, IndicesOptions indicesOptions) { this.jobId = Objects.requireNonNull(jobId); this.timeField = Objects.requireNonNull(timeField); this.indices = indices.toArray(new String[indices.size()]); @@ -51,6 +50,6 @@ interface TimeAligner { this.headers = headers; this.hasAggregations = hasAggregations; this.histogramInterval = histogramInterval; - this.indicesOptions = indicesOptions == null ? SearchRequest.DEFAULT_INDICES_OPTIONS : indicesOptions; + this.indicesOptions = Objects.requireNonNull(indicesOptions); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java index e39f45fc2fe64..7ec6b3b65c381 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorContext.java @@ -5,9 +5,7 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor.scroll; -import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.common.Nullable; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -30,7 +28,7 @@ class ScrollDataExtractorContext { ScrollDataExtractorContext(String jobId, TimeBasedExtractedFields extractedFields, List indices, QueryBuilder query, List scriptFields, int scrollSize, long start, long end, - Map headers, @Nullable IndicesOptions indicesOptions) { + Map headers, IndicesOptions indicesOptions) { this.jobId = Objects.requireNonNull(jobId); this.extractedFields = Objects.requireNonNull(extractedFields); this.indices = indices.toArray(new String[indices.size()]); @@ -40,6 +38,6 @@ class ScrollDataExtractorContext { this.start = start; this.end = end; this.headers = headers; - this.indicesOptions = indicesOptions == null ? SearchRequest.DEFAULT_INDICES_OPTIONS : indicesOptions; + this.indicesOptions = Objects.requireNonNull(indicesOptions); } } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java index 2988dfedfa968..7bc995df5b56c 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorFactory.java @@ -88,10 +88,7 @@ public static void create(Client client, // Step 1. Get field capabilities necessary to build the information of how to extract fields FieldCapabilitiesRequest fieldCapabilitiesRequest = new FieldCapabilitiesRequest(); - fieldCapabilitiesRequest.indices(datafeed.getIndices().toArray(new String[0])); - if (datafeed.getIndicesOptions() != null) { - fieldCapabilitiesRequest.indicesOptions(datafeed.getIndicesOptions()); - } + fieldCapabilitiesRequest.indices(datafeed.getIndices().toArray(new String[0])).indicesOptions(datafeed.getIndicesOptions()); // We need capabilities for all fields matching the requested fields' parents so that we can work around // multi-fields that are not in source. String[] requestFields = job.allInputFields() diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java index ba6a30e0d35e2..4b79abdfef662 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/rest/datafeeds/RestPutDatafeedAction.java @@ -46,13 +46,7 @@ public String getName() { @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { String datafeedId = restRequest.param(DatafeedConfig.ID.getPreferredName()); - IndicesOptions indicesOptions = null; - if (restRequest.hasParam("expand_wildcards") - || restRequest.hasParam("ignore_unavailable") - || restRequest.hasParam("allow_no_indices") - || restRequest.hasParam("ignore_throttled")) { - indicesOptions = IndicesOptions.fromRequest(restRequest, SearchRequest.DEFAULT_INDICES_OPTIONS); - } + IndicesOptions indicesOptions = IndicesOptions.fromRequest(restRequest, SearchRequest.DEFAULT_INDICES_OPTIONS); XContentParser parser = restRequest.contentParser(); PutDatafeedAction.Request putDatafeedRequest = PutDatafeedAction.Request.parseRequest(datafeedId, indicesOptions, parser); putDatafeedRequest.timeout(restRequest.paramAsTime("timeout", putDatafeedRequest.timeout())); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java index fad7b539f41e2..3c986b764a0aa 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java @@ -7,6 +7,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -75,10 +76,19 @@ public void testSelectNode_GivenJobIsOpened() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); + PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).selectNode(); assertEquals("node_id", result.getExecutorNode()); - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).checkDatafeedTaskCanBeCreated(); } public void testSelectNode_GivenJobIsOpening() { @@ -91,10 +101,19 @@ public void testSelectNode_GivenJobIsOpening() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); + PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).selectNode(); assertEquals("node_id", result.getExecutorNode()); - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).checkDatafeedTaskCanBeCreated(); } public void testNoJobTask() { @@ -107,15 +126,23 @@ public void testNoJobTask() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); + PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).selectNode(); assertNull(result.getExecutorNode()); assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id], because the job's [job_id] state is " + "[closed] while state [opened] is required")); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) - .checkDatafeedTaskCanBeCreated()); + () -> new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " + "[cannot start datafeed [datafeed_id], because the job's [job_id] state is [closed] while state [opened] is required]")); } @@ -131,15 +158,23 @@ public void testSelectNode_GivenJobFailedOrClosed() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); + PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).selectNode(); assertNull(result.getExecutorNode()); assertEquals("cannot start datafeed [datafeed_id], because the job's [job_id] state is [" + jobState + "] while state [opened] is required", result.getExplanation()); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) - .checkDatafeedTaskCanBeCreated()); + () -> new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " + "[cannot start datafeed [datafeed_id], because the job's [job_id] state is [" + jobState + "] while state [opened] is required]")); @@ -160,13 +195,22 @@ public void testShardUnassigned() { givenClusterState("foo", 1, 0, states); - PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); + PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).selectNode(); assertNull(result.getExecutorNode()); assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id] because index [foo] " + "does not have all primary shards active yet.")); - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).checkDatafeedTaskCanBeCreated(); } public void testShardNotAllActive() { @@ -185,13 +229,22 @@ public void testShardNotAllActive() { givenClusterState("foo", 2, 0, states); - PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); + PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).selectNode(); assertNull(result.getExecutorNode()); assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id] because index [foo] " + "does not have all primary shards active yet.")); - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).checkDatafeedTaskCanBeCreated(); } public void testIndexDoesntExist() { @@ -204,17 +257,32 @@ public void testIndexDoesntExist() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); + PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).selectNode(); assertNull(result.getExecutorNode()); - assertThat(result.getExplanation(), equalTo("cannot start datafeed [datafeed_id] because index [not_foo] " + - "does not exist, is closed, or is still initializing.")); + assertThat(result.getExplanation(), + equalTo("cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and " + + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, " + + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, " + + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true]] with exception [no such index [not_foo]]")); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) - .checkDatafeedTaskCanBeCreated()); + () -> new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " - + "[cannot start datafeed [datafeed_id] because index [not_foo] does not exist, is closed, or is still initializing.]")); + + "[cannot start datafeed [datafeed_id] because it failed resolving " + + "indices given [not_foo] and indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, " + + "expand_wildcards_open=true, expand_wildcards_closed=false, expand_wildcards_hidden=false, " + + "allow_aliases_to_multiple_indices=true, forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true]] " + + "with exception [no such index [not_foo]]]")); } public void testRemoteIndex() { @@ -227,8 +295,12 @@ public void testRemoteIndex() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); + PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).selectNode(); assertNotNull(result.getExecutorNode()); } @@ -245,15 +317,23 @@ public void testSelectNode_jobTaskStale() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); + PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).selectNode(); assertNull(result.getExecutorNode()); assertEquals("cannot start datafeed [datafeed_id], because the job's [job_id] state is stale", result.getExplanation()); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) - .checkDatafeedTaskCanBeCreated()); + () -> new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " + "[cannot start datafeed [datafeed_id], because the job's [job_id] state is stale]")); @@ -261,9 +341,19 @@ public void testSelectNode_jobTaskStale() { addJobTask(job.getId(), "node_id1", JobState.OPENED, tasksBuilder); tasks = tasksBuilder.build(); givenClusterState("foo", 1, 0); - result = new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); + result = new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).selectNode(); assertEquals("node_id1", result.getExecutorNode()); - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).checkDatafeedTaskCanBeCreated(); + new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).checkDatafeedTaskCanBeCreated(); } public void testSelectNode_GivenJobOpeningAndIndexDoesNotExist() { @@ -280,10 +370,17 @@ public void testSelectNode_GivenJobOpeningAndIndexDoesNotExist() { givenClusterState("foo", 1, 0); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) - .checkDatafeedTaskCanBeCreated()); + () -> new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), containsString("No node found to start datafeed [datafeed_id], allocation explanation " - + "[cannot start datafeed [datafeed_id] because index [not_foo] does not exist, is closed, or is still initializing.]")); + + "[cannot start datafeed [datafeed_id] because it failed resolving indices given [not_foo] and " + + "indices_options [IndicesOptions[ignore_unavailable=false, allow_no_indices=true, expand_wildcards_open=true, " + + "expand_wildcards_closed=false, expand_wildcards_hidden=false, allow_aliases_to_multiple_indices=true, " + + "forbid_closed_indices=true, ignore_aliases=false, ignore_throttled=true]] with exception [no such index [not_foo]]]")); } public void testSelectNode_GivenMlUpgradeMode() { @@ -297,8 +394,12 @@ public void testSelectNode_GivenMlUpgradeMode() { givenClusterState("foo", 1, 0); - PersistentTasksCustomMetaData.Assignment result = - new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null).selectNode(); + PersistentTasksCustomMetaData.Assignment result = new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).selectNode(); assertThat(result, equalTo(MlTasks.AWAITING_UPGRADE)); } @@ -314,8 +415,12 @@ public void testCheckDatafeedTaskCanBeCreated_GivenMlUpgradeMode() { givenClusterState("foo", 1, 0); ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new DatafeedNodeSelector(clusterState, resolver, df.getId(), df.getJobId(), df.getIndices(), null) - .checkDatafeedTaskCanBeCreated()); + () -> new DatafeedNodeSelector(clusterState, + resolver, + df.getId(), + df.getJobId(), + df.getIndices(), + SearchRequest.DEFAULT_INDICES_OPTIONS).checkDatafeedTaskCanBeCreated()); assertThat(e.getMessage(), equalTo("Could not start datafeed [datafeed_id] as indices are being upgraded")); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java index 3708f2052d22c..72c3edb33cd21 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/aggregation/AggregationDataExtractorTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.ml.datafeed.extractor.aggregation; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; @@ -272,7 +273,7 @@ public void testExtractionGivenInitSearchResponseEncounteredUnavailableShards() private AggregationDataExtractorContext createContext(long start, long end) { return new AggregationDataExtractorContext(jobId, timeField, fields, indices, query, aggs, start, end, true, - Collections.emptyMap(), null); + Collections.emptyMap(), SearchRequest.DEFAULT_INDICES_OPTIONS); } @SuppressWarnings("unchecked") diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java index 68ef6f8f41476..657545d5f3e8f 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/chunked/ChunkedDataExtractorTests.java @@ -565,7 +565,8 @@ private ChunkedDataExtractorContext createContext(long start, long end) { private ChunkedDataExtractorContext createContext(long start, long end, boolean hasAggregations, Long histogramInterval) { return new ChunkedDataExtractorContext(jobId, timeField, indices, query, scrollSize, start, end, chunkSpan, - ChunkedDataExtractorFactory.newIdentityTimeAligner(), Collections.emptyMap(), hasAggregations, histogramInterval, null); + ChunkedDataExtractorFactory.newIdentityTimeAligner(), Collections.emptyMap(), hasAggregations, histogramInterval, + SearchRequest.DEFAULT_INDICES_OPTIONS); } private static class StubSubExtractor implements DataExtractor { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java index e198b46621aae..76e86a9a82170 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/extractor/scroll/ScrollDataExtractorTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.search.ClearScrollRequest; import org.elasticsearch.action.search.ClearScrollResponse; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; @@ -444,7 +445,7 @@ public void testDomainSplitScriptField() throws IOException { List sFields = Arrays.asList(withoutSplit, withSplit); ScrollDataExtractorContext context = new ScrollDataExtractorContext(jobId, extractedFields, indices, - query, sFields, scrollSize, 1000, 2000, Collections.emptyMap(), null); + query, sFields, scrollSize, 1000, 2000, Collections.emptyMap(), SearchRequest.DEFAULT_INDICES_OPTIONS); TestDataExtractor extractor = new TestDataExtractor(context); @@ -490,7 +491,7 @@ public void testDomainSplitScriptField() throws IOException { private ScrollDataExtractorContext createContext(long start, long end) { return new ScrollDataExtractorContext(jobId, extractedFields, indices, query, scriptFields, scrollSize, start, end, - Collections.emptyMap(), null); + Collections.emptyMap(), SearchRequest.DEFAULT_INDICES_OPTIONS); } private SearchResponse createEmptySearchResponse() { From 70caa0d60089458217a5da84048f8417f16b944c Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 26 Feb 2020 14:13:43 -0500 Subject: [PATCH 3/5] addressing PR comments --- .../core/ml/datafeed/DatafeedConfig.java | 2 +- .../core/ml/datafeed/DatafeedUpdate.java | 3 +- .../core/ml/datafeed/DatafeedConfigTests.java | 26 ++++++++------- .../core/ml/datafeed/DatafeedUpdateTests.java | 33 +++++++++++++------ 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index 1fb94356ecb05..b7755df7b1202 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -158,7 +158,7 @@ private static ObjectParser createParser(boolean ignoreUnknownFie DELAYED_DATA_CHECK_CONFIG); parser.declareInt(Builder::setMaxEmptySearches, MAX_EMPTY_SEARCHES); parser.declareObject(Builder::setIndicesOptions, - (p, c) -> IndicesOptions.fromMap(p.map(), new IndicesOptions(IndicesOptions.Option.NONE, IndicesOptions.WildcardStates.NONE)), + (p, c) -> IndicesOptions.fromMap(p.map(), SearchRequest.DEFAULT_INDICES_OPTIONS), INDICES_OPTIONS); return parser; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java index f1c6f98857b8d..a983b31b24934 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.core.ml.datafeed; import org.elasticsearch.Version; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; @@ -81,7 +82,7 @@ public class DatafeedUpdate implements Writeable, ToXContentObject { DatafeedConfig.DELAYED_DATA_CHECK_CONFIG); PARSER.declareInt(Builder::setMaxEmptySearches, DatafeedConfig.MAX_EMPTY_SEARCHES); PARSER.declareObject(Builder::setIndicesOptions, - (p, c) -> IndicesOptions.fromMap(p.map(), new IndicesOptions(IndicesOptions.Option.NONE, IndicesOptions.WildcardStates.NONE)), + (p, c) -> IndicesOptions.fromMap(p.map(), SearchRequest.DEFAULT_INDICES_OPTIONS), DatafeedConfig.INDICES_OPTIONS); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java index 0cf14ba937e0b..12beacb5f63e2 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -58,6 +59,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import static org.elasticsearch.xpack.core.ml.utils.QueryProviderTests.createRandomValidQueryProvider; @@ -143,11 +145,12 @@ private static DatafeedConfig.Builder createRandomizedDatafeedConfigBuilder(Stri if (randomBoolean()) { builder.setMaxEmptySearches(randomIntBetween(10, 100)); } - builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean())); + builder.setIndicesOptions(IndicesOptions.fromParameters( + randomFrom(IndicesOptions.WildcardStates.values()).name().toLowerCase(Locale.ROOT), + Boolean.toString(randomBoolean()), + Boolean.toString(randomBoolean()), + Boolean.toString(randomBoolean()), + SearchRequest.DEFAULT_INDICES_OPTIONS)); return builder; } @@ -759,7 +762,7 @@ public void testSerializationOfComplexAggs() throws IOException { xContentType); DatafeedConfig parsedDatafeedConfig = doParseInstance(parser); - assertEquals(datafeedConfig.getAggregations(), parsedDatafeedConfig.getAggregations()); + assertEquals(datafeedConfig, parsedDatafeedConfig); // Assert that the parsed versions of our aggs and queries work as well assertEquals(aggBuilder, parsedDatafeedConfig.getParsedAggregations(xContentRegistry())); @@ -948,11 +951,12 @@ protected DatafeedConfig mutateInstance(DatafeedConfig instance) throws IOExcept } break; case 11: - builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean())); + builder.setIndicesOptions(IndicesOptions.fromParameters( + randomFrom(IndicesOptions.WildcardStates.values()).name().toLowerCase(Locale.ROOT), + Boolean.toString(instance.getIndicesOptions().ignoreUnavailable() == false), + Boolean.toString(instance.getIndicesOptions().allowNoIndices() == false), + Boolean.toString(instance.getIndicesOptions().ignoreThrottled() == false), + SearchRequest.DEFAULT_INDICES_OPTIONS)); break; default: throw new AssertionError("Illegal randomisation branch"); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java index 88e644721242c..9c1e135551bb5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java @@ -7,6 +7,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; +import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -50,6 +51,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import static org.elasticsearch.xpack.core.ml.datafeed.AggProviderTests.createRandomValidAggProvider; import static org.elasticsearch.xpack.core.ml.utils.QueryProviderTests.createRandomValidQueryProvider; @@ -110,11 +112,12 @@ public static DatafeedUpdate createRandomized(String datafeedId, @Nullable Dataf builder.setMaxEmptySearches(randomBoolean() ? -1 : randomIntBetween(10, 100)); } if (randomBoolean()) { - builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean())); + builder.setIndicesOptions(IndicesOptions.fromParameters( + randomFrom(IndicesOptions.WildcardStates.values()).name().toLowerCase(Locale.ROOT), + Boolean.toString(randomBoolean()), + Boolean.toString(randomBoolean()), + Boolean.toString(randomBoolean()), + SearchRequest.DEFAULT_INDICES_OPTIONS)); } return builder.build(); } @@ -444,11 +447,21 @@ protected DatafeedUpdate mutateInstance(DatafeedUpdate instance) throws IOExcept } break; case 11: - builder.setIndicesOptions(IndicesOptions.fromOptions(randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean(), - randomBoolean())); + if (instance.getIndicesOptions() != null) { + builder.setIndicesOptions(IndicesOptions.fromParameters( + randomFrom(IndicesOptions.WildcardStates.values()).name().toLowerCase(Locale.ROOT), + Boolean.toString(instance.getIndicesOptions().ignoreUnavailable() == false), + Boolean.toString(instance.getIndicesOptions().allowNoIndices() == false), + Boolean.toString(instance.getIndicesOptions().ignoreThrottled() == false), + SearchRequest.DEFAULT_INDICES_OPTIONS)); + } else { + builder.setIndicesOptions(IndicesOptions.fromParameters( + randomFrom(IndicesOptions.WildcardStates.values()).name().toLowerCase(Locale.ROOT), + Boolean.toString(randomBoolean()), + Boolean.toString(randomBoolean()), + Boolean.toString(randomBoolean()), + SearchRequest.DEFAULT_INDICES_OPTIONS)); + } break; default: throw new AssertionError("Illegal randomisation branch"); From 1c9cd9519592bf91256d41be96e4d51da422c7d7 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Wed, 26 Feb 2020 14:22:22 -0500 Subject: [PATCH 4/5] reverting bad assertion --- .../xpack/core/ml/datafeed/DatafeedConfigTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java index 12beacb5f63e2..659aa387f8c6d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java @@ -774,7 +774,7 @@ public void testSerializationOfComplexAggs() throws IOException { datafeedConfig.writeTo(output); try(StreamInput streamInput = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) { DatafeedConfig streamedDatafeedConfig = new DatafeedConfig(streamInput); - assertEquals(datafeedConfig.getAggregations(), streamedDatafeedConfig.getAggregations()); + assertEquals(datafeedConfig, streamedDatafeedConfig); // Assert that the parsed versions of our aggs and queries work as well assertEquals(aggBuilder, streamedDatafeedConfig.getParsedAggregations(xContentRegistry())); From e21eae70a07010911470e566a2709bc2d36415be Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Thu, 27 Feb 2020 08:03:20 -0500 Subject: [PATCH 5/5] mute for bwc --- .../test/mixed_cluster/40_ml_datafeed_crud.yml | 9 +++------ .../test/old_cluster/40_ml_datafeed_crud.yml | 10 +++------- .../test/upgraded_cluster/40_ml_datafeed_crud.yml | 3 +++ 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml index e84bd217bd257..3f89d59f6a608 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/40_ml_datafeed_crud.yml @@ -1,10 +1,7 @@ setup: - - do: - cluster.health: - wait_for_status: green - wait_for_nodes: 3 - # wait for long enough that we give delayed unassigned shards to stop being delayed - timeout: 70s + - skip: + reason: "Waiting for backport of https://github.com/elastic/elasticsearch/pull/52793" + version: all --- "Test old cluster datafeed without aggs": diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml index 491435f43fb93..b46cf39684bf0 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/40_ml_datafeed_crud.yml @@ -1,11 +1,7 @@ setup: - - do: - cluster.health: - wait_for_status: green - wait_for_nodes: 3 - # wait for long enough that we give delayed unassigned shards to stop being delayed - timeout: 70s - + - skip: + reason: "Waiting for backport of https://github.com/elastic/elasticsearch/pull/52793" + version: all --- "Put job and datafeed without aggs in old cluster": diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml index b3a538093242f..35a899a6dc971 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/40_ml_datafeed_crud.yml @@ -1,4 +1,7 @@ setup: + - skip: + reason: "Waiting for backport of https://github.com/elastic/elasticsearch/pull/52793" + version: all - do: cluster.health: wait_for_status: green