Merge remote-tracking branch 'upstream/main' into deprecate_range_params

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -25,13 +25,15 @@
 import org.elasticsearch.compute.operator.EvalOperator;
 import org.elasticsearch.compute.operator.Operator;
 import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.xpack.esql.core.expression.Expression;
 import org.elasticsearch.xpack.esql.core.expression.FieldAttribute;
 import org.elasticsearch.xpack.esql.core.expression.Literal;
 import org.elasticsearch.xpack.esql.core.expression.predicate.regex.RLikePattern;
 import org.elasticsearch.xpack.esql.core.tree.Source;
 import org.elasticsearch.xpack.esql.core.type.DataType;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.evaluator.EvalMapper;
+import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case;
 import org.elasticsearch.xpack.esql.expression.function.scalar.date.DateTrunc;
 import org.elasticsearch.xpack.esql.expression.function.scalar.math.Abs;
 import org.elasticsearch.xpack.esql.expression.function.scalar.multivalue.MvMin;
@@ -53,6 +55,7 @@
 
 import java.time.Duration;
 import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -91,6 +94,8 @@ public class EvalBenchmark {
  "abs",
  "add",
  "add_double",
+ "case_1_eager",
+ "case_1_lazy",
  "date_trunc",
  "equal_to_const",
  "long_equal_to_long",
@@ -125,6 +130,18 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
  layout(doubleField)
  ).get(driverContext);
  }
+ case "case_1_eager", "case_1_lazy" -> {
+ FieldAttribute f1 = longField();
+ FieldAttribute f2 = longField();
+ Expression condition = new Equals(Source.EMPTY, f1, new Literal(Source.EMPTY, 1L, DataType.LONG));
+ Expression lhs = f1;
+ Expression rhs = f2;
+ if (operation.endsWith("lazy")) {
+ lhs = new Add(Source.EMPTY, lhs, new Literal(Source.EMPTY, 1L, DataType.LONG));
+ rhs = new Add(Source.EMPTY, rhs, new Literal(Source.EMPTY, 1L, DataType.LONG));
+ }
+ yield EvalMapper.toEvaluator(new Case(Source.EMPTY, condition, List.of(lhs, rhs)), layout(f1, f2)).get(driverContext);
+ }
  case "date_trunc" -> {
  FieldAttribute timestamp = new FieldAttribute(
  Source.EMPTY,
@@ -216,6 +233,28 @@ private static void checkExpected(String operation, Page actual) {
  }
  }
  }
+ case "case_1_eager" -> {
+ LongVector f1 = actual.<LongBlock>getBlock(0).asVector();
+ LongVector f2 = actual.<LongBlock>getBlock(1).asVector();
+ LongVector result = actual.<LongBlock>getBlock(2).asVector();
+ for (int i = 0; i < BLOCK_LENGTH; i++) {
+ long expected = f1.getLong(i) == 1 ? f1.getLong(i) : f2.getLong(i);
+ if (result.getLong(i) != expected) {
+ throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + result.getLong(i) + "]");
+ }
+ }
+ }
+ case "case_1_lazy" -> {
+ LongVector f1 = actual.<LongBlock>getBlock(0).asVector();
+ LongVector f2 = actual.<LongBlock>getBlock(1).asVector();
+ LongVector result = actual.<LongBlock>getBlock(2).asVector();
+ for (int i = 0; i < BLOCK_LENGTH; i++) {
+ long expected = 1 + (f1.getLong(i) == 1 ? f1.getLong(i) : f2.getLong(i));
+ if (result.getLong(i) != expected) {
+ throw new AssertionError("[" + operation + "] expected [" + expected + "] but was [" + result.getLong(i) + "]");
+ }
+ }
+ }
  case "date_trunc" -> {
  LongVector v = actual.<LongBlock>getBlock(1).asVector();
  long oneDay = TimeValue.timeValueHours(24).millis();
@@ -280,6 +319,15 @@ private static Page page(String operation) {
  }
  yield new Page(builder.build());
  }
+ case "case_1_eager", "case_1_lazy" -> {
+ var f1 = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
+ var f2 = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
+ for (int i = 0; i < BLOCK_LENGTH; i++) {
+ f1.appendLong(i);
+ f2.appendLong(-i);
+ }
+ yield new Page(f1.build(), f2.build());
+ }
  case "long_equal_to_long" -> {
  var lhs = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);
  var rhs = blockFactory.newLongBlockBuilder(BLOCK_LENGTH);

catalog-info.yaml

@@ -125,7 +125,7 @@ spec:
  ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
  SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
  SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
- branch_configuration: lucene_snapshot lucene_snapshot_10
+ branch_configuration: lucene_snapshot
  default_branch: lucene_snapshot
  teams:
  elasticsearch-team: {}
@@ -142,10 +142,6 @@ spec:
  branch: lucene_snapshot
  cronline: "0 2 * * * America/New_York"
  message: "Builds a new lucene snapshot 1x per day"
- Periodically on lucene_snapshot_10:
- branch: lucene_snapshot_10
- cronline: "0 2 * * * America/New_York"
- message: "Builds a new lucene snapshot 1x per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -173,7 +169,6 @@ spec:
  ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
  SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
  SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
- branch_configuration: lucene_snapshot lucene_snapshot_10
  default_branch: lucene_snapshot
  teams:
  elasticsearch-team: {}
@@ -190,10 +185,6 @@ spec:
  branch: lucene_snapshot
  cronline: "0 6 * * * America/New_York"
  message: "Merges main into lucene_snapshot branch 1x per day"
- Periodically on lucene_snapshot_10:
- branch: lucene_snapshot_10
- cronline: "0 6 * * * America/New_York"
- message: "Merges main into lucene_snapshot_10 branch 1x per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -221,7 +212,7 @@ spec:
  ELASTIC_SLACK_NOTIFICATIONS_ENABLED: "true"
  SLACK_NOTIFICATIONS_CHANNEL: "#lucene"
  SLACK_NOTIFICATIONS_ALL_BRANCHES: "true"
- branch_configuration: lucene_snapshot lucene_snapshot_10
+ branch_configuration: lucene_snapshot
  default_branch: lucene_snapshot
  teams:
  elasticsearch-team: {}
@@ -238,10 +229,6 @@ spec:
  branch: lucene_snapshot
  cronline: "0 9,12,15,18 * * * America/New_York"
  message: "Runs tests against lucene_snapshot branch several times per day"
- Periodically on lucene_snapshot_10:
- branch: lucene_snapshot_10
- cronline: "0 9,12,15,18 * * * America/New_York"
- message: "Runs tests against lucene_snapshot_10 branch several times per day"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1

docs/changelog/111684.yaml

@@ -0,0 +1,5 @@
+pr: 111684
+summary: Write downloaded model parts async
+area: Machine Learning
+type: enhancement
+issues: []

docs/changelog/112295.yaml

@@ -0,0 +1,5 @@
+pr: 112295
+summary: "ESQL: Speed up CASE for some parameters"
+area: ES|QL
+type: enhancement
+issues: []

docs/changelog/112405.yaml

@@ -0,0 +1,6 @@
+pr: 112405
+summary: Improve date expression/remote handling in index names
+area: Search
+type: bug
+issues:
+ - 112243

docs/changelog/113051.yaml

@@ -0,0 +1,5 @@
+pr: 113051
+summary: Add Search Inference ID To Semantic Text Mapping
+area: Mapping
+type: enhancement
+issues: []

docs/changelog/113425.yaml

@@ -0,0 +1,5 @@
+pr: 113425
+summary: Add `ensureGreen` test method for use with `adminClient`
+area: Infra/Metrics
+type: enhancement
+issues: []

docs/changelog/113462.yaml

@@ -0,0 +1,5 @@
+pr: 113462
+summary: Suppress merge-on-recovery for older indices
+area: CRUD
+type: enhancement
+issues: []

docs/reference/indices/index-templates.asciidoc

@@ -44,6 +44,7 @@ following index patterns:
 - `metrics-*-*`
 - `synthetics-*-*`
 - `profiling-*`
+- `security_solution-*-*`
 // end::built-in-index-template-patterns[]
 
 {fleet-guide}/fleet-overview.html[{agent}] uses these templates to create

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamAutoshardingIT.java

@@ -502,7 +502,7 @@ static void putComposableIndexTemplate(String id, List<String> patterns, @Nullab
  request.indexTemplate(
  ComposableIndexTemplate.builder()
  .indexPatterns(patterns)
- .template(new Template(settings, null, null, null))
+ .template(Template.builder().settings(settings))
  .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
  .build()
  );

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java

@@ -2441,7 +2441,13 @@ static void putComposableIndexTemplate(
  request.indexTemplate(
  ComposableIndexTemplate.builder()
  .indexPatterns(patterns)
- .template(new Template(settings, mappings == null ? null : CompressedXContent.fromJSON(mappings), aliases, lifecycle))
+ .template(
+ Template.builder()
+ .settings(settings)
+ .mappings(mappings == null ? null : CompressedXContent.fromJSON(mappings))
+ .aliases(aliases)
+ .lifecycle(lifecycle)
+ )
  .metadata(metadata)
  .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, withFailureStore))
  .build()

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/ResolveClusterDataStreamIT.java

@@ -453,7 +453,7 @@ void putComposableIndexTemplate(Client client, String id, List<String> patterns,
  request.indexTemplate(
  ComposableIndexTemplate.builder()
  .indexPatterns(patterns)
- .template(new Template(null, null, aliases, null))
+ .template(Template.builder().aliases(aliases))
  .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
  .build()
  );

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/CrudSystemDataStreamLifecycleIT.java

@@ -201,12 +201,10 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
  ComposableIndexTemplate.builder()
  .indexPatterns(List.of(".test-data-stream"))
  .template(
- new Template(
- Settings.EMPTY,
- mappings,
- null,
- DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build()
- )
+ Template.builder()
+ .settings(Settings.EMPTY)
+ .mappings(mappings)
+ .lifecycle(DataStreamLifecycle.newBuilder().dataRetention(randomMillisUpToYear9999()).build())
  )
  .dataStreamTemplate(new DataStreamTemplate())
  .build(),

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java

@@ -345,7 +345,7 @@ public void testOriginationDate() throws Exception {
  request.indexTemplate(
  ComposableIndexTemplate.builder()
  .indexPatterns(List.of("index_*"))
- .template(new Template(null, CompressedXContent.fromJSON(mapping), null, null))
+ .template(Template.builder().mappings(CompressedXContent.fromJSON(mapping)))
  .build()
  );
  client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet();
@@ -1221,7 +1221,12 @@ static void putComposableIndexTemplate(
  request.indexTemplate(
  ComposableIndexTemplate.builder()
  .indexPatterns(patterns)
- .template(new Template(settings, mappings == null ? null : CompressedXContent.fromJSON(mappings), null, lifecycle))
+ .template(
+ Template.builder()
+ .settings(settings)
+ .mappings(mappings == null ? null : CompressedXContent.fromJSON(mappings))
+ .lifecycle(lifecycle)
+ )
  .metadata(metadata)
  .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate(false, false, withFailureStore))
  .build()
@@ -1268,14 +1273,12 @@ public Collection<SystemDataStreamDescriptor> getSystemDataStreamDescriptors() {
  .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
  .indexPatterns(List.of(DataStream.BACKING_INDEX_PREFIX + SYSTEM_DATA_STREAM_NAME + "*"))
  .template(
- new Template(
- Settings.EMPTY,
- null,
- null,
- DataStreamLifecycle.newBuilder()
- .dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
- .build()
- )
+ Template.builder()
+ .settings(Settings.EMPTY)
+ .lifecycle(
+ DataStreamLifecycle.newBuilder()
+ .dataRetention(TimeValue.timeValueDays(SYSTEM_DATA_STREAM_RETENTION_DAYS))
+ )
  )
  .build(),
  Map.of(),

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/ExplainDataStreamLifecycleIT.java

@@ -445,7 +445,12 @@ static void putComposableIndexTemplate(
  request.indexTemplate(
  ComposableIndexTemplate.builder()
  .indexPatterns(patterns)
- .template(new Template(settings, mappings == null ? null : CompressedXContent.fromJSON(mappings), null, lifecycle))
+ .template(
+ Template.builder()
+ .settings(settings)
+ .mappings(mappings == null ? null : CompressedXContent.fromJSON(mappings))
+ .lifecycle(lifecycle)
+ )
  .metadata(metadata)
  .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
  .build()

modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleFixtures.java

@@ -114,7 +114,12 @@ static void putComposableIndexTemplate(
  request.indexTemplate(
  ComposableIndexTemplate.builder()
  .indexPatterns(patterns)
- .template(new Template(settings, mappings == null ? null : CompressedXContent.fromJSON(mappings), null, lifecycle))
+ .template(
+ Template.builder()
+ .settings(settings)
+ .mappings(mappings == null ? null : CompressedXContent.fromJSON(mappings))
+ .lifecycle(lifecycle)
+ )
  .metadata(metadata)
  .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
  .build()

modules/dot-prefix-validation/build.gradle

@@ -23,3 +23,7 @@ restResources {
 tasks.named('yamlRestTest') {
  usesDefaultDistribution()
 }
+
+tasks.named('yamlRestCompatTest') {
+ usesDefaultDistribution()
+}