Introduce runtime section in mappings

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/get/TransportGetFieldMappingsIndexAction.java

@@ -39,9 +39,9 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.IndexService;
-import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.mapper.DocumentMapper;
 import org.elasticsearch.index.mapper.Mapper;
+import org.elasticsearch.index.mapper.MappingLookup;
 import org.elasticsearch.index.shard.ShardId;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.threadpool.ThreadPool;
@@ -155,6 +155,8 @@ private static Map<String, FieldMappingMetadata> findFieldMappings(Predicate<Str
  if (documentMapper == null) {
  return Collections.emptyMap();
  }
+ //TODO the logic here needs to be reworked to also include runtime fields. Though matching is against mappers rather
+ // than field types, and runtime fields are mixed with ordinary fields in FieldTypeLookup
  Map<String, FieldMappingMetadata> fieldMappings = new HashMap<>();
  final MappingLookup allFieldMappers = documentMapper.mappers();
  for (String field : request.fields()) {

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -278,15 +278,7 @@ public final FieldMapper merge(Mapper mergeWith) {
  Conflicts conflicts = new Conflicts(name());
  builder.merge((FieldMapper) mergeWith, conflicts);
  conflicts.check();
- return builder.build(parentPath(name()));
- }
-
- private static ContentPath parentPath(String name) {
- int endPos = name.lastIndexOf(".");
- if (endPos == -1) {
- return new ContentPath(0);
- }
- return new ContentPath(name.substring(0, endPos));
+ return builder.build(Builder.parentPath(name()));
  }
 
  protected void checkIncomingMergeType(FieldMapper mergeWith) {
@@ -482,7 +474,7 @@ public List<String> copyToFields() {
  /**
  * Serializes a parameter
  */
- protected interface Serializer<T> {
+ public interface Serializer<T> {
  void serialize(XContentBuilder builder, String name, T value) throws IOException;
  }
 
@@ -931,7 +923,7 @@ protected String buildFullName(ContentPath contentPath) {
  /**
  * Writes the current builder parameter values as XContent
  */
- protected final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
+ public final void toXContent(XContentBuilder builder, boolean includeDefaults) throws IOException {
  for (Parameter<?> parameter : getParameters()) {
  parameter.toXContent(builder, includeDefaults);
  }
@@ -1011,6 +1003,14 @@ public final void parse(String name, ParserContext parserContext, Map<String, Ob
  validate();
  }
 
+ protected static ContentPath parentPath(String name) {
+ int endPos = name.lastIndexOf(".");
+ if (endPos == -1) {
+ return new ContentPath(0);
+ }
+ return new ContentPath(name.substring(0, endPos));
+ }
+
  // These parameters were previously *always* parsed by TypeParsers#parseField(), even if they
  // made no sense; if we've got here, that means that they're not declared on a current mapper,
  // and so we emit a deprecation warning rather than failing a previously working mapping.

server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java

@@ -32,6 +32,7 @@
 /**
  * An immutable container for looking up {@link MappedFieldType}s by their name.
  */
+//TODO Does this need to implement Iterable? It's used in two places
 final class FieldTypeLookup implements Iterable<MappedFieldType> {
 
  private final Map<String, MappedFieldType> fullNameToFieldType = new HashMap<>();
@@ -46,8 +47,10 @@ final class FieldTypeLookup implements Iterable<MappedFieldType> {
  private final Map<String, Set<String>> fieldToCopiedFields = new HashMap<>();
  private final DynamicKeyFieldTypeLookup dynamicKeyLookup;
 
+ //TODO ideally the constructor would not take mappers, but MappedFieldTypes and an external method does the conversion
  FieldTypeLookup(Collection<FieldMapper> fieldMappers,
- Collection<FieldAliasMapper> fieldAliasMappers) {
+ Collection<FieldAliasMapper> fieldAliasMappers,
+ Collection<RuntimeFieldType> runtimeFieldTypes) {
  Map<String, DynamicKeyFieldMapper> dynamicKeyMappers = new HashMap<>();
 
  for (FieldMapper fieldMapper : fieldMappers) {
@@ -77,6 +80,11 @@ final class FieldTypeLookup implements Iterable<MappedFieldType> {
  fullNameToFieldType.put(aliasName, fullNameToFieldType.get(path));
  }
 
+ for (RuntimeFieldType runtimeFieldType : runtimeFieldTypes) {
+ //this will override concrete fields with runtime fields that have the same name
+ fullNameToFieldType.put(runtimeFieldType.name(), runtimeFieldType);
+ }
+
  this.dynamicKeyLookup = new DynamicKeyFieldTypeLookup(dynamicKeyMappers, aliasToConcreteName);
  }
 

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -59,6 +59,7 @@ class ParserContext {
 
  private final Function<String, SimilarityProvider> similarityLookupService;
  private final Function<String, TypeParser> typeParsers;
+ private final Function<String, RuntimeFieldType.Parser> runtimeTypeParsers;
  private final Version indexVersionCreated;
  private final Supplier<QueryShardContext> queryShardContextSupplier;
  private final DateFormatter dateFormatter;
@@ -69,6 +70,7 @@ class ParserContext {
 
  public ParserContext(Function<String, SimilarityProvider> similarityLookupService,
  Function<String, TypeParser> typeParsers,
+ Function<String, RuntimeFieldType.Parser> runtimeTypeParsers,
  Version indexVersionCreated,
  Supplier<QueryShardContext> queryShardContextSupplier,
  DateFormatter dateFormatter,
@@ -78,6 +80,7 @@ public ParserContext(Function<String, SimilarityProvider> similarityLookupServic
  BooleanSupplier idFieldDataEnabled) {
  this.similarityLookupService = similarityLookupService;
  this.typeParsers = typeParsers;
+ this.runtimeTypeParsers = runtimeTypeParsers;
  this.indexVersionCreated = indexVersionCreated;
  this.queryShardContextSupplier = queryShardContextSupplier;
  this.dateFormatter = dateFormatter;
@@ -132,6 +135,8 @@ public DateFormatter getDateFormatter() {
 
  protected Function<String, TypeParser> typeParsers() { return typeParsers; }
 
+ protected Function<String, RuntimeFieldType.Parser> runtimeTypeParsers() { return runtimeTypeParsers; }
+
  protected Function<String, SimilarityProvider> similarityLookupService() { return similarityLookupService; }
 
  /**
@@ -147,8 +152,9 @@ public ParserContext createMultiFieldContext(ParserContext in) {
 
  static class MultiFieldParserContext extends ParserContext {
  MultiFieldParserContext(ParserContext in) {
- super(in.similarityLookupService, in.typeParsers, in.indexVersionCreated, in.queryShardContextSupplier,
- in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings, in.idFieldDataEnabled);
+ super(in.similarityLookupService, in.typeParsers, in.runtimeTypeParsers, in.indexVersionCreated,
+ in.queryShardContextSupplier, in.dateFormatter, in.scriptService, in.indexAnalyzers, in.indexSettings,
+ in.idFieldDataEnabled);
  }
 
  @Override

server/src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -125,8 +125,8 @@ public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers,
  this.mapperRegistry = mapperRegistry;
  Function<DateFormatter, Mapper.TypeParser.ParserContext> parserContextFunction =
  dateFormatter -> new Mapper.TypeParser.ParserContext(similarityService::getSimilarity, mapperRegistry.getMapperParsers()::get,
- indexVersionCreated, queryShardContextSupplier, dateFormatter, scriptService, indexAnalyzers, indexSettings,
- idFieldDataEnabled);
+ mapperRegistry.getRuntimeFieldTypeParsers()::get, indexVersionCreated, queryShardContextSupplier, dateFormatter,
+ scriptService, indexAnalyzers, indexSettings, idFieldDataEnabled);
  this.documentParser = new DocumentParser(xContentRegistry, parserContextFunction);
  Map<String, MetadataFieldMapper.TypeParser> metadataMapperParsers =
  mapperRegistry.getMetadataMapperParsers(indexSettings.getIndexVersionCreated());

server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java

@@ -32,6 +32,7 @@
 import java.util.Map;
 import java.util.stream.Stream;
 
+//TODO does this need to be iterable? It is not easy to track who's relying on that
 public final class MappingLookup implements Iterable<Mapper> {
 
  /** Full field name to mapper */
@@ -51,24 +52,24 @@ public static MappingLookup fromMapping(Mapping mapping) {
  newFieldMappers.add(metadataMapper);
  }
  }
- collect(mapping.root, newObjectMappers, newFieldMappers, newFieldAliasMappers);
- return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers, mapping.metadataMappers.length);
+ for (Mapper child : mapping.root) {
+ collect(child, newObjectMappers, newFieldMappers, newFieldAliasMappers);
+ }
+ return new MappingLookup(newFieldMappers, newObjectMappers, newFieldAliasMappers,
+ mapping.root.runtimeFieldTypes(), mapping.metadataMappers.length);
  }
 
  private static void collect(Mapper mapper, Collection<ObjectMapper> objectMappers,
  Collection<FieldMapper> fieldMappers,
  Collection<FieldAliasMapper> fieldAliasMappers) {
- if (mapper instanceof RootObjectMapper) {
- // root mapper isn't really an object mapper
- } else if (mapper instanceof ObjectMapper) {
+ if (mapper instanceof ObjectMapper) {
  objectMappers.add((ObjectMapper)mapper);
  } else if (mapper instanceof FieldMapper) {
  fieldMappers.add((FieldMapper)mapper);
  } else if (mapper instanceof FieldAliasMapper) {
  fieldAliasMappers.add((FieldAliasMapper) mapper);
  } else {
- throw new IllegalStateException("Unrecognized mapper type [" +
- mapper.getClass().getSimpleName() + "].");
+ throw new IllegalStateException("Unrecognized mapper type [" + mapper.getClass().getSimpleName() + "].");
  }
 
  for (Mapper child : mapper) {
@@ -79,6 +80,7 @@ private static void collect(Mapper mapper, Collection<ObjectMapper> objectMapper
  public MappingLookup(Collection<FieldMapper> mappers,
  Collection<ObjectMapper> objectMappers,
  Collection<FieldAliasMapper> aliasMappers,
+ Collection<RuntimeFieldType> runtimeFieldTypes,
  int metadataFieldCount) {
  Map<String, Mapper> fieldMappers = new HashMap<>();
  Map<String, Analyzer> indexAnalyzers = new HashMap<>();
@@ -115,7 +117,7 @@ public MappingLookup(Collection<FieldMapper> mappers,
  }
  }
 
- this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers);
+ this.fieldTypeLookup = new FieldTypeLookup(mappers, aliasMappers, runtimeFieldTypes);
 
  this.fieldMappers = Collections.unmodifiableMap(fieldMappers);
  this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers);