Add hybridfs store type

docs/reference/index-modules/store.asciidoc

@@ -40,7 +40,7 @@ The following sections lists all the different storage types supported.
 `fs`::
 
 Default file system implementation. This will pick the best implementation
-depending on the operating environment, which is currently `mmapfs` on all
+depending on the operating environment, which is currently `hybridfs` on all
 supported systems but is subject to change.
 
 [[simplefs]]`simplefs`::
@@ -67,12 +67,22 @@ process equal to the size of the file being mapped. Before using this
 class, be sure you have allowed plenty of
 <<vm-max-map-count,virtual address space>>.
 
+[[hybridfs]]`hybridfs`::
+
+The `hybridfs` type is a hybrid of `niofs` and `mmapfs`, which chooses the best
+file system type for each type of file based on the read access pattern.
+Currently only the Lucene term dictionary, norms and doc values files are
+memory mapped. All other files are opened using Lucene `NIOFSDirectory`.
+Similarly to `mmapfs` be sure you have allowed plenty of
+<<vm-max-map-count,virtual address space>>.
+
 [[allow-mmapfs]]
-You can restrict the use of the `mmapfs` store type via the setting
-`node.store.allow_mmapfs`. This is a boolean setting indicating whether or not
-`mmapfs` is allowed. The default is to allow `mmapfs`. This setting is useful,
-for example, if you are in an environment where you can not control the ability
-to create a lot of memory maps so you need disable the ability to use `mmapfs`.
+You can restrict the use of the `mmapfs` and the related `hybridfs` store type
+via the setting `node.store.allow_mmapfs`. This is a boolean setting indicating
+whether or not memory-mapping is allowed. The default is to allow it. This
+setting is useful, for example, if you are in an environment where you can not
+control the ability to create a lot of memory maps so you need disable the
+ability to use memory-mapping.
 
 === Pre-loading data into the file system cache
 

server/src/main/java/org/elasticsearch/index/IndexModule.java

@@ -302,6 +302,7 @@ public static boolean isBuiltinType(String storeType) {
 
 
  public enum Type {
+ HYBRIDFS("hybridfs"),
  NIOFS("niofs"),
  MMAPFS("mmapfs"),
  SIMPLEFS("simplefs"),
@@ -330,7 +331,7 @@ public String getSettingsKey() {
  public static Type fromSettingsKey(final String key) {
  final Type type = TYPES.get(key);
  if (type == null) {
- throw new IllegalArgumentException("no matching type for [" + key + "]");
+ throw new IllegalArgumentException("no matching store type for [" + key + "]");
  }
  return type;
  }
@@ -356,7 +357,7 @@ public interface IndexSearcherWrapperFactory {
 
  public static Type defaultStoreType(final boolean allowMmapfs) {
  if (allowMmapfs && Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
- return Type.MMAPFS;
+ return Type.HYBRIDFS;
  } else if (Constants.WINDOWS) {
  return Type.SIMPLEFS;
  } else {

server/src/main/java/org/elasticsearch/index/store/FsDirectoryService.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.index.store;
 
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
 import org.apache.lucene.store.FileSwitchDirectory;
 import org.apache.lucene.store.LockFactory;
 import org.apache.lucene.store.MMapDirectory;
@@ -30,17 +31,25 @@
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.util.set.Sets;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.shard.ShardPath;
 
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
 
 public class FsDirectoryService extends DirectoryService {
+ /*
+ * We are mmapping norms, docvalues as well as term dictionaries, all other files are served through NIOFS
+ * this provides good random access performance while not creating unnecessary mmaps for files like stored
+ * fields etc.
+ */
+ private static final Set<String> PRIMARY_EXTENSIONS = Collections.unmodifiableSet(Sets.newHashSet("nvd", "dvd", "tim"));
 
  protected final IndexStore indexStore;
  public static final Setting<LockFactory> INDEX_LOCK_FACTOR_SETTING = new Setting<>("index.store.fs.fs_lock", "native", (s) -> {
@@ -78,27 +87,36 @@ public Directory newDirectory() throws IOException {
  protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
  final String storeType =
  indexSettings.getSettings().get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey());
+ IndexModule.Type type;
  if (IndexModule.Type.FS.match(storeType)) {
- final IndexModule.Type type =
- IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings()));
- switch (type) {
- case MMAPFS:
- return new MMapDirectory(location, lockFactory);
- case SIMPLEFS:
- return new SimpleFSDirectory(location, lockFactory);
- case NIOFS:
- return new NIOFSDirectory(location, lockFactory);
- default:
- throw new AssertionError("unexpected built-in store type [" + type + "]");
- }
- } else if (IndexModule.Type.SIMPLEFS.match(storeType)) {
- return new SimpleFSDirectory(location, lockFactory);
- } else if (IndexModule.Type.NIOFS.match(storeType)) {
- return new NIOFSDirectory(location, lockFactory);
- } else if (IndexModule.Type.MMAPFS.match(storeType)) {
- return new MMapDirectory(location, lockFactory);
+ type = IndexModule.defaultStoreType(IndexModule.NODE_STORE_ALLOW_MMAPFS.get(indexSettings.getNodeSettings()));
+ } else {
+ type = IndexModule.Type.fromSettingsKey(storeType);
+ }
+ switch (type) {
+ case HYBRIDFS:
+ // Use Lucene defaults
+ final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
+ if (primaryDirectory instanceof MMapDirectory) {
+ return new FileSwitchDirectory(PRIMARY_EXTENSIONS, primaryDirectory, new NIOFSDirectory(location, lockFactory), true) {
+ @Override
+ public String[] listAll() throws IOException {
+ // Avoid doing listAll twice:
+ return primaryDirectory.listAll();
+ }
+ };
+ } else {
+ return primaryDirectory;
+ }
+ case MMAPFS:
+ return new MMapDirectory(location, lockFactory);
+ case SIMPLEFS:
+ return new SimpleFSDirectory(location, lockFactory);
+ case NIOFS:
+ return new NIOFSDirectory(location, lockFactory);
+ default:
+ throw new AssertionError("unexpected built-in store type [" + type + "]");
  }
- throw new IllegalArgumentException("No directory found for type [" + storeType + "]");
  }
 
  private static Directory setPreload(Directory directory, Path location, LockFactory lockFactory,

server/src/test/java/org/elasticsearch/index/store/IndexStoreTests.java

@@ -19,6 +19,7 @@
 package org.elasticsearch.index.store;
 
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FileSwitchDirectory;
 import org.apache.lucene.store.MMapDirectory;
 import org.apache.lucene.store.NIOFSDirectory;
 import org.apache.lucene.store.NoLockFactory;
@@ -64,6 +65,9 @@ private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingV
  new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
  try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
  switch (type) {
+ case HYBRIDFS:
+ assertTrue(type + " " + directory.toString(), directory instanceof FileSwitchDirectory);
+ break;
  case NIOFS:
  assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
  break;
@@ -75,7 +79,7 @@ private void doTestStoreDirectory(Index index, Path tempDir, String typeSettingV
  break;
  case FS:
  if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
- assertTrue(directory.toString(), directory instanceof MMapDirectory);
+ assertTrue(directory.toString(), directory instanceof FileSwitchDirectory);
  } else if (Constants.WINDOWS) {
  assertTrue(directory.toString(), directory instanceof SimpleFSDirectory);
  } else {